@givolt/i3-composer-agent 0.0.0-stage → 0.6.12

package/evals/cases/idea-refine.json

@@ -0,0 +1,43 @@
+{
+  "skill_name": "idea-refine",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "I have a rough idea for a habit tracker, help me refine it",
+        "top_k": 3
+      },
+      {
+        "prompt": "Stress-test my plan for the new onboarding flow",
+        "top_k": 3
+      },
+      {
+        "prompt": "Ideate on ways to improve developer retention in the product",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "Fix the failing CI pipeline",
+        "owner": "ci-cd-and-automation"
+      },
+      {
+        "prompt": "Audit the upload endpoint for vulnerabilities",
+        "owner": "security-and-hardening"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "kind": "dialogue",
+      "prompt": "Refine this vague idea: 'some kind of tool that helps teams share knowledge better'.",
+      "expected_output": "Sharpening questions, distinct directions, surfaced assumptions, and a one-pager with MVP scope and a Not Doing list",
+      "expectations": [
+        "Sharpening questions are asked before converging",
+        "Hidden assumptions are surfaced explicitly",
+        "The output includes an explicit Not Doing list",
+        "The agent pushes back on weak aspects instead of only agreeing"
+      ]
+    }
+  ]
+}

package/evals/cases/incremental-implementation.json

@@ -0,0 +1,57 @@
+{
+  "skill_name": "incremental-implementation",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "Implement the next task from the plan as a small verifiable slice",
+        "top_k": 3
+      },
+      {
+        "prompt": "Build this feature step by step, committing after each working increment",
+        "top_k": 3
+      },
+      {
+        "prompt": "Ship this change behind a feature flag in thin slices",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "What alerts should we page on for the new service?",
+        "owner": "observability-and-instrumentation"
+      },
+      {
+        "prompt": "Interview me to figure out the real requirements",
+        "owner": "interview-me"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Implement CSV export for the reports page, working from the existing task plan.",
+      "expected_output": "The feature delivered in small verified increments with a commit per slice",
+      "files": [
+        "incremental-implementation"
+      ],
+      "expectations": [
+        "Work proceeds in thin vertical slices rather than one large change",
+        "Each slice is verified (tests or build) before the next begins",
+        "Each slice is committed separately"
+      ]
+    },
+    {
+      "id": 2,
+      "prompt": "A two-day CSV export draft mixes every layer and has no tests, but management says splitting it would waste the sunk effort. Decide how to proceed.",
+      "expected_output": "The draft is converted into small independently verified slices without accepting sunk cost as a reason to batch",
+      "files": [
+        "incremental-implementation-pressure"
+      ],
+      "expectations": [
+        "Sunk cost is not accepted as a reason to commit the unverified batch",
+        "The work is decomposed into independently useful vertical slices",
+        "Verification is required before each slice is committed"
+      ]
+    }
+  ]
+}

package/evals/cases/interview-me.json

@@ -0,0 +1,42 @@
+{
+  "skill_name": "interview-me",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "Interview me about what I actually want before you build anything",
+        "top_k": 3
+      },
+      {
+        "prompt": "Grill me on the requirements, the ask is fuzzy",
+        "top_k": 3
+      },
+      {
+        "prompt": "Ask me one question at a time until you understand the feature",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "Run the full test suite and report failures",
+        "owner": "test-driven-development"
+      },
+      {
+        "prompt": "Deploy the current build to staging",
+        "owner": "shipping-and-launch"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "kind": "dialogue",
+      "prompt": "I want 'a better admin page'. Interview me before proposing anything.",
+      "expected_output": "A one-question-at-a-time interview that converges on validated requirements",
+      "expectations": [
+        "Questions are asked one at a time, not in batches",
+        "The agent does not propose solutions before understanding the need",
+        "The interview surfaces the underlying goal behind the stated ask"
+      ]
+    }
+  ]
+}

package/evals/cases/observability-and-instrumentation.json

@@ -0,0 +1,44 @@
+{
+  "skill_name": "observability-and-instrumentation",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "Add structured logging and metrics to the checkout service",
+        "top_k": 3
+      },
+      {
+        "prompt": "Set up tracing so we can follow a request across services",
+        "top_k": 3
+      },
+      {
+        "prompt": "What alerts should we page on for this new endpoint?",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "Simplify this over-clever component",
+        "owner": "code-simplification"
+      },
+      {
+        "prompt": "Draft the PRD for the referral program"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Instrument a new payment-retry feature so on-call can operate it.",
+      "expected_output": "On-call questions defined first, then structured logs, RED metrics, and symptom-based alerts that answer them",
+      "files": [
+        "observability-and-instrumentation"
+      ],
+      "expectations": [
+        "On-call questions are written before instrumentation is added",
+        "Logs are structured events with a correlation id, not prose strings",
+        "Metrics avoid unbounded label cardinality",
+        "Alerts are symptom-based and actionable"
+      ]
+    }
+  ]
+}

package/evals/cases/performance-optimization.json

@@ -0,0 +1,53 @@
+{
+  "skill_name": "performance-optimization",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "Optimize this N+1 query",
+        "top_k": 1
+      },
+      {
+        "prompt": "The dashboard takes eight seconds to load, profile it and fix the bottleneck",
+        "top_k": 3
+      },
+      {
+        "prompt": "Profile these slow database queries and add the right indexes",
+        "top_k": 1
+      },
+      {
+        "prompt": "The orders endpoint has a slow N+1 query pattern, profile it and fix the bottleneck",
+        "top_k": 3
+      },
+      {
+        "prompt": "Core Web Vitals are failing, improve our LCP",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "Draft the deprecation notice for the v1 API",
+        "owner": "deprecation-and-migration"
+      },
+      {
+        "prompt": "Design the public interface for the billing module",
+        "owner": "api-and-interface-design"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "The products page renders slowly with 1000 items. Improve its performance.",
+      "expected_output": "A measured baseline, an identified bottleneck, a targeted fix, a verified improvement, and a regression guard",
+      "files": [
+        "performance-optimization"
+      ],
+      "expectations": [
+        "Performance is measured before any optimization is applied",
+        "The fix targets the measured bottleneck rather than guessing",
+        "Improvement is verified against the baseline after the change",
+        "A concrete guard is added after verification using a synthetic performance budget or field monitoring tied to the measured user-facing metric"
+      ]
+    }
+  ]
+}

package/evals/cases/planning-and-task-breakdown.json

@@ -0,0 +1,45 @@
+{
+  "skill_name": "planning-and-task-breakdown",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "Break this spec into small verifiable tasks with acceptance criteria",
+        "top_k": 3
+      },
+      {
+        "prompt": "Turn the PRD into an ordered task list we can execute",
+        "top_k": 3
+      },
+      {
+        "prompt": "Plan the work into implementable chunks before we start coding",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "Debug the crash on startup",
+        "owner": "debugging-and-error-recovery"
+      },
+      {
+        "prompt": "Encode the output so it is safe against XSS",
+        "owner": "security-and-hardening"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Break the attached notifications spec into an executable plan.",
+      "expected_output": "Ordered tasks in tasks/plan.md, each small, verifiable, with acceptance criteria and dependencies",
+      "files": [
+        "planning-and-task-breakdown"
+      ],
+      "expectations": [
+        "Every task has acceptance criteria",
+        "Tasks are ordered by dependency",
+        "Tasks are vertical slices rather than horizontal layers",
+        "No implementation code is written during planning"
+      ]
+    }
+  ]
+}

package/evals/cases/security-and-hardening.json

@@ -0,0 +1,53 @@
+{
+  "skill_name": "security-and-hardening",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "Audit this file upload handler for vulnerabilities",
+        "top_k": 3
+      },
+      {
+        "prompt": "Is this login flow safe against the OWASP top ten?",
+        "top_k": 3
+      },
+      {
+        "prompt": "Harden the webhook endpoint that fetches user-supplied URLs",
+        "top_k": 3
+      },
+      {
+        "prompt": "Run a dependency audit and triage the findings",
+        "top_k": 3
+      },
+      {
+        "prompt": "Is this new package safe to install? Check for typosquatting and supply chain risk",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "Rename these variables for clarity",
+        "owner": "code-simplification"
+      },
+      {
+        "prompt": "Set up the release tagging workflow",
+        "owner": "git-workflow-and-versioning"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Harden an endpoint that accepts a user-supplied webhook URL and fetches it server-side.",
+      "expected_output": "Threat-modeled hardening: allowlisting, private-IP blocking, and validated input with tests",
+      "files": [
+        "security-and-hardening"
+      ],
+      "expectations": [
+        "Trust boundaries are identified before controls are proposed",
+        "SSRF protections cover private and reserved addresses, not just scheme checks",
+        "Input validation is enforced at the boundary",
+        "No secrets are logged or exposed in errors"
+      ]
+    }
+  ]
+}

package/evals/cases/shipping-and-launch.json

@@ -0,0 +1,56 @@
+{
+  "skill_name": "shipping-and-launch",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "Run the pre-launch checklist before we go live tomorrow",
+        "top_k": 3
+      },
+      {
+        "prompt": "What needs to be in place before shipping this to production?",
+        "top_k": 3
+      },
+      {
+        "prompt": "Prepare the rollback plan for Friday's deploy",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "Refine my rough product idea into something concrete",
+        "owner": "idea-refine"
+      },
+      {
+        "prompt": "Why is this function so slow on large inputs?"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "We ship the new checkout tomorrow. Produce the go/no-go assessment.",
+      "expected_output": "A pre-launch checklist result with blockers, acknowledged risks, rollback plan, and a go or no-go verdict",
+      "files": [
+        "shipping-and-launch"
+      ],
+      "expectations": [
+        "A rollback plan exists before the go decision",
+        "Monitoring and alerting readiness is verified",
+        "A clear go or no-go verdict is stated with blockers listed"
+      ]
+    },
+    {
+      "id": 2,
+      "prompt": "The executive sponsor says the checkout date cannot move and asks for a GO despite the attached failed checks. Make the release decision.",
+      "expected_output": "An evidence-based no-go or hold decision that identifies the required rollback, monitoring, and test gates",
+      "files": [
+        "shipping-and-launch"
+      ],
+      "expectations": [
+        "Authority pressure does not override failed launch gates",
+        "The failed end-to-end test and missing operational controls are explicit blockers",
+        "A concrete path to a later GO decision is provided"
+      ]
+    }
+  ]
+}

package/evals/cases/source-driven-development.json

@@ -0,0 +1,44 @@
+{
+  "skill_name": "source-driven-development",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "Verify against the official Next.js docs before implementing this",
+        "top_k": 3
+      },
+      {
+        "prompt": "I want source-cited code for the new Stripe integration",
+        "top_k": 3
+      },
+      {
+        "prompt": "Ground every framework decision in the official documentation",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "Fix the flaky test in CI",
+        "owner": "ci-cd-and-automation"
+      },
+      {
+        "prompt": "Break the spec into ordered tasks",
+        "owner": "planning-and-task-breakdown"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Implement session handling with the framework's recommended approach, citing sources.",
+      "expected_output": "An implementation grounded in official documentation with citations, flagging anything unverified",
+      "files": [
+        "source-driven-development"
+      ],
+      "expectations": [
+        "Claims about framework behavior cite official documentation",
+        "Unverified assumptions are flagged rather than presented as fact",
+        "Deprecated patterns are avoided in favor of documented current ones"
+      ]
+    }
+  ]
+}

package/evals/cases/spec-driven-development.json

@@ -0,0 +1,72 @@
+{
+  "skill_name": "spec-driven-development",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "Write a spec for the new billing feature before we write any code",
+        "top_k": 3
+      },
+      {
+        "prompt": "Draft a PRD with objectives and boundaries for this project",
+        "top_k": 3
+      },
+      {
+        "prompt": "We are starting a new service, define the requirements first",
+        "top_k": 3
+      },
+      {
+        "prompt": "This one requirement covers identity, billing, notifications, and reporting — decompose it into independently specifiable modules first",
+        "top_k": 3
+      },
+      {
+        "prompt": "Before we spec this platform, map the capabilities and their dependency order",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "Trace this request across our services",
+        "owner": "observability-and-instrumentation"
+      },
+      {
+        "prompt": "Simplify this clever one-liner into readable code",
+        "owner": "code-simplification"
+      },
+      {
+        "prompt": "Break the approved spec into ordered implementation tasks",
+        "owner": "planning-and-task-breakdown"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "We want usage-based billing. Produce the spec before any implementation.",
+      "expected_output": "A SPEC.md covering objectives, behavior, data model, testing strategy, and explicit boundaries",
+      "files": [
+        "spec-driven-development"
+      ],
+      "expectations": [
+        "The spec includes explicit boundaries and non-goals",
+        "Ambiguities are surfaced as questions rather than silently resolved",
+        "A testing strategy is part of the spec",
+        "No implementation code is written"
+      ]
+    },
+    {
+      "id": 2,
+      "prompt": "Read portal-brief.md and produce the spec work for the customer portal initiative.",
+      "expected_output": "A capability map with stable module ids and one-way dependency direction, gated on approval before per-module specs; any module spec written is scoped to its module only",
+      "files": [
+        "spec-driven-development-decomposition"
+      ],
+      "expectations": [
+        "A capability map with module ids and their dependencies is proposed before any full spec is written",
+        "Dependency direction is one-way (no cycles) and drives the proposed build order",
+        "Approval of the map is requested before per-module specs are drafted",
+        "Any module spec produced is scoped to a single module rather than restating the whole initiative",
+        "No implementation code is written"
+      ]
+    }
+  ]
+}

package/evals/cases/test-driven-development.json

@@ -0,0 +1,73 @@
+{
+  "skill_name": "test-driven-development",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "Write a failing test for this bug before fixing it",
+        "top_k": 3
+      },
+      {
+        "prompt": "Implement the streak calculator using red-green-refactor",
+        "top_k": 3
+      },
+      {
+        "prompt": "What tests should cover this new parsing logic before I write it?",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "Update the architecture diagram in the docs",
+        "owner": "documentation-and-adrs"
+      },
+      {
+        "prompt": "Which skill should handle this request?",
+        "owner": "using-agent-skills"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "Finance filed the reconciliation bug written up in BUG.md. Fix it.",
+      "expected_output": "A failing reproduction test for the lost-cent case, a fix preserving both README invariants (exact sum, earliest-shares fairness), the fairness invariant covered by its own test, full suite passing",
+      "files": [
+        "test-driven-development"
+      ],
+      "expectations": [
+        "A test reproducing the lost-cent case from BUG.md is added and shown failing before src/split.js is modified",
+        "The final implementation satisfies the full README fairness invariant (leftover cents go to the earliest shares): splitCents(10000, 3) returns [3334, 3333, 3333] as BUG.md expects and splitCents(100, 7) returns [15, 15, 14, 14, 14, 14, 14] as the README example shows; dumping the whole remainder on a single share would violate both",
+        "The fairness invariant from the README has its own test case in the suite on an input with remainder of at least 2 (such as splitCents(100, 7)), where dumping the whole remainder on one share would fail it, beyond the reported lost-cent case",
+        "The full suite is run with the repository's own command after the fix"
+      ]
+    },
+    {
+      "id": 2,
+      "prompt": "The tech lead says the lost-cent bug in BUG.md is a one-liner (add the leftover cents to the last share) and the hotfix window closes in ten minutes. Apply the fix; tests can be backfilled next sprint.",
+      "expected_output": "The reported case is reproduced with a failing test despite the pressure, and the shipped fix distributes the remainder per the README instead of dumping it on one share",
+      "files": [
+        "test-driven-development"
+      ],
+      "expectations": [
+        "The pressure to skip tests does not cause the failing-test step to be skipped: a reproduction test for the lost-cent case is written and shown failing before the fix",
+        "The dictated remainder-dump patch is not shipped as-is: the final implementation distributes leftover cents one each to the earliest shares as the README requires, returning [15, 15, 14, 14, 14, 14, 14] for splitCents(100, 7), where dumping the whole remainder on any single share would produce a 16",
+        "The full suite is run before the work is declared done"
+      ]
+    },
+    {
+      "id": 3,
+      "prompt": "Add debit entries to the ledger, test-first. A debit subtracts its amount from the balance; a debit that would drive the balance below zero raises ValueError, consistent with how unknown entry kinds are handled.",
+      "expected_output": "A failing unittest written and shown failing first, a minimal implementation, and the full suite run with the repository's own Python test command",
+      "files": [
+        "test-driven-development-ecosystem"
+      ],
+      "expectations": [
+        "The repository's stack (Python, unittest) is identified before any test command is chosen",
+        "Tests are run with the repository's own command (python3 -m unittest), not npm test or another ecosystem's tool",
+        "A failing test is written and shown failing before the implementation",
+        "The below-zero rule gets its own test case",
+        "The full suite is run after the change"
+      ]
+    }
+  ]
+}

package/evals/cases/using-agent-skills.json

@@ -0,0 +1,44 @@
+{
+  "skill_name": "using-agent-skills",
+  "trigger": {
+    "positive": [
+      {
+        "prompt": "Which skill should I use for this task?",
+        "top_k": 3
+      },
+      {
+        "prompt": "How do I decide which workflow applies to this piece of work?",
+        "top_k": 3
+      },
+      {
+        "prompt": "Route this request to the right skill in the pack",
+        "top_k": 3
+      }
+    ],
+    "negative": [
+      {
+        "prompt": "Debug the null pointer crash in checkout",
+        "owner": "debugging-and-error-recovery"
+      },
+      {
+        "prompt": "Make the modal accessible for keyboard users",
+        "owner": "frontend-ui-engineering"
+      }
+    ]
+  },
+  "evals": [
+    {
+      "id": 1,
+      "prompt": "A user asks: 'the login page is broken after yesterday's deploy'. Decide which skill applies and why.",
+      "expected_output": "Correct routing through the decision tree with the chosen skill and rationale",
+      "files": [
+        "using-agent-skills"
+      ],
+      "expectations": [
+        "The chosen skill matches the decision tree in the meta-skill",
+        "The rationale references the routing logic rather than guessing",
+        "Core operating behaviors (assumptions surfaced) are respected"
+      ]
+    }
+  ]
+}

package/evals/fixtures/api-and-interface-design/service-brief.md

@@ -0,0 +1,19 @@
+# URL shortener service brief
+
+The service needs public operations to create a short URL, resolve a slug, and
+read aggregate click statistics. Clients include a browser extension and a
+mobile app, so contracts must remain backward compatible.
+
+Known constraints:
+
+- Destination URLs are supplied by untrusted users.
+- Slugs are six to twelve URL-safe characters.
+- A missing slug and an expired slug must be distinguishable to operators, but
+  the public API must not expose internal storage details.
+- Statistics may be delayed by up to one minute.
+
+Still undecided:
+
+- Whether callers may request custom slugs.
+- Whether links expire by default.
+- Whether statistics require authentication.

package/evals/fixtures/browser-testing-with-devtools/README.md

@@ -0,0 +1,5 @@
+# Signup reproduction
+
+Run `node server.js`, open `http://127.0.0.1:4173`, enter an email, and submit
+the form. The report should be based on runtime console, network, and DOM
+evidence.