@mmerterden/multi-agent-pipeline 12.6.0 → 12.8.0

package/pipeline/eval/golden-tasks/07-backend-node-idempotency/task.json

@@ -1,12 +0,0 @@
-{
-  "title": "Add idempotency keys to the payments POST endpoint",
-  "description": "POST /payments can double-charge on client retry. Accept an Idempotency-Key header, store the first response keyed by it, and replay that response for duplicate keys within a TTL.",
-  "inputType": "free-text",
-  "expectedStack": "backend",
-  "expectedLanguage": "JavaScript",
-  "expectedFramework": "Express",
-  "expectedBlockers": 1,
-  "expectedDeferrals": 0,
-  "baseBranch": "main",
-  "taskBranch": "feature/payments-idempotency"
-}

package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-1-analysis.json

@@ -1,25 +0,0 @@
-{
-  "stack": {
-    "primary": "ios",
-    "language": "Swift 6",
-    "framework": "SwiftUI"
-  },
-  "touchedAreas": [
-    {
-      "path": "Sources/Auth/TokenStore.swift",
-      "why": "Wrap the Keychain read in an LAContext biometric evaluation before returning the token"
-    },
-    {
-      "path": "Sources/Auth/TokenStoreTests.swift",
-      "why": "Cover the gated read: success after auth, failure/cancel returns no token"
-    }
-  ],
-  "risks": [
-    {
-      "risk": "Biometric evaluation is async and can fail/cancel; a wrong fallback could either lock users out or defeat the gate",
-      "severity": "medium",
-      "mitigation": "Define explicit behavior for cancel / not-enrolled / lockout, and test each path"
-    }
-  ],
-  "summary": "Add a biometric (LAContext) check in front of the existing Keychain-backed TokenStore read. Security-sensitive surface: the correctness hinges on the failure-path policy (cancel / not-enrolled / lockout), which is exactly the kind of judgment call where two same-family reviewers approving is not independent proof."
-}

package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-2-plan.json

@@ -1,31 +0,0 @@
-{
-  "approach": "TDD-first. Add failing tests for the gated read: a successful biometric evaluation returns the token; a cancelled or failed evaluation returns no token and does not fall back to an ungated read (RED). Wrap the existing Keychain read in an LAContext evaluation, keeping the TokenStore API unchanged (GREEN). Define explicit behavior for cancel / not-enrolled / lockout (refactor).",
-  "tasks": [
-    {
-      "id": "T1",
-      "title": "Add failing biometric-gated read tests",
-      "type": "test",
-      "files": ["Sources/Auth/TokenStoreTests.swift"],
-      "dependsOn": [],
-      "acceptanceCriteria": [
-        "success after auth returns the stored token",
-        "cancelled / failed auth returns nil and never reads the Keychain ungated",
-        "evaluation runs against an injectable LAContext double"
-      ]
-    },
-    {
-      "id": "T2",
-      "title": "Gate the Keychain read behind LAContext",
-      "type": "code",
-      "files": ["Sources/Auth/TokenStore.swift"],
-      "dependsOn": ["T1"],
-      "acceptanceCriteria": [
-        "TokenStore public API is unchanged",
-        "Token is only returned after a successful biometric evaluation",
-        "cancel / not-enrolled / lockout each have explicit, tested behavior"
-      ]
-    }
-  ],
-  "approved": true,
-  "userFeedback": null
-}

package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-4-review.json

@@ -1,12 +0,0 @@
-[
-  {
-    "reviewer": "opus",
-    "findings": [],
-    "approved": true
-  },
-  {
-    "reviewer": "sonnet",
-    "findings": [],
-    "approved": true
-  }
-]

package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/expected/phase-4-triage.json

@@ -1,18 +0,0 @@
-{
-  "accepted": [],
-  "deferred": [],
-  "rejected": [],
-  "approved": true,
-  "consensus": {
-    "reviewerCount": 2,
-    "verdict": "unverified",
-    "disagreements": [
-      {
-        "file": "Sources/Auth/TokenStore.swift",
-        "line": 0,
-        "issue": "biometric failure-path policy (cancel / not-enrolled / lockout) on a Keychain-gated read",
-        "note": "Both reviewers approved, but this is a security/auth judgment surface - the agreement is unverified. A human should confirm the cancel / not-enrolled / lockout behavior before trusting the pass."
-      }
-    ]
-  }
-}

package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/metadata.json

@@ -1,14 +0,0 @@
-{
-  "focus": "Consensus surfacing (schema v3.1.0). Both reviewers APPROVE on a security/auth surface, so triage has zero accepted/deferred/rejected - but the agreement is flagged `unverified` (same-base-model approval on a judgment-heavy surface is not treated as proof) and a disagreement note is surfaced. Verifies the consensus block passes validation with approved=true + verdict=unverified.",
-  "exercises": [
-    "Phase 4 both reviewers approve (no findings) on a biometric/Keychain change",
-    "Triage short-circuits to empty buckets, approved=true",
-    "Triage attaches consensus { reviewerCount: 2, verdict: unverified, disagreements: [...] }",
-    "validate-triage accepts the consensus block and its cross-checks pass (unverified + approved=true is valid)"
-  ],
-  "regressionsCaught": [
-    "If validate-triage stops accepting a consensus block, this fixture fails",
-    "If the consensus cross-check wrongly rejects unverified+approved=true, this fixture fails",
-    "If a unanimous-pass were emitted on a security surface (instead of unverified), the intent of consensus surfacing is lost - the metadata documents the expected verdict"
-  ]
-}

package/pipeline/eval/golden-tasks/08-ios-auth-consensus-unverified/task.json

@@ -1,12 +0,0 @@
-{
-  "title": "Add a biometric gate before reading the Keychain auth token",
-  "description": "Require a Face ID / Touch ID check (LAContext) before TokenStore returns the auth token, so a stolen unlocked device can't silently reuse the session. Keep the existing TokenStore API.",
-  "inputType": "free-text",
-  "expectedStack": "ios",
-  "expectedLanguage": "Swift",
-  "expectedFramework": "SwiftUI",
-  "expectedBlockers": 0,
-  "expectedDeferrals": 0,
-  "baseBranch": "develop",
-  "taskBranch": "feature/biometric-token-gate"
-}

package/pipeline/eval/golden-tasks/README.md

@@ -1,65 +0,0 @@
-# Golden tasks  -  regression fixtures for the full pipeline (v6.2.F+)
-
-## Purpose
-
-Each subdirectory here captures one **complete pipeline run as a reference trace**. A golden task fixture answers: *"If the pipeline ran on this input, what shape of outputs should Phase 1-4 produce?"*
-
-The runner (`pipeline/scripts/eval-golden-tasks.mjs`) does **not** invoke a real model  -  that would require API keys, add non-determinism, and make CI flaky. Instead it validates each fixture against the pipeline's schemas and checks internal consistency (Phase 2 todos reference Phase 1 files, Phase 4 findings reference files that exist in the diff, etc.).
-
-This is a **contract regression test**, same category as `eval-triage.mjs`. A live-model evaluation harness is a separate (and expensive) artifact, deferred to a future release.
-
-## Fixture shape
-
-```
-NN-short-kebab-name/
-  task.json             -  input (task description, expected stack, target repo metadata)
-  expected/
-    phase-1-analysis.json     -  must conform to analysis-output.schema.json
-    phase-2-plan.json         -  must conform to planning-output.schema.json
-    phase-4-review.json       -  must conform to reviewer-output.schema.json (one entry per reviewer)
-    phase-4-triage.json       -  must conform to triage-output.schema.json
-  metadata.json         -  fixture-level notes (why this task is interesting, which phase it exercises most)
-```
-
-`expected/` files are hand-curated reference outputs. They are not literal outputs of a past run  -  they are what a correct pipeline **should** produce on this input. When the spec changes (e.g. a new field added to analysis-output.schema.json), fixtures are updated to match.
-
-## When to add a new fixture
-
-Add one when you discover a class of task that the pipeline should handle but no existing fixture covers. Examples:
-- A different primary stack (add Python / Node.js / Go fixtures once the runner supports them)
-- A task shape that historically caused regressions (e.g. multi-repo, figma-only, compliance-heavy)
-- An edge case in Phase 4 (e.g. a task where all reviewers approve but triage downgrades)
-
-Do **not** add fixtures just to inflate the count. The runner walks every fixture on `npm test`  -  each one adds seconds to CI. Quality > quantity.
-
-## Running
-
-```bash
-node pipeline/scripts/eval-golden-tasks.mjs                # all fixtures
-node pipeline/scripts/eval-golden-tasks.mjs --case 01-...  # one fixture
-node pipeline/scripts/eval-golden-tasks.mjs --json         # CI-friendly output
-```
-
-Exit codes: `0` all pass · `1` one or more fail · `2` usage / setup error.
-
-## What the runner asserts (v6.2.F scaffolding)
-
-1. **Schema conformance**: each `expected/phase-*.json` validates against its paired schema via the existing `validate-*.mjs` scripts.
-2. **Internal consistency**: Phase 2 `todos[].touchedFiles[]` is a subset of Phase 1 `touchedAreas[].path`. Phase 4 `findings[].file` is a subset of Phase 2 planned files OR the current diff's files.
-3. **Stack match**: `phase-1-analysis.json.stack.primary` equals `task.json.expectedStack`.
-4. **No invention**: Phase 4 `findings[]` either (a) match Phase 2 planned files, or (b) surface via the `triage.rejected[]` bucket with a reason.
-
-## What the runner does NOT do (yet)
-
-- Invoke a real Claude/Copilot agent. That's a separate multi-model eval, future work.
-- Score output "quality"  -  no rubric, no judge. Only contract checks.
-- Replay timing / token budgets. Tracker and budget smokes cover that orthogonally.
-
----
-
-### Seed fixtures shipped with v6.2.F
-
-| # | Name | Focus |
-|---|------|-------|
-| 01 | `01-ios-bugfix-darkmode` | SwiftUI bugfix path  -  single repo, 2 files touched, 1 blocker finding caught, 0 deferrals |
-| 02 | `02-android-feature-compose` | Kotlin/Compose feature path  -  3 files, 1 blocker, 1 deferred out-of-scope finding |

package/pipeline/eval/intent-cases.json

@@ -1,40 +0,0 @@
-{
-  "_readme": "Labeled cases for the Phase 0 conceptual-vs-edit intent guard (pipeline/lib/classify-intent.sh). eval-intent.mjs runs each input through the classifier and reports accuracy. The DANGEROUS error is a real task read as 'question' (work silently skipped) or a clear question read as 'task' (a worktree spun up for nothing); 'ambiguous' is the safe middle and is accepted for genuinely vague input. Add real misclassifications here as they are found - this is the regression set.",
-  "cases": [
-    { "input": "fix the login crash on launch", "expected": "task" },
-    { "input": "refactor the cache layer", "expected": "task" },
-    { "input": "add a retry to the network client", "expected": "task" },
-    { "input": "LoginView dark mode fix", "expected": "task" },
-    { "input": "implement biometric unlock for the keychain", "expected": "task" },
-    { "input": "remove the deprecated analytics call", "expected": "task" },
-    { "input": "can you split this file into two", "expected": "task" },
-    { "input": "could you rename TokenStore to SessionStore", "expected": "task" },
-    { "input": "please add a loading spinner to the list", "expected": "task" },
-    { "input": "login ekranina retry ekle", "expected": "task" },
-    { "input": "cache katmanini yeniden duzenle", "expected": "task" },
-    { "input": "su crash'i duzelt", "expected": "task" },
-    { "input": "how does the login flow work", "expected": "question" },
-    { "input": "what does this function return", "expected": "question" },
-    { "input": "why is the navigation so slow", "expected": "question" },
-    { "input": "should we use Keychain or UserDefaults", "expected": "question" },
-    { "input": "does it support offline mode", "expected": "question" },
-    { "input": "should we enable caching here", "expected": "question" },
-    { "input": "which architecture is better for this screen", "expected": "question" },
-    { "input": "explain the auth module", "expected": "question" },
-    { "input": "compare MVVM and TCA for this", "expected": "question" },
-    { "input": "is this thread-safe", "expected": "question" },
-    { "input": "nasil calisiyor bu akis", "expected": "question" },
-    { "input": "neden bu kadar yavas", "expected": "question" },
-    { "input": "Keychain mi UserDefaults mi kullanmaliyim", "expected": "question" },
-    { "input": "bu thread-safe mi", "expected": "question" },
-
-    { "input": "fix the bug?", "expected": "task" },
-    { "input": "add retry logic, ok?", "expected": "task" },
-    { "input": "can you explain how to add a retry", "expected": "question" },
-    { "input": "could you tell me which file to update", "expected": "question" },
-    { "input": "would you say we should refactor this", "expected": "question" },
-    { "input": "what happens if I remove this", "expected": "question" },
-    { "input": "bu fonksiyonu nasil refactor ederim", "expected": "question" },
-    { "input": "hangi mimariyi kullanmaliyim", "expected": "question" }
-  ]
-}

package/pipeline/eval/run-metrics-fixture.json

@@ -1,46 +0,0 @@
-{
-  "taskId": "PROJ-101",
-  "status": "complete",
-  "currentPhase": 7,
-  "projects": [
-    { "project": "my-ios-app", "buildStatus": { "ok": true, "attempts": 1 } }
-  ],
-  "reviewIterations": [
-    {
-      "reviewers": [
-        { "reviewer": "opus", "findings": [
-          { "severity": "blocking", "file": "Auth/Session.swift", "line": 54, "issue": "token logged in plaintext", "fix": "redact" },
-          { "severity": "suggestion", "file": "Auth/Session.swift", "line": 12, "issue": "rename helper", "fix": "rename" }
-        ], "approved": false },
-        { "reviewer": "sonnet", "findings": [
-          { "severity": "important", "file": "Auth/Session.swift", "line": 70, "issue": "no migration for legacy token", "fix": "add migration" }
-        ], "approved": false }
-      ],
-      "triage": {
-        "accepted": [
-          { "severity": "blocking", "file": "Auth/Session.swift", "line": 54, "issue": "token logged in plaintext", "fix": "redact the token in the log", "reviewer": "opus" }
-        ],
-        "deferred": [
-          { "finding": { "severity": "important", "file": "Auth/Session.swift", "line": 70, "issue": "no migration for legacy token" }, "reason": "follow-up PR" }
-        ],
-        "rejected": [
-          { "finding": { "severity": "suggestion", "file": "Auth/Session.swift", "line": 12, "issue": "rename helper" }, "reason": "style only" }
-        ],
-        "approved": false
-      }
-    },
-    {
-      "reviewers": [
-        { "reviewer": "opus", "findings": [], "approved": true },
-        { "reviewer": "sonnet", "findings": [], "approved": true }
-      ],
-      "triage": {
-        "accepted": [],
-        "deferred": [],
-        "rejected": [],
-        "approved": true,
-        "consensus": { "reviewerCount": 2, "verdict": "unanimous-pass" }
-      }
-    }
-  ]
-}

package/pipeline/eval/triage/01-empty-findings/expected.json

@@ -1,6 +0,0 @@
-{
-  "accepted": [],
-  "deferred": [],
-  "rejected": [],
-  "approved": true
-}

package/pipeline/eval/triage/01-empty-findings/input.json

@@ -1,5 +0,0 @@
-{
-  "rawFindings": [],
-  "scope": "Add a Cancel button to the OrderConfirmation screen.",
-  "diffSummary": "1 file changed, 12 insertions, 0 deletions"
-}

package/pipeline/eval/triage/01-empty-findings/notes.md

@@ -1,7 +0,0 @@
-# Case 01  -  Empty findings short-circuit
-
-Both reviewers returned zero findings. Triage MUST short-circuit and emit `{approved: true}` directly, no agent call.
-
-Guards against: a future change that incorrectly invokes the triage agent on empty input (wasted Opus call + nondeterministic empty-result variation).
-
-Validator exit code: 0 (clean).

package/pipeline/eval/triage/02-real-blocker/expected.json

@@ -1,15 +0,0 @@
-{
-  "accepted": [
-    {
-      "severity": "blocking",
-      "file": "src/api/auth.ts",
-      "line": 87,
-      "issue": "Password comparison uses string equality, vulnerable to timing attacks",
-      "fix": "Use a constant-time comparison (crypto.timingSafeEqual)",
-      "reviewer": "opus"
-    }
-  ],
-  "deferred": [],
-  "rejected": [],
-  "approved": false
-}

package/pipeline/eval/triage/02-real-blocker/input.json

@@ -1,14 +0,0 @@
-{
-  "rawFindings": [
-    {
-      "severity": "blocking",
-      "file": "src/api/auth.ts",
-      "line": 87,
-      "issue": "Password comparison uses string equality, vulnerable to timing attacks",
-      "fix": "Use a constant-time comparison (crypto.timingSafeEqual)",
-      "reviewer": "opus"
-    }
-  ],
-  "scope": "Implement password-based login endpoint POST /api/auth/login.",
-  "diffSummary": "src/api/auth.ts: +43 -2"
-}

package/pipeline/eval/triage/02-real-blocker/notes.md

@@ -1,7 +0,0 @@
-# Case 02  -  In-scope blocker accepted
-
-A real security issue (timing-attack-vulnerable password comparison) lands in the auth module being modified. Triage MUST accept it as blocking; `approved: false` so Phase 5/6 are gated.
-
-Guards against: triage incorrectly deferring or rejecting a clearly in-scope security blocker because the reviewer's `severity` looked over-strict.
-
-Validator exit code: 0 (clean  -  accepted + approved:false is consistent because there IS a blocker).

package/pipeline/eval/triage/03-out-of-scope-defer/expected.json

@@ -1,18 +0,0 @@
-{
-  "accepted": [],
-  "deferred": [
-    {
-      "finding": {
-        "severity": "important",
-        "file": "src/api/payments.ts",
-        "line": 220,
-        "issue": "Refund flow doesn't log the user-id of the actor, breaking audit trail",
-        "fix": "Add userId to the refund-log entry",
-        "reviewer": "sonnet"
-      },
-      "reason": "out of scope: task is OrderConfirmation Cancel button; payments module is explicitly out of scope per Phase 2 plan. Real issue, surfaced in Phase 7 follow-ups."
-    }
-  ],
-  "rejected": [],
-  "approved": true
-}

package/pipeline/eval/triage/03-out-of-scope-defer/input.json

@@ -1,14 +0,0 @@
-{
-  "rawFindings": [
-    {
-      "severity": "important",
-      "file": "src/api/payments.ts",
-      "line": 220,
-      "issue": "Refund flow doesn't log the user-id of the actor, breaking audit trail",
-      "fix": "Add userId to the refund-log entry",
-      "reviewer": "sonnet"
-    }
-  ],
-  "scope": "Add a Cancel button to the OrderConfirmation screen. Touches src/screens/OrderConfirmation.tsx only. Payments module is out of scope.",
-  "diffSummary": "src/screens/OrderConfirmation.tsx: +12 -0"
-}

package/pipeline/eval/triage/03-out-of-scope-defer/notes.md

@@ -1,10 +0,0 @@
-# Case 03  -  Real issue, out of scope → defer
-
-A reviewer correctly flagged a real audit-logging gap, but it's in a module the current task doesn't touch. Triage MUST defer (NOT accept, NOT reject) so it surfaces in Phase 7 follow-ups without blocking this task.
-
-Guards against:
-
-- triage accepting it and dragging unrelated work into Phase 3 rework
-- triage rejecting it and silently losing a real issue
-
-`approved: true` because no accepted blocker remains.

package/pipeline/eval/triage/04-false-positive-reject/expected.json

@@ -1,18 +0,0 @@
-{
-  "accepted": [],
-  "deferred": [],
-  "rejected": [
-    {
-      "finding": {
-        "severity": "blocking",
-        "file": "src/utils/parseDate.ts",
-        "line": 14,
-        "issue": "Function may receive null input causing TypeError",
-        "fix": "Add a null check at the top of parseDate",
-        "reviewer": "sonnet"
-      },
-      "reason": "false positive: Phase 1 analysis confirmed all callers validate non-null before calling parseDate. Adding an internal null check would be defensive duplication and obscure intent."
-    }
-  ],
-  "approved": true
-}

package/pipeline/eval/triage/04-false-positive-reject/input.json

@@ -1,14 +0,0 @@
-{
-  "rawFindings": [
-    {
-      "severity": "blocking",
-      "file": "src/utils/parseDate.ts",
-      "line": 14,
-      "issue": "Function may receive null input causing TypeError",
-      "fix": "Add a null check at the top of parseDate",
-      "reviewer": "sonnet"
-    }
-  ],
-  "scope": "Refactor parseDate to support ISO 8601 with timezone. Existing callers already validate non-null at the call sites (verified in Phase 1 analysis).",
-  "diffSummary": "src/utils/parseDate.ts: +18 -6"
-}

package/pipeline/eval/triage/04-false-positive-reject/notes.md

@@ -1,10 +0,0 @@
-# Case 04  -  False positive → reject
-
-A reviewer flagged a "blocking" null-safety issue, but Phase 1 analysis already proved all callers validate input. Adding an internal null check would be defensive duplication that hides intent. Triage MUST reject with a clear reason.
-
-Guards against:
-
-- triage blindly accepting any "blocking" tag without scope check (the v2.2.0 evaluation specifically called this out as a risk)
-- triage deferring a non-issue (would clutter Phase 7 follow-ups with noise)
-
-Also exercises the over-rejection guard: 1/1 = 100% rejected, but `total < 5` so the guard does NOT trip (validator exits 0, not 2).

package/pipeline/eval/triage/05-mixed-classification/expected.json

@@ -1,43 +0,0 @@
-{
-  "accepted": [
-    {
-      "severity": "blocking",
-      "file": "src/api/checkout.ts",
-      "line": 102,
-      "issue": "Race condition: order total computed twice without lock, can produce inconsistent invoice",
-      "fix": "Compute total once inside the existing transaction block",
-      "reviewer": "opus"
-    },
-    {
-      "severity": "important",
-      "file": "src/api/checkout.ts",
-      "line": 145,
-      "issue": "Hard-coded currency 'USD' won't work for non-US tenants",
-      "fix": "Read currency from tenant config",
-      "reviewer": "sonnet"
-    },
-    {
-      "severity": "suggestion",
-      "file": "src/api/checkout.ts",
-      "line": 200,
-      "issue": "Magic number 60 in retry backoff",
-      "fix": "Extract to RETRY_BACKOFF_SECONDS constant",
-      "reviewer": "sonnet"
-    }
-  ],
-  "deferred": [
-    {
-      "finding": {
-        "severity": "important",
-        "file": "src/notifications/email.ts",
-        "line": 33,
-        "issue": "Email template missing for refund-success path",
-        "fix": "Add refund_success.html and wire it up",
-        "reviewer": "sonnet"
-      },
-      "reason": "out of scope: notifications module is owned by another team per Phase 0 scope. Real issue, surfaced in Phase 7 follow-ups."
-    }
-  ],
-  "rejected": [],
-  "approved": false
-}

package/pipeline/eval/triage/05-mixed-classification/input.json

@@ -1,38 +0,0 @@
-{
-  "rawFindings": [
-    {
-      "severity": "blocking",
-      "file": "src/api/checkout.ts",
-      "line": 102,
-      "issue": "Race condition: order total computed twice without lock, can produce inconsistent invoice",
-      "fix": "Compute total once inside the existing transaction block",
-      "reviewer": "opus"
-    },
-    {
-      "severity": "important",
-      "file": "src/api/checkout.ts",
-      "line": 145,
-      "issue": "Hard-coded currency 'USD' won't work for non-US tenants",
-      "fix": "Read currency from tenant config",
-      "reviewer": "sonnet"
-    },
-    {
-      "severity": "important",
-      "file": "src/notifications/email.ts",
-      "line": 33,
-      "issue": "Email template missing for refund-success path",
-      "fix": "Add refund_success.html and wire it up",
-      "reviewer": "sonnet"
-    },
-    {
-      "severity": "suggestion",
-      "file": "src/api/checkout.ts",
-      "line": 200,
-      "issue": "Magic number 60 in retry backoff",
-      "fix": "Extract to RETRY_BACKOFF_SECONDS constant",
-      "reviewer": "sonnet"
-    }
-  ],
-  "scope": "Add tenant-aware checkout flow. Touches src/api/checkout.ts and tenant config. Notifications module is owned by another team.",
-  "diffSummary": "src/api/checkout.ts: +180 -45, src/config/tenants.ts: +12 -0"
-}

package/pipeline/eval/triage/05-mixed-classification/notes.md

@@ -1,17 +0,0 @@
-# Case 05  -  Mixed classification (the realistic case)
-
-4 raw findings spanning all 3 outcomes:
-
-- 1 blocking + 1 important + 1 suggestion in the in-scope checkout module → **accepted**
-- 1 important in the out-of-scope notifications module → **deferred**
-- 0 rejected (no false positives in this fixture)
-
-`approved: false` because there's an accepted blocker.
-
-Guards against:
-
-- triage forgetting to classify some findings (every raw finding must end up in exactly one bucket)
-- triage merging suggestions into deferred (suggestions in scope still go to accepted, just at lower priority)
-- triage hallucinating new findings not in raw input (validator strips these  -  case has 4 in, 4 out across all buckets)
-
-This is the case most likely to catch a regression in the triage prompt structure.

package/pipeline/eval/triage/06-severity-mismatch/expected.json

@@ -1,15 +0,0 @@
-{
-  "accepted": [
-    {
-      "severity": "blocking",
-      "file": "src/api/auth.ts",
-      "line": 120,
-      "issue": "Fallback to admin role when token validation fails",
-      "fix": "Return 401 instead of falling back to admin role",
-      "reviewer": "sonnet"
-    }
-  ],
-  "deferred": [],
-  "rejected": [],
-  "approved": false
-}

package/pipeline/eval/triage/06-severity-mismatch/input.json

@@ -1,14 +0,0 @@
-{
-  "rawFindings": [
-    {
-      "severity": "suggestion",
-      "file": "src/api/auth.ts",
-      "line": 120,
-      "issue": "Fallback to admin role when token validation fails",
-      "fix": "Return 401 instead of falling back to admin role",
-      "reviewer": "sonnet"
-    }
-  ],
-  "scope": "Add JWT token validation to the auth middleware.",
-  "diffSummary": "src/api/auth.ts: +28 -3"
-}

package/pipeline/eval/triage/06-severity-mismatch/notes.md

@@ -1,9 +0,0 @@
-# Case 06  -  Severity mismatch (suggestion → blocking)
-
-Reviewer tagged a real security issue as `severity: "suggestion"`  -  falling back to the admin role when token validation fails is a privilege-escalation vector, not a cosmetic nit.
-
-Triage MUST recognize the actual severity from the issue description and bump it to `blocking`. `approved: false` because the accepted finding is now blocking.
-
-Guards against: blind trust of reviewer severity tags. The triage agent must evaluate the described behavior, not just echo the label.
-
-Validator exit code: 0 (accepted + approved:false is consistent because there IS a blocker).

package/pipeline/eval/triage/07-duplicate-reviewers/expected.json

@@ -1,27 +0,0 @@
-{
-  "accepted": [
-    {
-      "severity": "blocking",
-      "file": "src/checkout.ts",
-      "line": 88,
-      "issue": "Missing transaction wrapper around multi-table writes in checkout",
-      "fix": "Wrap the checkout persistence logic in a database transaction",
-      "reviewer": "opus"
-    }
-  ],
-  "deferred": [],
-  "rejected": [
-    {
-      "finding": {
-        "severity": "blocking",
-        "file": "src/checkout.ts",
-        "line": 88,
-        "issue": "Checkout writes to orders and inventory tables without a transaction boundary",
-        "fix": "Use a single transaction for all database writes in the checkout flow",
-        "reviewer": "sonnet"
-      },
-      "reason": "duplicate of accepted[0]: same file, same line, same root issue (missing transaction wrapper). Opus version retained by convention."
-    }
-  ],
-  "approved": false
-}