@mmerterden/multi-agent-pipeline 12.6.0 → 12.8.0

package/pipeline/eval/triage/07-duplicate-reviewers/input.json

@@ -1,22 +0,0 @@
-{
-  "rawFindings": [
-    {
-      "severity": "blocking",
-      "file": "src/checkout.ts",
-      "line": 88,
-      "issue": "Missing transaction wrapper around multi-table writes in checkout",
-      "fix": "Wrap the checkout persistence logic in a database transaction",
-      "reviewer": "opus"
-    },
-    {
-      "severity": "blocking",
-      "file": "src/checkout.ts",
-      "line": 88,
-      "issue": "Checkout writes to orders and inventory tables without a transaction boundary",
-      "fix": "Use a single transaction for all database writes in the checkout flow",
-      "reviewer": "sonnet"
-    }
-  ],
-  "scope": "Implement checkout flow with order creation and inventory update.",
-  "diffSummary": "src/checkout.ts: +95 -10"
-}

package/pipeline/eval/triage/07-duplicate-reviewers/notes.md

@@ -1,9 +0,0 @@
-# Case 07  -  Duplicate findings from two reviewers
-
-Both Opus and Sonnet flagged the same issue (same file, same line, same root cause) but with slightly different wording. Triage must deduplicate, not double-count.
-
-Convention: when two reviewers agree, keep the Opus finding in `accepted` and move the Sonnet duplicate to `rejected` with a reason linking it to the accepted entry.
-
-Guards against: double-classification when reviewers agree. Without dedup, the developer would see the same fix twice and the finding count would overstate the real issue count.
-
-Validator exit code: 0 (1/2 rejected is under the 80% over-rejection threshold).

package/pipeline/eval/triage/08-style-misclassified/expected.json

@@ -1,18 +0,0 @@
-{
-  "accepted": [],
-  "deferred": [],
-  "rejected": [
-    {
-      "finding": {
-        "severity": "blocking",
-        "file": "src/utils/format.ts",
-        "line": 14,
-        "issue": "Function name `fmt` is too short, should be `formatCurrency`",
-        "fix": "Rename fmt to formatCurrency for readability",
-        "reviewer": "sonnet"
-      },
-      "reason": "stylistic preference, not a correctness issue. Reviewer over-tagged severity. Naming was reviewed in PR #234 and approved by team."
-    }
-  ],
-  "approved": true
-}

package/pipeline/eval/triage/08-style-misclassified/input.json

@@ -1,14 +0,0 @@
-{
-  "rawFindings": [
-    {
-      "severity": "blocking",
-      "file": "src/utils/format.ts",
-      "line": 14,
-      "issue": "Function name `fmt` is too short, should be `formatCurrency`",
-      "fix": "Rename fmt to formatCurrency for readability",
-      "reviewer": "sonnet"
-    }
-  ],
-  "scope": "Add currency formatting utility for multi-locale support.",
-  "diffSummary": "src/utils/format.ts: +22 -0"
-}

package/pipeline/eval/triage/08-style-misclassified/notes.md

@@ -1,9 +0,0 @@
-# Case 08  -  Stylistic blocker misclassification
-
-Reviewer tagged a naming preference as `severity: "blocking"`. Renaming `fmt` to `formatCurrency` is a style opinion, not a correctness or security issue.
-
-Triage MUST reject this: the blocking severity does not match the nature of the finding. The reason should cite that it is stylistic and that the reviewer over-tagged.
-
-Guards against: accepting a stylistic nitpick because it has a "blocking" tag. Severity must be evaluated by substance, not blindly trusted.
-
-Validator exit code: 0 (1/1 = 100% rejected but total < 5 findings, so over-rejection guard does not trip).

package/pipeline/eval/triage/09-cascading-finding/expected.json

@@ -1,23 +0,0 @@
-{
-  "accepted": [
-    {
-      "severity": "blocking",
-      "file": "src/tests/setup.ts",
-      "line": 5,
-      "issue": "Test setup uses Math.random() without a fixed seed causing flaky results",
-      "fix": "Replace Math.random() with a seeded PRNG for reproducible test data",
-      "reviewer": "sonnet"
-    },
-    {
-      "severity": "blocking",
-      "file": "src/tests/setup.ts",
-      "line": 12,
-      "issue": "Tests fail intermittently in CI due to non-deterministic ordering",
-      "fix": "Fixing the unseeded Math.random() in setup.ts (accepted[0]) should resolve this symptom",
-      "reviewer": "opus"
-    }
-  ],
-  "deferred": [],
-  "rejected": [],
-  "approved": false
-}

package/pipeline/eval/triage/09-cascading-finding/input.json

@@ -1,22 +0,0 @@
-{
-  "rawFindings": [
-    {
-      "severity": "blocking",
-      "file": "src/tests/setup.ts",
-      "line": 12,
-      "issue": "Tests fail intermittently in CI due to non-deterministic ordering",
-      "fix": "Ensure deterministic test execution by fixing random seed initialization",
-      "reviewer": "opus"
-    },
-    {
-      "severity": "blocking",
-      "file": "src/tests/setup.ts",
-      "line": 5,
-      "issue": "Test setup uses Math.random() without a fixed seed causing flaky results",
-      "fix": "Replace Math.random() with a seeded PRNG for reproducible test data",
-      "reviewer": "sonnet"
-    }
-  ],
-  "scope": "Stabilize test infrastructure to eliminate flaky CI runs.",
-  "diffSummary": "src/tests/setup.ts: +8 -2, src/tests/helpers.ts: +4 -1"
-}

package/pipeline/eval/triage/09-cascading-finding/notes.md

@@ -1,9 +0,0 @@
-# Case 09  -  Cascading finding (symptom + root cause)
-
-Two findings describe the same underlying problem at different levels: (a) the symptom ("tests fail intermittently") and (b) the root cause ("Math.random() without a fixed seed").
-
-Triage must accept BOTH but order them upstream-first: the root cause (b) comes before the symptom (a) in the accepted array. The fix text for (a) references (b) to document the cascade  -  fixing the seed should resolve the intermittent failures.
-
-Guards against: treating symptom and root cause as independent issues. The cascade relationship must be explicit in the fix text so the developer addresses root cause first.
-
-Validator exit code: 0 (both accepted with blocking severity, approved:false is consistent).

package/pipeline/eval/triage/10-deferred-crossref/expected.json

@@ -1,18 +0,0 @@
-{
-  "accepted": [],
-  "deferred": [
-    {
-      "finding": {
-        "severity": "important",
-        "file": "src/payments/refund.ts",
-        "line": 55,
-        "issue": "Refund flow doesn't emit audit event for compliance tracking",
-        "fix": "Add an audit event emission after successful refund processing",
-        "reviewer": "opus"
-      },
-      "reason": "out of scope for current task (checkout session creation); HOWEVER this should be addressed before PROJ-5678 (next sprint refund work) because that task assumes audit events exist for refund operations."
-    }
-  ],
-  "rejected": [],
-  "approved": true
-}

package/pipeline/eval/triage/10-deferred-crossref/input.json

@@ -1,14 +0,0 @@
-{
-  "rawFindings": [
-    {
-      "severity": "important",
-      "file": "src/payments/refund.ts",
-      "line": 55,
-      "issue": "Refund flow doesn't emit audit event for compliance tracking",
-      "fix": "Add an audit event emission after successful refund processing",
-      "reviewer": "opus"
-    }
-  ],
-  "scope": "Implement checkout session creation endpoint. Touches src/checkout/ only.",
-  "diffSummary": "src/checkout/session.ts: +45 -0, src/checkout/types.ts: +12 -0"
-}

package/pipeline/eval/triage/10-deferred-crossref/notes.md

@@ -1,9 +0,0 @@
-# Case 10  -  Deferred with cross-reference
-
-A real issue (missing audit event in refund flow) that is out of scope for the current task (checkout session creation). However, the next planned task (PROJ-5678, refund work) assumes audit events already exist.
-
-Triage must defer  -  not reject  -  because the issue is real. The reason must capture the cross-task dependency: deferring is correct, but the reason needs to surface that PROJ-5678 will be affected.
-
-Guards against: losing inter-task dependencies when deferring. A bare "out of scope" reason is insufficient; the cross-reference to the downstream task ensures Phase 7 captures this for backlog prioritization.
-
-Validator exit code: 0 (deferred + approved:true is consistent because no blocking items in accepted).

package/pipeline/eval/triage/11-vercel-token-leak-blocker/expected.json

@@ -1,27 +0,0 @@
-{
-  "accepted": [
-    {
-      "severity": "blocking",
-      "file": "scripts/deploy.sh",
-      "line": 14,
-      "issue": "Script invokes `vercel deploy --token=$VERCEL_TOKEN`. On retry the Vercel CLI prints failed argv verbatim  -  the token leaks into the conversation transcript and CI logs. Same shape as the v7.9.1 incident that forced a token rotation.",
-      "fix": "Replace with the safe wrapper: `bash $HOME/.claude/scripts/vercel-deploy.sh deploy --prod`. The wrapper refuses --token argv, reads VERCEL_TOKEN from env, and pipes stdout/stderr through a redact filter that scrubs vcp_... patterns.",
-      "reviewer": "opus"
-    }
-  ],
-  "deferred": [],
-  "rejected": [
-    {
-      "finding": {
-        "severity": "important",
-        "file": "scripts/deploy.sh",
-        "line": 14,
-        "issue": "Token passed as positional argument allows shell history (zsh, bash) to capture it across sessions.",
-        "fix": "Use env var; never argv.",
-        "reviewer": "sonnet"
-      },
-      "reason": "duplicate  -  same root cause as the accepted blocking finding; merging would dilute the actionable line"
-    }
-  ],
-  "approved": false
-}

package/pipeline/eval/triage/11-vercel-token-leak-blocker/input.json

@@ -1,22 +0,0 @@
-{
-  "rawFindings": [
-    {
-      "severity": "blocking",
-      "file": "scripts/deploy.sh",
-      "line": 14,
-      "issue": "Script invokes `vercel deploy --token=$VERCEL_TOKEN`. On retry the Vercel CLI prints failed argv verbatim  -  the token leaks into the conversation transcript and CI logs. Same shape as the v7.9.1 incident that forced a token rotation.",
-      "fix": "Replace with the safe wrapper: `bash $HOME/.claude/scripts/vercel-deploy.sh deploy --prod`. The wrapper refuses --token argv, reads VERCEL_TOKEN from env, and pipes stdout/stderr through a redact filter that scrubs vcp_... patterns.",
-      "reviewer": "opus"
-    },
-    {
-      "severity": "important",
-      "file": "scripts/deploy.sh",
-      "line": 14,
-      "issue": "Token passed as positional argument allows shell history (zsh, bash) to capture it across sessions.",
-      "fix": "Use env var; never argv.",
-      "reviewer": "sonnet"
-    }
-  ],
-  "scope": "Add a one-shot deploy script that wraps `vercel deploy` for the marketing site.",
-  "diffSummary": "1 file changed, 28 insertions, 0 deletions"
-}

package/pipeline/eval/triage/11-vercel-token-leak-blocker/notes.md

@@ -1,14 +0,0 @@
-# Case 11  -  Vercel token-leak as a blocker (v8.0.0)
-
-Two reviewers flag the same `vercel deploy --token=$VERCEL_TOKEN` call. Triage MUST:
-
-1. **Accept** the Opus finding  -  it's a real production-affecting blocker, identical in shape to the v7.9.1 incident that forced a token rotation.
-2. **Reject** the Sonnet finding as a duplicate  -  same root cause (token in argv), additional reviewer would dilute the actionable line. The fix is the same code change either way.
-3. Set `approved: false` because at least one accepted blocking remains.
-
-Guards against:
-- Future regressions where token-leak findings get classified as `suggestion` or `style` (they are blocking  -  secret leaks reach production logs and force rotation).
-- Triage adding both reviewers' findings to `accepted` and producing two redundant "fix this" lines for the same line of code.
-- Approving the diff while a blocker is open.
-
-Validator exit code: 0 (clean  -  no contradictions).

package/pipeline/eval/triage/README.md

@@ -1,54 +0,0 @@
-# Triage Eval Suite
-
-Semantic regression detection for Phase 4 triage. Each fixture pair (`input.json` + `expected.json`) tests a specific behavior the triage agent must preserve when its prompt or model changes.
-
-## How it works
-
-Each test case lives in its own subdirectory:
-
-```
-pipeline/eval/triage/
-├── README.md
-├── 01-empty-findings/
-│   ├── input.json        -  what reviewers returned (raw findings, scope, diff)
-│   ├── expected.json     -  what the triage step MUST produce, normalized
-│   └── notes.md          -  why this case matters
-├── 02-real-blocker/
-├── 03-out-of-scope-defer/
-├── 04-false-positive-reject/
-├── ...
-```
-
-Run all cases:
-
-```bash
-node pipeline/scripts/eval-triage.mjs
-```
-
-Filter to one case:
-
-```bash
-node pipeline/scripts/eval-triage.mjs --case 03-out-of-scope-defer
-```
-
-`--json` produces machine-readable output for CI consumption.
-
-## Pass criteria
-
-For each case the runner:
-
-1. Validates `expected.json` against `triage-output.schema.json` (smoke-level sanity).
-2. Runs `validate-triage.mjs` against `expected.json` and asserts the documented exit code.
-3. Compares classification membership: each finding in `input.json.rawFindings` must appear in exactly one of `accepted | deferred | rejected` in `expected.json` (no leaks, no inventions).
-
-This is **not** a model-quality check (we don't run a real triage agent in CI  -  that requires API keys and is non-deterministic). It's a **contract regression check**: the fixtures encode "for input X, the spec says triage MUST produce Y", and any prompt/spec change that breaks the mapping fails CI.
-
-## Adding new cases
-
-When a new triage edge case is discovered in production:
-
-1. Capture the actual reviewer findings + diff context into a new `pipeline/eval/triage/<NN>-<short-name>/input.json`.
-2. Hand-curate `expected.json` (or paste a known-good triage output and verify by hand).
-3. Add `notes.md` explaining the failure mode this case guards against.
-4. Run the suite locally; `eval-triage.mjs` should pass with your new case.
-5. Commit. CI will keep it green going forward.

package/pipeline/scripts/benchmark-phase-0.sh

@@ -1,128 +0,0 @@
-#!/usr/bin/env bash
-# benchmark-phase-0.sh  -  measures Phase 0 latency (single-repo + multi-repo)
-# and writes a baseline / comparison row to benchmarks/v3.7-phase-0.jsonl.
-#
-# Usage:
-#   benchmark-phase-0.sh                   # run once, append result
-#   benchmark-phase-0.sh --runs 5          # run N times, report median
-#   benchmark-phase-0.sh --label baseline  # tag the row (e.g. "baseline" / "post-refactor")
-#   benchmark-phase-0.sh --multi-repo      # exercise the multi-repo path
-#
-# What it measures (proxy  -  actual Phase 0 needs a live task):
-#   1. Preferences load (read + JSON parse)
-#   2. Schema validation (validate prefs against prefs.schema.json  -  node + ajv if present)
-#   3. Project scan ($HOME maxdepth 2 with marker globs  -  same as Phase 0 Step 2)
-#   4. Branch listing (no remote  -  uses local $HOME repo if available)
-#   5. Recents read + TTL prune simulation
-#
-# Best-effort: missing tools (node, jq) cause the section to skip rather than fail.
-
-set -uo pipefail
-
-RUNS=1
-LABEL="adhoc"
-MULTI=false
-while [ "$#" -gt 0 ]; do
-  case "$1" in
-    --runs) RUNS="$2"; shift 2 ;;
-    --label) LABEL="$2"; shift 2 ;;
-    --multi-repo) MULTI=true; shift ;;
-    *) shift ;;
-  esac
-done
-
-PREFS="$HOME/.claude/multi-agent-preferences.json"
-SCHEMA="$HOME/.claude/schemas/prefs.schema.json"
-OUT_DIR="$HOME/.claude/benchmarks"
-OUT="$OUT_DIR/v3.7-phase-0.jsonl"
-mkdir -p "$OUT_DIR"
-
-# Portable nanosecond timer using python (date -u +%N is GNU-only)
-now_ms() {
-  python3 -c 'import time; print(int(time.time()*1000))' 2>/dev/null \
-    || echo "$(($(date +%s) * 1000))"
-}
-
-bench_one() {
-  local total_start=$(now_ms)
-
-  # 1. prefs load
-  local t1=$(now_ms)
-  if [ -f "$PREFS" ]; then
-    cat "$PREFS" >/dev/null 2>&1
-    local prefs_load_ms=$(( $(now_ms) - t1 ))
-  else
-    local prefs_load_ms=0
-  fi
-
-  # 2. schema validate (best-effort)
-  local schema_ms=0
-  if [ -f "$PREFS" ] && [ -f "$SCHEMA" ] && command -v node >/dev/null 2>&1; then
-    local t2=$(now_ms)
-    node -e "JSON.parse(require('fs').readFileSync('$PREFS','utf8'))" 2>/dev/null
-    schema_ms=$(( $(now_ms) - t2 ))
-  fi
-
-  # 3. project scan
-  local t3=$(now_ms)
-  find "$HOME" -maxdepth 2 -type f \( \
-    -name "Package.swift" -o -name "package.json" -o -name "build.gradle" \
-    -o -name "Cargo.toml" -o -name "go.mod" -o -name "pom.xml" \
-  \) 2>/dev/null | head -50 >/dev/null
-  local scan_ms=$(( $(now_ms) - t3 ))
-
-  # 4. branch listing (use first git repo found in $HOME, if any)
-  local branch_ms=0
-  local repo=""
-  repo=$(find "$HOME" -maxdepth 3 -type d -name ".git" 2>/dev/null | head -1 | xargs -I {} dirname {})
-  if [ -n "$repo" ]; then
-    local t4=$(now_ms)
-    git -C "$repo" branch -r 2>/dev/null | head -50 >/dev/null
-    branch_ms=$(( $(now_ms) - t4 ))
-  fi
-
-  # 5. recents read + TTL prune sim
-  local t5=$(now_ms)
-  if [ -f "$PREFS" ] && command -v node >/dev/null 2>&1; then
-    node -e "
-      const p = JSON.parse(require('fs').readFileSync('$PREFS','utf8'));
-      const cutoff = Date.now() - 15*86400*1000;
-      const rb = p.global?.recentBranches || {};
-      let kept = 0, pruned = 0;
-      for (const proj of Object.keys(rb)) {
-        const arr = rb[proj] || [];
-        for (const b of arr) {
-          if (b && b.lastUsed && Date.parse(b.lastUsed) >= cutoff) kept++;
-          else pruned++;
-        }
-      }
-    " 2>/dev/null
-  fi
-  local recents_ms=$(( $(now_ms) - t5 ))
-
-  local total_ms=$(( $(now_ms) - total_start ))
-  local mode="single"; [ "$MULTI" = true ] && mode="multi"
-  printf '{"ts":"%s","label":"%s","mode":"%s","prefs_load_ms":%d,"schema_ms":%d,"project_scan_ms":%d,"branch_list_ms":%d,"recents_ms":%d,"total_ms":%d}\n' \
-    "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$LABEL" "$mode" \
-    "$prefs_load_ms" "$schema_ms" "$scan_ms" "$branch_ms" "$recents_ms" "$total_ms"
-}
-
-echo "Running $RUNS bench iteration(s)  -  label=$LABEL mode=$([ "$MULTI" = true ] && echo multi || echo single)"
-TOTALS=()
-for i in $(seq 1 "$RUNS"); do
-  ROW=$(bench_one)
-  echo "  $ROW"
-  echo "$ROW" >> "$OUT"
-  TOTAL=$(echo "$ROW" | grep -oE '"total_ms":[0-9]+' | grep -oE '[0-9]+')
-  TOTALS+=("$TOTAL")
-done
-
-# Median across runs
-if [ "${#TOTALS[@]}" -gt 1 ]; then
-  SORTED=$(printf '%s\n' "${TOTALS[@]}" | sort -n)
-  MEDIAN=$(echo "$SORTED" | awk -v n="${#TOTALS[@]}" 'NR == int((n+1)/2)')
-  echo ""
-  echo "median total_ms: $MEDIAN (across $RUNS runs)"
-fi
-
-echo "Wrote to: $OUT"

package/pipeline/scripts/check-md-links.mjs

@@ -1,84 +0,0 @@
-#!/usr/bin/env node
-// Internal markdown link checker for the user-facing docs surface and the
-// pipeline instruction tree.
-//
-// Scope: root *.md + docs/**/*.md + examples/**/*.md + pipeline/**/*.md.
-// Verifies that every relative link target ([text](./path) or
-// [text](path#anchor)) resolves to an existing file or directory. Skipped:
-// external (http/https/mailto) links, pure #anchors, template links whose
-// path contains a {placeholder} variable, and links through shell
-// variables ($HOME, ...) - except $HOME/.claude/multi-agent-refs/, which
-// is checked against its repo source (pipeline/multi-agent-refs/) so a
-// ref rename cannot leave stale runtime links behind. This is a drift
-// gate for file moves/renames, not a network crawler.
-//
-// Excluded trees: pipeline/skills/shared/external/ is vendored community
-// skill content - several packs (the Apple HIG set) index upstream
-// references/ companion files that are deliberately not vendored here.
-
-import { readFileSync, existsSync, readdirSync, statSync } from "node:fs";
-import { dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-
-const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..");
-
-function mdFiles(dir, acc = []) {
-  for (const entry of readdirSync(dir)) {
-    if (entry === "node_modules" || entry.startsWith(".")) continue;
-    const full = join(dir, entry);
-    if (statSync(full).isDirectory()) mdFiles(full, acc);
-    else if (entry.endsWith(".md")) acc.push(full);
-  }
-  return acc;
-}
-
-const targets = [
-  ...readdirSync(repoRoot).filter((f) => f.endsWith(".md")).map((f) => join(repoRoot, f)),
-  ...["docs", "examples", "pipeline"].filter((d) => existsSync(join(repoRoot, d))).flatMap((d) => mdFiles(join(repoRoot, d))),
-  // CHANGELOG-archive.md is frozen history - it legitimately references docs
-  // that were deleted in later releases.
-]
-  .filter((f) => !f.endsWith("CHANGELOG-archive.md"))
-  // Vendored community skill packs (see header note).
-  .filter((f) => !f.includes(join("pipeline", "skills", "shared", "external") + "/"));
-
-const LINK_RE = /\[[^\]]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
-const errors = [];
-
-for (const file of targets) {
-  // Drop fenced blocks and inline code spans - links inside code are
-  // illustrative placeholders, not navigation.
-  const text = readFileSync(file, "utf-8")
-    .replace(/```[\s\S]*?```/g, "")
-    .replace(/`[^`\n]*`/g, "");
-  for (const match of text.matchAll(LINK_RE)) {
-    const raw = match[1];
-    if (/^(https?:|mailto:|#)/.test(raw)) continue;
-    // Template placeholders ({fileKey}, {slug}, ...) resolve at runtime,
-    // never on disk - skip them.
-    if (raw.includes("{") || raw.includes("}")) continue;
-    const target = raw.split("#")[0];
-    if (target === "") continue;
-    let resolved;
-    const refsPrefix = "$HOME/.claude/multi-agent-refs/";
-    if (target.startsWith(refsPrefix)) {
-      // Runtime install path with a repo source - check the source tree.
-      resolved = join(repoRoot, "pipeline", "multi-agent-refs", target.slice(refsPrefix.length));
-    } else if (target.includes("$") || target.startsWith("~")) {
-      // Other shell-variable / home paths resolve at runtime only.
-      continue;
-    } else {
-      resolved = resolve(dirname(file), decodeURIComponent(target));
-    }
-    if (!existsSync(resolved)) {
-      errors.push(`${file.slice(repoRoot.length + 1)}: broken link -> ${raw}`);
-    }
-  }
-}
-
-if (errors.length > 0) {
-  console.error(`check-md-links: ${errors.length} broken internal link(s)`);
-  for (const e of errors) console.error("  " + e);
-  process.exit(1);
-}
-console.log(`check-md-links: ${targets.length} files OK`);