@mmerterden/multi-agent-pipeline 16.2.2 → 16.4.0

package/pipeline/scripts/plan-coverage-gate.mjs

@@ -0,0 +1,219 @@
+#!/usr/bin/env node
+// plan-coverage-gate.mjs  -  the plan's own work must be accounted for before commit.
+//
+// Phase 4 answers "is what changed correct" and Step 1.45 answers "did every
+// planned TEST land". Nothing answered "did every planned TASK land". The
+// criteria manifest's denominator is rule IDs, not plan steps, and the only
+// place a step's status surfaced was render-work-summary.sh at Phase 7 - a
+// report, printed after the commit. So a plan with seven steps could ship five
+// and read as done.
+//
+// Pattern source: github/spec-kit templates/commands/converge.md. Two of its
+// properties are copied deliberately:
+//   - it assesses the CURRENT STATE, not a diff. A file the plan promised is
+//     either on disk or it is not; git history does not enter into it.
+//   - a clean run writes and says nothing. No empty "Convergence" section, no
+//     noise to scroll past.
+// What is not copied: spec-kit appends remediation tasks to tasks.md. This gate
+// only reports and fails; appending work to a plan that the user already
+// approved is a Phase 2 decision, not a gate's.
+//
+// A step is accounted for when it is `completed`, or `skipped` with a
+// skipReason, or `failed` with a failureReason. A bare `pending` /
+// `in_progress` is the gap this exists to catch: not "we decided not to do it"
+// but "nobody noticed it was still open".
+//
+// Usage:
+//   plan-coverage-gate.mjs --state <agent-state.json> [--analysis <doc.md>] [--root <dir>] [--json]
+//   plan-coverage-gate.mjs --todos <plan-todos.json> [--analysis <doc.md>] [--root <dir>] [--json]
+//
+// Exit codes: 0 clean, 1 coverage gap, 2 usage / parse error.
+
+import { readFileSync, existsSync } from "node:fs";
+import { join, isAbsolute } from "node:path";
+
+export function auditTodos(todos) {
+  const gaps = [];
+  const accounted = [];
+  for (const todo of Array.isArray(todos) ? todos : []) {
+    const id = todo?.id ?? "(no id)";
+    const task = todo?.task ?? todo?.title ?? "(no task text)";
+    const status = todo?.status;
+    if (status === "completed") {
+      accounted.push({ id, status });
+    } else if (status === "skipped") {
+      if (String(todo?.skipReason ?? "").trim()) accounted.push({ id, status });
+      else gaps.push({ id, task, status, why: "skipped with no skipReason" });
+    } else if (status === "failed") {
+      if (String(todo?.failureReason ?? "").trim()) accounted.push({ id, status });
+      else gaps.push({ id, task, status, why: "failed with no failureReason" });
+    } else {
+      gaps.push({ id, task, status: status ?? "(absent)", why: "never reached a terminal status" });
+    }
+  }
+  return { gaps, accounted };
+}
+
+// Section 14 rows tagged `Add new` name files the analysis promised would
+// exist. The tag vocabulary is Locked 16 and identical in both languages, so
+// the heading is matched on the number rather than on either title.
+export function filesToAdd(markdown) {
+  const lines = String(markdown).split("\n");
+  const start = lines.findIndex((l) => /^#{1,3}\s*14\.\s/.test(l));
+  if (start === -1) return null;
+  const rows = [];
+  for (let i = start + 1; i < lines.length; i++) {
+    const line = lines[i];
+    if (/^#{1,3}\s*(1[5-9]|2[0-9])\.\s/.test(line)) break;
+    if (!line.trim().startsWith("|")) continue;
+    const cells = line
+      .split("|")
+      .slice(1, -1)
+      .map((c) => c.trim());
+    if (cells.length < 2) continue;
+    const path = cells[0].replace(/`/g, "").trim();
+    const tag = cells[1].replace(/`/g, "").trim();
+    if (!path || /^-+$/.test(path)) continue;
+    if (path.includes("<") || tag.includes("<")) continue;
+    // No header-row filter: the tag column of the header reads "Etiket / Tag",
+    // which the `Add new` test below already rejects. The filter that used to be
+    // here matched on the PATH column instead and would have skipped a real row
+    // whose path begins with a `File/` directory.
+    if (!/^add new$/i.test(tag)) continue;
+    rows.push(path);
+  }
+  return rows;
+}
+
+function readJson(path) {
+  return JSON.parse(readFileSync(path, "utf8"));
+}
+
+const isMain = process.argv[1] && import.meta.url === `file://${process.argv[1]}`;
+if (isMain) {
+  const args = process.argv.slice(2);
+  const flag = (name) => {
+    const i = args.indexOf(name);
+    return i === -1 ? undefined : args[i + 1];
+  };
+  // Repeatable: `state.analysis.docPath[]` is an array, one document per
+  // platform. Reading only the first would check one platform's promised files
+  // and report the verdict as if it covered the run.
+  const flagAll = (name) => {
+    const out = [];
+    for (let i = 0; i < args.length; i++) {
+      if (args[i] === name && args[i + 1] !== undefined) out.push(args[i + 1]);
+    }
+    return out;
+  };
+
+  if (args.includes("--help") || args.length === 0) {
+    process.stdout.write(
+      [
+        "usage: plan-coverage-gate.mjs --state <agent-state.json> [--analysis <doc.md>] [--root <dir>] [--json]",
+        "       plan-coverage-gate.mjs --todos <plan-todos.json> [--analysis <doc.md>] [--root <dir>] [--json]",
+        "",
+        "Fails when a plan step never reached a terminal status, when a skip or",
+        "failure carries no reason, or when an analysis Section 14 `Add new` file",
+        "is missing from the tree. Run before Phase 6 commit.",
+        "",
+      ].join("\n"),
+    );
+    process.exit(args.includes("--help") ? 0 : 2);
+  }
+
+  const statePath = flag("--state");
+  const todosPath = flag("--todos");
+  const analysisPaths = flagAll("--analysis");
+  const root = flag("--root") ?? process.cwd();
+  const asJson = args.includes("--json");
+
+  if (!statePath && !todosPath) {
+    process.stderr.write("plan-coverage-gate: pass --state or --todos\n");
+    process.exit(2);
+  }
+
+  let todos;
+  try {
+    if (todosPath) {
+      const parsed = readJson(todosPath);
+      todos = parsed?.todos ?? parsed;
+    } else {
+      const state = readJson(statePath);
+      todos = state?.plan?.todos ?? state?.plan?.todoList?.todos ?? null;
+    }
+  } catch (e) {
+    process.stderr.write(`plan-coverage-gate: cannot read the plan: ${e.message}\n`);
+    process.exit(2);
+  }
+
+  // No plan at all is not a pass, and neither is an empty one: a Phase 2 that
+  // produced no steps, or a state whose todos got cleared, would otherwise read
+  // as "0/0 accounted for" and let the commit through. A Short run has no Phase
+  // 2, and the caller is expected to skip this gate for those modes rather than
+  // let it report clean.
+  if (!Array.isArray(todos) || todos.length === 0) {
+    const msg = Array.isArray(todos)
+      ? "the plan has zero steps - that is an unusable plan, not a clean one; skip this gate explicitly for modes with no Phase 2"
+      : "no plan todos found - if this mode has no Phase 2, skip this gate explicitly";
+    if (asJson)
+      process.stdout.write(`${JSON.stringify({ verdict: "unusable", reason: msg }, null, 2)}\n`);
+    else process.stderr.write(`plan-coverage-gate: ${msg}\n`);
+    process.exit(2);
+  }
+
+  const { gaps, accounted } = auditTodos(todos);
+
+  let missingFiles = [];
+  let filesVerdict;
+  if (analysisPaths.length) {
+    const allRows = [];
+    const docsWithoutSection = [];
+    for (const docPath of analysisPaths) {
+      let rows = null;
+      try {
+        rows = filesToAdd(readFileSync(docPath, "utf8"));
+      } catch (e) {
+        process.stderr.write(`plan-coverage-gate: cannot read the analysis doc: ${e.message}\n`);
+        process.exit(2);
+      }
+      if (rows === null) docsWithoutSection.push(docPath);
+      else allRows.push(...rows);
+    }
+    const unique = [...new Set(allRows)];
+    missingFiles = unique.filter((p) => !existsSync(isAbsolute(p) ? p : join(root, p)));
+    const docNote = docsWithoutSection.length
+      ? `; ${docsWithoutSection.length} doc(s) carried no Section 14`
+      : "";
+    filesVerdict = unique.length
+      ? `${unique.length - missingFiles.length}/${unique.length} promised files present across ${analysisPaths.length} doc(s)${docNote}`
+      : `no Section 14 rows in ${analysisPaths.length} doc(s)${docNote}`;
+  } else {
+    filesVerdict = "skipped (no --analysis given)";
+  }
+
+  const failed = gaps.length > 0 || missingFiles.length > 0;
+  const result = {
+    verdict: failed ? "gap" : "clean",
+    todos: { total: todos.length, accounted: accounted.length, gaps },
+    filesToAdd: { verdict: filesVerdict, missing: missingFiles },
+  };
+
+  if (asJson) {
+    process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+  } else if (failed) {
+    process.stderr.write(
+      `plan coverage: ${gaps.length} unaccounted step(s), ${missingFiles.length} missing file(s)\n`,
+    );
+    for (const g of gaps) process.stderr.write(`  ${g.id} [${g.status}] ${g.task} - ${g.why}\n`);
+    for (const f of missingFiles)
+      process.stderr.write(`  missing file promised by analysis Section 14: ${f}\n`);
+    process.stderr.write(`  files-to-add: ${filesVerdict}\n`);
+  } else {
+    // spec-kit's rule: a clean pass says one line and writes nothing.
+    process.stdout.write(
+      `plan coverage: ${accounted.length}/${todos.length} steps accounted for; files-to-add: ${filesVerdict}\n`,
+    );
+  }
+  process.exit(failed ? 1 : 0);
+}

package/pipeline/scripts/run-metrics.mjs

@@ -15,6 +15,12 @@
  *   firstPassClean                        - iteration 1 had 0 accepted BLOCKING
  *   accepted/deferred/rejected (final)    - triage outcome
  *   reviewFindingsRaw -> acceptedRatio    - signal-to-noise of the reviewers
+ *   reviewSignal.perReviewer              - the same ratio per reviewer model, so
+ *                                           "which model is worth dispatching" stops
+ *                                           being a guess. Needs the anonymization
+ *                                           label map to attribute accepted findings;
+ *                                           without it the raw counts still land and
+ *                                           attribution reports "unavailable".
  *   consensusVerdict                      - unanimous-* / split / unverified
  *   buildPassed                           - per-repo build outcome
  *
@@ -79,9 +85,42 @@ if (iterations.length) {
 let rawFindings = 0;
 let acceptedAll = 0;
 let classifiedAll = 0;
+// Per-reviewer tallies. `unknown` is its own bucket: a reviewer entry written
+// before the shape was typed must not be folded into a named model, or the
+// ratio reads as coverage that was never measured.
+const rawByModel = new Map();
+// Separate denominator: only iterations that carried a label map can contribute
+// an accepted count, so mixing them with map-less iterations understates every
+// reviewer's ratio. A resumed run that upgraded mid-flight is exactly that case.
+const rawAttributableByModel = new Map();
+const acceptedByModel = new Map();
+let mappedIterations = 0;
+let acceptedUnattributed = 0;
 for (const it of iterations) {
+  const labelToModel = it?.anonymizationMap?.labelToModel || it?.triage?.labelToModel || null;
   for (const r of Array.isArray(it.reviewers) ? it.reviewers : []) {
-    rawFindings += Array.isArray(r?.findings) ? r.findings.length : 0;
+    const n = Array.isArray(r?.findings) ? r.findings.length : 0;
+    rawFindings += n;
+    const model = typeof r?.model === "string" && r.model ? r.model : "unknown";
+    rawByModel.set(model, (rawByModel.get(model) || 0) + n);
+    if (labelToModel)
+      rawAttributableByModel.set(model, (rawAttributableByModel.get(model) || 0) + n);
+  }
+  if (labelToModel) mappedIterations += 1;
+  {
+    const acceptedList = Array.isArray(it?.triage?.accepted) ? it.triage.accepted : [];
+    for (const f of acceptedList) {
+      const model = labelToModel ? labelToModel[f?.foundBy] : undefined;
+      // Every accepted finding lands in exactly one bucket, so
+      // acceptedAll === sum(perReviewer.accepted) + acceptedUnattributed holds.
+      // Two things arrive here: a deterministic-gate finding, which carries no
+      // foundBy by design, and any finding from an iteration that had no map.
+      if (!model) {
+        acceptedUnattributed += 1;
+        continue;
+      }
+      acceptedByModel.set(model, (acceptedByModel.get(model) || 0) + 1);
+    }
   }
   const t = it.triage || {};
   acceptedAll += Array.isArray(t.accepted) ? t.accepted.length : 0;
@@ -125,6 +164,32 @@ const metrics = {
     classified: classifiedAll,
     acceptedAll,
     acceptedRatio: rawFindings > 0 ? Math.round((acceptedAll / rawFindings) * 100) / 100 : null,
+    perReviewer: [...rawByModel.entries()]
+      .sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))
+      .map(([model, raw]) => {
+        const rawAttributable = rawAttributableByModel.get(model) || 0;
+        const accepted = acceptedByModel.get(model) || 0;
+        return {
+          model,
+          rawFindings: raw,
+          rawAttributable,
+          accepted: mappedIterations > 0 ? accepted : null,
+          // Denominator is the attributable raw count, never the total: a run
+          // whose second iteration lost its map would otherwise read as if the
+          // reviewer produced noise it was never credited for.
+          acceptedRatio:
+            mappedIterations > 0 && rawAttributable > 0
+              ? Math.round((accepted / rawAttributable) * 100) / 100
+              : null,
+        };
+      }),
+    // "map" means the mapping existed, not that it resolved something: zero
+    // accepted findings under a present map is a real 0, and reporting it as
+    // "unavailable" is the absent-vs-zero conflation this metric set exists to
+    // avoid. A number, not a "1/2" string, because metrics.jsonl is aggregated.
+    perReviewerAttribution: mappedIterations > 0 ? "map" : "unavailable",
+    mappedIterations,
+    acceptedUnattributed,
   },
   consensusVerdict: lastTriage.consensus ? lastTriage.consensus.verdict : null,
   buildPassed,

package/pipeline/scripts/validate-analysis-doc.mjs

@@ -95,10 +95,15 @@ function parseFrontMatter(text) {
 }
 
 function sectionBody(lines, keywords, level = 2) {
-  const head = new RegExp(`^#{${level}}\\s+\\d+(\\.\\d+)*\\.?\\s`);
+  // Opened by a heading at exactly `level`; closed by the next heading at that
+  // level OR ANY SHALLOWER one. A sub-section that is the last of its parent has
+  // no sibling after it, so terminating only on equal depth runs the body into
+  // the following section and audits its rows as if they belonged here.
+  const open = new RegExp(`^#{${level}}\\s+\\d+(\\.\\d+)*\\.?\\s`);
+  const close = new RegExp(`^#{1,${level}}\\s`);
   let start = -1;
   for (let i = 0; i < lines.length; i++) {
-    if (head.test(lines[i]) && keywords.some((k) => lines[i].includes(k))) {
+    if (open.test(lines[i]) && keywords.some((k) => lines[i].includes(k))) {
       start = i;
       break;
     }
@@ -106,7 +111,7 @@ function sectionBody(lines, keywords, level = 2) {
   if (start < 0) return null;
   let end = lines.length;
   for (let i = start + 1; i < lines.length; i++) {
-    if (head.test(lines[i])) {
+    if (close.test(lines[i])) {
       end = i;
       break;
     }

package/pipeline/skills/shared/core/multi-agent-channels/SKILL.md

@@ -120,7 +120,7 @@ PR: {prUrl}
 # Scenario 2
 ```
 
-First line: PR URL (one-click jump). POST `/rest/api/2/issue/{id}/comment` with heredoc + `jq --rawfile` + `curl --data-binary @file`. Real newlines, no HTML entities.
+First line: PR URL (one-click jump). Run the converted body through `node "$HOME/.claude/scripts/jira-wiki-escape.mjs"` before POST  -  Jira turns `:)` `(x)` `(!)` `(/)` into emoticon images, including inside `{{monospace}}`. POST `/rest/api/2/issue/{id}/comment` with heredoc + `jq --rawfile` + `curl --data-binary @file` from the escaped file. Real newlines, no HTML entities.
 
 ### Confluence page
 

package/pipeline/skills/shared/core/multi-agent-create-jira/SKILL.md

@@ -40,7 +40,7 @@ A `C` section fires on: Design Reference → Figma URL; Screenshots → pasted i
    - **Figma** (`FIGMA_URL` set)  -  3-tier chain (MCP → REST `figma` PAT → user screenshot); standalone command, MCP allowed here.
    - **Swagger** (`SWAGGER_URL` set or contract pasted)  -  fetch the spec, extract ONLY the referenced endpoints (paths/operationIds/tags named in `FREE_TEXT`, else the pointed-at group): method + path + one-line summary + key request/response fields. Never crawl the whole spec. Fetch failure → link-only or omit.
    - **Screenshots**  -  staged for the step 8 attachment opt-in, referenced inline with `!file|thumbnail!`.
-7. **Draft + clarifying questions**  -  compose summary (`{minedPrefix} {title}`, 255 trunc) + description from the type's **standard template** in Jira wiki markup (`h3.`), auto-sizing conditional sections, humanizer pass, in `outputLanguage`; body to a file for `jq -n --rawfile` (UTF-8 verbatim, `--data-binary @file`, no re-encode). Ask ONLY genuinely unknown fields: component, epic (when usage rate is high), priority (when norm ambiguous), labels, assignee, sprint vs backlog, required customs, and any always-present section still empty (Bug Steps/Environment, or a Task/Story with no derivable Scope/AC). Never invent content the user did not supply.
+7. **Draft + clarifying questions**  -  compose summary (`{minedPrefix} {title}`, 255 trunc) + description from the type's **standard template** in Jira wiki markup (`h3.`), auto-sizing conditional sections, humanizer pass, in `outputLanguage`; body to a file for `jq -n --rawfile` (UTF-8 verbatim, `--data-binary @file`, no re-encode), passed through `node "$HOME/.claude/scripts/jira-wiki-escape.mjs"` first  -  the description field expands `:)` `(x)` `(!)` `(/)` into emoticon images. Ask ONLY genuinely unknown fields: component, epic (when usage rate is high), priority (when norm ambiguous), labels, assignee, sprint vs backlog, required customs, and any always-present section still empty (Bug Steps/Environment, or a Task/Story with no derivable Scope/AC). Never invent content the user did not supply.
 8. **Full preview + approval gate (never skipped)**  -  render the complete issue (all fields + full description + attachment plan, uploads default off). Ask: `Approve` / `Edit` (apply → re-render → re-ask, loop) / `Cancel` (nothing created). No flag or mode bypasses this gate.
 9. **Create**  -  `POST /rest/api/2/issue` with `jq -n --rawfile` payload + `curl --data-binary @file`. 400 field errors → show verbatim, re-ask those fields, re-preview.
 10. **Post-create** (each failure-isolated: warn + continue): sprint chosen → `POST /rest/agile/1.0/sprint/{id}/issue` `{"issues":["KEY"]}`; Figma → `POST /rest/api/2/issue/{KEY}/remotelink`; Swagger → `remotelink` with title "API contract"; screenshots opt-in → `POST /rest/api/2/issue/{KEY}/attachments` with `X-Atlassian-Token: no-check`.