@mmerterden/multi-agent-pipeline 20.23.2 → 20.25.0

package/pipeline/schemas/plan-todos.schema.json

@@ -27,8 +27,8 @@
         "properties": {
           "id": {
             "type": "string",
-            "pattern": "^t[0-9]+$",
-            "description": "Stable identifier (t1, t2, ...). Used in dependency edges + progress lines."
+            "pattern": "^[tT][0-9]+$",
+            "description": "Stable identifier: t1, t2 when written directly, T1, T2 when converted from a planning-output document. Used in dependency edges and progress lines."
           },
           "task": {
             "type": "string",
@@ -43,7 +43,7 @@
             "type": "array",
             "default": [],
             "description": "Array of todo IDs that must reach `completed` (or `skipped`) before this one can move to `in_progress`. Empty for parallelizable steps.",
-            "items": { "type": "string", "pattern": "^t[0-9]+$" }
+            "items": { "type": "string", "pattern": "^[tT][0-9]+$" }
           },
           "startedAt": { "type": "string", "format": "date-time" },
           "completedAt": { "type": "string", "format": "date-time" },
@@ -58,10 +58,111 @@
             "enum": ["Reuse", "Add new", "Modify"],
             "description": "Carried from the analysis Section 14 row this step came from (Locked 16). Phase 2 reads it as an instruction: a Reuse step binds the existing file and must not write a new one. Phase 3 checks the diff against it. Absent when the step did not originate from a Files-to-Add row."
           },
+          "type": {
+            "type": "string",
+            "enum": ["test", "code", "refactor", "docs", "chore"],
+            "description": "Carried from the planning-output task type. A docs or chore todo is exempt from the TDD evidence check (lib/tdd-evidence.mjs)."
+          },
+          "tdd": {
+            "type": "object",
+            "additionalProperties": false,
+            "description": "TDD evidence, written by plan-todos.sh tdd-red / tdd-green / complete through lib/tdd-evidence.mjs. complete reads it according to prefs.global.tddEvidence.mode.",
+            "properties": {
+              "red": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["at", "failed", "total"],
+                "description": "The run before the change: at least one failing test, or none when outcome is already-passing.",
+                "properties": {
+                  "at": { "type": "string", "format": "date-time" },
+                  "failed": { "type": "integer", "minimum": 0 },
+                  "total": { "type": "integer", "minimum": 1 },
+                  "tests": {
+                    "type": "array",
+                    "items": { "type": "string" },
+                    "description": "Failing test ids the todo named; every failing id when it named none; empty when the results carry no names."
+                  },
+                  "testsUnverifiable": {
+                    "type": "boolean",
+                    "description": "The todo named tests (--tests) but the results name no failing test, so the names went unchecked. Recorded in warn and off mode; enforce refuses the run instead."
+                  }
+                }
+              },
+              "green": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["at", "failed", "total"],
+                "description": "The run after the change, with no failures.",
+                "properties": {
+                  "at": { "type": "string", "format": "date-time" },
+                  "failed": { "const": 0 },
+                  "total": { "type": "integer", "minimum": 1 }
+                }
+              },
+              "outcome": {
+                "type": "string",
+                "enum": ["red", "verified", "already-passing", "exempt", "unverified"],
+                "description": "red: RED recorded. verified: RED then GREEN. already-passing: the behaviour existed and one passing run stands for both. exempt: no evidence needed. unverified: completed in warn mode without evidence; Phase 3 notes it."
+              },
+              "reason": {
+                "type": "string",
+                "description": "Why the outcome is already-passing, exempt or unverified."
+              },
+              "selfDeclared": {
+                "type": "boolean",
+                "description": "The exemption came from the todo itself (complete --no-tests), not from its type, the task type or the test policy. Phase 3 notes it."
+              }
+            }
+          },
           "estimatedMinutes": {
             "type": "integer",
             "minimum": 1,
             "description": "Optional Phase 1 estimate for the step. Phase 5 compares against actual duration."
+          },
+          "files": {
+            "type": "array",
+            "items": { "type": "string" },
+            "description": "The planning-output task's files[], carried by plan-todos.sh set. The drift check compares the files a step changed against it."
+          },
+          "creates": {
+            "type": "array",
+            "items": { "type": "string" },
+            "description": "The planning-output task's creates[], carried by plan-todos.sh set. The drift check counts these files as planned, like files[]."
+          },
+          "drift": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["outside", "notDone", "at"],
+            "description": "Set by plan-todos.sh drift when this step ended off-plan: the files it changed outside files[] and the number of planned items it did not do.",
+            "properties": {
+              "outside": { "type": "array", "items": { "type": "string" } },
+              "notDone": { "type": "integer", "minimum": 0 },
+              "at": { "type": "string", "format": "date-time" }
+            }
+          },
+          "needsReplan": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["because", "at"],
+            "description": "Set by plan-todos.sh drift on every pending step that depends, directly or transitively, on a step that drifted. Until confirmedAt is set, task-graph.mjs reports the step as waiting (reason needs-replan) and plan-todos.sh start refuses it; confirm-replan sets confirmedAt.",
+            "properties": {
+              "because": {
+                "type": "string",
+                "pattern": "^[tT][0-9]+$",
+                "description": "The step that drifted."
+              },
+              "at": { "type": "string", "format": "date-time" },
+              "reason": {
+                "type": "string",
+                "description": "What the drift was: files outside the plan, planned items not done, or a stated reason."
+              },
+              "confirmedAt": {
+                "type": "string",
+                "format": "date-time",
+                "description": "When someone confirmed the step's plan still holds."
+              },
+              "note": { "type": "string", "description": "What the confirmation decided." }
+            }
           }
         }
       }

package/pipeline/schemas/planning-output.schema.json

@@ -50,6 +50,52 @@
             "type": "array",
             "items": { "type": "string" },
             "description": "Analysis requirement ids this task implements or tests (BR-<slug>-NN, FG-NN). spec-consistency-gate.mjs reads them, with any id in the title or acceptance criteria, to check that every requirement has a task and a test and that every task traces to a requirement."
+          },
+          "creates": {
+            "type": "array",
+            "items": { "type": "string", "minLength": 1 },
+            "description": "Files this task brings into existence. validate-planning.mjs fails a plan in which two tasks create the same file, and warns when a task that edits a created file is not ordered after its creator."
+          },
+          "removes": {
+            "type": "array",
+            "items": { "type": "string", "minLength": 1 },
+            "description": "Files or symbols this task deletes, or renames away (the old name). A name with a '/' or a file extension is a file. validate-planning.mjs warns when another task edits, consumes or creates what this task removes."
+          },
+          "produces": {
+            "type": "array",
+            "items": { "type": "string", "minLength": 1 },
+            "description": "Interfaces or symbols this task introduces or changes the shape of. validate-planning.mjs warns when a task that consumes one is not ordered after its producer."
+          },
+          "consumes": {
+            "type": "array",
+            "items": { "type": "string", "minLength": 1 },
+            "description": "Interfaces or symbols this task calls or relies on without changing them. Paired against other tasks' produces and removes."
+          },
+          "complexity": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["score"],
+            "description": "Grounded size of the task, written by plan-complexity.mjs --write. A task with recommendSplit is split in Phase 1 before approval.",
+            "properties": {
+              "score": { "type": "integer", "minimum": 1, "maximum": 10 },
+              "recommendSplit": {
+                "type": "boolean",
+                "description": "score is above prefs.global.planComplexity.splitAbove."
+              },
+              "factors": {
+                "type": "object",
+                "additionalProperties": false,
+                "description": "The counts the score was built from. fanIn, crossModuleEdges and notIndexed are null when no code graph was available.",
+                "properties": {
+                  "files": { "type": "integer", "minimum": 0 },
+                  "fanIn": { "type": ["integer", "null"], "minimum": 0 },
+                  "modules": { "type": "integer", "minimum": 0 },
+                  "crossModuleEdges": { "type": ["integer", "null"], "minimum": 0 },
+                  "testSurface": { "type": "integer", "minimum": 0 },
+                  "notIndexed": { "type": ["integer", "null"], "minimum": 0 }
+                }
+              }
+            }
           }
         }
       }

package/pipeline/schemas/prefs.schema.json

@@ -568,7 +568,7 @@
             "fableEnabled": {
               "type": "boolean",
               "default": false,
-              "description": "Whether the fable rung is available at all on Claude Code. Ships OFF: the top rung is the most expensive thing a run can reach, and a cost control that is on by default is not a control. false makes every persona that declares preferredModel: fable dispatch on opus from the first call, with no dispatch attempt on fable and no error to recover from  -  a cost control, not a fallback. Claude Code only: Copilot CLI does not offer Fable 5, and on Codex CLI the fable rung means gpt-5.6 @ xhigh, which this switch deliberately does not touch. Turning it off also collapses the Phase 3 Claude Code reviewer panel from three to two (Reviewer 1 lands on opus, which Reviewer 2 already holds), and consensus.reviewerCount records 2."
+              "description": "Whether the fable rung is available at all on Claude Code. Ships OFF: the top rung is the most expensive thing a run can reach, and a cost control that is on by default is not a control. false makes every persona that declares preferredModel: fable dispatch on opus from the first call, with no dispatch attempt on fable and no error to recover from  -  a cost control, not a fallback. Claude Code only: Copilot CLI does not offer Fable, and on Codex CLI the fable rung means gpt-5.6 @ xhigh, which this switch deliberately does not touch. Turning it off also collapses the Phase 3 Claude Code reviewer panel from three to two (Reviewer 1 lands on opus, which Reviewer 2 already holds), and consensus.reviewerCount records 2."
             },
             "onDispatchError": {
               "type": "boolean",
@@ -577,6 +577,24 @@
             }
           }
         },
+        "reworkEscalation": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "Rework model escalation (multi-agent-refs/features/rework-escalation.md). On the last rework round the cap allows, or when review-delta.mjs reports findings still present after a fix, scripts/rework-escalation.mjs moves that one dev dispatch one rung up the model-fallback ladder, as a fresh agent given the findings and the prior report as files. It never escalates against the cost budget (warn or exceeded), never re-upgrades a run that fell back, never reaches fable while modelFallback.fableEnabled is off, and never passes maxTier; a refusal is recorded with its reason.",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "default": true,
+              "description": "Master switch. On by default: it fires only on the rounds where a failure costs most, moves one rung, and stops at maxTier. false keeps every rework round on the dev dispatch's rung."
+            },
+            "maxTier": {
+              "type": "string",
+              "enum": ["fable", "opus", "sonnet", "haiku"],
+              "default": "opus",
+              "description": "Highest rung an escalation may reach. fable also needs modelFallback.fableEnabled true."
+            }
+          }
+        },
         "modelRouting": {
           "type": "object",
           "description": "Policy-driven model selection. Ships DISABLED and changes no dispatch behaviour while enabled is false. Full contract and the reason `scope` has no host-session member: pipeline/schemas/route-config.schema.json. Written by /multi-agent:route-on, cleared of its on-state (not its rules) by /multi-agent:route-off, reported by /multi-agent:route-status.",
@@ -1151,6 +1169,19 @@
             }
           }
         },
+        "tddEvidence": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "Whether completing a plan todo needs a recorded RED run and GREEN run (plan-todos.sh tdd-red / tdd-green). Applies where Phase 2 iterates plan todos (prefs.global.planTodos.enabled); without it Phase 2 walks tasks[] and nothing is recorded. Docs and chore todos, component tasks, test policy none and complete --no-tests are exempt and recorded with a reason.",
+          "properties": {
+            "mode": {
+              "type": "string",
+              "enum": ["off", "warn", "enforce"],
+              "default": "warn",
+              "description": "off: no check. warn: complete warns and records tdd.outcome unverified, which Phase 3 notes. enforce: complete refuses a todo without the evidence."
+            }
+          }
+        },
         "planTodos": {
           "type": "object",
           "additionalProperties": false,
@@ -1163,10 +1194,24 @@
             }
           }
         },
+        "planComplexity": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "Grounded complexity score per Phase 1 plan task (pipeline/scripts/plan-complexity.mjs): counted from the files a task touches, their code-graph fan-in, the modules they span and their test surface, never from the planner's opinion. Phase 1 splits a task scoring above the threshold before the plan is approved.",
+          "properties": {
+            "splitAbove": {
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 10,
+              "default": 7,
+              "description": "A task whose score (1-10) is above this gets recommendSplit and is split before plan approval. Lower it to get smaller tasks; 10 never splits."
+            }
+          }
+        },
         "clarifyAmbiguous": {
           "type": "object",
           "additionalProperties": false,
-          "description": "v8.6+  -  Phase 0 Step 9 clarifying-question loop. Before Phase 1 starts, a cheap Haiku classifier scores task ambiguity (0-10) and emits up to N questions if score < threshold. Cost: ~$0.0025 per Haiku call. Off by default  -  flip on for teams burned by ambiguity-driven rework or when working on cross-team issues where the spec lives in someone else's head.",
+          "description": "v8.6+  -  Phase 0 Step 9 clarifying-question loop. Before Phase 1 starts, a cheap Haiku classifier scores task ambiguity (0-10) and emits up to N questions if score < threshold. Cost: ~$0.0004 per Haiku call. Off by default  -  flip on for teams burned by ambiguity-driven rework or when working on cross-team issues where the spec lives in someone else's head.",
           "properties": {
             "enabled": {
               "type": "boolean",
@@ -1793,6 +1838,75 @@
             "tailLines": 20
           }
         },
+        "editSyntaxGate": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "Parse the file an edit just wrote and tell the model when it no longer parses: swiftc -parse, node --check, python3 py_compile (bytecode to a temp dir), JSON.parse, bash -n, chosen by extension, each under a 2 s deadline. Kotlin and every other extension are skipped. The hook is pipeline/scripts/edit-syntax-gate.sh: PostToolUse on Claude Code (install/templates/claude-hooks.json) and on Codex apply_patch (codex-hooks.json, registered by host-hooks.mjs). Copilot CLI is not registered: as of 1.0.45 it discards postToolUse output. It never blocks: the message arrives as PostToolUse context and the hook exits 0. A parser that is not installed or runs past the deadline means no message.",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "default": false,
+              "description": "false = the hook is a no-op. true = every edited file with a supported extension is parsed after the edit."
+            }
+          },
+          "default": {
+            "enabled": false
+          }
+        },
+        "mutationProbe": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "Phase 3 optional review signal: pipeline/scripts/mutation-probe.mjs makes small textual mutants (a flipped comparison, a negated boolean literal, a swapped + / -, a changed returned constant) on lines the diff added or changed, runs the test command against each in a disposable worktree, and reports every mutant the tests did not catch as a finding. The caller's checkout is never written.",
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "default": false,
+              "description": "false = Phase 3 skips the probe. true = Phase 3 runs it when a test command is known and merges each survivor as an important finding (the tests do not catch that change)."
+            },
+            "maxMutants": {
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 50,
+              "default": 5,
+              "description": "Upper bound on mutants per run. Each one costs a full test-command run, so the bound is the cost bound."
+            },
+            "timeoutSec": {
+              "type": "integer",
+              "minimum": 5,
+              "maximum": 3600,
+              "default": 120,
+              "description": "Deadline for one test-command run. A mutant whose run times out is reported as timed out, not as a survivor."
+            }
+          },
+          "default": {
+            "enabled": false,
+            "maxMutants": 5,
+            "timeoutSec": 120
+          }
+        },
+        "depBump": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "/multi-agent:deps (multi-agent-refs/features/dep-bump.md): bump one dependency, build and test, fix only the breakage the bump caused, review and open a PR. pipeline/scripts/dep-bump.mjs applies the bump and reads these settings through its config command.",
+          "properties": {
+            "maxFixRounds": {
+              "type": "integer",
+              "minimum": 0,
+              "maximum": 5,
+              "default": 2,
+              "description": "Fix rounds a red build or test run may take after the bump. When the cap is reached and the run is still red, the worktree is rolled back to the recorded base and the run reports what broke instead of committing. 0 means a bump that does not pass as-is is rolled back at once."
+            },
+            "queryRegistry": {
+              "type": "boolean",
+              "default": true,
+              "description": "true = the candidate list asks the package registry for newer versions through the ecosystem's own tool (npm view, git ls-remote, Maven or PyPI metadata). A failed query is noted per package and the list falls back to the manifests, lockfiles and local caches. false = offline listing only."
+            }
+          },
+          "default": {
+            "maxFixRounds": 2,
+            "queryRegistry": true
+          }
+        },
         "bulkRead": {
           "type": "object",
           "additionalProperties": false,
@@ -1915,7 +2029,7 @@
               },
               "localRepo": {
                 "type": "string",
-                "description": "Repo that localPath is relative to, e.g. $HOME/multi-agent-plugins. Without it localPath reads as relative to the current repo, which is how both entries came to point at the UPSTREAM tree and stayed wrong through a rename: nothing in the config said which checkout held the derived copies."
+                "description": "Repo that localPath is relative to, e.g. $HOME/multi-agent-plugins. Without it localPath reads as relative to the current repo, so an entry can silently point at the UPSTREAM tree: nothing else in the config says which checkout holds the derived copies."
               },
               "upstreamMarketplace": {
                 "type": "string",
@@ -2276,6 +2390,11 @@
               "type": "string",
               "default": "Relates",
               "description": "The link type between a story and its channel clones, by this site's name."
+            },
+            "dependencyLinkType": {
+              "type": "string",
+              "default": "Blocks",
+              "description": "The link type written from a prerequisite story to the story that depends on it (a `Depends on:` clause in the analysis document), by this site's name. The prerequisite is the link's source."
             }
           }
         },

package/pipeline/scripts/_code-graph.mjs

@@ -212,8 +212,8 @@ const IMPORT_LINE_RE = /^[ \t]*import[ \t].*$/gm;
  *
  * Swift's submodule form - `import class DesignKit.PrimaryButton` - otherwise
  * feeds the `class` definition pattern, which registers the MODULE name as a
- * declared type. On a large Swift app that alone lifted two module names into
- * the top ten god-nodes, which is how the bug surfaced. Imports are extracted
+ * declared type. On a large Swift app that alone lifts module names into the
+ * top ten god-nodes. Imports are extracted
  * before this runs, so nothing is lost.
  *
  * @param {string} stripped - output of stripCode

package/pipeline/scripts/_jsonl.mjs

@@ -0,0 +1,151 @@
+/**
+ * @file _jsonl.mjs  -  read a JSON-lines file one record at a time, by line number.
+ *
+ * Session transcripts are append-only logs that can run to hundreds of
+ * megabytes, are sometimes cut mid-line by a crash, and occasionally carry a
+ * record far larger than the rest (a whole file pasted into a tool result).
+ * So the reader streams, stops at a byte cap, refuses a single line over a
+ * per-line cap without parsing it, and reports every line it could not use
+ * instead of throwing. Line numbers are physical lines of the file, blank ones
+ * included, so `file:line` points at what an editor shows.
+ *
+ * Each yielded row is `{ line, record }` for a parsed object, or
+ * `{ line, reason }` for a line that was not used: `parse` (not JSON),
+ * `not-object` (JSON, but not an object), `oversized` (over the per-line cap)
+ * or `truncated` (the line the byte cap cut through). `stats` is filled in as
+ * the file is read.
+ *
+ * @module pipeline/scripts/_jsonl
+ */
+
+import { createReadStream, statSync } from "node:fs";
+
+export const DEFAULT_MAX_BYTES = 256 * 1024 * 1024;
+export const DEFAULT_MAX_LINE_BYTES = 8 * 1024 * 1024;
+
+/**
+ * @param {string} path
+ * @param {{maxBytes?:number, maxLineBytes?:number, stats?:object}} [opts]
+ */
+export async function* readJsonl(path, opts = {}) {
+  const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
+  const maxLineBytes = opts.maxLineBytes ?? DEFAULT_MAX_LINE_BYTES;
+  const stats = opts.stats ?? {};
+  Object.assign(stats, {
+    lines: 0,
+    malformed: 0,
+    oversized: 0,
+    truncated: false,
+    unreadable: false,
+    bytes: 0,
+  });
+
+  let size;
+  try {
+    const st = statSync(path);
+    if (!st.isFile()) throw new Error("not a regular file");
+    size = st.size;
+  } catch {
+    stats.unreadable = true;
+    return;
+  }
+  const capped = size > maxBytes;
+  stats.bytes = Math.min(size, maxBytes);
+  if (stats.bytes === 0) return;
+
+  const classify = (n, text) => {
+    let value;
+    try {
+      value = JSON.parse(text);
+    } catch {
+      stats.malformed++;
+      return { line: n, reason: "parse" };
+    }
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+      stats.malformed++;
+      return { line: n, reason: "not-object" };
+    }
+    return { line: n, record: value };
+  };
+
+  // A line becomes {n, text} or {n, oversized}. Bytes of the current line are
+  // kept only while it is within the cap; past it they are dropped as they
+  // arrive, so a single huge line costs one chunk of memory, not its length.
+  // One line is held back so the last one is known before it is classified:
+  // when the byte cap cut the file, that line is incomplete.
+  const NL = 0x0a;
+  let parts = [];
+  let partBytes = 0;
+  let over = false;
+  let n = 0;
+  let held = null;
+  const finish = () => {
+    n++;
+    let line;
+    if (over) line = { n, oversized: true };
+    else {
+      let text = Buffer.concat(parts, partBytes).toString("utf8");
+      if (text.endsWith("\r")) text = text.slice(0, -1);
+      line = Buffer.byteLength(text) > maxLineBytes ? { n, oversized: true } : { n, text };
+    }
+    parts = [];
+    partBytes = 0;
+    over = false;
+    return line;
+  };
+  const emit = (line) => {
+    if (line.oversized) {
+      stats.oversized++;
+      return { line: line.n, reason: "oversized" };
+    }
+    return line.text.trim() ? classify(line.n, line.text) : null;
+  };
+  const append = (buf) => {
+    if (over || buf.length === 0) return;
+    if (partBytes + buf.length > maxLineBytes + 1) {
+      over = true;
+      parts = [];
+      partBytes = 0;
+      return;
+    }
+    parts.push(buf);
+    partBytes += buf.length;
+  };
+
+  const input = createReadStream(path, { start: 0, end: stats.bytes - 1 });
+  try {
+    for await (const chunk of input) {
+      let from = 0;
+      for (let at = chunk.indexOf(NL); at !== -1; at = chunk.indexOf(NL, from)) {
+        append(chunk.subarray(from, at));
+        from = at + 1;
+        const line = finish();
+        if (held) {
+          const row = emit(held);
+          if (row) yield row;
+        }
+        held = line;
+      }
+      append(chunk.subarray(from));
+    }
+  } catch {
+    stats.unreadable = true;
+  }
+  if (partBytes > 0 || over) {
+    const line = finish();
+    if (held) {
+      const row = emit(held);
+      if (row) yield row;
+    }
+    held = line;
+  }
+  stats.lines = n;
+  if (!held || (!held.oversized && !held.text.trim())) return;
+  if (capped) {
+    stats.truncated = true;
+    yield { line: held.n, reason: "truncated" };
+    return;
+  }
+  const row = emit(held);
+  if (row) yield row;
+}