@mmerterden/multi-agent-pipeline 20.23.2 → 20.25.0

package/pipeline/scripts/capture-flush.sh

@@ -5,14 +5,14 @@
 #
 # WHY THIS EXISTS
 #
-# Every persistent write used to live in Phase 5: triage ingest, the learnings
-# ledger distill, the knowledge-base append, the code-graph refresh. And Phase 5
-# is, by the pipeline's own admission in features/code-graph.md, the phase a run
-# is LEAST likely to reach. A run killed in Phase 3, a session that hits its
-# context ceiling, a crash after review - each one threw away everything it had
-# established, and the next run on the same repo rediscovered it from scratch.
+# The persistent writes are triage ingest, the learnings ledger distill, the
+# knowledge-base append and the code-graph refresh. Phase 5 is, per
+# features/code-graph.md, the phase a run is LEAST likely to reach. A run killed
+# in Phase 3, a session that hits its context ceiling, a crash after review -
+# if the writes waited for Phase 5, each would throw away everything it had
+# established, and the next run on the same repo would rediscover it.
 #
-# So the writes move here, and Phase 5 becomes the LAST flush rather than the
+# So the writes happen here, and Phase 5 is the LAST flush rather than the
 # only one. Phase boundaries call this, and so does SessionEnd. Nothing about
 # the trigger depends on a model noticing that a moment qualifies: it hangs on
 # a phase transition and on process exit, both objectively visible without any

package/pipeline/scripts/code-graph-rules/android.json

@@ -110,5 +110,5 @@
     "Unit",
     "Any"
   ],
-  "note": "Kotlin declaration modifiers are not separate patterns: `data class Foo`, `sealed class Foo` and `annotation class Foo` all still contain `class Foo`, so one pattern covers them. `enum` allows an optional `class` between keyword and name, which is the only shape that differs between Java (`enum Foo`) and Kotlin (`enum class Foo`). `object` matches the named form only - `companion object {` and the anonymous `object : Iface {` carry no name and correctly produce nothing. Imports are fully qualified on the JVM, so the pattern captures the final segment: `import com.example.ui.Button` resolves against the file named Button.kt, which is how the engine links a file-level import edge. A star import (`import com.example.ui.*`) captures nothing, deliberately - it names a package, and pointing every star import at an arbitrary file in that package would invent an edge nobody wrote. Pattern ORDER is load-bearing: `enum class Cabin` matches both the enum pattern and the class pattern, and the first match to register the name decides its kind, so the more specific patterns are listed first. With class first, every Kotlin enum was filed as a class and the report's symbol-kind census was wrong."
+  "note": "Kotlin declaration modifiers are not separate patterns: `data class Foo`, `sealed class Foo` and `annotation class Foo` all still contain `class Foo`, so one pattern covers them. `enum` allows an optional `class` between keyword and name, which is the only shape that differs between Java (`enum Foo`) and Kotlin (`enum class Foo`). `object` matches the named form only - `companion object {` and the anonymous `object : Iface {` carry no name and correctly produce nothing. Imports are fully qualified on the JVM, so the pattern captures the final segment: `import com.example.ui.Button` resolves against the file named Button.kt, which is how the engine links a file-level import edge. A star import (`import com.example.ui.*`) captures nothing, deliberately - it names a package, and pointing every star import at an arbitrary file in that package would invent an edge nobody wrote. Pattern ORDER is load-bearing: `enum class Cabin` matches both the enum pattern and the class pattern, and the first match to register the name decides its kind, so the more specific patterns are listed first. With class first, every Kotlin enum would be filed as a class and the report's symbol-kind census would be wrong."
 }

package/pipeline/scripts/cost-analyze.mjs

@@ -46,20 +46,12 @@
  *   2   usage error
  */
 
-import {
-  createReadStream,
-  existsSync,
-  mkdirSync,
-  readdirSync,
-  readFileSync,
-  statSync,
-  writeFileSync,
-} from "node:fs";
-import { createInterface } from "node:readline";
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { costUsd } from "./_cost.mjs";
+import { readJsonl } from "./_jsonl.mjs";
 import { runMain } from "../lib/fatal.mjs";
 import { messages, outputLanguage } from "../lib/user-facing.mjs";
 
@@ -176,30 +168,37 @@ const asNumber = (v, fallback) => {
 const TABLE = JSON.parse(readFileSync(join(HERE, "cost-table.json"), "utf-8"));
 
 /**
- * modelId -> rung. The transcripts carry the wire value ("claude-opus-5"); the
- * price table is keyed by rung. Matching on modelId rather than on the rung
- * name is what keeps a generation move a one-line data edit, which is the
- * contract cost-table.json states for itself.
+ * Wire ID -> rate. Transcripts carry the wire value ("claude-opus-5-5"); the
+ * price table is keyed by rung, so each rung's `modelId` and `copilotModelId`
+ * map to its rate, `legacy` adds previous-generation IDs at their own rates,
+ * and `models` overrides the rung rate for a dispatched ID whose model is not
+ * the rung's (Copilot CLI's older models bill at their own rate). Matching on
+ * the ID rather than the rung name keeps a generation move a data edit, which
+ * is the contract cost-table.json states for itself.
  */
-const RUNG_BY_MODEL_ID = new Map(
-  Object.entries(TABLE.prices || {}).map(([rung, p]) => [p.modelId, rung]),
-);
+const RATE_BY_MODEL_ID = new Map([
+  ...Object.entries(TABLE.legacy || {}),
+  ...Object.values(TABLE.prices || {}).flatMap((p) =>
+    [p.modelId, p.copilotModelId].filter(Boolean).map((id) => [id, p]),
+  ),
+  ...Object.entries(TABLE.models || {}),
+]);
+const IDS_LONGEST_FIRST = [...RATE_BY_MODEL_ID.keys()].sort((a, b) => b.length - a.length);
 
 function rateFor(modelId) {
   if (!modelId) return null;
-  const rung = RUNG_BY_MODEL_ID.get(modelId);
-  if (rung) return TABLE.prices[rung];
-  // A dated snapshot id (an alias plus a release date suffix) prices as its
-  // alias - written as a shape rather than as a literal, because
+  const exact = RATE_BY_MODEL_ID.get(modelId);
+  if (exact) return exact;
+  // A dated snapshot id (a known id, a hyphen, a release date) prices as that
+  // id; the longest known id wins, so a current id is never read as a shorter
+  // legacy one. Written as a shape rather than as a literal, because
   // smoke-model-fallback.sh reads every model id in this tree and requires it
-  // to exist in cost-table.json, and an EXAMPLE id is exactly the kind of thing
-  // that would make that gate wrong about what ships. An
-  // id nobody knows prices as nothing and is COUNTED as unpriced rather than
-  // folded in at zero.
-  for (const [id, rung] of RUNG_BY_MODEL_ID) {
-    if (modelId.startsWith(id)) return TABLE.prices[rung];
-  }
-  return null;
+  // to exist in cost-table.json. An id nobody knows prices as nothing and is
+  // COUNTED as unpriced rather than folded in at zero.
+  const id = IDS_LONGEST_FIRST.find(
+    (known) => modelId.startsWith(`${known}-`) || modelId.startsWith(`${known}[`),
+  );
+  return id ? RATE_BY_MODEL_ID.get(id) : null;
 }
 
 /**
@@ -263,20 +262,10 @@ async function readSeries({ days }) {
   const unpricedModels = new Set();
 
   for (const f of files) {
-    let rl;
-    try {
-      rl = createInterface({ input: createReadStream(f.path), crlfDelay: Infinity });
-    } catch {
-      continue;
-    }
-    for await (const line of rl) {
-      if (!line || line.charCodeAt(0) !== 123) continue;
-      let d;
-      try {
-        d = JSON.parse(line);
-      } catch {
-        continue;
-      }
+    // Uncapped: a cost total that silently stops at a byte cap under-reports.
+    for await (const row of readJsonl(f.path, { maxBytes: Infinity, maxLineBytes: Infinity })) {
+      const d = row.record;
+      if (!d) continue;
       const msg = d.message;
       const u = msg?.usage;
       if (!u) continue;

package/pipeline/scripts/cost-table.json

@@ -1,38 +1,54 @@
 {
-  "_readme": "Per-model unit prices in USD per million tokens. Source: Anthropic public pricing (Claude-family rows verified 2026-07-28 against the current model table; the gpt-* rows remain approximate). Update when Anthropic publishes new tiers. Rung names (fable / opus / sonnet / haiku) are the pipeline's stable identifiers and `modelId` is the wire value they currently resolve to - dispatch reads the rung, so a generation move is a `modelId` edit here plus the phase specs, never a rename of the rungs. `provider` says which layer a rung belongs to: an `anthropic` rung is reachable from every call site; `free-chain` (the `free` rung) only from bulk-read and delegate, where the multi-agent-toolkit's verified free-model chain answers and the caller's default takes over when it escalates; any other provider from no call site. model-dispatch.sh enforces this. Unknown models render USD as ' - ' and emit a footnote  -  never block PR-body generation. cacheReadPerMtok is the discounted rate for prompt-cache hits (~10% of inPerMtok); the renderer prices a phase's tokens_cached at this rate when the tracker records it, so resume/cache reuse is visible in the ledger.",
+  "_readme": "Per-model unit prices in USD per million tokens. Source: Anthropic public pricing (Claude-family rows verified 2026-10-14 against the current model table; the gpt-* rows remain approximate). Update when Anthropic publishes new tiers. Rung names (fable / opus / sonnet / haiku) are the pipeline's stable identifiers and `modelId` is the wire value they currently resolve to - dispatch reads the rung, so a generation move is a `modelId` edit here plus the phase specs, never a rename of the rungs. `copilotModelId`, where present, is the name Copilot CLI dispatches for the rung: one of the dotted IDs `copilot help config` lists (Copilot rejects any other as not available), the newest of the rung's family there. A rung without one (fable) is not offered on Copilot CLI, where its personas take the opus rung. smoke-model-fallback.sh checks the IDs against `copilot help config` when the CLI is installed. `provider` says which layer a rung belongs to: an `anthropic` rung is reachable from every call site; `free-chain` (the `free` rung) only from bulk-read and delegate, where the multi-agent-toolkit's verified free-model chain answers and the caller's default takes over when it escalates; any other provider from no call site. model-dispatch.sh enforces this. Unknown models render USD as ' - ' and emit a footnote  -  never block PR-body generation. cacheReadPerMtok is the discounted rate for prompt-cache hits, published per model and well below inPerMtok; the renderer prices a phase's tokens_cached at this rate when the tracker records it, so resume/cache reuse is visible in the ledger.",
   "schemaVersion": "1.1.0",
+  "_legacyReadme": "`legacy` prices previous-generation wire IDs a transcript may still name, keyed by the exact ID (a dated snapshot of one prices as it). Dispatch never reads it; cost-analyze consults it after the current `modelId` values and the `models` map. Official Anthropic prices per million tokens.",
+  "legacy": {
+    "claude-fable-5": { "inPerMtok": 10.0, "outPerMtok": 50.0, "cacheReadPerMtok": 1.0 },
+    "claude-opus-5": { "inPerMtok": 5.0, "outPerMtok": 25.0, "cacheReadPerMtok": 0.5 },
+    "claude-sonnet-5": { "inPerMtok": 2.0, "outPerMtok": 10.0, "cacheReadPerMtok": 0.2 },
+    "claude-haiku-4-5": { "inPerMtok": 1.0, "outPerMtok": 5.0, "cacheReadPerMtok": 0.1 }
+  },
+  "_modelsReadme": "`models` prices wire IDs a rung dispatches whose model is not the rung's `modelId`: Copilot CLI's `copilotModelId` values name an older model than the rung's, billed at that model's own rate. Keyed by the exact ID; cost-analyze consults it before the rung rates, so a Copilot CLI session is priced at its own model. Official Anthropic prices per million tokens, verified 2026-10-14.",
+  "models": {
+    "claude-opus-4.7": { "inPerMtok": 5.0, "outPerMtok": 25.0, "cacheReadPerMtok": 0.5 },
+    "claude-sonnet-4.6": { "inPerMtok": 3.0, "outPerMtok": 15.0, "cacheReadPerMtok": 0.3 },
+    "claude-haiku-4.5": { "inPerMtok": 1.0, "outPerMtok": 5.0, "cacheReadPerMtok": 0.1 }
+  },
   "prices": {
     "fable": {
       "inPerMtok": 10.0,
       "outPerMtok": 50.0,
-      "cacheReadPerMtok": 1.0,
-      "modelId": "claude-fable-5",
+      "cacheReadPerMtok": 0.25,
+      "modelId": "claude-fable-5-1",
       "provider": "anthropic",
-      "note": "Top tier (restored v10.6.0)  -  architects, Reviewer 1, triage. Verified against Anthropic pricing 2026-07-02."
+      "note": "Top tier - architects, Reviewer 1 and triage on Claude Code. Claude Fable 5.1, verified against Anthropic pricing 2026-10-14."
     },
     "opus": {
-      "inPerMtok": 5.0,
-      "outPerMtok": 25.0,
-      "cacheReadPerMtok": 0.5,
-      "modelId": "claude-opus-5",
+      "inPerMtok": 4.0,
+      "outPerMtok": 20.0,
+      "cacheReadPerMtok": 0.2,
+      "modelId": "claude-opus-5-5",
+      "copilotModelId": "claude-opus-4.7",
       "provider": "anthropic",
-      "note": "Second tier - Reviewer 1 and triage on Copilot CLI, and the opus rung of the fable -> opus -> sonnet fallback ladder. Same rate as the Opus 4.8 it replaces, so the ledger needed no reprice on the generation move. Claude Opus 5 draws on a rate-limit pool SEPARATE from the combined Opus 4.x pool - moving traffic here neither frees headroom on the old bucket nor inherits it."
+      "note": "Second tier - Reviewer 2 on Claude Code, Reviewer 1 and triage on Copilot CLI, and the opus rung of the fable -> opus -> sonnet fallback ladder. Claude Opus 5.5, verified against Anthropic pricing 2026-10-14."
     },
     "sonnet": {
-      "inPerMtok": 3.0,
-      "outPerMtok": 15.0,
-      "cacheReadPerMtok": 0.3,
-      "modelId": "claude-sonnet-5",
+      "inPerMtok": 2.0,
+      "outPerMtok": 10.0,
+      "cacheReadPerMtok": 0.1,
+      "modelId": "claude-sonnet-5-5",
+      "copilotModelId": "claude-sonnet-4.6",
       "provider": "anthropic",
-      "note": "Floor tier for Claude-family dispatch - Reviewer 3 on both hosts, and the terminal rung of the fallback ladder. Priced at the standard 3/15 rather than the 2/10 introductory rate that runs through 2026-08-31: over-reporting during the intro window is the safe direction for a cost ledger, and it needs no dated edit when the intro ends."
+      "note": "Floor tier for Claude-family dispatch - Reviewer 3 on both hosts, the dev phase, and the terminal rung of the fallback ladder. Claude Sonnet 5.5, verified against Anthropic pricing 2026-10-14."
     },
     "haiku": {
-      "inPerMtok": 1.0,
-      "outPerMtok": 5.0,
-      "cacheReadPerMtok": 0.1,
-      "modelId": "claude-haiku-4-5",
+      "inPerMtok": 0.5,
+      "outPerMtok": 2.5,
+      "cacheReadPerMtok": 0.05,
+      "modelId": "claude-haiku-5-5",
+      "copilotModelId": "claude-haiku-4.5",
       "provider": "anthropic",
-      "note": "Speed tier - task-clarifier and other latency-sensitive dispatches, plus the terminal rung of the fallback ladder. Named by its alias like every other rung here rather than by a dated snapshot, so the four rungs stay comparable at a glance."
+      "note": "Speed tier - task-clarifier and other latency-sensitive dispatches, plus the terminal rung of the fallback ladder. Claude Haiku 5.5 is priced in two tiers by prompt length (in 0.10, cache-read 0.01, out 0.50 up to 100k input tokens; in 0.50, cache-read 0.05, out 2.50 above); this table holds one rate per rung, so it carries the over-100k tier: over-reporting is the safe direction for a cost ledger. Verified against Anthropic pricing 2026-10-14."
     },
     "gpt-5.4": {
       "inPerMtok": 10.0,

package/pipeline/scripts/decisions-log.mjs

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+/**
+ * decisions-log.mjs  -  append to and render `state.decisions[]`, the list of
+ * choices an autopilot run made where an attended run would have asked.
+ *
+ * Writers: a picker resolved without asking (picker-contract.md, autopilot),
+ * a split consensus taken on the triage verdict, a model fallback taken
+ * without the date-gate question, a rework dispatched one rung up
+ * (rework-escalation.mjs). gate-ledger.mjs park and review-decision-gate.mjs
+ * write their own entries through lib/decisions.mjs.
+ *
+ * `add` writes only while the quality gates are active (lib/unattended.mjs
+ * gatesActive): in an attended run the person answered, so there is nothing to
+ * report back. Writes take the write-state lock through gate-ledger.mjs
+ * updateState.
+ *
+ * `render` prints the markdown section Phase 5 puts in its report, redacted,
+ * and nothing at all for a run with no entries.
+ *
+ * Usage:
+ *   decisions-log.mjs add --what <text> --why <text> --cost-if-wrong <text> --phase <0-5>
+ *                         [--state <path>] [--repo <dir>]
+ *   decisions-log.mjs render [--state <path>] [--repo <dir>]
+ *
+ * Exit: 0 written, skipped (attended) or rendered · 2 usage · 3 no state or I/O error
+ *
+ * @module pipeline/scripts/decisions-log
+ */
+
+import { writeFileSync } from "node:fs";
+
+import { decisionEntry, renderDecisions, withDecision } from "../lib/decisions.mjs";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+import { readRegularJson } from "../lib/regular-file.mjs";
+import { gateContext, resolveStatePath, updateState } from "./gate-ledger.mjs";
+
+class UsageError extends Error {}
+
+function parse(argv) {
+  const opts = {};
+  for (let i = 1; i < argv.length; i++) {
+    const a = argv[i];
+    if (!a.startsWith("--")) throw new UsageError(`unexpected argument ${a}`);
+    const v = argv[++i];
+    if (v === undefined) throw new UsageError(`${a} needs a value`);
+    opts[a.slice(2)] = v;
+  }
+  return opts;
+}
+
+/**
+ * Append one entry when the gates are active. Resolves to whether it wrote.
+ *
+ * @param {{what: string, why: string, costIfWrong: string, phase: number}} input
+ * @param {{statePath?: string|null, repo?: string|null, env?: Record<string,string|undefined>}} [opts]
+ */
+export async function addDecision(
+  input,
+  { statePath = null, repo = null, env = process.env } = {},
+) {
+  const entry = decisionEntry(input);
+  const ctx = gateContext({ statePath, repo, env });
+  if (!ctx.active || !ctx.statePath) return { written: false, entry };
+  await updateState(ctx.statePath, (s) => withDecision(s, entry));
+  return { written: true, entry, statePath: ctx.statePath };
+}
+
+async function main(argv) {
+  const mode = argv[0];
+  const opts = parse(argv);
+  if (mode === "add") {
+    let input;
+    try {
+      input = decisionEntry({
+        what: opts.what,
+        why: opts.why,
+        costIfWrong: opts["cost-if-wrong"],
+        phase: opts.phase,
+      });
+    } catch (err) {
+      throw new UsageError(err.message);
+    }
+    await addDecision(input, { statePath: opts.state || null, repo: opts.repo || process.cwd() });
+    return 0;
+  }
+  if (mode === "render") {
+    const path = resolveStatePath({ statePath: opts.state, repo: opts.repo || process.cwd() });
+    if (!path) {
+      process.stderr.write("decisions-log: no run state found\n");
+      return 3;
+    }
+    const state = readRegularJson(path, null);
+    if (!state || typeof state !== "object") {
+      process.stderr.write(`decisions-log: cannot read ${path}\n`);
+      return 3;
+    }
+    const md = renderDecisions(state.decisions);
+    if (md) writeFileSync(1, md);
+    return 0;
+  }
+  throw new UsageError("mode must be add or render");
+}
+
+if (invokedDirectly(import.meta.url)) {
+  main(process.argv.slice(2)).then(
+    (code) => process.exit(code),
+    (err) => {
+      process.stderr.write(`decisions-log: ${err.message}\n`);
+      if (err instanceof UsageError) {
+        process.stderr.write(
+          "usage: decisions-log.mjs add --what t --why t --cost-if-wrong t --phase n [--state p] | render [--state p]\n",
+        );
+        process.exit(2);
+      }
+      process.exit(3);
+    },
+  );
+}