@mmerterden/multi-agent-pipeline 20.23.1 → 20.24.0

package/pipeline/scripts/analysis-story-tree.mjs

@@ -50,6 +50,7 @@
  *
  * Usage:
  *   analysis-story-tree.mjs <analysis.md> [--json] [--brief] [--prefs FILE] [--page-url URL]
+ *   analysis-story-tree.mjs --save-subtasks "<type>[,<type>...]"|none [--prefs FILE]
  *
  * Exit: 0 planned, 2 nothing to plan, 3 usage, 4 the document is not issue-ready.
  */
@@ -60,6 +61,8 @@ import { createHash } from "node:crypto";
 import { homedir } from "node:os";
 import { dirname, join } from "node:path";
 import { hostCommand } from "../lib/user-facing.mjs";
+import { lockTimeoutFrom, updateJsonFileSync } from "../lib/json-file-lock.mjs";
+import { analyzeGraph, cyclicComponentOf } from "../lib/task-graph.mjs";
 import { resolveConstitution } from "./constitution.mjs";
 import { gateContext, recordQuietly } from "./gate-ledger.mjs";
 import {
@@ -81,7 +84,11 @@ const PREFS_PATH =
     : join(homedir(), ".claude", "multi-agent-preferences.json");
 const pageUrlFlag = argv.indexOf("--page-url");
 const PAGE_URL = pageUrlFlag !== -1 ? argv[pageUrlFlag + 1] : null;
-const valueIdx = new Set([prefsFlag, pageUrlFlag].filter((i) => i !== -1).map((i) => i + 1));
+const saveFlag = argv.indexOf("--save-subtasks");
+const SAVE_SUBTASKS = saveFlag !== -1 ? (argv[saveFlag + 1] ?? "") : null;
+const valueIdx = new Set(
+  [prefsFlag, pageUrlFlag, saveFlag].filter((i) => i !== -1).map((i) => i + 1),
+);
 const file = argv.find((a, i) => !a.startsWith("--") && !valueIdx.has(i));
 
 function die(msg, code) {
@@ -162,6 +169,9 @@ export function identityLabel(prefix, docId, sourceIds) {
  *   issueTree.epicLinkFieldId    the epic-link field, when not discovered
  *   issueTree.cloneByChannel     one linked story per channel in channelComponents
  *   issueTree.cloneLinkType      the link type between a story and its clones
+ *   issueTree.storyPanels        story section panel colours, or false for none
+ *   issueTree.subtaskRolesDeclined  the person chose no sub-tasks; do not ask again
+ *   issueTree.dependencyLinkType  the link type written for a story dependency
  */
 const ISSUE_TREE_SETTINGS = [
   "issueTree.enabled",
@@ -177,6 +187,9 @@ const ISSUE_TREE_SETTINGS = [
   "issueTree.epicLinkFieldId",
   "issueTree.cloneByChannel",
   "issueTree.cloneLinkType",
+  "issueTree.storyPanels",
+  "issueTree.subtaskRolesDeclined",
+  "issueTree.dependencyLinkType",
 ];
 
 function readPrefs() {
@@ -206,6 +219,100 @@ export function subtaskTypeFor(setting, role) {
   return null;
 }
 
+const DEPENDS_ON = /\bdepends on:([^|\n]*)/gi;
+
+/**
+ * The `Depends on: <ids>` clauses of a document's requirement rows.
+ *
+ * A clause belongs to the table row it sits in, in any cell, so a trailing
+ * column under any header works and so does a note appended to a cell. The
+ * row's subject is the first requirement id left once the clause is removed.
+ * Clauses outside a table row are not read: without a row there is no subject.
+ *
+ * `stripped` is the text with every read clause removed. Ids are defined by
+ * where they appear, so a reference left in place would define the id it
+ * points at, and a dependency on a requirement nobody wrote would plan a story
+ * for it instead of being reported.
+ *
+ * @returns {{deps: Map<string, Set<string>>, stripped: string}}
+ */
+export function dependencyClauses(text, corporate) {
+  const atom = corporate ? ATOM_CORP : ATOM_GLOBAL;
+  const deps = new Map();
+  const lines = text.split("\n");
+  for (const [i, line] of lines.entries()) {
+    if (!/^\s*\|/.test(line) || !/\bdepends on:/i.test(line)) continue;
+    const refs = [];
+    const rest = line.replace(DEPENDS_ON, (clause, ids) => {
+      const found = ids.match(atom);
+      if (!found) return clause;
+      refs.push(...found.map((x) => x.toUpperCase()));
+      return "";
+    });
+    const subject = (rest.match(atom) || [])[0]?.toUpperCase();
+    if (!subject || !refs.length) continue;
+    lines[i] = rest;
+    for (const r of refs) {
+      if (r === subject) continue;
+      if (!deps.has(subject)) deps.set(subject, new Set());
+      deps.get(subject).add(r);
+    }
+  }
+  return { deps, stripped: lines.join("\n") };
+}
+
+/**
+ * Requirement dependencies, read as story-to-story edges and checked as a
+ * graph. A story depends on another when one of its source ids depends on a
+ * source id of the other; two ids of one story give no edge. A dependency on
+ * an id no story covers, and a cycle between stories, are problems: the plan
+ * reports them and is not `ok`. An edge whose two stories sit in one strongly
+ * connected component carries `onCycle: true`; such an edge has no order to
+ * record, so it is never linked and never counted as a write.
+ */
+function storyDependencies(stories, reqDeps) {
+  const coverers = new Map();
+  for (const s of stories)
+    for (const id of s.sourceIds) {
+      if (!coverers.has(id)) coverers.set(id, []);
+      coverers.get(id).push(s);
+    }
+  const edges = new Map();
+  const missing = [];
+  for (const s of stories) {
+    for (const id of s.sourceIds) {
+      for (const dep of reqDeps.get(id) || []) {
+        const targets = coverers.get(dep);
+        if (!targets) {
+          missing.push({ id, dependsOn: dep });
+          continue;
+        }
+        for (const t of targets) {
+          if (t === s) continue;
+          const key = `${s.label}>${t.label}`;
+          if (!edges.has(key))
+            edges.set(key, { dependent: s.label, prerequisite: t.label, via: [] });
+          const via = `${dep} -> ${id}`;
+          if (!edges.get(key).via.includes(via)) edges.get(key).via.push(via);
+        }
+      }
+    }
+  }
+  for (const s of stories) {
+    const on = [...edges.values()].filter((e) => e.dependent === s.label);
+    if (on.length) s.dependsOn = on.map((e) => e.prerequisite);
+  }
+  const graph = stories.map((s) => ({ id: s.label, deps: s.dependsOn || [], status: "pending" }));
+  const { cycles } = analyzeGraph(graph);
+  const component = cyclicComponentOf(graph);
+  for (const e of edges.values()) {
+    const c = component.get(e.dependent);
+    if (c !== undefined && c === component.get(e.prerequisite)) e.onCycle = true;
+  }
+  const unique = [...new Map(missing.map((m) => [`${m.id}>${m.dependsOn}`, m])).values()];
+  return { edges: [...edges.values()], cycles, missing: unique };
+}
+
 /** Every open placeholder that makes a document unfit to become work. */
 export function openItems(text) {
   const found = [];
@@ -258,10 +365,11 @@ export function coverageOf(defined, nodes, corporate = false) {
   };
 }
 
-export function plan(text, prefs, { pageUrl = null } = {}) {
-  const fm = frontMatter(text);
+export function plan(source, prefs, { pageUrl = null } = {}) {
+  const fm = frontMatter(source);
   const profile = (fm.profile || "global").toLowerCase();
   const corporate = profile === "corporate";
+  const { deps: reqDeps, stripped: text } = dependencyClauses(source, corporate);
   const secs = sections(text);
 
   const atomRe = corporate ? ATOM_CORP : ATOM_GLOBAL;
@@ -388,6 +496,7 @@ export function plan(text, prefs, { pageUrl = null } = {}) {
     profile,
     pageUrl: pageUrl || fm.confluence_url || null,
     redesign: String(fm.redesign || "").toLowerCase() === "true",
+    panels: prefs.storyPanels,
   };
   for (const story of stories) {
     const b = buildStoryBody(story, bodyCtx);
@@ -412,6 +521,30 @@ export function plan(text, prefs, { pageUrl = null } = {}) {
   }
 
   const coverage = coverageOf(defined, stories, corporate);
+  const dependencies = reqDeps.size ? storyDependencies(stories, reqDeps) : null;
+  if (dependencies) {
+    const titleOf = new Map(stories.map((s) => [s.label, s.group]));
+    const linked = new Set(dependencies.edges.map((e) => `${e.dependent}>${e.prerequisite}`));
+    const isLoop = (c) => c.every((l, i) => linked.has(`${l}>${c[(i + 1) % c.length]}`));
+    const problems = [
+      ...dependencies.cycles.map((c) =>
+        isLoop(c)
+          ? `dependency cycle: ${[c[0], ...[...c].reverse()].map((l) => titleOf.get(l)).join(" -> ")}`
+          : `dependency cycle among: ${c.map((l) => titleOf.get(l)).join(", ")}`,
+      ),
+      ...dependencies.missing.map(
+        (m) => `${m.id} depends on ${m.dependsOn}, which no story covers`,
+      ),
+    ];
+    if (problems.length) {
+      coverage.dependencyProblems = problems;
+      if (coverage.verdict === "ok") coverage.verdict = "incomplete";
+      if (coverage.verdict === "incomplete" && !coverage.reason) {
+        coverage.reason = `${problems.length} dependency problem(s): ${problems.join("; ")}`;
+      }
+    }
+  }
+  const linkType = prefs.dependencyLinkType ?? "Blocks";
 
   return {
     document: {
@@ -434,12 +567,17 @@ export function plan(text, prefs, { pageUrl = null } = {}) {
       epicLinkFieldId: prefs.epicLinkFieldId ?? null,
       cloneByChannel: prefs.cloneByChannel === true,
       cloneLinkType: prefs.cloneLinkType ?? "Relates",
+      storyPanels: prefs.storyPanels ?? null,
+      subtaskRolesDeclined: prefs.subtaskRolesDeclined === true,
+      ...(dependencies || prefs.dependencyLinkType !== undefined
+        ? { dependencyLinkType: linkType }
+        : {}),
     },
     nodes: stories,
-    writeCount: stories.reduce(
-      (n, s) => n + 1 + s.subtasks.length + 2 * (s.clones?.length || 0),
-      0,
-    ),
+    ...(dependencies ? { dependencies } : {}),
+    writeCount:
+      stories.reduce((n, s) => n + 1 + s.subtasks.length + 2 * (s.clones?.length || 0), 0) +
+      (dependencies?.edges.filter((e) => !e.onCycle).length || 0),
   };
 }
 
@@ -448,7 +586,9 @@ export function plan(text, prefs, { pageUrl = null } = {}) {
  *
  * @returns {object}  a spec-consistency-gate result
  */
-function consistencyOf(text, nodes) {
+function consistencyOf(source, nodes) {
+  const corporate = (frontMatter(source).profile || "").toLowerCase() === "corporate";
+  const text = dependencyClauses(source, corporate).stripped;
   const constitution = resolveConstitution({
     repo: dirname(file),
     feature: frontMatter(text).feature || null,
@@ -464,7 +604,50 @@ function consistencyOf(text, nodes) {
   });
 }
 
+/**
+ * Save the sub-tasks a person chose for every story: each chosen site type
+ * becomes a role named after it and mapped to it, or "none" records that the
+ * question was answered with no sub-tasks.
+ */
+function saveSubtasks(choice) {
+  const names = String(choice)
+    .split(",")
+    .map((n) => n.trim())
+    .filter(Boolean);
+  const none = names.length === 1 && names[0].toLowerCase() === "none";
+  if (!names.length || names.some((n) => n.length > 100 || /[\n{}]/.test(n)))
+    return die("analysis-story-tree: --save-subtasks takes type names or none", 3);
+  try {
+    updateJsonFileSync(
+      PREFS_PATH,
+      (j) => {
+        if (!j || typeof j !== "object" || Array.isArray(j)) return undefined;
+        j.global = j.global || {};
+        const t = { ...(j.global.issueTree || {}) };
+        if (none) {
+          t.subtaskRoles = [];
+          t.subtaskRolesDeclined = true;
+        } else {
+          t.subtaskRoles = names;
+          t.subtaskIssueType = Object.fromEntries(names.map((n) => [n, n]));
+          delete t.subtaskRolesDeclined;
+        }
+        j.global.issueTree = t;
+        return j;
+      },
+      { mode: 0o600, followSymlink: true, timeoutMs: lockTimeoutFrom(process.env) },
+    );
+  } catch (e) {
+    return die(`analysis-story-tree: could not write ${PREFS_PATH}: ${e.message}`, 3);
+  }
+  process.stdout.write(
+    none ? "issueTree: no sub-tasks\n" : `issueTree: sub-tasks ${names.join(", ")}\n`,
+  );
+  return null;
+}
+
 async function main() {
+  if (SAVE_SUBTASKS !== null) return saveSubtasks(SAVE_SUBTASKS);
   if (!file)
     return die(
       "usage: analysis-story-tree.mjs <analysis.md> [--json] [--brief] [--prefs FILE] [--page-url URL]",
@@ -550,12 +733,18 @@ async function main() {
   );
   if (c.uncovered.length) process.stdout.write(`  not in any story: ${c.uncovered.join(", ")}\n`);
   if (c.invented.length) process.stdout.write(`  cited but undefined: ${c.invented.join(", ")}\n`);
+  for (const p of c.dependencyProblems || []) process.stdout.write(`  ${p}\n`);
   const k = result.consistency;
   process.stdout.write(
     `consistency: ${k.verdict} (${k.advisory ? "advisory, " : ""}${consistencyDetail(k)})\n`,
   );
   for (const line of describe(k)) process.stdout.write(`${line}\n`);
-  process.stdout.write(`\n${result.writeCount} issue(s) would be created:\n`);
+  const linkCount =
+    result.nodes.reduce((n, s) => n + (s.clones?.length || 0), 0) +
+    (result.dependencies?.edges.filter((e) => !e.onCycle).length || 0);
+  process.stdout.write(
+    `\n${result.writeCount - linkCount} issue(s) and ${linkCount} link(s) would be created:\n`,
+  );
   for (const s of result.nodes) {
     process.stdout.write(`  ${result.prefsUsed.storyIssueType}  ${s.title}\n`);
     process.stdout.write(
@@ -579,6 +768,18 @@ async function main() {
       );
     }
   }
+  const d = result.dependencies;
+  if (d) {
+    const group = new Map(result.nodes.map((n) => [n.label, n.group]));
+    process.stdout.write(
+      `\ndependencies: ${d.edges.length} ${result.prefsUsed.dependencyLinkType} link(s), the prerequisite first:\n`,
+    );
+    for (const e of d.edges) {
+      process.stdout.write(
+        `  ${group.get(e.prerequisite)} -> ${group.get(e.dependent)}  (${e.via.join(", ")})  [${e.prerequisite} -> ${e.dependent}]${e.onCycle ? "  not linked: on a dependency cycle" : ""}\n`,
+      );
+    }
+  }
   // Every field's provenance, so a wrong setting is visible here rather than in
   // Jira after the writes have happened.
   process.stdout.write("\nfrom prefs.global.issueTree:\n");

package/pipeline/scripts/autopilot-publish.mjs

@@ -13,7 +13,9 @@
  *   1. The request, the state and the test log are read as bounded regular
  *      files (lib/regular-file.mjs): a FIFO, a device or an oversized file is
  *      refused, never read. The request is read once and validated in process
- *      (pr-request.mjs validatePrRequest) on those same bytes.
+ *      (pr-request.mjs validatePrRequest) on those same bytes. The state's
+ *      `decisions[]` is appended to the body as one redacted section
+ *      (lib/decisions.mjs), so the PR lists what the run chose unasked.
  *   2. The repository comes from the runner's own config (the queue item's
  *      `nameWithOwner`), never from the request. A request naming another
  *      repository is refused, not redirected. The base is the configured
@@ -104,6 +106,7 @@ import {
 import { tmpdir } from "node:os";
 import { dirname, isAbsolute, join } from "node:path";
 
+import { appendDecisionsSection } from "../lib/decisions.mjs";
 import { invokedDirectly } from "../lib/invoked-directly.mjs";
 import { scan } from "../lib/outbound-gate.mjs";
 import { prRequestPath, prRequestsDir, prVerdictPath } from "../lib/pr-request-location.mjs";
@@ -123,6 +126,7 @@ const GIT_TIMEOUT_MS = 60 * 1000;
 const NET_TIMEOUT_MS = 5 * 60 * 1000;
 // A valid request is at most a 256-character title and a 65536-character body.
 const REQUEST_MAX_BYTES = 1024 * 1024;
+const BODY_MAX_CHARS = 65536;
 const STATE_MAX_BYTES = 16 * 1024 * 1024;
 const LOG_MAX_BYTES = 64 * 1024 * 1024;
 const VERIFY_TIMEOUT_SEC = 30 * 60;
@@ -1268,6 +1272,13 @@ export async function publish({ sessionId, statePath, item, config, env = proces
     }
 
     req = readRequest(reqPath);
+    // What the run decided in place of a person goes into the PR it opens. The
+    // section is redacted when rendered and passes the outbound gate below with
+    // the rest of the body.
+    req = {
+      ...req,
+      body: appendDecisionsSection(req.body, state.decisions, { maxChars: BODY_MAX_CHARS }),
+    };
 
     const repoCfg = (config?.repos || []).find((r) => r && r.nameWithOwner === repo);
     if (!repo || !repoCfg)

package/pipeline/scripts/cost-analyze.mjs

@@ -46,20 +46,12 @@
  *   2   usage error
  */
 
-import {
-  createReadStream,
-  existsSync,
-  mkdirSync,
-  readdirSync,
-  readFileSync,
-  statSync,
-  writeFileSync,
-} from "node:fs";
-import { createInterface } from "node:readline";
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { costUsd } from "./_cost.mjs";
+import { readJsonl } from "./_jsonl.mjs";
 import { runMain } from "../lib/fatal.mjs";
 import { messages, outputLanguage } from "../lib/user-facing.mjs";
 
@@ -263,20 +255,10 @@ async function readSeries({ days }) {
   const unpricedModels = new Set();
 
   for (const f of files) {
-    let rl;
-    try {
-      rl = createInterface({ input: createReadStream(f.path), crlfDelay: Infinity });
-    } catch {
-      continue;
-    }
-    for await (const line of rl) {
-      if (!line || line.charCodeAt(0) !== 123) continue;
-      let d;
-      try {
-        d = JSON.parse(line);
-      } catch {
-        continue;
-      }
+    // Uncapped: a cost total that silently stops at a byte cap under-reports.
+    for await (const row of readJsonl(f.path, { maxBytes: Infinity, maxLineBytes: Infinity })) {
+      const d = row.record;
+      if (!d) continue;
       const msg = d.message;
       const u = msg?.usage;
       if (!u) continue;

package/pipeline/scripts/decisions-log.mjs

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+/**
+ * decisions-log.mjs  -  append to and render `state.decisions[]`, the list of
+ * choices an autopilot run made where an attended run would have asked.
+ *
+ * Writers: a picker resolved without asking (picker-contract.md, autopilot),
+ * a split consensus taken on the triage verdict, a model fallback taken
+ * without the date-gate question, a rework dispatched one rung up
+ * (rework-escalation.mjs). gate-ledger.mjs park and review-decision-gate.mjs
+ * write their own entries through lib/decisions.mjs.
+ *
+ * `add` writes only while the quality gates are active (lib/unattended.mjs
+ * gatesActive): in an attended run the person answered, so there is nothing to
+ * report back. Writes take the write-state lock through gate-ledger.mjs
+ * updateState.
+ *
+ * `render` prints the markdown section Phase 5 puts in its report, redacted,
+ * and nothing at all for a run with no entries.
+ *
+ * Usage:
+ *   decisions-log.mjs add --what <text> --why <text> --cost-if-wrong <text> --phase <0-5>
+ *                         [--state <path>] [--repo <dir>]
+ *   decisions-log.mjs render [--state <path>] [--repo <dir>]
+ *
+ * Exit: 0 written, skipped (attended) or rendered · 2 usage · 3 no state or I/O error
+ *
+ * @module pipeline/scripts/decisions-log
+ */
+
+import { writeFileSync } from "node:fs";
+
+import { decisionEntry, renderDecisions, withDecision } from "../lib/decisions.mjs";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+import { readRegularJson } from "../lib/regular-file.mjs";
+import { gateContext, resolveStatePath, updateState } from "./gate-ledger.mjs";
+
+class UsageError extends Error {}
+
+function parse(argv) {
+  const opts = {};
+  for (let i = 1; i < argv.length; i++) {
+    const a = argv[i];
+    if (!a.startsWith("--")) throw new UsageError(`unexpected argument ${a}`);
+    const v = argv[++i];
+    if (v === undefined) throw new UsageError(`${a} needs a value`);
+    opts[a.slice(2)] = v;
+  }
+  return opts;
+}
+
+/**
+ * Append one entry when the gates are active. Resolves to whether it wrote.
+ *
+ * @param {{what: string, why: string, costIfWrong: string, phase: number}} input
+ * @param {{statePath?: string|null, repo?: string|null, env?: Record<string,string|undefined>}} [opts]
+ */
+export async function addDecision(
+  input,
+  { statePath = null, repo = null, env = process.env } = {},
+) {
+  const entry = decisionEntry(input);
+  const ctx = gateContext({ statePath, repo, env });
+  if (!ctx.active || !ctx.statePath) return { written: false, entry };
+  await updateState(ctx.statePath, (s) => withDecision(s, entry));
+  return { written: true, entry, statePath: ctx.statePath };
+}
+
+async function main(argv) {
+  const mode = argv[0];
+  const opts = parse(argv);
+  if (mode === "add") {
+    let input;
+    try {
+      input = decisionEntry({
+        what: opts.what,
+        why: opts.why,
+        costIfWrong: opts["cost-if-wrong"],
+        phase: opts.phase,
+      });
+    } catch (err) {
+      throw new UsageError(err.message);
+    }
+    await addDecision(input, { statePath: opts.state || null, repo: opts.repo || process.cwd() });
+    return 0;
+  }
+  if (mode === "render") {
+    const path = resolveStatePath({ statePath: opts.state, repo: opts.repo || process.cwd() });
+    if (!path) {
+      process.stderr.write("decisions-log: no run state found\n");
+      return 3;
+    }
+    const state = readRegularJson(path, null);
+    if (!state || typeof state !== "object") {
+      process.stderr.write(`decisions-log: cannot read ${path}\n`);
+      return 3;
+    }
+    const md = renderDecisions(state.decisions);
+    if (md) writeFileSync(1, md);
+    return 0;
+  }
+  throw new UsageError("mode must be add or render");
+}
+
+if (invokedDirectly(import.meta.url)) {
+  main(process.argv.slice(2)).then(
+    (code) => process.exit(code),
+    (err) => {
+      process.stderr.write(`decisions-log: ${err.message}\n`);
+      if (err instanceof UsageError) {
+        process.stderr.write(
+          "usage: decisions-log.mjs add --what t --why t --cost-if-wrong t --phase n [--state p] | render [--state p]\n",
+        );
+        process.exit(2);
+      }
+      process.exit(3);
+    },
+  );
+}