@mmerterden/multi-agent-pipeline 20.30.0 → 20.34.0

package/pipeline/scripts/debt-ledger.mjs

@@ -0,0 +1,323 @@
+#!/usr/bin/env node
+/**
+ * @file debt-ledger.mjs  -  the known limits a repository carries, in one list.
+ *
+ * Three sources feed it, each a limit somebody accepted on purpose:
+ *
+ *   shortcut   a `shortcut: <limit>, <when to replace>` marker a diff adds
+ *              (features/scope-check.md, "Known limits in the code")
+ *   deferred   a triage finding deferred as out of scope
+ *   parked     a finding parked by rework-rulings.mjs at the rework cap
+ *
+ * Storage: $HOME/.claude/memory/multi-agent/<repo-slug>/debt.jsonl, append-only,
+ * one row per item; the same item from the same task is written once. Every
+ * row passes the secret scan before it is written (_secret-scan.mjs).
+ *
+ * Subcommands:
+ *   markers  --patch <unified diff|->         print the markers the diff adds (no write)
+ *   add      --patch <diff> --task <id>       record those markers
+ *   from-run --triage <path> [--state <agent-state.json>] --task <id>
+ *                                             record deferred findings and parked rulings
+ *   list     [--open] [--json]                rows, newest first
+ *   resolve  --id <id>                        mark one row resolved (appends a resolution row)
+ *   [--repo-slug <slug>] on every subcommand; else $MULTI_AGENT_REPO_SLUG, then the git repo name.
+ *
+ * Exit: 0 done · 1 IO error · 2 nothing to record or unknown id · 64 usage
+ *
+ * @module pipeline/scripts/debt-ledger
+ */
+
+import { execFileSync } from "node:child_process";
+import { appendFileSync, existsSync, mkdirSync, readFileSync, realpathSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { shortId } from "./_retrieval.mjs";
+import { parseUnifiedDiff } from "./_unified-diff.mjs";
+import { checkedSlug, memoryExcluded } from "./_memory-policy.mjs";
+import { openScanner, stripPrivate } from "./_secret-scan.mjs";
+
+const COMMENT_OPENERS = ["//", "#", "/*", "<!--", "--", ";", "*"];
+const MAX_LINE = 2000;
+const MAX_TEXT = 300;
+
+/**
+ * True when position `at` of the line sits inside a string literal. With
+ * `apostrophes: false` a single quote opens nothing: before a `//` or `#`
+ * comment it is as often a lifetime or a word's apostrophe as a string.
+ */
+function insideString(line, at, apostrophes = true) {
+  let quote = null;
+  for (let i = 0; i < at; i++) {
+    const c = line[i];
+    if (quote) {
+      if (c === "\\") i++;
+      else if (c === quote) quote = null;
+    } else if (c === '"' || c === "`" || (apostrophes && c === "'")) {
+      quote = c;
+    }
+  }
+  return quote !== null;
+}
+
+/**
+ * The text after `shortcut:` when it opens a comment (only spaces between the
+ * comment opener and the marker, the opener outside any string), else null.
+ * `--` counts only at the start of the line, so a CLI flag is not a comment.
+ * Lowercase only, as documented.
+ */
+function markerIn(line) {
+  const at = line.indexOf("shortcut:");
+  if (at === -1) return null;
+  const before = line.slice(0, at).trimEnd();
+  const opener = COMMENT_OPENERS.find((o) => before.endsWith(o));
+  if (!opener) return null;
+  const openAt = before.length - opener.length;
+  if (opener === "--" && before.trim() !== "--") return null;
+  if (insideString(line, openAt, opener !== "//" && opener !== "#")) return null;
+  return line.slice(at + "shortcut:".length).trim();
+}
+
+/** Markers added by a unified diff: [{file, line, text}]. */
+export function markersIn(patch) {
+  const out = [];
+  for (const [file, diff] of parseUnifiedDiff(patch)) {
+    for (const { line, text: raw } of diff.addedAt) {
+      const marker = markerIn(raw.slice(0, MAX_LINE));
+      if (marker === null) continue;
+      let text = marker;
+      for (const close of ["*/", "-->"])
+        if (text.endsWith(close)) text = text.slice(0, -close.length);
+      text = text.trimEnd();
+      if (text) out.push({ file, line, text });
+    }
+  }
+  return out;
+}
+
+/** Deferred findings and parked rulings of one run: [{kind, file, line, text}]. */
+export function runItems(triage, state) {
+  const items = [];
+  for (const d of triage?.deferred ?? []) {
+    const f = d?.finding ?? {};
+    if (!f.issue) continue;
+    items.push({
+      kind: "deferred",
+      file: f.file ?? null,
+      line: Number.isInteger(f.line) ? f.line : null,
+      text: `${f.issue}${d.reason ? ` (deferred: ${d.reason})` : ""}`,
+    });
+  }
+  const iterations = Array.isArray(state?.reviewIterations) ? state.reviewIterations : [];
+  for (const r of iterations.at(-1)?.rulings ?? []) {
+    if (r?.ruling !== "parked" || !r.issue) continue;
+    items.push({
+      kind: "parked",
+      file: r.file ?? null,
+      line: Number.isInteger(r.line) ? r.line : null,
+      text: `${r.issue} (${r.reason})`,
+    });
+  }
+  return items;
+}
+
+const rowId = (r) => shortId("D", r.kind, r.file ?? "", r.text);
+
+function parseFlags(argv) {
+  const flags = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (!a.startsWith("--")) continue;
+    const next = argv[i + 1];
+    if (next !== undefined && !next.startsWith("--")) {
+      flags[a.slice(2)] = next;
+      i++;
+    } else flags[a.slice(2)] = true;
+  }
+  return flags;
+}
+
+function repoSlug(flags) {
+  return checkedSlug(rawRepoSlug(flags));
+}
+
+function rawRepoSlug(flags) {
+  if (typeof flags["repo-slug"] === "string") return flags["repo-slug"];
+  if (process.env.MULTI_AGENT_REPO_SLUG) return process.env.MULTI_AGENT_REPO_SLUG;
+  try {
+    const common = execFileSync(
+      "git",
+      ["rev-parse", "--path-format=absolute", "--git-common-dir"],
+      {
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "ignore"],
+      },
+    ).trim();
+    const root = basename(common) === ".git" ? dirname(common) : common.replace(/\.git$/, "");
+    if (basename(root)) return basename(root);
+  } catch {
+    /* not a git repo */
+  }
+  return "default";
+}
+
+const ledgerPath = (slug) =>
+  join(homedir(), ".claude", "memory", "multi-agent", slug, "debt.jsonl");
+
+function readRows(slug) {
+  const p = ledgerPath(slug);
+  if (!existsSync(p)) return [];
+  return readFileSync(p, "utf8")
+    .split("\n")
+    .filter((l) => l.trim())
+    .flatMap((l) => {
+      try {
+        return [JSON.parse(l)];
+      } catch {
+        return [];
+      }
+    });
+}
+
+/** Current state per id: the last row wins, so a resolution closes its item. */
+function current(rows) {
+  const byId = new Map();
+  for (const r of rows) byId.set(r.id, { ...byId.get(r.id), ...r });
+  return [...byId.values()];
+}
+
+function record(slug, items, task) {
+  if (memoryExcluded(slug)) return { added: 0, droppedBySecretScan: 0, excluded: true };
+  const p = ledgerPath(slug);
+  const seen = new Set(readRows(slug).map((r) => `${r.id}::${r.task}`));
+  const scan = openScanner();
+  let added = 0;
+  let droppedBySecretScan = 0;
+  try {
+    for (const it of items) {
+      const whole = stripPrivate(it.text).replace(/\s+/g, " ").trim();
+      if (!whole) continue;
+      // Scanned whole and cut afterwards: a token across the cut would
+      // otherwise be shortened below the scanner's minimum and stored.
+      const text = whole.slice(0, MAX_TEXT);
+      const row = { kind: it.kind, file: it.file, line: it.line, text };
+      const id = rowId(row);
+      if (seen.has(`${id}::${task}`)) continue;
+      if (!scan.isClean(`${whole}\n${it.file ?? ""}`)) {
+        droppedBySecretScan++;
+        continue;
+      }
+      mkdirSync(dirname(p), { recursive: true });
+      appendFileSync(
+        p,
+        JSON.stringify({ v: 1, id, ts: new Date().toISOString(), task, status: "open", ...row }) +
+          "\n",
+      );
+      seen.add(`${id}::${task}`);
+      added++;
+    }
+  } finally {
+    scan.close();
+  }
+  return { added, droppedBySecretScan };
+}
+
+function readJson(path) {
+  try {
+    return JSON.parse(readFileSync(path, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+function readPatch(path) {
+  return path === "-" ? readFileSync(0, "utf8") : readFileSync(path, "utf8");
+}
+
+function main(argv) {
+  const sub = argv[0];
+  const flags = parseFlags(argv.slice(1));
+  const out = (doc) => process.stdout.write(JSON.stringify(doc) + "\n");
+  const slug = repoSlug(flags);
+  const task = typeof flags.task === "string" ? flags.task : null;
+
+  if (sub === "markers" || sub === "add") {
+    if (typeof flags.patch !== "string") return usage();
+    const markers = markersIn(readPatch(flags.patch));
+    if (sub === "markers") {
+      out({ markers });
+      return 0;
+    }
+    if (!task) return usage();
+    const r = record(
+      slug,
+      markers.map((m) => ({ kind: "shortcut", ...m })),
+      task,
+    );
+    out({ ok: true, slug, ...r });
+    return r.added ? 0 : 2;
+  }
+  if (sub === "from-run") {
+    if (typeof flags.triage !== "string" || !task) return usage();
+    const triage = readJson(flags.triage);
+    if (!triage) {
+      console.error(`debt-ledger: triage unreadable: ${flags.triage}`);
+      return 1;
+    }
+    const state = typeof flags.state === "string" ? readJson(flags.state) : null;
+    const r = record(slug, runItems(triage, state), task);
+    out({ ok: true, slug, ...r });
+    return r.added ? 0 : 2;
+  }
+  if (sub === "list") {
+    let rows = current(readRows(slug)).reverse();
+    if (flags.open) rows = rows.filter((r) => r.status === "open");
+    if (flags.json) out({ slug, rows });
+    else
+      for (const r of rows)
+        process.stdout.write(
+          `${r.id} ${r.status} ${r.kind} ${r.file ?? "-"}${r.line ? `:${r.line}` : ""} ${r.text}\n`,
+        );
+    return 0;
+  }
+  if (sub === "resolve") {
+    if (typeof flags.id !== "string") return usage();
+    if (!current(readRows(slug)).some((r) => r.id === flags.id)) {
+      console.error(`debt-ledger: no item ${flags.id}`);
+      return 2;
+    }
+    appendFileSync(
+      ledgerPath(slug),
+      JSON.stringify({ id: flags.id, status: "resolved", resolvedAt: new Date().toISOString() }) +
+        "\n",
+    );
+    out({ ok: true, slug, resolved: flags.id });
+    return 0;
+  }
+  return usage();
+}
+
+function usage() {
+  console.error(
+    "usage: debt-ledger.mjs markers --patch <diff|-> | add --patch <diff> --task <id> | from-run --triage <path> [--state <path>] --task <id> | list [--open] [--json] | resolve --id <id>",
+  );
+  return 64;
+}
+
+const invoked = () => {
+  try {
+    return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
+  } catch {
+    return false;
+  }
+};
+
+if (invoked()) {
+  try {
+    process.exitCode = main(process.argv.slice(2));
+  } catch (e) {
+    console.error(`debt-ledger: ${e.message}`);
+    process.exitCode = 1;
+  }
+}

package/pipeline/scripts/file-context.mjs

@@ -0,0 +1,261 @@
+#!/usr/bin/env node
+/**
+ * @file file-context.mjs  -  what the repository's memory says about a file,
+ * handed to the model right after it reads that file.
+ *
+ * PostToolUse on Read. For the file just read it looks up, in the per-repo
+ * stores under ~/.claude/memory/multi-agent/<repo-slug>/:
+ *
+ *   learnings-ledger.jsonl   durable facts and conventions whose scope covers the file,
+ *                            minus those the feedback suppressed (_memory-policy.mjs)
+ *   triage-corpus.jsonl      the latest review findings recorded against it
+ *   debt.jsonl               open known limits in it (debt-ledger.mjs)
+ *
+ * and returns at most MAX_LINES of them as PostToolUse additionalContext,
+ * inside an <untrusted-data> block: the rows hold model-written text, so they
+ * are evidence about the file, never an instruction.
+ *
+ * When it runs: inside a pipeline run bound to the session (_active-run.mjs),
+ * or anywhere when prefs.global.fileContext.enabled is true; never when that
+ * flag is false. Each file is answered once per session. The text passes the
+ * secret scan before it is handed over; a hit means nothing is said.
+ *
+ * `file-context.mjs --lines <path>` prints the same lines, secret-scanned and
+ * wrapped the same way, for a caller that already decided to show them
+ * (check-read-size.sh, when it turns a whole-file read away). It applies no
+ * session or run condition, but stays silent when the flag is false.
+ *
+ * Blocks nothing and prints nothing on any failure: a missing store, a slow
+ * scan or a malformed payload all end in exit 0 with no output.
+ *
+ * @module pipeline/scripts/file-context
+ */
+
+import { spawnSync } from "node:child_process";
+import {
+  existsSync,
+  lstatSync,
+  mkdirSync,
+  readFileSync,
+  realpathSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { basename, dirname, join, relative } from "node:path";
+
+import { wrapUntrusted } from "../lib/untrusted.mjs";
+import { findActiveRun } from "./_active-run.mjs";
+import { ledgerRowId, standing } from "./_memory-policy.mjs";
+import { openScanner } from "./_secret-scan.mjs";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+
+const MAX_INPUT = 256 * 1024;
+const MAX_LINES = 6;
+const MAX_LINE_CHARS = 220;
+const STORE_MAX_BYTES = 4 * 1024 * 1024;
+
+const prefsPath = (env) =>
+  env.MULTI_AGENT_PREFS || join(env.HOME || homedir(), ".claude", "multi-agent-preferences.json");
+
+/** off | on | run-only, from prefs.global.fileContext.enabled. */
+export function mode(env = process.env) {
+  try {
+    const flag = JSON.parse(readFileSync(prefsPath(env), "utf8"))?.global?.fileContext?.enabled;
+    if (flag === false) return "off";
+    if (flag === true) return "on";
+  } catch {
+    /* no prefs: the default */
+  }
+  return "run-only";
+}
+
+/** Repo slug and the file's path inside the repo, as the stores key them. */
+export function locate(filePath) {
+  const dir = dirname(filePath);
+  const git = (args) =>
+    spawnSync("git", ["-C", dir, ...args], { encoding: "utf8", timeout: 2000 }).stdout?.trim();
+  const top = git(["rev-parse", "--show-toplevel"]);
+  if (!top) return null;
+  const common = git(["rev-parse", "--path-format=absolute", "--git-common-dir"]) || "";
+  const root = basename(common) === ".git" ? dirname(common) : top;
+  // git answers with the resolved path (/private/var on macOS), so the file is
+  // resolved the same way before the two are compared.
+  return { slug: basename(root), rel: relative(realpathSync(top), realpathSync(filePath)) };
+}
+
+function rows(path) {
+  try {
+    if (statSync(path).size > STORE_MAX_BYTES) return [];
+    return readFileSync(path, "utf8")
+      .split("\n")
+      .filter((l) => l.trim())
+      .flatMap((l) => {
+        try {
+          return [JSON.parse(l)];
+        } catch {
+          return [];
+        }
+      });
+  } catch {
+    return [];
+  }
+}
+
+function scopeCovers(scope, rel) {
+  if (!scope || scope === "global") return false;
+  const dirOf = (g) =>
+    String(g)
+      .replace(/\/+$/, "")
+      .replace(/\/\*+$/, "");
+  const s = dirOf(scope);
+  return s === rel || rel.startsWith(s + "/");
+}
+
+const clip = (s) => {
+  const one = String(s ?? "")
+    .replace(/\s+/g, " ")
+    .trim();
+  return one.length > MAX_LINE_CHARS ? one.slice(0, MAX_LINE_CHARS - 3) + "..." : one;
+};
+
+/** The lines worth saying about one file, most specific first. */
+export function linesFor(storeDir, rel) {
+  const out = [];
+  const feedback = new Map();
+  for (const f of rows(join(storeDir, "learnings-feedback.jsonl"))) {
+    if (!feedback.has(f.id)) feedback.set(f.id, []);
+    feedback.get(f.id).push(f);
+  }
+  for (const r of rows(join(storeDir, "learnings-ledger.jsonl"))) {
+    if (!scopeCovers(r.scope, rel)) continue;
+    if (standing(feedback.get(ledgerRowId(r)) ?? []).suppressed) continue;
+    out.push(`${r.kind}: ${clip(r.statement)}`);
+  }
+  const findings = rows(join(storeDir, "triage-corpus.jsonl"))
+    .filter((r) => r.file === rel)
+    .slice(-3)
+    .reverse();
+  for (const f of findings) {
+    out.push(`past review (${f.classification}, ${f.severity}): ${clip(f.issue)}`);
+  }
+  const debt = new Map();
+  for (const r of rows(join(storeDir, "debt.jsonl"))) debt.set(r.id, { ...debt.get(r.id), ...r });
+  for (const d of debt.values()) {
+    if (d.status === "open" && d.file === rel) out.push(`known limit (${d.kind}): ${clip(d.text)}`);
+  }
+  return out.slice(0, MAX_LINES);
+}
+
+/** The cache directory exists, is a real directory and belongs to this user. */
+function ownDir(dir) {
+  try {
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    const st = lstatSync(dir);
+    return st.isDirectory() && (process.getuid === undefined || st.uid === process.getuid());
+  } catch {
+    return false;
+  }
+}
+
+function seenBefore(sessionId, rel, slug) {
+  if (!sessionId) return false;
+  const uid = process.getuid?.() ?? "user";
+  const dir = join(tmpdir(), `multi-agent-file-context-${uid}`);
+  const f = join(dir, `${String(sessionId).replace(/[^A-Za-z0-9_-]/g, "")}.json`);
+  if (!ownDir(dir)) return false;
+  let seen = [];
+  try {
+    seen = JSON.parse(readFileSync(f, "utf8"));
+  } catch {
+    /* first file this session */
+  }
+  const key = `${slug}:${rel}`;
+  if (seen.includes(key)) return true;
+  try {
+    writeFileSync(f, JSON.stringify([...seen, key].slice(-500)), { mode: 0o600 });
+  } catch {
+    /* a lost cache only means saying it twice */
+  }
+  return false;
+}
+
+/** The hook's answer for one payload, or null for silence. */
+export function answer(payload, env = process.env) {
+  if (payload?.tool_name !== "Read") return null;
+  const filePath = payload?.tool_input?.file_path;
+  if (typeof filePath !== "string" || !filePath || !existsSync(filePath)) return null;
+  const m = mode(env);
+  if (m === "off") return null;
+  if (m === "run-only" && !findActiveRun({ sessionId: payload.session_id, cwd: payload.cwd, env }))
+    return null;
+  const where = locate(filePath);
+  if (!where || where.rel.startsWith("..")) return null;
+  const storeDir = join(env.HOME || homedir(), ".claude", "memory", "multi-agent", where.slug);
+  const lines = linesFor(storeDir, where.rel);
+  if (!lines.length || seenBefore(payload.session_id, where.rel, where.slug)) return null;
+  const text = lines.map((l) => `- ${l}`).join("\n");
+  const scan = openScanner();
+  try {
+    if (!scan.isClean(text)) return null;
+  } finally {
+    scan.close();
+  }
+  return {
+    hookSpecificOutput: {
+      hookEventName: "PostToolUse",
+      additionalContext: wrapUntrusted(`memory:${where.slug}/${where.rel}`, text),
+    },
+  };
+}
+
+/**
+ * The scanned lines for one file inside an untrusted-data block, or "" when
+ * file context is off, there is nothing, or the scan refused it.
+ */
+export function blockForPath(filePath, env = process.env) {
+  if (mode(env) === "off" || !existsSync(filePath)) return "";
+  const where = locate(filePath);
+  if (!where || where.rel.startsWith("..")) return "";
+  const storeDir = join(env.HOME || homedir(), ".claude", "memory", "multi-agent", where.slug);
+  const text = linesFor(storeDir, where.rel)
+    .map((l) => `- ${l}`)
+    .join("\n");
+  if (!text) return "";
+  const scan = openScanner();
+  try {
+    return scan.isClean(text) ? wrapUntrusted(`memory:${where.slug}/${where.rel}`, text) : "";
+  } finally {
+    scan.close();
+  }
+}
+
+function main() {
+  const i = process.argv.indexOf("--lines");
+  if (i > 0) {
+    const block = process.argv[i + 1] ? blockForPath(process.argv[i + 1]) : "";
+    if (block) process.stdout.write(block);
+    return;
+  }
+  let raw;
+  try {
+    raw = readFileSync(0, "utf8");
+  } catch {
+    return;
+  }
+  if (!raw || raw.length > MAX_INPUT) return;
+  let payload;
+  try {
+    payload = JSON.parse(raw);
+  } catch {
+    return;
+  }
+  const out = answer(payload);
+  if (out) process.stdout.write(JSON.stringify(out) + "\n");
+}
+
+try {
+  if (invokedDirectly(import.meta.url)) main();
+} catch {
+  /* silent by contract */
+}

package/pipeline/scripts/file-context.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# file-context.sh  -  PostToolUse hook (Read) for the pipeline.
+#
+# Hands the model what the repository's memory says about the file it just
+# read: ledger facts, past review findings and open known limits. The logic is
+# in file-context.mjs; this wrapper exists because every hook the pipeline
+# registers is a `bash "$HOME/.<host>/scripts/<name>"` command.
+#
+# Exits 0 on every path: a missing node, a slow scan or any internal error
+# means no context, never a blocked tool call.
+
+set -u
+
+HERE="$(cd "$(dirname "$0")" 2>/dev/null && pwd || true)"
+CORE="$HERE/file-context.mjs"
+[ -f "$CORE" ] || exit 0
+command -v node >/dev/null 2>&1 || exit 0
+
+node "$CORE" 2>/dev/null || true
+exit 0

package/pipeline/scripts/gen-mode-dispatch.mjs

@@ -40,10 +40,12 @@ const CONTRACT = JSON.parse(
   readFileSync(new URL("../schemas/phases.json", import.meta.url), "utf8"),
 );
 const PHASES = CONTRACT.phases;
+// A mode may name its steps itself: the analysis command runs the six slots as
+// Intake, Evidence, Synthesis, Review, Publish and Report.
 const modePhases = (name) =>
   CONTRACT.modes[name].phases.map((id) => {
     const p = PHASES.find((x) => x.id === id);
-    return `${p.id}:${p.name}`;
+    return `${p.id}:${CONTRACT.modes[name].names?.[id] ?? p.name}`;
   });
 
 const PHASE_NAMES = PHASES.map((p) => `${p.id}:${p.name}`);