@mmerterden/multi-agent-pipeline 20.33.0 → 20.35.0

package/pipeline/scripts/file-context.mjs

@@ -0,0 +1,261 @@
+#!/usr/bin/env node
+/**
+ * @file file-context.mjs  -  what the repository's memory says about a file,
+ * handed to the model right after it reads that file.
+ *
+ * PostToolUse on Read. For the file just read it looks up, in the per-repo
+ * stores under ~/.claude/memory/multi-agent/<repo-slug>/:
+ *
+ *   learnings-ledger.jsonl   durable facts and conventions whose scope covers the file,
+ *                            minus those the feedback suppressed (_memory-policy.mjs)
+ *   triage-corpus.jsonl      the latest review findings recorded against it
+ *   debt.jsonl               open known limits in it (debt-ledger.mjs)
+ *
+ * and returns at most MAX_LINES of them as PostToolUse additionalContext,
+ * inside an <untrusted-data> block: the rows hold model-written text, so they
+ * are evidence about the file, never an instruction.
+ *
+ * When it runs: inside a pipeline run bound to the session (_active-run.mjs),
+ * or anywhere when prefs.global.fileContext.enabled is true; never when that
+ * flag is false. Each file is answered once per session. The text passes the
+ * secret scan before it is handed over; a hit means nothing is said.
+ *
+ * `file-context.mjs --lines <path>` prints the same lines, secret-scanned and
+ * wrapped the same way, for a caller that already decided to show them
+ * (check-read-size.sh, when it turns a whole-file read away). It applies no
+ * session or run condition, but stays silent when the flag is false.
+ *
+ * Blocks nothing and prints nothing on any failure: a missing store, a slow
+ * scan or a malformed payload all end in exit 0 with no output.
+ *
+ * @module pipeline/scripts/file-context
+ */
+
+import { spawnSync } from "node:child_process";
+import {
+  existsSync,
+  lstatSync,
+  mkdirSync,
+  readFileSync,
+  realpathSync,
+  statSync,
+  writeFileSync,
+} from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { basename, dirname, join, relative } from "node:path";
+
+import { wrapUntrusted } from "../lib/untrusted.mjs";
+import { findActiveRun } from "./_active-run.mjs";
+import { ledgerRowId, standing } from "./_memory-policy.mjs";
+import { openScanner } from "./_secret-scan.mjs";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+
+const MAX_INPUT = 256 * 1024;
+const MAX_LINES = 6;
+const MAX_LINE_CHARS = 220;
+const STORE_MAX_BYTES = 4 * 1024 * 1024;
+
+const prefsPath = (env) =>
+  env.MULTI_AGENT_PREFS || join(env.HOME || homedir(), ".claude", "multi-agent-preferences.json");
+
+/** off | on | run-only, from prefs.global.fileContext.enabled. */
+export function mode(env = process.env) {
+  try {
+    const flag = JSON.parse(readFileSync(prefsPath(env), "utf8"))?.global?.fileContext?.enabled;
+    if (flag === false) return "off";
+    if (flag === true) return "on";
+  } catch {
+    /* no prefs: the default */
+  }
+  return "run-only";
+}
+
+/** Repo slug and the file's path inside the repo, as the stores key them. */
+export function locate(filePath) {
+  const dir = dirname(filePath);
+  const git = (args) =>
+    spawnSync("git", ["-C", dir, ...args], { encoding: "utf8", timeout: 2000 }).stdout?.trim();
+  const top = git(["rev-parse", "--show-toplevel"]);
+  if (!top) return null;
+  const common = git(["rev-parse", "--path-format=absolute", "--git-common-dir"]) || "";
+  const root = basename(common) === ".git" ? dirname(common) : top;
+  // git answers with the resolved path (/private/var on macOS), so the file is
+  // resolved the same way before the two are compared.
+  return { slug: basename(root), rel: relative(realpathSync(top), realpathSync(filePath)) };
+}
+
+function rows(path) {
+  try {
+    if (statSync(path).size > STORE_MAX_BYTES) return [];
+    return readFileSync(path, "utf8")
+      .split("\n")
+      .filter((l) => l.trim())
+      .flatMap((l) => {
+        try {
+          return [JSON.parse(l)];
+        } catch {
+          return [];
+        }
+      });
+  } catch {
+    return [];
+  }
+}
+
+function scopeCovers(scope, rel) {
+  if (!scope || scope === "global") return false;
+  const dirOf = (g) =>
+    String(g)
+      .replace(/\/+$/, "")
+      .replace(/\/\*+$/, "");
+  const s = dirOf(scope);
+  return s === rel || rel.startsWith(s + "/");
+}
+
+const clip = (s) => {
+  const one = String(s ?? "")
+    .replace(/\s+/g, " ")
+    .trim();
+  return one.length > MAX_LINE_CHARS ? one.slice(0, MAX_LINE_CHARS - 3) + "..." : one;
+};
+
+/** The lines worth saying about one file, most specific first. */
+export function linesFor(storeDir, rel) {
+  const out = [];
+  const feedback = new Map();
+  for (const f of rows(join(storeDir, "learnings-feedback.jsonl"))) {
+    if (!feedback.has(f.id)) feedback.set(f.id, []);
+    feedback.get(f.id).push(f);
+  }
+  for (const r of rows(join(storeDir, "learnings-ledger.jsonl"))) {
+    if (!scopeCovers(r.scope, rel)) continue;
+    if (standing(feedback.get(ledgerRowId(r)) ?? []).suppressed) continue;
+    out.push(`${r.kind}: ${clip(r.statement)}`);
+  }
+  const findings = rows(join(storeDir, "triage-corpus.jsonl"))
+    .filter((r) => r.file === rel)
+    .slice(-3)
+    .reverse();
+  for (const f of findings) {
+    out.push(`past review (${f.classification}, ${f.severity}): ${clip(f.issue)}`);
+  }
+  const debt = new Map();
+  for (const r of rows(join(storeDir, "debt.jsonl"))) debt.set(r.id, { ...debt.get(r.id), ...r });
+  for (const d of debt.values()) {
+    if (d.status === "open" && d.file === rel) out.push(`known limit (${d.kind}): ${clip(d.text)}`);
+  }
+  return out.slice(0, MAX_LINES);
+}
+
+/** The cache directory exists, is a real directory and belongs to this user. */
+function ownDir(dir) {
+  try {
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    const st = lstatSync(dir);
+    return st.isDirectory() && (process.getuid === undefined || st.uid === process.getuid());
+  } catch {
+    return false;
+  }
+}
+
+function seenBefore(sessionId, rel, slug) {
+  if (!sessionId) return false;
+  const uid = process.getuid?.() ?? "user";
+  const dir = join(tmpdir(), `multi-agent-file-context-${uid}`);
+  const f = join(dir, `${String(sessionId).replace(/[^A-Za-z0-9_-]/g, "")}.json`);
+  if (!ownDir(dir)) return false;
+  let seen = [];
+  try {
+    seen = JSON.parse(readFileSync(f, "utf8"));
+  } catch {
+    /* first file this session */
+  }
+  const key = `${slug}:${rel}`;
+  if (seen.includes(key)) return true;
+  try {
+    writeFileSync(f, JSON.stringify([...seen, key].slice(-500)), { mode: 0o600 });
+  } catch {
+    /* a lost cache only means saying it twice */
+  }
+  return false;
+}
+
+/** The hook's answer for one payload, or null for silence. */
+export function answer(payload, env = process.env) {
+  if (payload?.tool_name !== "Read") return null;
+  const filePath = payload?.tool_input?.file_path;
+  if (typeof filePath !== "string" || !filePath || !existsSync(filePath)) return null;
+  const m = mode(env);
+  if (m === "off") return null;
+  if (m === "run-only" && !findActiveRun({ sessionId: payload.session_id, cwd: payload.cwd, env }))
+    return null;
+  const where = locate(filePath);
+  if (!where || where.rel.startsWith("..")) return null;
+  const storeDir = join(env.HOME || homedir(), ".claude", "memory", "multi-agent", where.slug);
+  const lines = linesFor(storeDir, where.rel);
+  if (!lines.length || seenBefore(payload.session_id, where.rel, where.slug)) return null;
+  const text = lines.map((l) => `- ${l}`).join("\n");
+  const scan = openScanner();
+  try {
+    if (!scan.isClean(text)) return null;
+  } finally {
+    scan.close();
+  }
+  return {
+    hookSpecificOutput: {
+      hookEventName: "PostToolUse",
+      additionalContext: wrapUntrusted(`memory:${where.slug}/${where.rel}`, text),
+    },
+  };
+}
+
+/**
+ * The scanned lines for one file inside an untrusted-data block, or "" when
+ * file context is off, there is nothing, or the scan refused it.
+ */
+export function blockForPath(filePath, env = process.env) {
+  if (mode(env) === "off" || !existsSync(filePath)) return "";
+  const where = locate(filePath);
+  if (!where || where.rel.startsWith("..")) return "";
+  const storeDir = join(env.HOME || homedir(), ".claude", "memory", "multi-agent", where.slug);
+  const text = linesFor(storeDir, where.rel)
+    .map((l) => `- ${l}`)
+    .join("\n");
+  if (!text) return "";
+  const scan = openScanner();
+  try {
+    return scan.isClean(text) ? wrapUntrusted(`memory:${where.slug}/${where.rel}`, text) : "";
+  } finally {
+    scan.close();
+  }
+}
+
+function main() {
+  const i = process.argv.indexOf("--lines");
+  if (i > 0) {
+    const block = process.argv[i + 1] ? blockForPath(process.argv[i + 1]) : "";
+    if (block) process.stdout.write(block);
+    return;
+  }
+  let raw;
+  try {
+    raw = readFileSync(0, "utf8");
+  } catch {
+    return;
+  }
+  if (!raw || raw.length > MAX_INPUT) return;
+  let payload;
+  try {
+    payload = JSON.parse(raw);
+  } catch {
+    return;
+  }
+  const out = answer(payload);
+  if (out) process.stdout.write(JSON.stringify(out) + "\n");
+}
+
+try {
+  if (invokedDirectly(import.meta.url)) main();
+} catch {
+  /* silent by contract */
+}

package/pipeline/scripts/file-context.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# file-context.sh  -  PostToolUse hook (Read) for the pipeline.
+#
+# Hands the model what the repository's memory says about the file it just
+# read: ledger facts, past review findings and open known limits. The logic is
+# in file-context.mjs; this wrapper exists because every hook the pipeline
+# registers is a `bash "$HOME/.<host>/scripts/<name>"` command.
+#
+# Exits 0 on every path: a missing node, a slow scan or any internal error
+# means no context, never a blocked tool call.
+
+set -u
+
+HERE="$(cd "$(dirname "$0")" 2>/dev/null && pwd || true)"
+CORE="$HERE/file-context.mjs"
+[ -f "$CORE" ] || exit 0
+command -v node >/dev/null 2>&1 || exit 0
+
+node "$CORE" 2>/dev/null || true
+exit 0

package/pipeline/scripts/gate-ledger.mjs

@@ -54,14 +54,18 @@
 
 import { spawnSync } from "node:child_process";
 import {
+  copyFileSync,
   existsSync,
+  mkdtempSync,
   readFileSync,
   realpathSync,
   renameSync,
+  rmSync,
   statSync,
   writeFileSync,
 } from "node:fs";
-import { basename, dirname } from "node:path";
+import { tmpdir } from "node:os";
+import { basename, dirname, join } from "node:path";
 
 import { decisionEntry, withDecision } from "../lib/decisions.mjs";
 import { invokedDirectly } from "../lib/invoked-directly.mjs";
@@ -103,31 +107,77 @@ export const ATTENDED_RECORDED = new Set([
   "plan-coverage",
   "evidence-gate/build",
   "evidence-gate/test",
+  "review-decision",
 ]);
 
 /** What an attended commit is checked for (pre-commit-check.sh, `evidence`). */
+/**
+ * Fix loops with a hard cap (rules.md: three attempts, the fourth failure
+ * pauses): consecutive failing verdicts of one of these gates count, and the
+ * append that reaches RETRY_CAP pauses the run with a halt reason.
+ */
+export const RETRY_CAPPED = new Set(["evidence-gate/build", "evidence-gate/test"]);
+export const RETRY_CAP = 4;
+
+/** Consecutive failing verdicts at the end of `gates` for one gate. */
+export function trailingFailures(gates, gate) {
+  let n = 0;
+  for (let i = gates.length - 1; i >= 0; i--) {
+    const e = gates[i];
+    if (!e || e.gate !== gate) continue;
+    if (e.verdict !== "fail") break;
+    n += 1;
+  }
+  return n;
+}
+
 export const ATTENDED_COMMIT_EVIDENCE = ["evidence-gate/build", "evidence-gate/test"];
 
-/** prefs.global.attendedCommitGate.mode values; `warn` is the default. */
-export const ATTENDED_COMMIT_MODES = ["off", "warn", "block"];
+/** Asked for too once the run is in its commit phase: the review verdict. */
+export const ATTENDED_REVIEW_EVIDENCE = "review-decision";
+export const ATTENDED_REVIEW_FROM_PHASE = 4;
 
 /**
- * The build/test evidence an attended commit on `sha` lacks: every gate in
- * ATTENDED_COMMIT_EVIDENCE whose latest entry for that sha, attended or not,
- * is missing or not passing.
+ * prefs.global.attendedCommitGate.mode values. `auto`, the default, blocks
+ * the commit when the run is bound to the committing session (the session
+ * driving it) and warns otherwise.
+ */
+export const ATTENDED_COMMIT_MODES = ["off", "warn", "block", "auto"];
+
+/**
+ * The evidence an attended commit on `sha` lacks: every gate in
+ * ATTENDED_COMMIT_EVIDENCE, plus the review verdict from the commit phase on,
+ * whose latest entry for that sha, attended or not, is missing or not passing.
+ * A checkpoint commit during development comes before any review and is held
+ * to build and test only.
  *
  * @param {object} state
  * @param {string} sha
  * @returns {string[]}
  */
-export function attendedEvidenceGaps(state, sha) {
-  const latest = latestByGate(state?.gates, sha);
-  return ATTENDED_COMMIT_EVIDENCE.filter((g) => {
+export function attendedEvidenceGaps(state, sha, { sameTree } = {}) {
+  const wanted = [...ATTENDED_COMMIT_EVIDENCE];
+  if (Number(state?.currentPhase) >= ATTENDED_REVIEW_FROM_PHASE)
+    wanted.push(ATTENDED_REVIEW_EVIDENCE);
+  const asked = (Array.isArray(state?.gates) ? state.gates : []).filter((e) =>
+    wanted.includes(e?.gate),
+  );
+  const latest = sameTree
+    ? latestByGateWhere(asked, (s) => s === sha || sameTree(s))
+    : latestByGate(asked, sha);
+  return wanted.filter((g) => {
     const e = latest.get(g);
     return !e || !PASSING.has(e.verdict);
   });
 }
 
+/** The mode that applies: an explicit off/warn/block, else block for a bound run. */
+export function effectiveCommitMode(pref, bound) {
+  const mode = ATTENDED_COMMIT_MODES.includes(pref) ? pref : "auto";
+  if (mode !== "auto") return mode;
+  return bound ? "block" : "warn";
+}
+
 const QUESTION_ID = /^[a-z][a-z0-9-]*$/;
 const OPTION_ID = /^[a-z0-9][a-z0-9+_-]*$/;
 const QUESTION_KEYS = ["id", "stepId", "text", "kind", "options"];
@@ -184,6 +234,54 @@ export class LedgerError extends Error {
  * HEAD of the repository at `dir`, or null outside one.
  * @param {string} dir
  */
+/**
+ * A predicate: does a recorded sha hold the content this commit records?
+ * Yes when its tree is the index's, or the tree `git add -A` would stage from
+ * the working tree (built in a throwaway copy of the index, so the real index
+ * is untouched). The hook runs before a `git add ... && git commit` line
+ * stages anything, and Phase 4's `git reset HEAD~1` leaves a checkpoint
+ * commit's new files untracked; both still match the evidence they were
+ * tested with, while a file added after the evidence does not. Each sha is
+ * asked once; anything git cannot answer is "no".
+ */
+export function sameTreeAs(dir) {
+  const run = (args, env) => {
+    const r = spawnSync("git", ["-C", dir, ...args], {
+      encoding: "utf8",
+      timeout: 20000,
+      env: env ? { ...process.env, ...env } : process.env,
+    });
+    return r.status === 0 ? r.stdout.trim() : null;
+  };
+  const indexTree = run(["write-tree"]);
+  let workTree = null;
+  const indexPath = run(["rev-parse", "--path-format=absolute", "--git-path", "index"]);
+  if (indexPath) {
+    const scratch = mkdtempSync(join(tmpdir(), "ma-tree-"));
+    const copy = join(scratch, "index");
+    try {
+      if (existsSync(indexPath)) copyFileSync(indexPath, copy);
+      if (run(["add", "-A"], { GIT_INDEX_FILE: copy }) !== null) {
+        workTree = run(["write-tree"], { GIT_INDEX_FILE: copy });
+      }
+    } catch {
+      workTree = null;
+    } finally {
+      rmSync(scratch, { recursive: true, force: true });
+    }
+  }
+  const targets = new Set([indexTree, workTree].filter(Boolean));
+  const seen = new Map();
+  return (sha) => {
+    if (!targets.size || typeof sha !== "string" || !/^[0-9a-f]{7,64}$/.test(sha)) return false;
+    if (!seen.has(sha)) {
+      const tree = run(["rev-parse", "--verify", "--quiet", `${sha}^{tree}`]);
+      seen.set(sha, Boolean(tree) && targets.has(tree));
+    }
+    return seen.get(sha);
+  };
+}
+
 export function headSha(dir) {
   const r = spawnSync("git", ["-C", dir, "rev-parse", "HEAD"], {
     encoding: "utf8",
@@ -399,11 +497,22 @@ export async function record(
     detail: detail && typeof detail === "object" ? detail : { value: detail },
     ...(attended ? { attended: true } : {}),
   };
-  await updateState(path, (s) => ({
-    ...s,
-    gates: [...(Array.isArray(s.gates) ? s.gates : []), entry],
-  }));
-  return { written: true, statePath: path, entry };
+  let capped = false;
+  await updateState(path, (s) => {
+    const gates = [...(Array.isArray(s.gates) ? s.gates : []), entry];
+    const next = { ...s, gates };
+    if (
+      RETRY_CAPPED.has(gate) &&
+      verdict === "fail" &&
+      trailingFailures(gates, gate) >= RETRY_CAP
+    ) {
+      capped = true;
+      next.status = "paused";
+      next.haltReason = `${phaseOf(s)}:retry-cap: ${gate} failed ${RETRY_CAP} times in a row; the fix loop stops here for the user`;
+    }
+    return next;
+  });
+  return { written: true, statePath: path, entry, capped };
 }
 
 /**
@@ -494,9 +603,14 @@ function phaseOf(state) {
  * Latest entry per gate for one sha. "Latest" is by `at`, then by position.
  */
 export function latestByGate(gates, sha) {
+  return latestByGateWhere(gates, (s) => s === sha);
+}
+
+/** latestByGate over the entries whose sha `accept` takes. */
+export function latestByGateWhere(gates, accept) {
   const out = new Map();
   (Array.isArray(gates) ? gates : []).forEach((e, i) => {
-    if (!e || typeof e !== "object" || e.sha !== sha || typeof e.gate !== "string") return;
+    if (!e || typeof e !== "object" || typeof e.gate !== "string" || !accept(e.sha)) return;
     const prev = out.get(e.gate);
     const t = Date.parse(e.at) || 0;
     if (!prev || t > prev.t || (t === prev.t && i > prev.i)) out.set(e.gate, { e, t, i });
@@ -570,6 +684,12 @@ async function main(argv) {
     // Gates off: nothing is written and nothing is said.
     if (!r.written && r.reason === attendedVerdict(opts.gate).reason) return 0;
     if (!r.written) process.stderr.write(`gate-ledger: nothing written: ${r.reason}\n`);
+    if (r.capped) {
+      process.stderr.write(
+        `gate-ledger: ${opts.gate} failed ${RETRY_CAP} times in a row; the run is paused (retry cap). Report what was tried and ask the user how to continue.\n`,
+      );
+      return 5;
+    }
     return r.written ? 0 : 3;
   }
   if (mode === "check") {
@@ -619,9 +739,8 @@ async function main(argv) {
     // mode that commits nothing, evidence present), 1 warn, 4 block. Never 2
     // or 3 for a missing run: outside the pipeline a commit is none of this
     // hook's business.
-    const pref = globalPref("attendedCommitGate.mode", "warn");
-    const gateMode = ATTENDED_COMMIT_MODES.includes(pref) ? pref : "warn";
-    if (gateMode === "off") return 0;
+    const pref = globalPref("attendedCommitGate.mode", "auto");
+    if (pref === "off") return 0;
     let session = "";
     if (opts["hook-input"] === "-") {
       try {
@@ -638,7 +757,11 @@ async function main(argv) {
     if (modeEntry(state)?.commits === false) return 0;
     const sha = opts.sha || headSha(repo);
     if (!sha) return 0;
-    const missing = attendedEvidenceGaps(state, sha);
+    const launched = process.env.MULTI_AGENT_SESSION_ID || "";
+    const bound =
+      (session && state.hostSessionId === session) || (launched && state.sessionId === launched);
+    const gateMode = effectiveCommitMode(pref, Boolean(bound));
+    const missing = attendedEvidenceGaps(state, sha, { sameTree: sameTreeAs(repo) });
     if (opts.json)
       writeFileSync(1, `${JSON.stringify({ statePath: path, sha, gateMode, missing })}\n`);
     if (!missing.length) return 0;
@@ -647,9 +770,9 @@ async function main(argv) {
       2,
       `${[
         `pre-commit-check: ${verb}: the run's gate ledger (${path}) has no passing ${missing.join(" / ")} for HEAD ${sha.slice(0, 12)}.`,
-        "  Run the build and the tests through evidence-gate.mjs (Phase 2 Gate 1) so the commit rests on recorded evidence.",
+        "  Run the build and the tests through evidence-gate.mjs (Phase 2 Gate 1), and in the commit phase the review verdict (review-decision-gate.mjs), so the commit rests on recorded evidence.",
         gateMode === "block"
-          ? "  prefs.global.attendedCommitGate.mode is block; set it to warn to commit anyway."
+          ? "  This session drives the run, so the commit waits for the evidence; prefs.global.attendedCommitGate.mode warn lets it through."
           : "  The commit goes ahead; prefs.global.attendedCommitGate.mode block refuses it instead.",
       ].join("\n")}\n`,
     );