@mmerterden/multi-agent-pipeline 20.23.2 → 20.24.0

package/pipeline/scripts/cost-analyze.mjs

@@ -46,20 +46,12 @@
  *   2   usage error
  */
 
-import {
-  createReadStream,
-  existsSync,
-  mkdirSync,
-  readdirSync,
-  readFileSync,
-  statSync,
-  writeFileSync,
-} from "node:fs";
-import { createInterface } from "node:readline";
+import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { costUsd } from "./_cost.mjs";
+import { readJsonl } from "./_jsonl.mjs";
 import { runMain } from "../lib/fatal.mjs";
 import { messages, outputLanguage } from "../lib/user-facing.mjs";
 
@@ -263,20 +255,10 @@ async function readSeries({ days }) {
   const unpricedModels = new Set();
 
   for (const f of files) {
-    let rl;
-    try {
-      rl = createInterface({ input: createReadStream(f.path), crlfDelay: Infinity });
-    } catch {
-      continue;
-    }
-    for await (const line of rl) {
-      if (!line || line.charCodeAt(0) !== 123) continue;
-      let d;
-      try {
-        d = JSON.parse(line);
-      } catch {
-        continue;
-      }
+    // Uncapped: a cost total that silently stops at a byte cap under-reports.
+    for await (const row of readJsonl(f.path, { maxBytes: Infinity, maxLineBytes: Infinity })) {
+      const d = row.record;
+      if (!d) continue;
       const msg = d.message;
       const u = msg?.usage;
       if (!u) continue;

package/pipeline/scripts/decisions-log.mjs

@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+/**
+ * decisions-log.mjs  -  append to and render `state.decisions[]`, the list of
+ * choices an autopilot run made where an attended run would have asked.
+ *
+ * Writers: a picker resolved without asking (picker-contract.md, autopilot),
+ * a split consensus taken on the triage verdict, a model fallback taken
+ * without the date-gate question, a rework dispatched one rung up
+ * (rework-escalation.mjs). gate-ledger.mjs park and review-decision-gate.mjs
+ * write their own entries through lib/decisions.mjs.
+ *
+ * `add` writes only while the quality gates are active (lib/unattended.mjs
+ * gatesActive): in an attended run the person answered, so there is nothing to
+ * report back. Writes take the write-state lock through gate-ledger.mjs
+ * updateState.
+ *
+ * `render` prints the markdown section Phase 5 puts in its report, redacted,
+ * and nothing at all for a run with no entries.
+ *
+ * Usage:
+ *   decisions-log.mjs add --what <text> --why <text> --cost-if-wrong <text> --phase <0-5>
+ *                         [--state <path>] [--repo <dir>]
+ *   decisions-log.mjs render [--state <path>] [--repo <dir>]
+ *
+ * Exit: 0 written, skipped (attended) or rendered · 2 usage · 3 no state or I/O error
+ *
+ * @module pipeline/scripts/decisions-log
+ */
+
+import { writeFileSync } from "node:fs";
+
+import { decisionEntry, renderDecisions, withDecision } from "../lib/decisions.mjs";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+import { readRegularJson } from "../lib/regular-file.mjs";
+import { gateContext, resolveStatePath, updateState } from "./gate-ledger.mjs";
+
+class UsageError extends Error {}
+
+function parse(argv) {
+  const opts = {};
+  for (let i = 1; i < argv.length; i++) {
+    const a = argv[i];
+    if (!a.startsWith("--")) throw new UsageError(`unexpected argument ${a}`);
+    const v = argv[++i];
+    if (v === undefined) throw new UsageError(`${a} needs a value`);
+    opts[a.slice(2)] = v;
+  }
+  return opts;
+}
+
+/**
+ * Append one entry when the gates are active. Resolves to whether it wrote.
+ *
+ * @param {{what: string, why: string, costIfWrong: string, phase: number}} input
+ * @param {{statePath?: string|null, repo?: string|null, env?: Record<string,string|undefined>}} [opts]
+ */
+export async function addDecision(
+  input,
+  { statePath = null, repo = null, env = process.env } = {},
+) {
+  const entry = decisionEntry(input);
+  const ctx = gateContext({ statePath, repo, env });
+  if (!ctx.active || !ctx.statePath) return { written: false, entry };
+  await updateState(ctx.statePath, (s) => withDecision(s, entry));
+  return { written: true, entry, statePath: ctx.statePath };
+}
+
+async function main(argv) {
+  const mode = argv[0];
+  const opts = parse(argv);
+  if (mode === "add") {
+    let input;
+    try {
+      input = decisionEntry({
+        what: opts.what,
+        why: opts.why,
+        costIfWrong: opts["cost-if-wrong"],
+        phase: opts.phase,
+      });
+    } catch (err) {
+      throw new UsageError(err.message);
+    }
+    await addDecision(input, { statePath: opts.state || null, repo: opts.repo || process.cwd() });
+    return 0;
+  }
+  if (mode === "render") {
+    const path = resolveStatePath({ statePath: opts.state, repo: opts.repo || process.cwd() });
+    if (!path) {
+      process.stderr.write("decisions-log: no run state found\n");
+      return 3;
+    }
+    const state = readRegularJson(path, null);
+    if (!state || typeof state !== "object") {
+      process.stderr.write(`decisions-log: cannot read ${path}\n`);
+      return 3;
+    }
+    const md = renderDecisions(state.decisions);
+    if (md) writeFileSync(1, md);
+    return 0;
+  }
+  throw new UsageError("mode must be add or render");
+}
+
+if (invokedDirectly(import.meta.url)) {
+  main(process.argv.slice(2)).then(
+    (code) => process.exit(code),
+    (err) => {
+      process.stderr.write(`decisions-log: ${err.message}\n`);
+      if (err instanceof UsageError) {
+        process.stderr.write(
+          "usage: decisions-log.mjs add --what t --why t --cost-if-wrong t --phase n [--state p] | render [--state p]\n",
+        );
+        process.exit(2);
+      }
+      process.exit(3);
+    },
+  );
+}

package/pipeline/scripts/edit-syntax-gate.mjs

@@ -0,0 +1,240 @@
+#!/usr/bin/env node
+/**
+ * edit-syntax-gate.mjs  -  parse the file an edit just wrote, report a syntax error.
+ *
+ * Called by edit-syntax-gate.sh as a PostToolUse hook on Edit / Write /
+ * MultiEdit. Picks one parser by extension, runs it under a short deadline and,
+ * when the file does not parse, prints the Claude Code PostToolUse context
+ * object so the model sees the error before its next step:
+ *
+ *   .swift              swiftc -parse (when swiftc is on PATH)
+ *   .js .mjs .cjs       node --check (a file that looks like JSX: skipped)
+ *   .py                 python3 py_compile, bytecode written to a temp dir
+ *   .json               JSON.parse after a UTF-8 BOM is dropped; files that
+ *                       are JSONC by convention (tsconfig / jsconfig, .vscode,
+ *                       .devcontainer, .eslintrc, turbo, deno) are skipped
+ *   .sh .bash           bash -n
+ *
+ * Kotlin has no parser that answers inside the deadline (kotlinc starts a JVM),
+ * so .kt / .kts are skipped, as is every other extension (.jsx and .jsonc
+ * among them).
+ *
+ * Silent on: the gate switched off (`editSyntaxGate.enabled` not true), a parser
+ * not installed, a parser that runs past the deadline, a file that is gone or
+ * too large, an unreadable payload. A missing tool is the user's toolchain, not
+ * the edit's fault, and a false alarm costs more than a missed one: the build
+ * still catches what this lets through.
+ *
+ * Exit 0 always.
+ *
+ * @module pipeline/scripts/edit-syntax-gate
+ */
+
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { basename, extname, join, resolve, sep } from "node:path";
+
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+
+const EDIT_TOOLS = new Set(["Edit", "Write", "MultiEdit"]);
+const DEADLINE_MS = 2000;
+const MAX_BYTES = 5 * 1024 * 1024;
+const MAX_LINES = 6;
+const MAX_CHARS = 800;
+const NODE_INTERNAL_FRAME = /^\s*at .*\(?node:internal[/:]/;
+const PREF = "editSyntaxGate.enabled";
+
+function prefsCandidates() {
+  const home = process.env.HOME || homedir();
+  return [
+    process.env.MULTI_AGENT_PREFS,
+    join(home, ".claude", "multi-agent-preferences.json"),
+    join(home, ".config", "multi-agent-pipeline", "multi-agent-preferences.json"),
+  ].filter(Boolean);
+}
+
+/** `prefs.global.editSyntaxGate.enabled` from the first prefs file that exists. */
+export function gateEnabled(paths = prefsCandidates()) {
+  for (const p of paths) {
+    if (!existsSync(p)) continue;
+    try {
+      let node = JSON.parse(readFileSync(p, "utf8"))?.global;
+      for (const key of PREF.split(".")) node = node?.[key];
+      return node === true;
+    } catch {
+      return false;
+    }
+  }
+  return false;
+}
+
+const JSONC_NAMES =
+  /^(?:(?:tsconfig|jsconfig)(?:\..+)?\.json|\.eslintrc\.json|turbo\.json|deno\.jsonc?|devcontainer\.json|\.devcontainer\.json)$/;
+
+function isJsonc(file) {
+  return (
+    JSONC_NAMES.test(basename(file)) ||
+    file.includes(`${sep}.vscode${sep}`) ||
+    file.includes(`${sep}.devcontainer${sep}`)
+  );
+}
+
+/** `text` with string literals and comments blanked, so only code is left. */
+function codeOnly(text) {
+  let out = "";
+  for (let i = 0; i < text.length; i++) {
+    const c = text[i];
+    if (c === "/" && text[i + 1] === "/") {
+      while (i < text.length && text[i] !== "\n") i++;
+      out += "\n";
+    } else if (c === "/" && text[i + 1] === "*") {
+      const end = text.indexOf("*/", i + 2);
+      i = end === -1 ? text.length : end + 1;
+      out += " ";
+    } else if (c === '"' || c === "'" || c === "`") {
+      for (i++; i < text.length && text[i] !== c; i++) {
+        if (text[i] === "\\") i++;
+        else if (c !== "`" && text[i] === "\n") break;
+      }
+      out += " ";
+    } else out += c;
+  }
+  return out;
+}
+
+const JSX_TAG = /<[A-Z][\w.]*(?=[\s/>])|<\/[A-Za-z]?/;
+
+/**
+ * True when the code (strings and comments aside) carries an element-like
+ * `<Capitalized` or a closing `</`. node --check rejects JSX, and a file that
+ * only looks like JSX is skipped rather than flagged.
+ */
+export function looksLikeJsx(text) {
+  return JSX_TAG.test(codeOnly(text));
+}
+
+function run(cmd, args, cwd) {
+  const r = spawnSync(cmd, args, {
+    cwd,
+    encoding: "utf8",
+    timeout: DEADLINE_MS,
+    killSignal: "SIGKILL",
+    stdio: ["ignore", "pipe", "pipe"],
+    env: { ...process.env, PYTHONDONTWRITEBYTECODE: "1" },
+  });
+  if (r.error || r.signal || r.status === null) return null;
+  if (r.status === 0) return { ok: true };
+  return { ok: false, output: `${r.stderr || ""}${r.stdout || ""}` };
+}
+
+/**
+ * Parse one file. `null` = not judged (no parser, parser missing, deadline);
+ * `{ok}` = parsed; `{ok: false, checker, output}` = syntax error.
+ */
+export function checkFile(file) {
+  const ext = extname(file).toLowerCase();
+  if (ext === ".json") {
+    if (isJsonc(file)) return null;
+    try {
+      JSON.parse(readFileSync(file, "utf8").replace(/^\uFEFF/, ""));
+      return { ok: true };
+    } catch (err) {
+      return { ok: false, checker: "JSON.parse", output: err.message };
+    }
+  }
+  const scratch = mkdtempSync(join(tmpdir(), "edit-syntax-gate-"));
+  try {
+    let checker;
+    let res;
+    if (ext === ".js" || ext === ".mjs" || ext === ".cjs") {
+      if (looksLikeJsx(readFileSync(file, "utf8"))) return null;
+      checker = "node --check";
+      res = run(process.execPath, ["--check", file], scratch);
+    } else if (ext === ".sh" || ext === ".bash") {
+      checker = "bash -n";
+      res = run("bash", ["-n", file], scratch);
+    } else if (ext === ".py") {
+      checker = "python3 py_compile";
+      res = run(
+        "python3",
+        [
+          "-I",
+          "-c",
+          [
+            "import py_compile, sys",
+            "try:",
+            "    py_compile.compile(sys.argv[1], cfile=sys.argv[2], doraise=True)",
+            "except py_compile.PyCompileError as e:",
+            "    sys.stderr.write(e.msg)",
+            "    sys.exit(1)",
+          ].join("\n"),
+          file,
+          join(scratch, "out.pyc"),
+        ],
+        scratch,
+      );
+    } else if (ext === ".swift") {
+      checker = "swiftc -parse";
+      res = run("swiftc", ["-parse", file], scratch);
+    } else {
+      return null;
+    }
+    if (!res) return null;
+    return res.ok ? res : { ...res, checker };
+  } finally {
+    rmSync(scratch, { recursive: true, force: true });
+  }
+}
+
+function brief(output) {
+  const lines = String(output || "")
+    .split("\n")
+    .map((l) => l.trimEnd())
+    .filter((l) => l && !NODE_INTERNAL_FRAME.test(l));
+  const text = lines.slice(0, MAX_LINES).join("\n");
+  return text.length > MAX_CHARS ? `${text.slice(0, MAX_CHARS)}...` : text;
+}
+
+/** The PostToolUse context object for a syntax error, or null. */
+export function message(file, result) {
+  if (!result || result.ok) return null;
+  return {
+    hookSpecificOutput: {
+      hookEventName: "PostToolUse",
+      additionalContext: `edit-syntax-gate: ${file} no longer parses (${result.checker}). Fix it before going on:\n${brief(result.output)}`,
+    },
+  };
+}
+
+async function main() {
+  let payload;
+  try {
+    payload = JSON.parse(readFileSync(0, "utf8"));
+  } catch {
+    return;
+  }
+  if (!EDIT_TOOLS.has(payload?.tool_name)) return;
+  const raw = payload?.tool_input?.file_path;
+  if (typeof raw !== "string" || !raw) return;
+  // The checkers run in a scratch directory, so a relative path is made
+  // absolute here, against the session's cwd when the payload names one.
+  const base = typeof payload.cwd === "string" && payload.cwd ? payload.cwd : process.cwd();
+  const file = resolve(base, raw);
+  if (!gateEnabled()) return;
+  let st;
+  try {
+    st = statSync(file);
+  } catch {
+    return;
+  }
+  if (!st.isFile() || st.size > MAX_BYTES) return;
+  const out = message(file, checkFile(file));
+  if (out) process.stdout.write(`${JSON.stringify(out)}\n`);
+}
+
+// Fail open: a gate bug must not surface as a hook error on every edit.
+if (invokedDirectly(import.meta.url)) {
+  const done = () => process.exit(0);
+  main().catch(done).then(done);
+}

package/pipeline/scripts/edit-syntax-gate.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# edit-syntax-gate.sh  -  PostToolUse syntax check for the multi-agent pipeline.
+#
+# Runs after an Edit / Write / MultiEdit and parses the edited file with the
+# fastest parser its extension has, so a broken file is reported in the turn
+# that broke it instead of at the next build. The parsing is in
+# edit-syntax-gate.mjs; this wrapper exists because every hook the pipeline
+# registers is a `bash "$HOME/.claude/scripts/<name>"` command.
+#
+# Contract (Claude Code PostToolUse hook):
+#   - Reads the tool-call JSON on stdin.
+#   - On a parse error prints {"hookSpecificOutput": {"hookEventName":
+#     "PostToolUse", "additionalContext": "..."}} on stdout, which the model
+#     reads as context for its next step.
+#   - Exits 0 on every path: the edit has already happened, so blocking would
+#     only hide the message.
+#
+# Inert unless `prefs.global.editSyntaxGate.enabled` is true. No node on PATH,
+# a missing parser, a timeout or any internal error means no message.
+
+set -u
+
+HERE="$(cd "$(dirname "$0")" 2>/dev/null && pwd || true)"
+CORE="$HERE/edit-syntax-gate.mjs"
+[ -f "$CORE" ] || exit 0
+command -v node >/dev/null 2>&1 || exit 0
+
+node "$CORE" 2>/dev/null || true
+exit 0