@mmerterden/multi-agent-pipeline 20.33.0 → 20.35.0

package/pipeline/scripts/code-graph-rules/android.json

@@ -38,10 +38,15 @@
       "kind": "class",
       "regex": "\\bclass\\s+([A-Z][A-Za-z0-9_]*)"
     },
+    {
+      "id": "composable",
+      "kind": "composable",
+      "regex": "\\bfun\\s+(?:<[^>]*>\\s*)?(?:[A-Za-z_][A-Za-z0-9_.]*\\.)?([A-Z][A-Za-z0-9_]*)\\s*\\("
+    },
     {
       "id": "fun",
       "kind": "fun",
-      "regex": "\\bfun\\s+(?:<[^>]*>\\s*)?([a-zA-Z_][A-Za-z0-9_]*)"
+      "regex": "\\bfun\\s+(?:<[^>]*>\\s*)?(?:[A-Za-z_][A-Za-z0-9_.]*\\.)?([a-z_][A-Za-z0-9_]*)"
     }
   ],
   "importPatterns": [
@@ -50,9 +55,9 @@
       "regex": "^\\s*import\\s+(?:static\\s+)?(?:[A-Za-z_][A-Za-z0-9_]*\\.)+([A-Z][A-Za-z0-9_]*)"
     }
   ],
-  "referenceKinds": ["class", "interface", "object", "enum", "record", "typealias"],
+  "referenceKinds": ["class", "interface", "object", "enum", "record", "typealias", "composable"],
   "supertypes": { "syntax": "colon", "kinds": ["class", "interface", "object", "enum"] },
-  "referenceKindsNote": "fun is deliberately absent, for the reason Swift's rules record for func: a bare lowercase name matched across files is almost never a call to that exact declaration, and on a large app the generic ones (invoke, build, create, get) take the top god-node slots purely because each happened to be declared once. Composables are the tempting exception - they are capitalised and behave like types - but they are declared with fun, so including them means including every other fun with them.",
+  "referenceKindsNote": "fun is deliberately absent, for the reason Swift's rules record for func: a bare lowercase name matched across files is almost never a call to that exact declaration, and on a large app the generic ones (invoke, build, create, get) take the top god-node slots purely because each happened to be declared once. A capitalised fun is the exception and gets its own kind, composable: Kotlin names ordinary functions in lowerCamelCase, so a capitalised fun is in practice a Compose UI function (or a factory named after the type it builds), and it is called by that exact name the way a type is.",
   "ignoredIdentifiers": [
     "this",
     "super",

package/pipeline/scripts/contract-server.mjs

@@ -305,7 +305,7 @@ import { runMain } from "../lib/fatal.mjs";
 import { invokedDirectly } from "../lib/invoked-directly.mjs";
 
 /** The contract as a whole; pipeline/contract/manifest.json carries the same. */
-export const CONTRACT_VERSION = "1.6.0";
+export const CONTRACT_VERSION = "1.7.0";
 /** The majors `?v=` accepts. A new major keeps the previous one here for one minor. */
 export const SUPPORTED_MAJORS = [1];
 export const BIND_HOST = "127.0.0.1";
@@ -907,20 +907,35 @@ async function control(args) {
 /**
  * For a read whose answer is the same for every caller: a request that
  * arrives while one runs gets that one's answer instead of starting another.
+ * With `limit`, at most `limit.max` distinct keys of `limit.group` run at
+ * once; one more is 429, like `oneAtATime`.
  */
 const inFlight = new Map();
-const shared = (keyOf, fn) => (ctx) => {
-  const key = keyOf(ctx);
-  if (!inFlight.has(key)) {
-    inFlight.set(
-      key,
-      Promise.resolve()
-        .then(() => fn(ctx))
-        .finally(() => inFlight.delete(key)),
-    );
-  }
-  return inFlight.get(key);
-};
+const groupRunning = new Map();
+const shared =
+  (keyOf, fn, limit = null) =>
+  (ctx) => {
+    const key = keyOf(ctx);
+    if (!inFlight.has(key)) {
+      if (limit) {
+        const n = groupRunning.get(limit.group) ?? 0;
+        if (n >= limit.max) {
+          throw new HttpError(429, "busy", "too many of these are running; try again shortly");
+        }
+        groupRunning.set(limit.group, n + 1);
+      }
+      inFlight.set(
+        key,
+        Promise.resolve()
+          .then(() => fn(ctx))
+          .finally(() => {
+            inFlight.delete(key);
+            if (limit) groupRunning.set(limit.group, groupRunning.get(limit.group) - 1);
+          }),
+      );
+    }
+    return inFlight.get(key);
+  };
 
 /** One request at a time per key; a second one while the first runs is 429. */
 const running = new Set();
@@ -1513,6 +1528,33 @@ function evidenceRoute({ params }) {
   }
 }
 
+/**
+ * GET /v1/prs/reach: work-items.mjs reach as a child process, because loading
+ * a repository's graph is a synchronous parse of a file that can run to tens
+ * of megabytes, and no other route may wait behind it.
+ */
+async function prReachRoute() {
+  const r = await spawnAsync(process.execPath, [join(HERE, "work-items.mjs"), "reach"], {
+    env: process.env,
+    timeoutMs: 180_000,
+    maxBytes: 2 * 1024 * 1024,
+  });
+  let doc;
+  try {
+    doc = JSON.parse(r.stdout);
+  } catch {
+    doc = null;
+  }
+  if (r.status !== 0 || !doc || !Array.isArray(doc.items)) {
+    throw new HttpError(
+      502,
+      "upstream-failed",
+      "the reach of the pull requests could not be measured",
+    );
+  }
+  return doc;
+}
+
 function workError(err) {
   if (!(err instanceof WorkError)) return err;
   if (err.code === "no-run" || err.code === "bad-key")
@@ -1790,7 +1832,7 @@ export const ROUTES = [
     method: "GET",
     path: "/v1/autopilot",
     query: [],
-    handle: oneAtATime("autopilot-status", autopilotGet),
+    handle: shared(() => "autopilot-status", autopilotGet),
   },
   {
     method: "POST",
@@ -1870,13 +1912,26 @@ export const ROUTES = [
     method: "GET",
     path: "/v1/issues/{key}/maturity",
     query: [],
-    handle: oneAtATime("maturity", maturityRoute),
+    handle: shared(({ params }) => `maturity:${params.key}`, maturityRoute, {
+      group: "maturity",
+      max: 2,
+    }),
   },
   {
     method: "GET",
     path: "/v1/prs",
     query: [],
-    handle: oneAtATime("prs", ({ redact }) => pullRequests({ redact })),
+    handle: shared(
+      ({ redact }) => `prs:${redact ? 1 : 0}`,
+      ({ redact }) => pullRequests({ redact }),
+    ),
+  },
+  {
+    method: "GET",
+    path: "/v1/prs/reach",
+    query: [],
+    remote: "local-only",
+    handle: shared(() => "prs-reach", prReachRoute),
   },
   { method: "GET", path: "/v1/tools", query: [], handle: toolsRoute },
   {

package/pipeline/scripts/debt-ledger.mjs

@@ -0,0 +1,323 @@
+#!/usr/bin/env node
+/**
+ * @file debt-ledger.mjs  -  the known limits a repository carries, in one list.
+ *
+ * Three sources feed it, each a limit somebody accepted on purpose:
+ *
+ *   shortcut   a `shortcut: <limit>, <when to replace>` marker a diff adds
+ *              (features/scope-check.md, "Known limits in the code")
+ *   deferred   a triage finding deferred as out of scope
+ *   parked     a finding parked by rework-rulings.mjs at the rework cap
+ *
+ * Storage: $HOME/.claude/memory/multi-agent/<repo-slug>/debt.jsonl, append-only,
+ * one row per item; the same item from the same task is written once. Every
+ * row passes the secret scan before it is written (_secret-scan.mjs).
+ *
+ * Subcommands:
+ *   markers  --patch <unified diff|->         print the markers the diff adds (no write)
+ *   add      --patch <diff> --task <id>       record those markers
+ *   from-run --triage <path> [--state <agent-state.json>] --task <id>
+ *                                             record deferred findings and parked rulings
+ *   list     [--open] [--json]                rows, newest first
+ *   resolve  --id <id>                        mark one row resolved (appends a resolution row)
+ *   [--repo-slug <slug>] on every subcommand; else $MULTI_AGENT_REPO_SLUG, then the git repo name.
+ *
+ * Exit: 0 done · 1 IO error · 2 nothing to record or unknown id · 64 usage
+ *
+ * @module pipeline/scripts/debt-ledger
+ */
+
+import { execFileSync } from "node:child_process";
+import { appendFileSync, existsSync, mkdirSync, readFileSync, realpathSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { shortId } from "./_retrieval.mjs";
+import { parseUnifiedDiff } from "./_unified-diff.mjs";
+import { checkedSlug, memoryExcluded } from "./_memory-policy.mjs";
+import { openScanner, stripPrivate } from "./_secret-scan.mjs";
+
+const COMMENT_OPENERS = ["//", "#", "/*", "<!--", "--", ";", "*"];
+const MAX_LINE = 2000;
+const MAX_TEXT = 300;
+
+/**
+ * True when position `at` of the line sits inside a string literal. With
+ * `apostrophes: false` a single quote opens nothing: before a `//` or `#`
+ * comment it is as often a lifetime or a word's apostrophe as a string.
+ */
+function insideString(line, at, apostrophes = true) {
+  let quote = null;
+  for (let i = 0; i < at; i++) {
+    const c = line[i];
+    if (quote) {
+      if (c === "\\") i++;
+      else if (c === quote) quote = null;
+    } else if (c === '"' || c === "`" || (apostrophes && c === "'")) {
+      quote = c;
+    }
+  }
+  return quote !== null;
+}
+
+/**
+ * The text after `shortcut:` when it opens a comment (only spaces between the
+ * comment opener and the marker, the opener outside any string), else null.
+ * `--` counts only at the start of the line, so a CLI flag is not a comment.
+ * Lowercase only, as documented.
+ */
+function markerIn(line) {
+  const at = line.indexOf("shortcut:");
+  if (at === -1) return null;
+  const before = line.slice(0, at).trimEnd();
+  const opener = COMMENT_OPENERS.find((o) => before.endsWith(o));
+  if (!opener) return null;
+  const openAt = before.length - opener.length;
+  if (opener === "--" && before.trim() !== "--") return null;
+  if (insideString(line, openAt, opener !== "//" && opener !== "#")) return null;
+  return line.slice(at + "shortcut:".length).trim();
+}
+
+/** Markers added by a unified diff: [{file, line, text}]. */
+export function markersIn(patch) {
+  const out = [];
+  for (const [file, diff] of parseUnifiedDiff(patch)) {
+    for (const { line, text: raw } of diff.addedAt) {
+      const marker = markerIn(raw.slice(0, MAX_LINE));
+      if (marker === null) continue;
+      let text = marker;
+      for (const close of ["*/", "-->"])
+        if (text.endsWith(close)) text = text.slice(0, -close.length);
+      text = text.trimEnd();
+      if (text) out.push({ file, line, text });
+    }
+  }
+  return out;
+}
+
+/** Deferred findings and parked rulings of one run: [{kind, file, line, text}]. */
+export function runItems(triage, state) {
+  const items = [];
+  for (const d of triage?.deferred ?? []) {
+    const f = d?.finding ?? {};
+    if (!f.issue) continue;
+    items.push({
+      kind: "deferred",
+      file: f.file ?? null,
+      line: Number.isInteger(f.line) ? f.line : null,
+      text: `${f.issue}${d.reason ? ` (deferred: ${d.reason})` : ""}`,
+    });
+  }
+  const iterations = Array.isArray(state?.reviewIterations) ? state.reviewIterations : [];
+  for (const r of iterations.at(-1)?.rulings ?? []) {
+    if (r?.ruling !== "parked" || !r.issue) continue;
+    items.push({
+      kind: "parked",
+      file: r.file ?? null,
+      line: Number.isInteger(r.line) ? r.line : null,
+      text: `${r.issue} (${r.reason})`,
+    });
+  }
+  return items;
+}
+
+const rowId = (r) => shortId("D", r.kind, r.file ?? "", r.text);
+
+function parseFlags(argv) {
+  const flags = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (!a.startsWith("--")) continue;
+    const next = argv[i + 1];
+    if (next !== undefined && !next.startsWith("--")) {
+      flags[a.slice(2)] = next;
+      i++;
+    } else flags[a.slice(2)] = true;
+  }
+  return flags;
+}
+
+function repoSlug(flags) {
+  return checkedSlug(rawRepoSlug(flags));
+}
+
+function rawRepoSlug(flags) {
+  if (typeof flags["repo-slug"] === "string") return flags["repo-slug"];
+  if (process.env.MULTI_AGENT_REPO_SLUG) return process.env.MULTI_AGENT_REPO_SLUG;
+  try {
+    const common = execFileSync(
+      "git",
+      ["rev-parse", "--path-format=absolute", "--git-common-dir"],
+      {
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "ignore"],
+      },
+    ).trim();
+    const root = basename(common) === ".git" ? dirname(common) : common.replace(/\.git$/, "");
+    if (basename(root)) return basename(root);
+  } catch {
+    /* not a git repo */
+  }
+  return "default";
+}
+
+const ledgerPath = (slug) =>
+  join(homedir(), ".claude", "memory", "multi-agent", slug, "debt.jsonl");
+
+function readRows(slug) {
+  const p = ledgerPath(slug);
+  if (!existsSync(p)) return [];
+  return readFileSync(p, "utf8")
+    .split("\n")
+    .filter((l) => l.trim())
+    .flatMap((l) => {
+      try {
+        return [JSON.parse(l)];
+      } catch {
+        return [];
+      }
+    });
+}
+
+/** Current state per id: the last row wins, so a resolution closes its item. */
+function current(rows) {
+  const byId = new Map();
+  for (const r of rows) byId.set(r.id, { ...byId.get(r.id), ...r });
+  return [...byId.values()];
+}
+
+function record(slug, items, task) {
+  if (memoryExcluded(slug)) return { added: 0, droppedBySecretScan: 0, excluded: true };
+  const p = ledgerPath(slug);
+  const seen = new Set(readRows(slug).map((r) => `${r.id}::${r.task}`));
+  const scan = openScanner();
+  let added = 0;
+  let droppedBySecretScan = 0;
+  try {
+    for (const it of items) {
+      const whole = stripPrivate(it.text).replace(/\s+/g, " ").trim();
+      if (!whole) continue;
+      // Scanned whole and cut afterwards: a token across the cut would
+      // otherwise be shortened below the scanner's minimum and stored.
+      const text = whole.slice(0, MAX_TEXT);
+      const row = { kind: it.kind, file: it.file, line: it.line, text };
+      const id = rowId(row);
+      if (seen.has(`${id}::${task}`)) continue;
+      if (!scan.isClean(`${whole}\n${it.file ?? ""}`)) {
+        droppedBySecretScan++;
+        continue;
+      }
+      mkdirSync(dirname(p), { recursive: true });
+      appendFileSync(
+        p,
+        JSON.stringify({ v: 1, id, ts: new Date().toISOString(), task, status: "open", ...row }) +
+          "\n",
+      );
+      seen.add(`${id}::${task}`);
+      added++;
+    }
+  } finally {
+    scan.close();
+  }
+  return { added, droppedBySecretScan };
+}
+
+function readJson(path) {
+  try {
+    return JSON.parse(readFileSync(path, "utf8"));
+  } catch {
+    return null;
+  }
+}
+
+function readPatch(path) {
+  return path === "-" ? readFileSync(0, "utf8") : readFileSync(path, "utf8");
+}
+
+function main(argv) {
+  const sub = argv[0];
+  const flags = parseFlags(argv.slice(1));
+  const out = (doc) => process.stdout.write(JSON.stringify(doc) + "\n");
+  const slug = repoSlug(flags);
+  const task = typeof flags.task === "string" ? flags.task : null;
+
+  if (sub === "markers" || sub === "add") {
+    if (typeof flags.patch !== "string") return usage();
+    const markers = markersIn(readPatch(flags.patch));
+    if (sub === "markers") {
+      out({ markers });
+      return 0;
+    }
+    if (!task) return usage();
+    const r = record(
+      slug,
+      markers.map((m) => ({ kind: "shortcut", ...m })),
+      task,
+    );
+    out({ ok: true, slug, ...r });
+    return r.added ? 0 : 2;
+  }
+  if (sub === "from-run") {
+    if (typeof flags.triage !== "string" || !task) return usage();
+    const triage = readJson(flags.triage);
+    if (!triage) {
+      console.error(`debt-ledger: triage unreadable: ${flags.triage}`);
+      return 1;
+    }
+    const state = typeof flags.state === "string" ? readJson(flags.state) : null;
+    const r = record(slug, runItems(triage, state), task);
+    out({ ok: true, slug, ...r });
+    return r.added ? 0 : 2;
+  }
+  if (sub === "list") {
+    let rows = current(readRows(slug)).reverse();
+    if (flags.open) rows = rows.filter((r) => r.status === "open");
+    if (flags.json) out({ slug, rows });
+    else
+      for (const r of rows)
+        process.stdout.write(
+          `${r.id} ${r.status} ${r.kind} ${r.file ?? "-"}${r.line ? `:${r.line}` : ""} ${r.text}\n`,
+        );
+    return 0;
+  }
+  if (sub === "resolve") {
+    if (typeof flags.id !== "string") return usage();
+    if (!current(readRows(slug)).some((r) => r.id === flags.id)) {
+      console.error(`debt-ledger: no item ${flags.id}`);
+      return 2;
+    }
+    appendFileSync(
+      ledgerPath(slug),
+      JSON.stringify({ id: flags.id, status: "resolved", resolvedAt: new Date().toISOString() }) +
+        "\n",
+    );
+    out({ ok: true, slug, resolved: flags.id });
+    return 0;
+  }
+  return usage();
+}
+
+function usage() {
+  console.error(
+    "usage: debt-ledger.mjs markers --patch <diff|-> | add --patch <diff> --task <id> | from-run --triage <path> [--state <path>] --task <id> | list [--open] [--json] | resolve --id <id>",
+  );
+  return 64;
+}
+
+const invoked = () => {
+  try {
+    return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url));
+  } catch {
+    return false;
+  }
+};
+
+if (invoked()) {
+  try {
+    process.exitCode = main(process.argv.slice(2));
+  } catch (e) {
+    console.error(`debt-ledger: ${e.message}`);
+    process.exitCode = 1;
+  }
+}