@mmerterden/multi-agent-pipeline 20.33.0 → 20.35.0

package/pipeline/scripts/graph-links.mjs

@@ -0,0 +1,230 @@
+#!/usr/bin/env node
+/**
+ * @file graph-links.mjs  -  which files of one repository import another.
+ *
+ * Each repository keeps its own code graph (ADR-0010); a run that touches a
+ * main repository and its libraries needs to know where they meet. This joins
+ * graphs without merging them: a repository's import of a name it does not
+ * declare (a `module:` node) is matched against the names other repositories
+ * export, and every match becomes one link row.
+ *
+ * What a repository exports, by stack:
+ *   node      the `name` in package.json
+ *   python    top-level package directories (they hold __init__.py)
+ *   ios       target directories under Sources/
+ *   go        the module path in go.mod
+ *   android   top-level, non-nested type and composable declarations, since a
+ *             Kotlin import names a class
+ *   any       the repository directory name, when the stack names nothing
+ *
+ * A Kotlin import keeps only its last segment in the graph, and names like
+ * `Text` or `Result` exist in many packages. An android link therefore holds
+ * only the importing files whose `import <package>.<Name>` names the package
+ * the exporting file declares; both repositories are read from disk for it.
+ *
+ * Usage:
+ *   graph-links.mjs build --projects <a,b,...> [--out <path>]
+ *   graph-links.mjs affected --project <b> [--name <module-or-symbol>] [--links <path>]
+ *
+ *   Projects are directory names under ~/.claude/knowledge/. `build` writes
+ *   ~/.claude/knowledge/_links.json unless --out says otherwise. `affected`
+ *   lists the files in other repositories that import project b (or one of
+ *   its names), so a change there can be followed across.
+ *
+ * Exit: 0 done · 1 a graph or the link file is missing · 64 usage
+ *
+ * @module pipeline/scripts/graph-links
+ */
+
+import { existsSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { basename, join, resolve } from "node:path";
+
+import { loadGraph } from "./graph-query.mjs";
+import { invokedDirectly } from "../lib/invoked-directly.mjs";
+
+const KNOWLEDGE = () => join(homedir(), ".claude", "knowledge");
+const PROJECT_RE = /^[A-Za-z0-9._-]+$/;
+const ANDROID_KINDS = new Set(["class", "object", "interface", "enum", "composable", "typealias"]);
+
+function readText(p) {
+  try {
+    return readFileSync(p, "utf8");
+  } catch {
+    return null;
+  }
+}
+
+function dirs(p) {
+  try {
+    return readdirSync(p, { withFileTypes: true })
+      .filter((d) => d.isDirectory() && !d.name.startsWith("."))
+      .map((d) => d.name);
+  } catch {
+    return [];
+  }
+}
+
+/** name -> what it points at in this repository (a symbol id, or null for the repo). */
+export function exportsOf(graph) {
+  const out = new Map();
+  const root = graph.root || "";
+  if (graph.stack === "node") {
+    const pkg = readText(join(root, "package.json"));
+    try {
+      const name = JSON.parse(pkg ?? "null")?.name;
+      if (typeof name === "string" && name) out.set(name, null);
+    } catch {
+      /* no package name */
+    }
+  } else if (graph.stack === "python") {
+    for (const d of dirs(root)) if (existsSync(join(root, d, "__init__.py"))) out.set(d, null);
+  } else if (graph.stack === "ios") {
+    for (const d of dirs(join(root, "Sources"))) out.set(d, null);
+  } else if (graph.stack === "go") {
+    const mod = /^module\s+(\S+)/m.exec(readText(join(root, "go.mod")) ?? "");
+    if (mod) out.set(mod[1], null);
+  } else if (graph.stack === "android") {
+    for (const n of graph.nodes) {
+      if (n.kind === "symbol" && !n.nested && ANDROID_KINDS.has(n.symbolKind) && !out.has(n.name))
+        out.set(n.name, n.id);
+    }
+  }
+  if (!out.size && root) out.set(basename(root), null);
+  return out;
+}
+
+/** The package a Kotlin symbol's file declares, or null. */
+function kotlinPackage(root, symbolId, cache) {
+  const rel = symbolId.slice("sym:".length, symbolId.lastIndexOf("#"));
+  const key = `${root}\0${rel}`;
+  if (!cache.has(key)) {
+    const text = rel.includes("..") ? null : readText(join(root, rel));
+    cache.set(key, /^\s*package\s+([\w.]+)/m.exec(text ?? "")?.[1] ?? null);
+  }
+  return cache.get(key);
+}
+
+/** True when the file imports `name` from exactly that package. */
+function importsFrom(root, rel, pkg, name) {
+  if (rel.includes("..")) return false;
+  const text = readText(join(root, rel)) ?? "";
+  const esc = (v) => v.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+  return new RegExp(`^\\s*import\\s+${esc(pkg)}\\.${esc(name)}(?![\\w])`, "m").test(text);
+}
+
+/** The names an import could be asking for: itself, and its package prefix. */
+export function candidates(moduleName) {
+  const m = String(moduleName);
+  const set = new Set([m]);
+  if (m.startsWith("@")) set.add(m.split("/").slice(0, 2).join("/"));
+  else set.add(m.split(/[/.]/)[0]);
+  return [...set];
+}
+
+/** Link rows between every pair of the given graphs. */
+export function link(graphs) {
+  const exported = graphs.map((g) => ({
+    project: g.project,
+    graph: g.graph,
+    names: exportsOf(g.graph),
+  }));
+  const packages = new Map();
+  const rows = [];
+  for (const { project, graph } of graphs) {
+    const importers = new Map();
+    for (const e of graph.edges) {
+      if (e.kind !== "imports" || !e.to.startsWith("module:")) continue;
+      if (!importers.has(e.to)) importers.set(e.to, []);
+      importers.get(e.to).push(e.from.replace(/^file:/, ""));
+    }
+    for (const [mid, files] of importers) {
+      const name = mid.slice("module:".length);
+      for (const other of exported) {
+        if (other.project === project) continue;
+        const hit = candidates(name).find((c) => other.names.has(c));
+        if (hit === undefined) continue;
+        const target = other.names.get(hit);
+        let kept = [...new Set(files)].sort();
+        if (other.graph.stack === "android" && target) {
+          const pkg = kotlinPackage(other.graph.root || "", target, packages);
+          kept = pkg ? kept.filter((f) => importsFrom(graph.root || "", f, pkg, hit)) : [];
+        }
+        if (!kept.length) continue;
+        rows.push({ from: project, module: name, to: other.project, target, files: kept });
+      }
+    }
+  }
+  return rows.sort(
+    (a, b) =>
+      a.to.localeCompare(b.to) || a.from.localeCompare(b.from) || a.module.localeCompare(b.module),
+  );
+}
+
+function graphPath(project) {
+  if (!PROJECT_RE.test(project) || project === "." || project === "..")
+    throw new Error(`not a project name: ${project}`);
+  return join(KNOWLEDGE(), project, "code-graph.json");
+}
+
+function flag(argv, name) {
+  const i = argv.indexOf(`--${name}`);
+  return i >= 0 ? argv[i + 1] : undefined;
+}
+
+function main(argv) {
+  const sub = argv[0];
+  const linksPath = resolve(
+    flag(argv, "links") || flag(argv, "out") || join(KNOWLEDGE(), "_links.json"),
+  );
+  if (sub === "build") {
+    const projects = (flag(argv, "projects") || "").split(",").filter(Boolean);
+    if (projects.length < 2) return usage();
+    let graphs;
+    try {
+      graphs = projects.map((project) => ({ project, graph: loadGraph(graphPath(project)) }));
+    } catch (e) {
+      process.stderr.write(`graph-links: ${e.message}\n`);
+      return 1;
+    }
+    const rows = link(graphs);
+    const doc = { v: 1, builtAt: new Date().toISOString(), projects, links: rows };
+    writeFileSync(linksPath, JSON.stringify(doc, null, 2) + "\n", { mode: 0o600 });
+    process.stdout.write(JSON.stringify({ out: linksPath, projects, links: rows.length }) + "\n");
+    return 0;
+  }
+  if (sub === "affected") {
+    const project = flag(argv, "project");
+    if (!project) return usage();
+    const doc = readText(linksPath);
+    if (!doc) {
+      process.stderr.write(`graph-links: no link file at ${linksPath}; run build first\n`);
+      return 1;
+    }
+    const name = flag(argv, "name");
+    const hits = JSON.parse(doc).links.filter(
+      (r) =>
+        r.to === project &&
+        (!name ||
+          r.module === name ||
+          r.target?.endsWith(`#${name}`) ||
+          candidates(r.module).includes(name)),
+    );
+    process.stdout.write(
+      JSON.stringify({ project, name: name ?? null, importers: hits }, null, 2) + "\n",
+    );
+    return 0;
+  }
+  return usage();
+}
+
+function usage() {
+  process.stderr.write(
+    "usage: graph-links.mjs build --projects <a,b,...> [--out <path>] | affected --project <b> [--name <n>] [--links <path>]\n",
+  );
+  return 64;
+}
+
+if (invokedDirectly(import.meta.url)) {
+  process.exitCode = main(process.argv.slice(2));
+}

package/pipeline/scripts/graph-query.mjs

@@ -18,9 +18,16 @@
  *   --seeds N        How many ranked seeds to expand. Default: 8
  *   --json           Emit JSON instead of text
  *
+ *   graph-query --path <from> <to> [--max-depth N] [--directed] [--kind K,...]
+ *                    Shortest path between two symbols or files, by node id or
+ *                    by name. A name that matches more than one node is refused
+ *                    with the candidates, never guessed; a name that matches
+ *                    none gets its nearest names.
+ *
  * Exit codes:
- *   0  -  answered (possibly empty)
+ *   0  -  answered (possibly empty, or no path within the depth)
  *   1  -  graph missing or unreadable
+ *   3  -  a --path endpoint is ambiguous or unknown
  *   64 -  usage error
  *
  * @module pipeline/scripts/graph-query
@@ -93,6 +100,72 @@ export function rankNodes(graph, question) {
     .map((s) => ({ node: graph.nodes[s.index], score: s.score }));
 }
 
+const CANDIDATES = 5;
+
+/**
+ * One node for a name or id. Exact id first, then exact name, then the name
+ * case-insensitively. More than one match is refused with the candidates; no
+ * match returns the nearest names by the same ranking the query uses.
+ *
+ * @returns {{node: object} | {ambiguous: object[]} | {unknown: object[]}}
+ */
+export function resolveNode(graph, ref) {
+  const byId = graph.nodes.find((n) => n.id === ref);
+  if (byId) return { node: byId };
+  let matches = graph.nodes.filter((n) => n.name === ref);
+  if (!matches.length) {
+    const lower = String(ref).toLowerCase();
+    matches = graph.nodes.filter((n) => (n.name || "").toLowerCase() === lower);
+  }
+  if (matches.length === 1) return { node: matches[0] };
+  if (matches.length > 1) return { ambiguous: matches.slice(0, CANDIDATES) };
+  return {
+    unknown: rankNodes(graph, String(ref))
+      .slice(0, CANDIDATES)
+      .map((r) => r.node),
+  };
+}
+
+/**
+ * Breadth-first shortest path. Undirected unless `directed`, so a subclass
+ * reaches its base and the base reaches the subclass. External module nodes
+ * are endpoints only.
+ *
+ * @returns {{path: Array<{node: object, via: string|null}>} | {path: null}}
+ */
+export function shortestPath(
+  graph,
+  fromId,
+  toId,
+  { maxDepth = 8, directed = false, kinds = null } = {},
+) {
+  const byId = new Map(graph.nodes.map((n) => [n.id, n]));
+  const adj = adjacency(graph);
+  const prev = new Map([[fromId, null]]);
+  let frontier = [fromId];
+  for (let d = 0; d < maxDepth && frontier.length && !prev.has(toId); d++) {
+    const next = [];
+    for (const id of frontier) {
+      for (const n of adj.get(id) || []) {
+        if (directed && n.dir !== "out") continue;
+        if (kinds && !kinds.includes(n.kind)) continue;
+        if (prev.has(n.id)) continue;
+        prev.set(n.id, { from: id, via: `${n.kind}/${n.dir}` });
+        // An external module joins every file that imports it, so it ends a
+        // path but is never walked through.
+        if (byId.get(n.id)?.kind !== "module" || n.id === toId) next.push(n.id);
+      }
+    }
+    frontier = next;
+  }
+  if (!prev.has(toId)) return { path: null };
+  const path = [];
+  for (let id = toId; id !== null; id = prev.get(id)?.from ?? null) {
+    path.unshift({ node: byId.get(id), via: prev.get(id)?.via ?? null });
+  }
+  return { path };
+}
+
 function renderNode(node) {
   const where = node.path ? `${node.path}${node.line ? `:${node.line}` : ""}` : "-";
   const kind = node.symbolKind || node.kind;
@@ -157,8 +230,67 @@ export function traverse({ graph, question, budget, depth, seedCount }) {
   return { seeds, hits, truncated, tokens };
 }
 
+function pathMain(flags, positional, graph) {
+  if (positional.length !== 2) {
+    process.stderr.write(
+      "graph-query: --path needs two endpoints, e.g. --path LoginView AuthService\n",
+    );
+    return 64;
+  }
+  const ends = positional.map((ref) => ({ ref, ...resolveNode(graph, ref) }));
+  const bad = ends.filter((e) => !e.node);
+  if (bad.length) {
+    const doc = bad.map((e) => ({
+      ref: e.ref,
+      reason: e.ambiguous ? "ambiguous" : "unknown",
+      candidates: (e.ambiguous || e.unknown).map((n) => ({ id: n.id, line: renderNode(n) })),
+    }));
+    if (flags.json === true) process.stdout.write(JSON.stringify({ refused: doc }, null, 2) + "\n");
+    else
+      for (const d of doc) {
+        process.stdout.write(`${d.ref}: ${d.reason}; pass one of these ids:\n`);
+        for (const c of d.candidates) process.stdout.write(`  ${c.id}  ${c.line}\n`);
+      }
+    return 3;
+  }
+  const maxDepth = Math.min(Number(flags["max-depth"]) > 0 ? Number(flags["max-depth"]) : 8, 12);
+  const kinds = typeof flags.kind === "string" ? flags.kind.split(",") : null;
+  const { path } = shortestPath(graph, ends[0].node.id, ends[1].node.id, {
+    maxDepth,
+    directed: flags.directed === true,
+    kinds,
+  });
+  if (flags.json === true) {
+    const hops =
+      path?.map((h) => ({ id: h.node.id, line: renderNode(h.node), via: h.via })) ?? null;
+    process.stdout.write(
+      JSON.stringify({ from: ends[0].node.id, to: ends[1].node.id, path: hops }, null, 2) + "\n",
+    );
+    return 0;
+  }
+  if (!path) {
+    process.stdout.write(`no path within ${maxDepth} hops\n`);
+    return 0;
+  }
+  path.forEach((h, i) => process.stdout.write(`${printedLine(i, renderNode(h.node), h.via)}\n`));
+  return 0;
+}
+
 function main() {
   const { flags, positional } = parseFlags(process.argv.slice(2));
+  if (flags.path !== undefined) {
+    // parseFlags reads the first endpoint as the flag's value.
+    const ends = typeof flags.path === "string" ? [flags.path, ...positional] : positional;
+    const path = flags.graph && flags.graph !== true ? resolve(flags.graph) : defaultGraphPath();
+    let graph;
+    try {
+      graph = loadGraph(path);
+    } catch (e) {
+      process.stderr.write(`graph-query: ${e.message}\n`);
+      process.exit(1);
+    }
+    process.exit(pathMain(flags, ends, graph));
+  }
   const question = positional.join(" ").trim();
   if (!question) {
     process.stderr.write('graph-query: a question is required, e.g. graph-query "flight search"\n');

package/pipeline/scripts/graph-report.mjs

@@ -17,6 +17,8 @@
  *   --out <path>     Report path. Default: GRAPH_REPORT.md beside the graph
  *   --top N          Rows per ranked section. Default: 15
  *   --stdout         Print instead of writing
+ *   --diff <path>    Compare with another build of the same repository (the
+ *                    older one) and print what changed as JSON, writing nothing.
  *   --status         Print one header line and exit, writing nothing. This is
  *                    what `/multi-agent:graph status` runs: the graph is a
  *                    22MB JSON on a large repo, so "read the header yourself"
@@ -35,6 +37,7 @@ import { join, dirname, resolve } from "node:path";
 import { parseFlags } from "./graph-build.mjs";
 import { loadGraph, defaultGraphPath } from "./graph-query.mjs";
 import { loadRules } from "./_code-graph.mjs";
+import { clusters, graphDiff, importCycles, questions } from "./_graph-analysis.mjs";
 
 /**
  * Top-level source directory of a path, used as the module bucket.
@@ -223,6 +226,49 @@ export function render(graph, top) {
   }
   L.push("");
 
+  const cycles = importCycles(graph);
+  L.push(`## Import cycles`);
+  L.push("");
+  if (!cycles.length) {
+    L.push(`None: no files import each other, directly or around a loop.`);
+  } else {
+    L.push(
+      `${cycles.length} group(s) of files that import each other. A change to one reaches all of them.`,
+    );
+    L.push("");
+    for (const c of cycles.slice(0, top)) {
+      const shown = c
+        .slice(0, 6)
+        .map((f) => `\`${f.replace(/^file:/, "")}\``)
+        .join(", ");
+      L.push(`- ${c.length} files: ${shown}${c.length > 6 ? `, ... ${c.length - 6} more` : ""}`);
+    }
+  }
+  L.push("");
+
+  const groups = clusters(graph);
+  L.push(`## Clusters`);
+  L.push("");
+  L.push(
+    `Files more connected to each other than to the rest (label propagation over imports and references). ` +
+      `Cohesion is the share of a cluster's edges that stay inside it.`,
+  );
+  L.push("");
+  L.push(`| Files | Cohesion | Around | Where |`);
+  L.push(`|---|---|---|---|`);
+  for (const g of groups.slice(0, top)) {
+    L.push(`| ${g.files.length} | ${g.cohesion} | \`${g.id}\` | ${g.dirs} |`);
+  }
+  L.push("");
+
+  const asks = questions(graph, { hubs: s.hubs, cycles, groups });
+  if (asks.length) {
+    L.push(`## Questions worth asking`);
+    L.push("");
+    for (const q of asks) L.push(`- ${q}`);
+    L.push("");
+  }
+
   L.push(`## Modules`);
   L.push("");
   L.push(`| Module | Files | Symbols | Test files |`);
@@ -358,7 +404,14 @@ export function statusLine(graph, path) {
   );
 }
 
+const USAGE =
+  "usage: graph-report.mjs [--graph <path>] [--out <path>] [--top N] [--stdout] [--diff <older-graph>] [--status]\n";
+
 function main() {
+  if (process.argv.includes("--help") || process.argv.includes("-h")) {
+    process.stdout.write(USAGE);
+    return;
+  }
   const { flags } = parseFlags(process.argv.slice(2));
   const path = flags.graph && flags.graph !== true ? resolve(flags.graph) : defaultGraphPath();
 
@@ -375,6 +428,18 @@ function main() {
     return;
   }
 
+  if (flags.diff && flags.diff !== true) {
+    let before;
+    try {
+      before = loadGraph(resolve(flags.diff));
+    } catch (e) {
+      process.stderr.write(`graph-report: ${e.message}\n`);
+      process.exit(1);
+    }
+    process.stdout.write(JSON.stringify(graphDiff(before, graph), null, 2) + "\n");
+    return;
+  }
+
   const top = Number(flags.top) > 0 ? Number(flags.top) : 15;
   const markdown = render(graph, top);
 

package/pipeline/scripts/learn-from-transcripts.mjs

@@ -53,20 +53,14 @@
  * Exit: 0 ok (including "nothing found") | 1 usage
  */
 
-import {
-  existsSync,
-  mkdtempSync,
-  readFileSync,
-  readdirSync,
-  rmSync,
-  statSync,
-  writeFileSync,
-} from "node:fs";
+import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
 import { execFileSync } from "node:child_process";
 import { basename, dirname, join } from "node:path";
-import { homedir, tmpdir } from "node:os";
+import { homedir } from "node:os";
 import { fileURLToPath } from "node:url";
 import { runMain } from "../lib/fatal.mjs";
+import { memoryExcluded } from "./_memory-policy.mjs";
+import { dropSecrets } from "./_secret-scan.mjs";
 
 const HERE = dirname(fileURLToPath(import.meta.url));
 const argv = process.argv.slice(2);
@@ -382,41 +376,6 @@ function mine(events) {
   return found;
 }
 
-/* ----------------------------------------------------------------- privacy */
-
-/**
- * The single-source secret check. Candidates go to a temp file and through
- * pre-commit-check.sh, which owns the patterns; a hit is dropped, never written.
- */
-function dropSecrets(items) {
-  const scanner = existsSync(join(HERE, "pre-commit-check.sh"))
-    ? join(HERE, "pre-commit-check.sh")
-    : join(homedir(), ".claude", "scripts", "pre-commit-check.sh");
-  if (!existsSync(scanner)) {
-    return { kept: items, dropped: 0, scanned: false };
-  }
-  const dir = mkdtempSync(join(tmpdir(), "lft-"));
-  const kept = [];
-  let dropped = 0;
-  try {
-    for (const it of items) {
-      const f = join(dir, "candidate.txt");
-      writeFileSync(f, `${it.statement}\n${it.diagnosis || ""}\n${it.scope}\n`);
-      let clean = true;
-      try {
-        execFileSync("bash", [scanner, "--scan-file", f], { stdio: "ignore" });
-      } catch {
-        clean = false;
-      }
-      if (clean) kept.push(it);
-      else dropped++;
-    }
-  } finally {
-    rmSync(dir, { recursive: true, force: true });
-  }
-  return { kept, dropped, scanned: true };
-}
-
 /* -------------------------------------------------------------------- main */
 
 function main() {
@@ -462,12 +421,16 @@ function main() {
     uniq.push(it);
   }
 
-  const { kept, dropped, scanned } = dropSecrets(uniq);
+  const { kept, dropped, scanned } = dropSecrets(
+    uniq,
+    (it) => `${it.statement}\n${it.diagnosis || ""}\n${it.scope}`,
+  );
   const slug = repoSlug();
+  const excluded = memoryExcluded(slug);
 
   let written = 0;
   let deduped = 0;
-  if (APPLY && kept.length) {
+  if (APPLY && kept.length && !excluded) {
     const ledger = existsSync(join(HERE, "learnings-ledger.mjs"))
       ? join(HERE, "learnings-ledger.mjs")
       : join(homedir(), ".claude", "scripts", "learnings-ledger.mjs");
@@ -572,7 +535,7 @@ function main() {
   }
 
   let localFile = null;
-  if (APPLY && kept.length) localFile = writeLocalBlock(kept);
+  if (APPLY && kept.length && !excluded) localFile = writeLocalBlock(kept);
 
   // Record how many lessons the machine found, kept apart from the ones a model
   // proposed. Pooling them lets a "lessons learned" number climb while the half
@@ -595,6 +558,7 @@ function main() {
   const result = {
     status: "ok",
     repo: slug,
+    excluded,
     transcripts: files.length,
     toolResultsExamined: examined,
     candidates: uniq.length,