@mmerterden/multi-agent-pipeline 16.12.0 → 16.14.0

package/pipeline/scripts/_code-graph.mjs

@@ -0,0 +1,545 @@
+/**
+ * @file _code-graph.mjs  -  deterministic, LLM-free code graph for Phase 1 and Phase 7.
+ *
+ * Phase 1 narrows its Explore fan-out with this graph, and Phase 7 refreshes it
+ * instead of hand-writing architecture.md from a single task's window. The
+ * design follows graphify (Graphify-Labs/graphify): AST-quality extraction with
+ * zero LLM cost, one graph file, an incremental manifest gate, god-node hubs,
+ * and token-budgeted traversal as the query surface. The extraction itself is
+ * regex over comment-stripped source rather than tree-sitter - a tree-sitter
+ * binding would be an npm runtime dependency, which ADR-0004 forbids. That
+ * trade is deliberate: definitions and imports survive it, call graphs and type
+ * resolution do not, and neither consumer needs those.
+ *
+ * Definition and import patterns come from the rule files. References do NOT:
+ * a per-language reference regex is both brittle and redundant once every
+ * defined symbol name is known, so pass 2 intersects each file's identifier set
+ * with the global symbol index instead. That is language-agnostic and cannot
+ * drift from the definition patterns, because it is derived from them.
+ *
+ * Two passes, one read per file:
+ *   1. strip comments and strings, extract definitions + imports, keep the
+ *      file's identifier set (a Set of strings, not the source text)
+ *   2. intersect each identifier set with the symbol index to emit references
+ *
+ * Zero runtime dependencies (ADR-0004).
+ *
+ * @module pipeline/scripts/_code-graph
+ */
+
+import { readFileSync, existsSync, statSync, readdirSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { join, dirname, relative, sep } from "node:path";
+import { fileURLToPath } from "node:url";
+import { isExcludedBy } from "./_path-match.mjs";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+export const GRAPH_SCHEMA_VERSION = "1.0.0";
+
+/**
+ * Stack words the pipeline uses elsewhere, mapped to the rule-file id.
+ *
+ * The rule files are named after the toolchain (`node`), but the rest of the
+ * pipeline names the product surface (`web`), and a caller reasonably types
+ * whichever one they were last shown. Aliasing is safe only where the mapping
+ * is one-to-one: `backend` is deliberately absent, because it could be node,
+ * python or go and picking one would be a guess dressed as support.
+ */
+export const STACK_ALIASES = { web: "node", frontend: "node" };
+
+/**
+ * Canonical rule-file id for a stack word.
+ *
+ * @param {string} stack
+ * @returns {string}
+ */
+export function resolveStack(stack) {
+  return STACK_ALIASES[stack] || stack;
+}
+
+/**
+ * Load and merge the two rule files for a stack.
+ *
+ * `test-gap-rules/<stack>.json` is read directly rather than copied: it already
+ * owns sourceExtensions, excludePathGlobs and the test-path predicates, and a
+ * second copy would drift the first time a pattern changed.
+ *
+ * @param {string} stack - ios | android | node | python
+ * @param {string} [baseDir] - directory holding the two rule dirs
+ * @returns {object} merged rules
+ */
+export function loadRules(requested, baseDir = __dirname) {
+  const stack = resolveStack(requested);
+  const gapPath = join(baseDir, "test-gap-rules", `${stack}.json`);
+  const graphPath = join(baseDir, "code-graph-rules", `${stack}.json`);
+  if (requested === "backend") {
+    throw new Error(
+      "'backend' names a role, not a toolchain: pass node, python or go so the right rules load",
+    );
+  }
+  if (!existsSync(gapPath)) throw new Error(`no test-gap rules for stack '${stack}': ${gapPath}`);
+  if (!existsSync(graphPath))
+    throw new Error(`no code-graph rules for stack '${stack}': ${graphPath}`);
+
+  const gap = JSON.parse(readFileSync(gapPath, "utf8"));
+  const graph = JSON.parse(readFileSync(graphPath, "utf8"));
+
+  if (graph.stack !== stack || gap.stack !== stack) {
+    throw new Error(`rule file stack mismatch for '${stack}'`);
+  }
+  if (!Array.isArray(graph.definitionPatterns) || graph.definitionPatterns.length === 0) {
+    throw new Error(`code-graph rules for '${stack}' declare no definitionPatterns`);
+  }
+  if (!Array.isArray(graph.referenceKinds) || graph.referenceKinds.length === 0) {
+    throw new Error(`code-graph rules for '${stack}' declare no referenceKinds`);
+  }
+
+  return {
+    stack,
+    sourceExtensions: gap.sourceExtensions || [],
+    excludePathGlobs: gap.excludePathGlobs || [],
+    testPathSuffixes: gap.testPathSuffixes || [],
+    testPathContains: gap.testPathContains || [],
+    comments: graph.comments || {},
+    definitionPatterns: graph.definitionPatterns,
+    importPatterns: graph.importPatterns || [],
+    importSpecifiersAreStrings: graph.importSpecifiersAreStrings === true,
+    ignoredIdentifiers: new Set(graph.ignoredIdentifiers || []),
+    referenceKinds: new Set(graph.referenceKinds || []),
+  };
+}
+
+/**
+ * Remove comments and string literals so they cannot produce phantom edges.
+ *
+ * Character-scanned rather than regex-replaced: a regex pass cannot tell a `//`
+ * inside a string from a real comment, and that difference is exactly what
+ * makes reference edges noisy. Replaces removed spans with spaces so byte
+ * offsets, and therefore line numbers, are preserved.
+ *
+ * @param {string} text - source text
+ * @param {object} comments - rules.comments
+ * @returns {string} same length, comments and string bodies blanked
+ */
+export function stripCode(text, comments = {}) {
+  const line = comments.line || [];
+  const block = comments.block || [];
+  const quotes = comments.string || [];
+  const out = text.split("");
+
+  let i = 0;
+  const blank = (from, to) => {
+    for (let k = from; k < to && k < out.length; k++) {
+      if (out[k] !== "\n") out[k] = " ";
+    }
+  };
+
+  while (i < text.length) {
+    const rest = text.slice(i, i + 8);
+
+    const lineTok = line.find((t) => rest.startsWith(t));
+    if (lineTok) {
+      let end = text.indexOf("\n", i);
+      if (end === -1) end = text.length;
+      blank(i, end);
+      i = end;
+      continue;
+    }
+
+    const blockTok = block.find(([open]) => rest.startsWith(open));
+    if (blockTok) {
+      const [open, close] = blockTok;
+      let end = text.indexOf(close, i + open.length);
+      end = end === -1 ? text.length : end + close.length;
+      blank(i, end);
+      i = end;
+      continue;
+    }
+
+    const quoteTok = quotes.find((q) => rest.startsWith(q));
+    if (quoteTok) {
+      let j = i + quoteTok.length;
+      while (j < text.length) {
+        if (text[j] === "\\") {
+          j += 2;
+          continue;
+        }
+        if (text.startsWith(quoteTok, j)) {
+          j += quoteTok.length;
+          break;
+        }
+        if (text[j] === "\n" && quoteTok.length === 1) break;
+        j++;
+      }
+      blank(i, j);
+      i = j;
+      continue;
+    }
+
+    i++;
+  }
+
+  return out.join("");
+}
+
+/**
+ * First populated capture group of a match.
+ *
+ * Rule-file patterns may alternate (Swift's `import class Mod.Sym` needs one
+ * branch, plain `import Mod` another), which shifts the meaningful group.
+ * Reading m[1] alone silently dropped every match of a later branch.
+ *
+ * @param {RegExpMatchArray} m
+ * @returns {string|undefined}
+ */
+function captured(m) {
+  return m.slice(1).find((g) => g !== undefined);
+}
+
+const IMPORT_LINE_RE = /^[ \t]*import[ \t].*$/gm;
+
+/**
+ * Blank every import line, preserving offsets.
+ *
+ * Swift's submodule form - `import class DesignKit.PrimaryButton` - otherwise
+ * feeds the `class` definition pattern, which registers the MODULE name as a
+ * declared type. On a large Swift app that alone lifted two module names into
+ * the top ten god-nodes, which is how the bug surfaced. Imports are extracted
+ * before this runs, so nothing is lost.
+ *
+ * @param {string} stripped - output of stripCode
+ * @returns {string} same length, import lines blanked
+ */
+function blankImportLines(stripped) {
+  return stripped.replace(IMPORT_LINE_RE, (line) => " ".repeat(line.length));
+}
+
+const IDENTIFIER_RE = /[A-Za-z_][A-Za-z0-9_]*/g;
+
+/**
+ * Distinct identifiers in a piece of stripped source.
+ *
+ * @param {string} stripped - output of stripCode
+ * @returns {Set<string>}
+ */
+export function identifiers(stripped) {
+  const found = new Set();
+  for (const m of stripped.matchAll(IDENTIFIER_RE)) found.add(m[0]);
+  return found;
+}
+
+function lineOf(text, index) {
+  let line = 1;
+  for (let i = 0; i < index && i < text.length; i++) if (text[i] === "\n") line++;
+  return line;
+}
+
+/**
+ * Leading whitespace on the line holding this index.
+ *
+ * Nesting is read from the line's indentation, not from where the pattern
+ * happened to match: `public final class Foo` matches at `class`, seven columns
+ * in, and is still a top-level declaration. Indentation is the signal that holds
+ * across every language these rules cover - a brace language indents a nested
+ * type by convention, Python by grammar - which is why the test lives in the
+ * engine rather than in a per-stack rule.
+ *
+ * @param {string} text
+ * @param {number} index
+ * @returns {number}
+ */
+function indentOf(text, index) {
+  const start = text.lastIndexOf("\n", index - 1) + 1;
+  let i = start;
+  while (i < text.length && (text[i] === " " || text[i] === "\t")) i++;
+  return i - start;
+}
+
+/**
+ * Extract one file's definitions, imports and identifier set.
+ *
+ * @param {string} text - raw source
+ * @param {object} rules - merged rules from loadRules
+ * @returns {{definitions: Array, imports: string[], tokens: Set<string>}}
+ */
+export function extractFile(text, rules) {
+  const stripped = stripCode(text, rules.comments);
+
+  // Swift and the JVM name a module with a bare identifier, so the fully
+  // stripped body is the right text to read imports from. A JavaScript module
+  // specifier is a string literal, and stripping strings blanks it: run the
+  // import patterns over a text that kept its strings, or every import edge in
+  // that stack disappears with nothing to notice it. Comments are still gone
+  // either way, so an import written inside a comment never counts, and the
+  // definition and identifier passes below still read the fully stripped body.
+  const importSource = rules.importSpecifiersAreStrings
+    ? stripCode(text, { line: rules.comments.line, block: rules.comments.block })
+    : stripped;
+
+  const imports = [];
+  for (const pattern of rules.importPatterns) {
+    const re = new RegExp(pattern.regex, "gm");
+    for (const m of importSource.matchAll(re)) {
+      const mod = captured(m);
+      if (mod && !imports.includes(mod)) imports.push(mod);
+    }
+  }
+
+  const body = blankImportLines(stripped);
+  const definitions = [];
+  const seen = new Set();
+
+  for (const pattern of rules.definitionPatterns) {
+    const re = new RegExp(pattern.regex, "gm");
+    for (const m of body.matchAll(re)) {
+      const name = captured(m);
+      if (!name) continue;
+      const key = `${name}:${pattern.kind || pattern.id}`;
+      if (seen.has(key)) continue;
+      seen.add(key);
+      definitions.push({
+        name,
+        kind: pattern.kind || pattern.id,
+        patternId: pattern.id,
+        line: lineOf(body, m.index),
+        nested: indentOf(body, m.index) > 0,
+      });
+    }
+  }
+
+  return { definitions, imports, tokens: identifiers(body) };
+}
+
+/**
+ * Every source file under root, honouring .gitignore and excludePathGlobs.
+ *
+ * Uses `git ls-files` when root is a work tree, so .gitignore and
+ * .git/info/exclude are respected without reimplementing them; falls back to a
+ * directory walk otherwise.
+ *
+ * @param {string} root - absolute repo root
+ * @param {object} rules - merged rules
+ * @returns {string[]} repo-relative paths, sorted
+ */
+export function listSourceFiles(root, rules) {
+  let candidates;
+  try {
+    const out = execFileSync("git", ["-C", root, "ls-files", "-co", "--exclude-standard"], {
+      encoding: "utf8",
+      maxBuffer: 64 * 1024 * 1024,
+    });
+    candidates = out.split("\n").filter(Boolean);
+  } catch {
+    candidates = walkDir(root, root);
+  }
+
+  const isSource = (p) => {
+    if (!rules.sourceExtensions.some((ext) => p.endsWith(ext))) return false;
+    if (isExcludedBy(p, rules.excludePathGlobs)) return false;
+    return true;
+  };
+
+  return candidates.filter(isSource).sort();
+}
+
+function walkDir(dir, root, acc = []) {
+  let entries;
+  try {
+    entries = readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return acc;
+  }
+  for (const entry of entries) {
+    if (entry.name === ".git" || entry.name === "node_modules") continue;
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) walkDir(full, root, acc);
+    else if (entry.isFile()) acc.push(relative(root, full).split(sep).join("/"));
+  }
+  return acc;
+}
+
+/**
+ * Is this a test file per the stack's test-path predicates?
+ *
+ * Test files stay in the graph - they are how Phase 1 finds existing coverage -
+ * but they are tagged so the report can separate them from production hubs.
+ *
+ * @param {string} path - repo-relative path
+ * @param {object} rules - merged rules
+ * @returns {boolean}
+ */
+export function isTestPath(path, rules) {
+  if (rules.testPathSuffixes.some((s) => path.endsWith(s))) return true;
+  return rules.testPathContains.some((c) => path.includes(c));
+}
+
+export const fileNodeId = (path) => `file:${path}`;
+export const symbolNodeId = (path, name) => `sym:${path}#${name}`;
+
+/**
+ * Build the graph from a set of extracted files.
+ *
+ * Pass 2 resolves a reference only when the identifier maps to exactly one
+ * defined symbol. An ambiguous name (the same type declared in several files)
+ * would otherwise fan out to every candidate and turn the busiest names into
+ * false hubs, which is the failure that makes god-node output useless.
+ *
+ * @param {object} params
+ * @param {string} params.root - absolute repo root
+ * @param {string} params.stack
+ * @param {Map<string, object>} params.extracted - relPath -> extractFile result
+ * @param {object} params.rules
+ * @param {object} [params.manifest] - relPath -> {size, mtimeMs}
+ * @param {string|null} [params.baseCommit]
+ * @param {string} [params.generatedAt] - ISO timestamp, injected by the caller
+ * @returns {object} graph conforming to code-graph.schema.json
+ */
+export function buildGraph({
+  root,
+  stack,
+  extracted,
+  rules,
+  manifest = {},
+  baseCommit = null,
+  generatedAt,
+}) {
+  const nodes = new Map();
+  const edges = [];
+  const symbolsByName = new Map();
+
+  for (const [path, data] of extracted) {
+    const fid = fileNodeId(path);
+    nodes.set(fid, {
+      id: fid,
+      kind: "file",
+      name: path.split("/").pop(),
+      path,
+      isTest: isTestPath(path, rules),
+      degree: 0,
+    });
+    for (const def of data.definitions) {
+      const sid = symbolNodeId(path, def.name);
+      if (!nodes.has(sid)) {
+        nodes.set(sid, {
+          id: sid,
+          kind: "symbol",
+          name: def.name,
+          path,
+          line: def.line,
+          symbolKind: def.kind,
+          isTest: isTestPath(path, rules),
+          nested: def.nested === true,
+          degree: 0,
+        });
+      }
+      edges.push({ from: fid, to: sid, kind: "defines" });
+      // A nested declaration stays a node and keeps its defines edge, so it is
+      // still findable by name, but it never becomes a reference target. Sealed
+      // hierarchies name their cases after the concept they model - Icon, Color,
+      // Success, Disabled - and each is declared exactly once, so the ambiguity
+      // rule below does not catch them: every file that merely mentions the
+      // framework's Color would otherwise gain an edge to one app's nested case.
+      if (def.nested === true) continue;
+      if (!symbolsByName.has(def.name)) symbolsByName.set(def.name, []);
+      const bucket = symbolsByName.get(def.name);
+      if (!bucket.includes(sid)) bucket.push(sid);
+    }
+  }
+
+  const byBasename = new Map();
+  for (const path of extracted.keys()) {
+    const base = path
+      .split("/")
+      .pop()
+      .replace(/\.[^.]+$/, "");
+    if (!byBasename.has(base)) byBasename.set(base, []);
+    byBasename.get(base).push(path);
+  }
+
+  for (const [path, data] of extracted) {
+    const fid = fileNodeId(path);
+    for (const mod of data.imports) {
+      const targets = byBasename.get(mod);
+      if (targets && targets.length === 1) {
+        edges.push({ from: fid, to: fileNodeId(targets[0]), kind: "imports" });
+      } else {
+        const mid = `module:${mod}`;
+        if (!nodes.has(mid)) {
+          nodes.set(mid, { id: mid, kind: "module", name: mod, path: null, degree: 0 });
+        }
+        edges.push({ from: fid, to: mid, kind: "imports" });
+      }
+    }
+  }
+
+  for (const [path, data] of extracted) {
+    const fid = fileNodeId(path);
+    const emitted = new Set();
+    for (const token of data.tokens) {
+      if (rules.ignoredIdentifiers.has(token)) continue;
+      const candidates = symbolsByName.get(token);
+      if (!candidates || candidates.length !== 1) continue;
+      const sid = candidates[0];
+      const target = nodes.get(sid);
+      if (target.path === path) continue;
+      if (!rules.referenceKinds.has(target.symbolKind)) continue;
+      if (emitted.has(sid)) continue;
+      emitted.add(sid);
+      edges.push({ from: fid, to: sid, kind: "references" });
+    }
+  }
+
+  for (const edge of edges) {
+    if (nodes.has(edge.from)) nodes.get(edge.from).degree++;
+    if (nodes.has(edge.to)) nodes.get(edge.to).degree++;
+  }
+
+  return {
+    schemaVersion: GRAPH_SCHEMA_VERSION,
+    stack,
+    root,
+    baseCommit,
+    generatedAt,
+    stats: {
+      files: extracted.size,
+      nodes: nodes.size,
+      edges: edges.length,
+    },
+    nodes: [...nodes.values()],
+    edges,
+    manifest,
+  };
+}
+
+/**
+ * Manifest entry for one file, used as the incremental gate.
+ *
+ * @param {string} absPath
+ * @returns {{size: number, mtimeMs: number}}
+ */
+export function manifestEntry(absPath) {
+  const st = statSync(absPath);
+  return { size: st.size, mtimeMs: Math.floor(st.mtimeMs) };
+}
+
+/**
+ * Which files changed since the previous manifest?
+ *
+ * @param {string} root
+ * @param {string[]} files - repo-relative paths
+ * @param {object} prevManifest
+ * @returns {{changed: string[], removed: string[], manifest: object}}
+ */
+export function diffManifest(root, files, prevManifest = {}) {
+  const manifest = {};
+  const changed = [];
+  for (const path of files) {
+    const entry = manifestEntry(join(root, path));
+    manifest[path] = entry;
+    const prev = prevManifest[path];
+    if (!prev || prev.size !== entry.size || prev.mtimeMs !== entry.mtimeMs) changed.push(path);
+  }
+  const removed = Object.keys(prevManifest).filter((p) => !(p in manifest));
+  return { changed, removed, manifest };
+}

package/pipeline/scripts/_path-match.mjs

@@ -0,0 +1,87 @@
+/**
+ * @file _path-match.mjs  -  shared path-glob matching for the stack rule files.
+ *
+ * `excludePathGlobs` has more than one consumer now: test-gap-scan.mjs walks a
+ * diff, _code-graph.mjs walks a whole tree, and both must agree on which paths
+ * a rule file excludes. A second copy of the matcher would drift the first time
+ * a pattern shape changed - the same reason _stack-routing.mjs was extracted
+ * out of build-stack-plugins.mjs - and the two consumers would then silently
+ * disagree about what is in scope.
+ *
+ * Zero runtime dependencies (ADR-0004): the glob subset is hand-rolled rather
+ * than delegated to a micromatch-style library.
+ *
+ * @module pipeline/scripts/_path-match
+ */
+
+const DOUBLE_STAR_TOKEN = "\u0000";
+
+/**
+ * Translate a glob's body to regex source, without anchors.
+ *
+ * `**` is tokenized before the single-`*` pass so the `*` rewrite cannot mangle
+ * the already-translated `.*`, which broke cross-directory matches.
+ *
+ * @param {string} pattern
+ * @returns {string} regex source
+ */
+function globBody(pattern) {
+  return pattern
+    .replace(/\./g, "\\.")
+    .replace(/\*\*/g, DOUBLE_STAR_TOKEN)
+    .replace(/\*/g, "[^/]*")
+    .replace(new RegExp(DOUBLE_STAR_TOKEN, "g"), ".*");
+}
+
+/**
+ * Translate one rule-file glob to a fully anchored RegExp.
+ *
+ * @param {string} pattern - glob from a rule file's excludePathGlobs
+ * @returns {RegExp} anchored matcher
+ */
+export function globToRegExp(pattern) {
+  return new RegExp("^" + globBody(pattern) + "$");
+}
+
+/**
+ * Does this path match one rule-file glob?
+ *
+ * Four pattern shapes, in the order the rule files use them:
+ *
+ *   `build/`             a directory segment anywhere in the path
+ *   `*.egg-info/`        the same, with a wildcard in the segment
+ *   `**\/BuildConfig.java` a full-path pattern, anchored both ends
+ *   `*.d.ts`             a path suffix, with a wildcard
+ *   `.pb.swift`          a literal path suffix
+ *
+ * The two wildcard shapes used to fall through to a literal `endsWith` /
+ * `includes` test against the pattern text, which can never be true: nothing
+ * ends with the four characters `*.d.` followed by `ts`. `*.d.ts` in the node
+ * rules and `*.egg-info/` in the python rules were both silently inert, so
+ * generated typings and build residue reached every consumer of these rules.
+ *
+ * @param {string} path - repo-relative path
+ * @param {string} pattern - one glob
+ * @returns {boolean}
+ */
+export function matchesGlob(path, pattern) {
+  if (pattern.endsWith("/")) {
+    if (pattern.includes("*")) return new RegExp("(^|/)" + globBody(pattern)).test(path);
+    return path.includes(pattern) || path.startsWith(pattern);
+  }
+  if (pattern.includes("**")) return globToRegExp(pattern).test(path);
+  if (pattern.includes("*")) return new RegExp("(^|/)" + globBody(pattern) + "$").test(path);
+  return path.endsWith(pattern);
+}
+
+/**
+ * Is this path excluded by a rule file's excludePathGlobs?
+ *
+ * @param {string} path - repo-relative path
+ * @param {string[]|undefined} globs - rules.excludePathGlobs
+ * @returns {boolean} false when the rule file declares no globs
+ */
+export function isExcludedBy(path, globs) {
+  if (!Array.isArray(globs)) return false;
+  return globs.some((pattern) => matchesGlob(path, pattern));
+}