@mmerterden/multi-agent-pipeline 20.21.0 → 20.23.0

package/pipeline/scripts/_code-graph.mjs

@@ -32,10 +32,11 @@ import { execFileSync } from "node:child_process";
 import { join, dirname, relative, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { isExcludedBy } from "./_path-match.mjs";
+import { supertypesAt } from "./_supertypes.mjs";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-export const GRAPH_SCHEMA_VERSION = "1.0.0";
+export const GRAPH_SCHEMA_VERSION = "1.2.0";
 
 /**
  * Stack words the pipeline uses elsewhere, mapped to the rule-file id.
@@ -107,6 +108,13 @@ export function loadRules(requested, baseDir = __dirname) {
     importSpecifiersAreStrings: graph.importSpecifiersAreStrings === true,
     ignoredIdentifiers: new Set(graph.ignoredIdentifiers || []),
     referenceKinds: new Set(graph.referenceKinds || []),
+    supertypes: graph.supertypes
+      ? {
+          syntax: graph.supertypes.syntax,
+          kinds: new Set(graph.supertypes.kinds || []),
+          extensionPattern: graph.supertypes.extensionPattern || null,
+        }
+      : null,
   };
 }
 
@@ -256,6 +264,125 @@ function indentOf(text, index) {
   return i - start;
 }
 
+const leading = (line) => line.length - line.replace(/^[ \t]+/, "").length;
+const ATTRIBUTE_RE = /^[ \t]*@/;
+const CLOSER_RE = /^[ \t]*[}\])]/;
+const OPENS_BLOCK_RE = /(?:[{:=]|=>|->)[ \t]*$/;
+const DIRECTIVE_RE = /^#(?:if|elseif|else|endif)\b/;
+const STORED_PROPERTY_RE = /\b(?:var|let|val)\b/;
+
+const ATTRIBUTE_BLOCK_MAX = 80;
+const count = (text, re) => (text.match(re) || []).length;
+const bracketBalance = (text) =>
+  count(text, /\(/g) - count(text, /\)/g) + count(text, /\[/g) - count(text, /\]/g);
+const CLOSER_ONLY_RE = /^[ \t]*[)\]]+[ \t,]*$/;
+const DECLARES_RE =
+  /\b(?:func|init|class|struct|enum|protocol|actor|extension|typealias|def|fun|interface|object|var|let|val)\b/;
+
+/** What follows an attribute's argument list on its line; empty when the list runs on. */
+function afterArguments(rest) {
+  if (!/^\s*\(/.test(rest)) return rest;
+  let depth = 0;
+  for (let i = 0; i < rest.length; i++) {
+    if (rest[i] === "(" || rest[i] === "[") depth++;
+    else if ((rest[i] === ")" || rest[i] === "]") && --depth === 0) return rest.slice(i + 1);
+  }
+  return "";
+}
+
+/**
+ * The first line of an attribute whose arguments span several lines and close
+ * right above line `start` - `@available(iOS 16,` / `macOS 13, *)`, or a
+ * bracket list closed by `)` or `])` on its own line at the declaration's
+ * indentation - or null. The block opens at the declaration's indentation,
+ * keeps every other line deeper, balances its brackets, and is never a stored
+ * property's or another declaration that merely leads with an attribute.
+ */
+function attributeBlockStart(lines, start, indent) {
+  for (let k = start - 1; k >= 1 && start - k <= ATTRIBUTE_BLOCK_MAX; k--) {
+    const text = lines[k - 1];
+    if (!text.trim()) return null;
+    if (leading(text) === indent && ATTRIBUTE_RE.test(text)) {
+      const rest = afterArguments(text.replace(/^[ \t]*@[\w.]+/, ""));
+      if (STORED_PROPERTY_RE.test(rest) || text.includes("{") || DECLARES_RE.test(rest))
+        return null;
+      return bracketBalance(lines.slice(k - 1, start - 1).join("\n")) === 0 && k < start - 1
+        ? k
+        : null;
+    }
+    if (leading(text) > indent) continue;
+    if (k === start - 1 && leading(text) === indent && CLOSER_ONLY_RE.test(text)) continue;
+    return null;
+  }
+  return null;
+}
+
+/**
+ * Give every definition a span: `startLine` (the declaration line, or the first
+ * of the attribute or decorator lines directly above it, including one whose
+ * parenthesised arguments run over several lines, never a stored property)
+ * and `endLine` (the last line indented deeper than the declaration,
+ * plus a closing bracket at the declaration's own indentation when one
+ * follows). A closing bracket that opens the body again - the end of a
+ * multi-line parameter list, `) -> Int {`, `):`, `) : Base() {` - continues the
+ * span, and compiler directives in column 0 (`#if`, `#endif`) are skipped like
+ * blank lines. Code after the span, at the same or a lower indentation,
+ * belongs to no declaration until the next one starts.
+ *
+ * Indentation is the nesting signal the engine already trusts (see indentOf),
+ * so this needs no parser and holds for Python as well as the brace languages.
+ *
+ * @param {Array<{line: number, indent: number}>} definitions - mutated in place
+ * @param {string[]} lines - the stripped body, one entry per source line
+ */
+export function assignSpans(definitions, lines) {
+  for (const d of definitions) {
+    let start = d.line;
+    while (start > 1) {
+      const above = lines[start - 2];
+      if (
+        ATTRIBUTE_RE.test(above) &&
+        leading(above) === d.indent &&
+        !STORED_PROPERTY_RE.test(above)
+      ) {
+        start--;
+        continue;
+      }
+      const block = attributeBlockStart(lines, start, d.indent);
+      if (!block) break;
+      start = block;
+    }
+    d.startLine = start;
+  }
+  const ordered = [...definitions].sort((a, b) => a.line - b.line || a.indent - b.indent);
+  for (let i = 0; i < ordered.length; i++) {
+    const d = ordered[i];
+    let limit = lines.length;
+    for (let j = i + 1; j < ordered.length; j++) {
+      if (ordered[j].line > d.line && ordered[j].indent <= d.indent) {
+        limit = ordered[j].startLine - 1;
+        break;
+      }
+    }
+    let end = d.line;
+    for (let n = d.line + 1; n <= limit; n++) {
+      const text = lines[n - 1];
+      if (!text.trim() || DIRECTIVE_RE.test(text)) continue;
+      const ind = leading(text);
+      if (ind > d.indent) {
+        end = n;
+        continue;
+      }
+      if (ind === d.indent && CLOSER_RE.test(text)) {
+        end = n;
+        if (OPENS_BLOCK_RE.test(text)) continue;
+      }
+      break;
+    }
+    d.endLine = end;
+  }
+}
+
 /**
  * Extract one file's definitions, imports and identifier set.
  *
@@ -298,17 +425,34 @@ export function extractFile(text, rules) {
       const key = `${name}:${pattern.kind || pattern.id}`;
       if (seen.has(key)) continue;
       seen.add(key);
+      const kind = pattern.kind || pattern.id;
+      const st = rules.supertypes;
+      const supertypes =
+        st && st.kinds.has(kind) ? supertypesAt(body, m.index + m[0].length, st.syntax) : [];
       definitions.push({
         name,
-        kind: pattern.kind || pattern.id,
+        kind,
         patternId: pattern.id,
         line: lineOf(body, m.index),
+        indent: indentOf(body, m.index),
         nested: indentOf(body, m.index) > 0,
+        ...(supertypes.length ? { supertypes } : {}),
       });
     }
   }
 
-  return { definitions, imports, tokens: identifiers(body) };
+  const conformances = [];
+  if (rules.supertypes?.extensionPattern) {
+    for (const m of body.matchAll(new RegExp(rules.supertypes.extensionPattern, "gm"))) {
+      const type = captured(m)?.split(".").pop();
+      const supertypes = supertypesAt(body, m.index + m[0].length, rules.supertypes.syntax);
+      if (type && supertypes.length) conformances.push({ type, supertypes });
+    }
+  }
+
+  assignSpans(definitions, body.split("\n"));
+  for (const d of definitions) delete d.indent;
+  return { definitions, imports, tokens: identifiers(body), conformances };
 }
 
 /**
@@ -325,11 +469,12 @@ export function extractFile(text, rules) {
 export function listSourceFiles(root, rules) {
   let candidates;
   try {
-    const out = execFileSync("git", ["-C", root, "ls-files", "-co", "--exclude-standard"], {
+    const out = execFileSync("git", ["-C", root, "ls-files", "-z", "-co", "--exclude-standard"], {
       encoding: "utf8",
       maxBuffer: 64 * 1024 * 1024,
+      stdio: ["ignore", "pipe", "ignore"],
     });
-    candidates = out.split("\n").filter(Boolean);
+    candidates = out.split("\0").filter(Boolean);
   } catch {
     candidates = walkDir(root, root);
   }
@@ -371,7 +516,8 @@ function walkDir(dir, root, acc = []) {
  */
 export function isTestPath(path, rules) {
   if (rules.testPathSuffixes.some((s) => path.endsWith(s))) return true;
-  return rules.testPathContains.some((c) => path.includes(c));
+  const rooted = `/${path}`;
+  return rules.testPathContains.some((c) => rooted.includes(c));
 }
 
 export const fileNodeId = (path) => `file:${path}`;
@@ -427,6 +573,8 @@ export function buildGraph({
           name: def.name,
           path,
           line: def.line,
+          ...(def.endLine ? { endLine: def.endLine } : {}),
+          ...(def.startLine && def.startLine !== def.line ? { startLine: def.startLine } : {}),
           symbolKind: def.kind,
           isTest: isTestPath(path, rules),
           nested: def.nested === true,
@@ -473,6 +621,10 @@ export function buildGraph({
     }
   }
 
+  // Production code cannot depend on a type a test file declares, so a
+  // non-test file never gains an edge to one: a test helper named after a
+  // framework type (`protocol View` in a fake) would otherwise collect every
+  // file that mentions the framework's own.
   for (const [path, data] of extracted) {
     const fid = fileNodeId(path);
     const emitted = new Set();
@@ -483,6 +635,7 @@ export function buildGraph({
       const sid = candidates[0];
       const target = nodes.get(sid);
       if (target.path === path) continue;
+      if (target.isTest && !nodes.get(fid).isTest) continue;
       if (!rules.referenceKinds.has(target.symbolKind)) continue;
       if (emitted.has(sid)) continue;
       emitted.add(sid);
@@ -490,6 +643,35 @@ export function buildGraph({
     }
   }
 
+  // A subtype points at each supertype the graph defines once: its superclass,
+  // the protocols or interfaces it adopts, and conformances an extension adds.
+  // The file that names the supertype already references it; this edge says
+  // how, so a walk restricted to `inherits` answers "who implements this".
+  const unique = (name) => {
+    const c = symbolsByName.get(name);
+    return c && c.length === 1 ? c[0] : null;
+  };
+  const inherited = new Set();
+  const inherit = (from, superName) => {
+    const to = unique(superName);
+    const key = `${from}>${to}`;
+    if (!from || !to || from === to || inherited.has(key)) return;
+    if (nodes.get(to).isTest && !nodes.get(from).isTest) return;
+    inherited.add(key);
+    edges.push({ from, to, kind: "inherits" });
+  };
+  for (const [path, data] of extracted) {
+    for (const def of data.definitions) {
+      for (const sup of def.supertypes || []) inherit(symbolNodeId(path, def.name), sup);
+    }
+    const testFile = isTestPath(path, rules);
+    for (const c of data.conformances || []) {
+      const from = unique(c.type);
+      if (testFile && from && !nodes.get(from).isTest) continue;
+      for (const sup of c.supertypes) inherit(from, sup);
+    }
+  }
+
   for (const edge of edges) {
     if (nodes.has(edge.from)) nodes.get(edge.from).degree++;
     if (nodes.has(edge.to)) nodes.get(edge.to).degree++;

package/pipeline/scripts/_diff-impact.mjs

@@ -0,0 +1,319 @@
+/**
+ * @file _diff-impact.mjs  -  a diff, mapped onto the code graph.
+ *
+ * Three questions a reviewer or a planner asks of a change, answered from the
+ * graph without a model:
+ *
+ *   parseHunkRanges  which lines of each file the diff touched, on the side
+ *                    the graph was built from (graphSide)
+ *   changedSymbols   which declarations those lines fall in, innermost first,
+ *                    and which files the graph does not index at all
+ *   graphFreshness   how far the graph is behind the tree it describes
+ *
+ * The reverse walk itself is graph-affected.mjs; this module only produces its
+ * seeds. A graph built before definition spans existed (schema 1.0.0) has no
+ * `endLine`, so a hunk in such a file seeds the whole file and the file is
+ * listed in `fileLevel`: a coarser answer that says it is coarse.
+ *
+ * Zero runtime dependencies (ADR-0004).
+ *
+ * @module pipeline/scripts/_diff-impact
+ */
+
+import { execFileSync } from "node:child_process";
+import { existsSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { diffHeaderPath } from "./git-path.mjs";
+
+const HUNK_RE = /^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/;
+
+/** The path of a `---`/`+++` line: git-quoted or not, one-letter prefix removed. */
+function sidePath(raw, prefix) {
+  const path = diffHeaderPath(raw);
+  if (path === "/dev/null") return null;
+  return path.startsWith(prefix) ? path.slice(prefix.length) : path;
+}
+
+function addLine(set, line) {
+  set.add(Math.max(1, line));
+}
+
+function toRanges(lines) {
+  const sorted = [...lines].sort((a, b) => a - b);
+  const out = [];
+  for (const l of sorted) {
+    const last = out[out.length - 1];
+    if (last && l === last.end + 1) last.end = l;
+    else if (!last || l > last.end) out.push({ start: l, end: l });
+  }
+  return out;
+}
+
+/**
+ * Every file of a unified diff with the lines it touched on each side.
+ *
+ * New side: an added line counts as itself, and a deletion as the line now
+ * standing where the removed lines were. Old side: a removed line counts as
+ * itself, and an insertion as the old line standing where it went in. Either way a hunk that
+ * only adds or only removes still lands inside the declaration it changed.
+ * Hunks are counted from their headers, so concatenated `diff -u` output with
+ * no `diff --git` lines parses too, CRLF included, at any context size.
+ *
+ * @param {string} diffText - unified diff
+ * @returns {Array<{oldPath: string|null, newPath: string|null, oldRanges: Array<{start: number, end: number}>, newRanges: Array<{start: number, end: number}>}>}
+ */
+export function parseDiffFiles(diffText) {
+  const files = [];
+  let cur = null;
+  let pendingOld;
+  let oldLine = 0;
+  let newLine = 0;
+  let oldLeft = 0;
+  let newLeft = 0;
+  for (const line of String(diffText).split("\n")) {
+    const raw = line.endsWith("\r") ? line.slice(0, -1) : line;
+    if (oldLeft > 0 || newLeft > 0) {
+      if (raw.startsWith("+")) {
+        addLine(cur.newLines, newLine);
+        addLine(cur.oldLines, oldLine);
+        newLine++;
+        newLeft--;
+        continue;
+      }
+      if (raw.startsWith("-")) {
+        addLine(cur.oldLines, oldLine);
+        addLine(cur.newLines, newLine);
+        oldLine++;
+        oldLeft--;
+        continue;
+      }
+      if (raw.startsWith(" ") || raw === "") {
+        oldLine++;
+        newLine++;
+        oldLeft--;
+        newLeft--;
+        continue;
+      }
+      if (raw.startsWith("\\")) continue;
+      oldLeft = 0;
+      newLeft = 0;
+    }
+    if (raw.startsWith("diff --git ")) {
+      pendingOld = undefined;
+      continue;
+    }
+    if (raw.startsWith("--- ")) {
+      pendingOld = sidePath(raw.slice(4), "a/");
+      continue;
+    }
+    if (raw.startsWith("+++ ")) {
+      cur = {
+        oldPath: pendingOld === undefined ? null : pendingOld,
+        newPath: sidePath(raw.slice(4), "b/"),
+        oldLines: new Set(),
+        newLines: new Set(),
+      };
+      files.push(cur);
+      pendingOld = undefined;
+      continue;
+    }
+    const hunk = HUNK_RE.exec(raw);
+    if (hunk && cur) {
+      const oldCount = hunk[2] === undefined ? 1 : Number(hunk[2]);
+      const newCount = hunk[4] === undefined ? 1 : Number(hunk[4]);
+      oldLine = Number(hunk[1]);
+      newLine = Number(hunk[3]);
+      oldLeft = oldCount;
+      newLeft = newCount;
+      if (newCount === 0) addLine(cur.newLines, newLine);
+      if (oldCount === 0) addLine(cur.oldLines, oldLine);
+    }
+  }
+  return files
+    .filter((f) => f.oldPath || f.newPath)
+    .map((f) => ({
+      oldPath: f.oldPath,
+      newPath: f.newPath,
+      oldRanges: toRanges(f.oldLines),
+      newRanges: toRanges(f.newLines),
+    }));
+}
+
+/**
+ * Line ranges per file on one side of the diff. `new` (default) keys by the
+ * new path; a deleted file falls back to its old path. `old` keys by the old
+ * path; an added file falls back to its new path. Pick the side the graph was
+ * built from (`graphSide`): its spans are line numbers in that tree.
+ *
+ * @param {string} diffText - unified diff
+ * @param {"new"|"old"} [side]
+ * @returns {Map<string, Array<{start: number, end: number}>>} in diff order
+ */
+export function parseHunkRanges(diffText, side = "new") {
+  const out = new Map();
+  for (const f of parseDiffFiles(diffText)) {
+    const path = side === "old" ? f.oldPath || f.newPath : f.newPath || f.oldPath;
+    const ranges = side === "old" ? f.oldRanges : f.newRanges;
+    if (ranges.length) out.set(path, ranges);
+  }
+  return out;
+}
+
+/**
+ * Declarations a set of ranges falls in.
+ *
+ * `changed` holds the innermost declaration per range - what a person would
+ * call "the function I edited". `seeds` adds every enclosing declaration, since
+ * references in the graph point at types, not at the methods inside them, and a
+ * walk seeded with the method alone would find no dependents. A range that hits
+ * no declaration (top-level code) seeds its file.
+ *
+ * @param {object} graph
+ * @param {Map<string, Array<{start: number, end: number}>>} ranges
+ * @returns {{changed: object[], seeds: object[], fileLevel: string[], notIndexed: string[]}}
+ */
+export function changedSymbols(graph, ranges) {
+  const files = new Map();
+  const symbolsByPath = new Map();
+  for (const n of graph.nodes) {
+    if (n.kind === "file") files.set(n.path, n);
+    else if (n.kind === "symbol" && n.path) {
+      if (!symbolsByPath.has(n.path)) symbolsByPath.set(n.path, []);
+      symbolsByPath.get(n.path).push(n);
+    }
+  }
+
+  const changed = new Map();
+  const seeds = new Map();
+  const fileLevel = [];
+  const notIndexed = [];
+
+  for (const [path, list] of ranges) {
+    const fileNode = files.get(path);
+    if (!fileNode) {
+      notIndexed.push(path);
+      continue;
+    }
+    const symbols = symbolsByPath.get(path) || [];
+    const spanned = symbols.filter((s) => Number.isInteger(s.endLine));
+    if (symbols.length && !spanned.length) {
+      fileLevel.push(path);
+      seeds.set(fileNode.id, fileNode);
+      continue;
+    }
+    for (const r of list) {
+      const containing = spanned
+        .filter((s) => (s.startLine ?? s.line) <= r.end && s.endLine >= r.start)
+        .sort((a, b) => b.line - a.line);
+      if (!containing.length) {
+        seeds.set(fileNode.id, fileNode);
+        continue;
+      }
+      const inner = containing[0];
+      changed.set(inner.id, inner);
+      for (const s of containing) seeds.set(s.id, s);
+    }
+  }
+
+  return { changed: [...changed.values()], seeds: [...seeds.values()], fileLevel, notIndexed };
+}
+
+function gitOut(root, args) {
+  try {
+    return execFileSync("git", ["-C", root, ...args], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+    }).trim();
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * How far the graph is behind the tree. `changedFiles` compares each indexed
+ * file's size and mtime with the manifest written at build time, so an edit
+ * that was never committed counts; `commitMoved` compares the build commit with
+ * HEAD. Either makes the graph stale - an answer from it is about the tree as
+ * it was, and callers say so instead of presenting it as current.
+ *
+ * @param {object} graph
+ * @param {{root?: string}} [opts] - defaults to the root recorded in the graph
+ * @returns {{stale: boolean, commitMoved: boolean|null, baseCommit: string|null, head: string|null, changedFiles: string[], removedFiles: string[]}}
+ */
+export function graphFreshness(graph, { root = graph.root } = {}) {
+  const manifest = graph.manifest || {};
+  const changedFiles = [];
+  const removedFiles = [];
+  for (const [path, prev] of Object.entries(manifest)) {
+    const abs = join(root, path);
+    if (!existsSync(abs)) {
+      removedFiles.push(path);
+      continue;
+    }
+    const st = statSync(abs);
+    if (st.size !== prev.size || Math.floor(st.mtimeMs) !== prev.mtimeMs) changedFiles.push(path);
+  }
+  const head = root ? gitOut(root, ["rev-parse", "HEAD"]) : null;
+  const baseCommit = graph.baseCommit || null;
+  const commitMoved = head && baseCommit ? head !== baseCommit : null;
+  return {
+    stale: Boolean(commitMoved) || changedFiles.length > 0 || removedFiles.length > 0,
+    commitMoved,
+    baseCommit,
+    head,
+    changedFiles: changedFiles.sort(),
+    removedFiles: removedFiles.sort(),
+  };
+}
+
+/**
+ * Which side of a diff the graph's line numbers describe. For a commit range
+ * (`range`, `<base>..<head>`) the graph's build commit decides: built at the
+ * head, the new side; at the base, the old side. Otherwise a graph that matches
+ * the tree at `root` (same HEAD, no file edited since the build) describes the
+ * new side; any other graph was built before the change - on the base checkout,
+ * or before the edit - and its spans are old-side line numbers.
+ *
+ * @param {object} graph
+ * @param {{root?: string, range?: string}} [opts]
+ * @returns {"new"|"old"}
+ */
+export function graphSide(graph, opts = {}) {
+  const parts = typeof opts.range === "string" ? opts.range.split("..") : [];
+  if (parts.length === 2 && opts.root && graph.baseCommit && parts.every((p) => REF_RE.test(p))) {
+    const sha = (ref) => gitOut(opts.root, ["rev-parse", "--verify", "--quiet", `${ref}^{commit}`]);
+    if (sha(parts[1]) === graph.baseCommit) return "new";
+    if (sha(parts[0]) === graph.baseCommit) return "old";
+  }
+  return graphFreshness(graph, opts).stale ? "old" : "new";
+}
+
+const REF_RE = /^(?!-)[A-Za-z0-9._/~^@{}-]{1,200}$/;
+
+/**
+ * The unified diff for `<base>` (working tree against base, uncommitted work
+ * included) or `<base>..<head>` (two commits), with no context lines.
+ *
+ * @param {string} root
+ * @param {string} spec
+ * @returns {string}
+ */
+export function diffFor(root, spec) {
+  const parts = String(spec).split("..");
+  if (parts.length > 2 || parts.some((p) => !REF_RE.test(p))) {
+    throw new Error(`not a ref or ref range: ${JSON.stringify(String(spec).slice(0, 80))}`);
+  }
+  const args = [
+    "-C",
+    root,
+    "diff",
+    "-U0",
+    "--no-color",
+    "--no-ext-diff",
+    "--src-prefix=a/",
+    "--dst-prefix=b/",
+    "--end-of-options",
+    ...parts,
+  ];
+  return execFileSync("git", args, { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+}