@dreamtree-org/graphify 1.3.0 → 1.5.0

package/dist/index.cjs

@@ -44,15 +44,18 @@ __export(index_exports, {
   collectFiles: () => collectFiles,
   deserializeGraph: () => deserializeGraph,
   diffGraphs: () => diffGraphs,
+  expandRetrieval: () => expandRetrieval,
   explainNode: () => explainNode,
   exportGraph: () => exportGraph,
   extract: () => extract,
   extractMysql: () => extractMysql,
   globToRegExp: () => globToRegExp,
   graphForDirectory: () => graphForDirectory,
+  ingestDocument: () => ingestDocument,
   isTestFile: () => isTestFile,
   loadGraph: () => loadGraph,
   loadResults: () => loadResults,
+  mergeExtractionsInto: () => mergeExtractionsInto,
   mergeGraphs: () => mergeGraphs,
   queryGraph: () => queryGraph,
   rankForTask: () => rankForTask,
@@ -70,6 +73,7 @@ __export(index_exports, {
   shortestPath: () => shortestPath,
   testsForChangedFiles: () => testsForChangedFiles,
   testsForNode: () => testsForNode,
+  updateCorpusGraph: () => updateCorpusGraph,
   validateExtraction: () => validateExtraction,
   validateRules: () => validateRules
 });
@@ -2028,13 +2032,15 @@ var RelationSchema = import_zod.z.enum([
   "references",
   "contains",
   "method",
-  "re_exports"
+  "re_exports",
+  "follows"
 ]);
 var GraphNodeSchema = import_zod.z.object({
   id: import_zod.z.string().min(1, "node id must be non-empty"),
   label: import_zod.z.string(),
   sourceFile: import_zod.z.string(),
-  sourceLocation: import_zod.z.string()
+  sourceLocation: import_zod.z.string(),
+  kind: import_zod.z.string().optional()
 });
 var GraphEdgeSchema = import_zod.z.object({
   source: import_zod.z.string().min(1, "edge source must be non-empty"),
@@ -2082,16 +2088,27 @@ function buildGraph(extractions) {
   const validated = extractions.map(
     (extraction, index) => validateExtraction(extraction, `extraction #${index}`)
   );
+  mergeExtractionsInto(graph, validated);
+  return graph;
+}
+function mergeExtractionsInto(graph, validated) {
   const allNodes = validated.flatMap((extraction) => extraction.nodes);
   const allEdges = validated.flatMap((extraction) => extraction.edges);
   const sortedNodes = [...allNodes].sort((a, b) => a.id.localeCompare(b.id));
   for (const node of sortedNodes) {
-    if (graph.hasNode(node.id)) continue;
-    graph.addNode(node.id, {
+    const attributes = {
       label: node.label,
       sourceFile: node.sourceFile,
-      sourceLocation: node.sourceLocation
-    });
+      sourceLocation: node.sourceLocation,
+      ...node.kind !== void 0 ? { kind: node.kind } : {}
+    };
+    if (graph.hasNode(node.id)) {
+      if (graph.getNodeAttribute(node.id, "sourceFile") === "<unknown>") {
+        graph.mergeNodeAttributes(node.id, attributes);
+      }
+      continue;
+    }
+    graph.addNode(node.id, attributes);
   }
   const sortedEdges = [...allEdges].sort(
     (a, b) => a.source.localeCompare(b.source) || a.target.localeCompare(b.target) || a.relation.localeCompare(b.relation)
@@ -2110,9 +2127,402 @@ function buildGraph(extractions) {
       confidence: edge.confidence
     });
   }
+}
+
+// src/ingest/document.ts
+var import_node_crypto = require("crypto");
+var posix2 = __toESM(require("path/posix"), 1);
+var DEFAULT_TARGET_TOKENS = 400;
+var DEFAULT_MAX_TOKENS = 512;
+var CHUNK_LABEL_LEN = 60;
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function contentHash(content) {
+  const normalized = content.replace(/\r\n/g, "\n").trim();
+  return (0, import_node_crypto.createHash)("sha256").update(normalized, "utf8").digest("hex");
+}
+function chunkLabel(content) {
+  const collapsed = content.replace(/\s+/g, " ").trim();
+  return collapsed.length > CHUNK_LABEL_LEN ? `${collapsed.slice(0, CHUNK_LABEL_LEN - 1)}\u2026` : collapsed;
+}
+function parseBlocks(text, honorHeadings) {
+  const lines = text.replace(/\r\n/g, "\n").split("\n");
+  const blocks = [];
+  let paragraph = [];
+  let paragraphLine = 0;
+  let inFence = false;
+  const flush = () => {
+    if (paragraph.length === 0) return;
+    blocks.push({ type: "paragraph", text: paragraph.join("\n"), depth: 0, line: paragraphLine });
+    paragraph = [];
+  };
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (/^(```|~~~)/.test(line.trim())) {
+      if (paragraph.length === 0) paragraphLine = i + 1;
+      paragraph.push(line);
+      inFence = !inFence;
+      continue;
+    }
+    if (inFence) {
+      paragraph.push(line);
+      continue;
+    }
+    const heading = honorHeadings ? /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line) : null;
+    if (heading) {
+      flush();
+      blocks.push({
+        type: "heading",
+        text: heading[2].trim(),
+        depth: heading[1].length,
+        line: i + 1
+      });
+      continue;
+    }
+    if (line.trim() === "") {
+      flush();
+      continue;
+    }
+    if (paragraph.length === 0) paragraphLine = i + 1;
+    paragraph.push(line);
+  }
+  flush();
+  return blocks;
+}
+function splitLongParagraph(text, maxTokens) {
+  const maxChars = maxTokens * 4;
+  const pieces = [];
+  let rest = text;
+  while (rest.length > maxChars) {
+    let cut = rest.lastIndexOf(" ", maxChars);
+    const newlineCut = rest.lastIndexOf("\n", maxChars);
+    cut = Math.max(cut, newlineCut);
+    if (cut <= 0) cut = maxChars;
+    pieces.push(rest.slice(0, cut));
+    rest = rest.slice(cut).trimStart();
+  }
+  if (rest.length > 0) pieces.push(rest);
+  return pieces;
+}
+function extractLinkTargets(content, sourceRef) {
+  const targets = /* @__PURE__ */ new Set();
+  const linkPattern = /\[[^\]]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
+  const dir = posix2.dirname(sourceRef);
+  for (const match of content.matchAll(linkPattern)) {
+    const raw = match[1].split("#")[0];
+    if (raw === "" || /^[a-z][a-z0-9+.-]*:/i.test(raw)) continue;
+    targets.add(posix2.normalize(posix2.join(dir === "." ? "" : dir, raw)));
+  }
+  return [...targets].sort((a, b) => a.localeCompare(b));
+}
+async function ingestDocument(input, options = {}) {
+  if (input.sourceRef.trim() === "") throw new Error("ingestDocument: sourceRef must be non-empty");
+  const targetTokens = options.chunking?.targetTokens ?? DEFAULT_TARGET_TOKENS;
+  const maxTokens = Math.max(options.chunking?.maxTokens ?? DEFAULT_MAX_TOKENS, targetTokens);
+  const overlapTokens = options.chunking?.overlapTokens ?? 0;
+  const metadata = input.metadata ?? {};
+  const sourceRef = input.sourceRef;
+  const honorHeadings = !/^text\/plain\b/i.test(input.mimeType ?? "");
+  const blocks = parseBlocks(input.text, honorHeadings);
+  const nodes = [];
+  const edges = [];
+  const documentId = sourceRef;
+  nodes.push({
+    id: documentId,
+    label: input.title ?? posix2.basename(sourceRef),
+    sourceFile: sourceRef,
+    sourceLocation: "L1",
+    kind: "document"
+  });
+  const sectionStack = [];
+  const sectionIdCounts = /* @__PURE__ */ new Map();
+  let sectionCount = 0;
+  const chunks = [];
+  let buffer = [];
+  let bufferLine = 0;
+  let bufferSection = null;
+  let previousChunkId = null;
+  const flushChunk = () => {
+    const content = buffer.join("\n\n");
+    buffer = [];
+    if (content.trim() === "") return;
+    const index = chunks.length;
+    const nodeId = `${sourceRef}::chunk:${String(index).padStart(4, "0")}`;
+    chunks.push({
+      nodeId,
+      content,
+      tokenEstimate: estimateTokens(content),
+      index,
+      sectionPath: bufferSection?.path ?? [],
+      contentHash: contentHash(content),
+      metadata: { ...metadata }
+    });
+    nodes.push({
+      id: nodeId,
+      label: chunkLabel(content),
+      sourceFile: sourceRef,
+      sourceLocation: `L${bufferLine}`,
+      kind: "chunk"
+    });
+    edges.push({
+      source: bufferSection?.id ?? documentId,
+      target: nodeId,
+      relation: "contains",
+      confidence: "EXTRACTED"
+    });
+    if (previousChunkId !== null) {
+      edges.push({ source: previousChunkId, target: nodeId, relation: "follows", confidence: "EXTRACTED" });
+    }
+    previousChunkId = nodeId;
+    for (const target of extractLinkTargets(content, sourceRef)) {
+      if (target === sourceRef) continue;
+      edges.push({ source: nodeId, target, relation: "references", confidence: "INFERRED" });
+    }
+    if (overlapTokens > 0) {
+      const tailChars = overlapTokens * 4;
+      const tail = content.length > tailChars ? content.slice(content.indexOf(" ", content.length - tailChars) + 1) : "";
+      if (tail.trim() !== "") {
+        buffer.push(tail);
+      }
+    }
+  };
+  for (const block of blocks) {
+    if (block.type === "heading") {
+      flushChunk();
+      buffer = [];
+      while (sectionStack.length > 0 && sectionStack[sectionStack.length - 1].depth >= block.depth) {
+        sectionStack.pop();
+      }
+      const parent = sectionStack[sectionStack.length - 1];
+      const path12 = [...parent?.path ?? [], block.text];
+      const baseId = `${sourceRef}::sec:${path12.join("/")}`;
+      const seen = sectionIdCounts.get(baseId) ?? 0;
+      sectionIdCounts.set(baseId, seen + 1);
+      const id = seen === 0 ? baseId : `${baseId}#${seen + 1}`;
+      nodes.push({
+        id,
+        label: block.text,
+        sourceFile: sourceRef,
+        sourceLocation: `L${block.line}`,
+        kind: "section"
+      });
+      edges.push({ source: parent?.id ?? documentId, target: id, relation: "contains", confidence: "EXTRACTED" });
+      sectionStack.push({ id, depth: block.depth, path: path12 });
+      sectionCount += 1;
+      bufferSection = sectionStack[sectionStack.length - 1];
+      bufferLine = block.line + 1;
+      continue;
+    }
+    const pieces = estimateTokens(block.text) > maxTokens ? splitLongParagraph(block.text, maxTokens) : [block.text];
+    for (const piece of pieces) {
+      const bufferTokens = estimateTokens(buffer.join("\n\n"));
+      if (buffer.length > 0 && bufferTokens + estimateTokens(piece) > targetTokens) {
+        flushChunk();
+      }
+      if (buffer.length === 0) bufferLine = block.line;
+      buffer.push(piece);
+    }
+  }
+  flushChunk();
+  let entities = 0;
+  if (options.semanticExtractor) {
+    const semantic = validateExtraction(
+      await options.semanticExtractor.extractSemantic(sourceRef, input.text),
+      `semantic pass for ${sourceRef}`
+    );
+    entities = semantic.nodes.length;
+    nodes.push(...semantic.nodes);
+    edges.push(...semantic.edges);
+  }
+  const extraction = validateExtraction({ nodes, edges }, `ingestDocument(${sourceRef})`);
+  return {
+    chunks,
+    extraction,
+    stats: {
+      sections: sectionCount,
+      entities,
+      tokenTotal: chunks.reduce((sum, chunk) => sum + chunk.tokenEstimate, 0)
+    }
+  };
+}
+
+// src/ingest/corpus.ts
+var import_graphology3 = __toESM(require("graphology"), 1);
+
+// src/cluster.ts
+var import_node_crypto2 = require("crypto");
+var import_graphology2 = __toESM(require("graphology"), 1);
+var import_graphology_communities_leiden = __toESM(require("@aflsolutions/graphology-communities-leiden"), 1);
+var import_graphology_communities_louvain = __toESM(require("graphology-communities-louvain"), 1);
+var FIXED_SEED = 1592594996;
+function mulberry32(seed) {
+  let a = seed >>> 0;
+  return function next() {
+    a = a + 1831565813 | 0;
+    let t = Math.imul(a ^ a >>> 15, 1 | a);
+    t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
+    return ((t ^ t >>> 14) >>> 0) / 4294967296;
+  };
+}
+function sortedWorkingCopy(graph) {
+  const copy = new import_graphology2.default({ type: graph.type, multi: graph.multi, allowSelfLoops: graph.allowSelfLoops });
+  const nodeIds = graph.nodes().sort((a, b) => a.localeCompare(b));
+  for (const id of nodeIds) {
+    copy.addNode(id, { ...graph.getNodeAttributes(id) });
+  }
+  const edgeEntries = [];
+  graph.forEachEdge((edge, attributes, source, target) => {
+    edgeEntries.push({ key: edge, source, target, attributes });
+  });
+  edgeEntries.sort(
+    (a, b) => a.source.localeCompare(b.source) || a.target.localeCompare(b.target) || String(a.attributes.relation).localeCompare(String(b.attributes.relation))
+  );
+  for (const entry of edgeEntries) {
+    copy.addEdgeWithKey(entry.key, entry.source, entry.target, { ...entry.attributes });
+  }
+  return copy;
+}
+function toUndirectedCopy(graph) {
+  const copy = new import_graphology2.default({ type: "undirected", multi: true, allowSelfLoops: graph.allowSelfLoops });
+  graph.forEachNode((nodeId, attributes) => copy.addNode(nodeId, { ...attributes }));
+  graph.forEachEdge((edgeKey, attributes, source, target) => {
+    copy.addEdgeWithKey(edgeKey, source, target, { ...attributes });
+  });
+  return copy;
+}
+function runAlgorithm(working, options) {
+  const rng = mulberry32(FIXED_SEED);
+  if (options.algorithm === "leiden") {
+    const undirected = toUndirectedCopy(working);
+    import_graphology_communities_leiden.default.assign(undirected, { rng, maxIterations: options.maxIterations ?? 0 });
+    undirected.forEachNode((nodeId, attributes) => {
+      working.setNodeAttribute(nodeId, "community", attributes.community);
+    });
+    return;
+  }
+  import_graphology_communities_louvain.default.assign(working, { rng });
+}
+function cluster(graph, options = {}) {
+  if (graph.order === 0) return graph;
+  const working = sortedWorkingCopy(graph);
+  runAlgorithm(working, options);
+  const hubs = /* @__PURE__ */ new Map();
+  working.forEachNode((nodeId) => {
+    const community = working.getNodeAttribute(nodeId, "community");
+    const degree = working.degree(nodeId);
+    const current = hubs.get(community);
+    if (!current || degree > current.degree) {
+      hubs.set(community, { id: nodeId, degree });
+    }
+  });
+  const membersByCommunity = /* @__PURE__ */ new Map();
+  working.forEachNode((nodeId) => {
+    const community = working.getNodeAttribute(nodeId, "community");
+    const list = membersByCommunity.get(community);
+    if (list) list.push(nodeId);
+    else membersByCommunity.set(community, [nodeId]);
+  });
+  const labelByCommunity = /* @__PURE__ */ new Map();
+  const hashByCommunity = /* @__PURE__ */ new Map();
+  for (const [community, members] of membersByCommunity) {
+    members.sort((a, b) => a.localeCompare(b));
+    hashByCommunity.set(community, (0, import_node_crypto2.createHash)("sha256").update(members.join("\n")).digest("hex"));
+    const hub = hubs.get(community);
+    const hubLabel = hub ? working.getNodeAttribute(hub.id, "label") : void 0;
+    labelByCommunity.set(community, hubLabel && hubLabel.length > 0 ? hubLabel : `community-${community}`);
+  }
+  working.forEachNode((nodeId) => {
+    const community = working.getNodeAttribute(nodeId, "community");
+    graph.mergeNodeAttributes(nodeId, {
+      community,
+      communityLabel: labelByCommunity.get(community),
+      communityHash: hashByCommunity.get(community)
+    });
+  });
   return graph;
 }
 
+// src/ingest/corpus.ts
+var CLUSTER_STALE_ATTR = "graphifyClusterStale";
+var DEFAULT_RECLUSTER_RATIO = 0.1;
+function assignLocalCommunities(graph) {
+  const communityMeta = /* @__PURE__ */ new Map();
+  let maxCommunity = -1;
+  graph.forEachNode((_id, attrs) => {
+    const community = attrs.community;
+    if (typeof community !== "number") return;
+    if (community > maxCommunity) maxCommunity = community;
+    if (!communityMeta.has(community)) {
+      communityMeta.set(community, {
+        label: attrs.communityLabel,
+        hash: attrs.communityHash
+      });
+    }
+  });
+  const unassigned = graph.filterNodes((_id, attrs) => attrs.community === void 0).sort((a, b) => a.localeCompare(b));
+  for (const id of unassigned) {
+    const votes = /* @__PURE__ */ new Map();
+    graph.forEachNeighbor(id, (_neighbor, attrs) => {
+      const community = attrs.community;
+      if (typeof community === "number") votes.set(community, (votes.get(community) ?? 0) + 1);
+    });
+    let adopted = null;
+    let bestVotes = 0;
+    for (const [community, count] of votes) {
+      if (count > bestVotes || count === bestVotes && adopted !== null && community < adopted) {
+        adopted = community;
+        bestVotes = count;
+      }
+    }
+    if (adopted === null) {
+      adopted = ++maxCommunity;
+      communityMeta.set(adopted, {
+        label: graph.getNodeAttribute(id, "label") ?? id
+      });
+    }
+    const meta = communityMeta.get(adopted);
+    graph.mergeNodeAttributes(id, {
+      community: adopted,
+      communityLabel: meta?.label ?? `community-${adopted}`,
+      ...meta?.hash !== void 0 ? { communityHash: meta.hash } : {}
+    });
+  }
+}
+function updateCorpusGraph(graph, extraction, options = {}) {
+  const validated = validateExtraction(extraction, "updateCorpusGraph");
+  const target = graph ?? new import_graphology3.default({ type: "directed", multi: true, allowSelfLoops: true });
+  const sourceFiles = new Set(validated.nodes.map((node) => node.sourceFile));
+  const stale = target.filterNodes((_id, attrs) => sourceFiles.has(attrs.sourceFile));
+  for (const id of stale) target.dropNode(id);
+  const extractionIds = new Set(validated.nodes.map((node) => node.id));
+  const addedReal = validated.nodes.filter((node) => !target.hasNode(node.id)).length;
+  const newPlaceholders = /* @__PURE__ */ new Set();
+  for (const edge of validated.edges) {
+    for (const endpoint of [edge.source, edge.target]) {
+      if (!extractionIds.has(endpoint) && !target.hasNode(endpoint)) newPlaceholders.add(endpoint);
+    }
+  }
+  mergeExtractionsInto(target, [validated]);
+  const orphaned = target.filterNodes(
+    (id, attrs) => attrs.sourceFile === "<unknown>" && target.degree(id) === 0
+  );
+  for (const id of orphaned) target.dropNode(id);
+  const churn = stale.length + addedReal + newPlaceholders.size + orphaned.length;
+  const staleCount = (Number(target.getAttribute(CLUSTER_STALE_ATTR)) || 0) + churn;
+  const mode = options.recluster ?? "auto";
+  const ratio = options.reclusterRatio ?? DEFAULT_RECLUSTER_RATIO;
+  const neverClustered = target.order > 0 && target.findNode((_id, attrs) => attrs.community !== void 0) === void 0;
+  if (mode === true || mode === "auto" && (neverClustered || staleCount > target.order * ratio)) {
+    cluster(target, { algorithm: options.algorithm });
+    target.setAttribute(CLUSTER_STALE_ATTR, 0);
+  } else {
+    if (mode === "auto") assignLocalCommunities(target);
+    target.setAttribute(CLUSTER_STALE_ATTR, staleCount);
+  }
+  return target;
+}
+
 // src/resolve.ts
 var fs9 = __toESM(require("fs/promises"), 1);
 var path5 = __toESM(require("path"), 1);
@@ -2307,99 +2717,6 @@ function resolveCrossFileReferences(extractions, options = {}) {
   return { resolvedEndpoints, droppedPlaceholders };
 }
 
-// src/cluster.ts
-var import_node_crypto = require("crypto");
-var import_graphology2 = __toESM(require("graphology"), 1);
-var import_graphology_communities_leiden = __toESM(require("@aflsolutions/graphology-communities-leiden"), 1);
-var import_graphology_communities_louvain = __toESM(require("graphology-communities-louvain"), 1);
-var FIXED_SEED = 1592594996;
-function mulberry32(seed) {
-  let a = seed >>> 0;
-  return function next() {
-    a = a + 1831565813 | 0;
-    let t = Math.imul(a ^ a >>> 15, 1 | a);
-    t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t;
-    return ((t ^ t >>> 14) >>> 0) / 4294967296;
-  };
-}
-function sortedWorkingCopy(graph) {
-  const copy = new import_graphology2.default({ type: graph.type, multi: graph.multi, allowSelfLoops: graph.allowSelfLoops });
-  const nodeIds = graph.nodes().sort((a, b) => a.localeCompare(b));
-  for (const id of nodeIds) {
-    copy.addNode(id, { ...graph.getNodeAttributes(id) });
-  }
-  const edgeEntries = [];
-  graph.forEachEdge((edge, attributes, source, target) => {
-    edgeEntries.push({ key: edge, source, target, attributes });
-  });
-  edgeEntries.sort(
-    (a, b) => a.source.localeCompare(b.source) || a.target.localeCompare(b.target) || String(a.attributes.relation).localeCompare(String(b.attributes.relation))
-  );
-  for (const entry of edgeEntries) {
-    copy.addEdgeWithKey(entry.key, entry.source, entry.target, { ...entry.attributes });
-  }
-  return copy;
-}
-function toUndirectedCopy(graph) {
-  const copy = new import_graphology2.default({ type: "undirected", multi: true, allowSelfLoops: graph.allowSelfLoops });
-  graph.forEachNode((nodeId, attributes) => copy.addNode(nodeId, { ...attributes }));
-  graph.forEachEdge((edgeKey, attributes, source, target) => {
-    copy.addEdgeWithKey(edgeKey, source, target, { ...attributes });
-  });
-  return copy;
-}
-function runAlgorithm(working, options) {
-  const rng = mulberry32(FIXED_SEED);
-  if (options.algorithm === "leiden") {
-    const undirected = toUndirectedCopy(working);
-    import_graphology_communities_leiden.default.assign(undirected, { rng, maxIterations: options.maxIterations ?? 0 });
-    undirected.forEachNode((nodeId, attributes) => {
-      working.setNodeAttribute(nodeId, "community", attributes.community);
-    });
-    return;
-  }
-  import_graphology_communities_louvain.default.assign(working, { rng });
-}
-function cluster(graph, options = {}) {
-  if (graph.order === 0) return graph;
-  const working = sortedWorkingCopy(graph);
-  runAlgorithm(working, options);
-  const hubs = /* @__PURE__ */ new Map();
-  working.forEachNode((nodeId) => {
-    const community = working.getNodeAttribute(nodeId, "community");
-    const degree = working.degree(nodeId);
-    const current = hubs.get(community);
-    if (!current || degree > current.degree) {
-      hubs.set(community, { id: nodeId, degree });
-    }
-  });
-  const membersByCommunity = /* @__PURE__ */ new Map();
-  working.forEachNode((nodeId) => {
-    const community = working.getNodeAttribute(nodeId, "community");
-    const list = membersByCommunity.get(community);
-    if (list) list.push(nodeId);
-    else membersByCommunity.set(community, [nodeId]);
-  });
-  const labelByCommunity = /* @__PURE__ */ new Map();
-  const hashByCommunity = /* @__PURE__ */ new Map();
-  for (const [community, members] of membersByCommunity) {
-    members.sort((a, b) => a.localeCompare(b));
-    hashByCommunity.set(community, (0, import_node_crypto.createHash)("sha256").update(members.join("\n")).digest("hex"));
-    const hub = hubs.get(community);
-    const hubLabel = hub ? working.getNodeAttribute(hub.id, "label") : void 0;
-    labelByCommunity.set(community, hubLabel && hubLabel.length > 0 ? hubLabel : `community-${community}`);
-  }
-  working.forEachNode((nodeId) => {
-    const community = working.getNodeAttribute(nodeId, "community");
-    graph.mergeNodeAttributes(nodeId, {
-      community,
-      communityLabel: labelByCommunity.get(community),
-      communityHash: hashByCommunity.get(community)
-    });
-  });
-  return graph;
-}
-
 // src/analyze.ts
 var ABSOLUTE_DEGREE_FLOOR = 10;
 var MEAN_DEGREE_MULTIPLIER = 3;
@@ -2577,7 +2894,7 @@ var fs12 = __toESM(require("fs/promises"), 1);
 var path6 = __toESM(require("path"), 1);
 
 // src/security.ts
-var import_node_crypto2 = require("crypto");
+var import_node_crypto3 = require("crypto");
 var dns = __toESM(require("dns"), 1);
 var http = __toESM(require("http"), 1);
 var https = __toESM(require("https"), 1);
@@ -2652,12 +2969,12 @@ function escapeHtml(text) {
 var fs11 = __toESM(require("fs/promises"), 1);
 
 // src/store/serialize.ts
-var import_graphology3 = __toESM(require("graphology"), 1);
+var import_graphology4 = __toESM(require("graphology"), 1);
 function serializeGraph(graph) {
   return graph.export();
 }
 function deserializeGraph(data) {
-  return import_graphology3.default.from(data);
+  return import_graphology4.default.from(data);
 }
 
 // src/store/localFile.ts
@@ -2819,7 +3136,7 @@ var fs14 = __toESM(require("fs/promises"), 1);
 var path8 = __toESM(require("path"), 1);
 
 // src/extractionCache.ts
-var import_node_crypto3 = require("crypto");
+var import_node_crypto4 = require("crypto");
 var fs13 = __toESM(require("fs/promises"), 1);
 var path7 = __toESM(require("path"), 1);
 var ExtractionCache = class {
@@ -2828,7 +3145,7 @@ var ExtractionCache = class {
     this.dir = path7.join(outDir, "cache");
   }
   key(relFile, content) {
-    return (0, import_node_crypto3.createHash)("sha256").update(relFile).update("\0").update(content).digest("hex");
+    return (0, import_node_crypto4.createHash)("sha256").update(relFile).update("\0").update(content).digest("hex");
   }
   async get(key) {
     try {
@@ -3200,6 +3517,141 @@ function explainNode(graph, query) {
   };
 }
 
+// src/retrieval.ts
+var DEFAULT_TOKEN_BUDGET2 = 2e3;
+var DEFAULT_MAX_HOPS = 2;
+function resolveSeeds(graph, hits) {
+  const seeds = /* @__PURE__ */ new Map();
+  for (const hit of hits) {
+    let id = null;
+    if (hit.nodeId !== void 0 && graph.hasNode(hit.nodeId)) {
+      id = hit.nodeId;
+    } else if (hit.text !== void 0) {
+      id = resolveNode(graph, hit.text)?.id ?? null;
+    }
+    const key = id ?? hit.nodeId ?? hit.text ?? "";
+    if (key === "") continue;
+    const existing = seeds.get(key);
+    if (existing) {
+      if (hit.score !== void 0 && (existing.score === void 0 || hit.score > existing.score)) {
+        existing.score = hit.score;
+      }
+      continue;
+    }
+    seeds.set(key, { id: key, score: hit.score, inGraph: id !== null, text: hit.text });
+  }
+  return [...seeds.values()];
+}
+function expandRetrieval(graph, hits, options = {}) {
+  const tokenBudget = options.tokenBudget ?? DEFAULT_TOKEN_BUDGET2;
+  const maxHops = options.maxHops ?? DEFAULT_MAX_HOPS;
+  const relations = options.relations ? new Set(options.relations) : null;
+  const seeds = resolveSeeds(graph, hits);
+  const graphSeeds = seeds.filter((s) => s.inGraph);
+  const seedScore = new Map(graphSeeds.map((s) => [s.id, s.score]));
+  const hops = /* @__PURE__ */ new Map();
+  const via = /* @__PURE__ */ new Map();
+  const frontier = [];
+  let spentTokens = 0;
+  for (const seed of graphSeeds) {
+    hops.set(seed.id, 0);
+    frontier.push(seed.id);
+    spentTokens += nodeTokenCost(graph, seed.id);
+  }
+  let truncated = false;
+  while (frontier.length > 0) {
+    const current = frontier.shift();
+    const currentHops = hops.get(current);
+    if (currentHops >= maxHops) continue;
+    const candidates = [];
+    graph.forEachEdge(current, (_key, attrs, source, target) => {
+      const relation = String(attrs.relation);
+      if (relations && !relations.has(relation)) return;
+      const neighbor = source === current ? target : source;
+      candidates.push({
+        neighbor,
+        edge: { source, target, relation, confidence: String(attrs.confidence) }
+      });
+    });
+    candidates.sort((a, b) => a.neighbor.localeCompare(b.neighbor));
+    for (const { neighbor, edge } of candidates) {
+      if (hops.has(neighbor)) continue;
+      const cost = nodeTokenCost(graph, neighbor);
+      if (spentTokens + cost > tokenBudget) {
+        truncated = true;
+        continue;
+      }
+      spentTokens += cost;
+      hops.set(neighbor, currentHops + 1);
+      via.set(neighbor, edge);
+      frontier.push(neighbor);
+    }
+  }
+  const degree = (id) => graph.degree(id);
+  const nodes = [...hops.entries()].map(([id, hopCount]) => {
+    const attrs = graph.getNodeAttributes(id);
+    return {
+      id,
+      label: attrs.label ?? id,
+      ...attrs.kind !== void 0 ? { kind: String(attrs.kind) } : {},
+      sourceFile: attrs.sourceFile ?? "",
+      sourceLocation: attrs.sourceLocation ?? "",
+      hops: hopCount,
+      ...via.has(id) ? { via: via.get(id) } : {},
+      ...hopCount === 0 && seedScore.get(id) !== void 0 ? { seedScore: seedScore.get(id) } : {}
+    };
+  });
+  for (const seed of seeds) {
+    if (seed.inGraph) continue;
+    nodes.push({
+      id: seed.id,
+      label: seed.text ?? seed.id,
+      sourceFile: "",
+      sourceLocation: "",
+      hops: 0,
+      ...seed.score !== void 0 ? { seedScore: seed.score } : {}
+    });
+  }
+  nodes.sort((a, b) => {
+    if (a.hops !== b.hops) return a.hops - b.hops;
+    if (a.hops === 0) {
+      return (b.seedScore ?? -Infinity) - (a.seedScore ?? -Infinity) || a.id.localeCompare(b.id);
+    }
+    const degreeDiff = (graph.hasNode(b.id) ? degree(b.id) : 0) - (graph.hasNode(a.id) ? degree(a.id) : 0);
+    return degreeDiff || a.id.localeCompare(b.id);
+  });
+  const included = /* @__PURE__ */ new Set([...hops.keys()]);
+  const edges = [];
+  graph.forEachEdge((_key, attrs, source, target) => {
+    if (!included.has(source) || !included.has(target)) return;
+    const relation = String(attrs.relation);
+    if (relations && !relations.has(relation)) return;
+    edges.push({ source, target, relation, confidence: String(attrs.confidence) });
+  });
+  edges.sort(
+    (a, b) => a.source.localeCompare(b.source) || a.target.localeCompare(b.target) || a.relation.localeCompare(b.relation)
+  );
+  const communities = /* @__PURE__ */ new Map();
+  for (const id of included) {
+    const attrs = graph.getNodeAttributes(id);
+    const community = attrs.community;
+    if (community === void 0) continue;
+    const entry = communities.get(community) ?? {
+      id: community,
+      label: attrs.communityLabel ?? String(community),
+      seedCount: 0
+    };
+    if (hops.get(id) === 0) entry.seedCount += 1;
+    communities.set(community, entry);
+  }
+  return {
+    nodes,
+    edges,
+    communities: [...communities.values()].sort((a, b) => b.seedCount - a.seedCount || a.id - b.id),
+    truncated
+  };
+}
+
 // src/impact.ts
 var DEPENDENCY_RELATIONS = /* @__PURE__ */ new Set([
   "calls",
@@ -3265,7 +3717,7 @@ function affectedBy(graph, nodeQuery, options = {}) {
 }
 
 // src/merge.ts
-var import_graphology4 = __toESM(require("graphology"), 1);
+var import_graphology5 = __toESM(require("graphology"), 1);
 var CONFIDENCE_RANK3 = { EXTRACTED: 3, INFERRED: 2, AMBIGUOUS: 1 };
 function strongerConfidence2(a, b) {
   return CONFIDENCE_RANK3[a] >= CONFIDENCE_RANK3[b] ? a : b;
@@ -3282,7 +3734,7 @@ function mergeGraphs(entries) {
   if (duplicate !== void 0) {
     throw new Error(`Duplicate project name in merge: "${duplicate}" \u2014 every entry needs a unique name.`);
   }
-  const merged = new import_graphology4.default({ type: "directed", multi: true, allowSelfLoops: true });
+  const merged = new import_graphology5.default({ type: "directed", multi: true, allowSelfLoops: true });
   const sorted = [...entries].sort((a, b) => a.name.localeCompare(b.name));
   for (const { name, graph } of sorted) {
     const nodeIds = [...graph.nodes()].sort((a, b) => a.localeCompare(b));
@@ -3468,7 +3920,7 @@ async function addViewEdges(query, schema, viewNames, tableNames, edges) {
 }
 
 // src/memory.ts
-var import_node_crypto4 = require("crypto");
+var import_node_crypto5 = require("crypto");
 var fs15 = __toESM(require("fs/promises"), 1);
 var path10 = __toESM(require("path"), 1);
 var OUTCOMES = /* @__PURE__ */ new Set(["useful", "dead_end", "corrected"]);
@@ -3490,7 +3942,7 @@ async function saveResult(entry, memoryDir, now = /* @__PURE__ */ new Date()) {
   validateResult(entry);
   await fs15.mkdir(memoryDir, { recursive: true });
   const saved = { ...entry, savedAt: now.toISOString() };
-  const hash = (0, import_node_crypto4.createHash)("sha256").update(JSON.stringify(saved)).digest("hex").slice(0, 8);
+  const hash = (0, import_node_crypto5.createHash)("sha256").update(JSON.stringify(saved)).digest("hex").slice(0, 8);
   const stamp = saved.savedAt.replace(/[:.]/g, "-");
   const filePath = path10.join(memoryDir, `result-${stamp}-${hash}.json`);
   await fs15.writeFile(filePath, JSON.stringify(saved, null, 2), "utf-8");
@@ -4005,15 +4457,18 @@ function checkRules(graph, config) {
   collectFiles,
   deserializeGraph,
   diffGraphs,
+  expandRetrieval,
   explainNode,
   exportGraph,
   extract,
   extractMysql,
   globToRegExp,
   graphForDirectory,
+  ingestDocument,
   isTestFile,
   loadGraph,
   loadResults,
+  mergeExtractionsInto,
   mergeGraphs,
   queryGraph,
   rankForTask,
@@ -4031,6 +4486,7 @@ function checkRules(graph, config) {
   shortestPath,
   testsForChangedFiles,
   testsForNode,
+  updateCorpusGraph,
   validateExtraction,
   validateRules
 });