@stravica-ai/digital-operator-pa 0.1.7 → 0.1.10

package/dist/engineering-pa-core/src/knowledge/retrieval/graph-builder.mjs

@@ -1,186 +1,260 @@
 #!/usr/bin/env node
 // graph-builder.mjs — Builds a knowledge graph from forward links and backlinks.
 //
-// Reads forward links from markdown files + backlinks from .backlinks.json.
+// Reads forward links from markdown files, machine-readable index.json hubs,
+// and backlinks from .backlinks.json.
 // Output: operator/knowledge/.graph.json
 //
 // CLI: node graph-builder.mjs [--root /path/to/repo]
+//
+// Link extraction and resolution live in ./links.mjs, shared with
+// backlink-indexer.mjs — see that file for the resolution contract.
+//
+// LIBRARY + CLI, same split as backlink-indexer.mjs: `buildKnowledgeGraph()` is
+// the importable body, `main()` is the CLI face, and the top-level run is
+// behind a directly-invoked guard.
 
-import { readdir, readFile, writeFile } from 'node:fs/promises';
-import { join, resolve, dirname, relative, normalize } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import { realpathSync } from 'node:fs';
+import { join, resolve, dirname, relative } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import {
+  buildFileIndex,
+  buildIndexHubs,
+  deriveDomain,
+  extractLinks,
+  isIndexJson,
+  scanDirsFor,
+  toPosix,
+  walkFiles,
+} from './links.mjs';
+import { writeJsonAtomic } from './write-atomic.mjs';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const DEFAULT_ROOT = resolve(__dirname, '../../../..');
 
 const USAGE = `
-graph-builder — Build a knowledge graph from markdown links and backlinks
+graph-builder — Build a knowledge graph from markdown links, index hubs and backlinks
 
 Usage:
   node graph-builder.mjs [options]
 
 Options:
   --root <path>    Repository root (default: 4 levels up from this script)
+  --quiet          Suppress the per-link diagnostics report
   --help, -h       Show this help
 
 Reads forward links from .md files in operator/knowledge/ and core/knowledge/.
+Reads document references from every index.json under those trees.
 Reads backlinks from operator/knowledge/.backlinks.json.
 
 Output: operator/knowledge/.graph.json
 `.trim();
 
-// ── Helpers ─────────────────────────────────────────────────────────
-
-async function walkDir(dir) {
-  const results = [];
-  let entries;
-  try {
-    entries = await readdir(dir, { withFileTypes: true });
-  } catch (err) {
-    if (err.code === 'ENOENT') return results;
-    throw err;
-  }
-  for (const entry of entries) {
-    const full = join(dir, entry.name);
-    if (entry.isDirectory()) {
-      if (entry.name.startsWith('.')) continue;
-      results.push(...await walkDir(full));
-    } else if (entry.isFile() && entry.name.endsWith('.md')) {
-      results.push(full);
-    }
-  }
-  return results;
-}
-
 function extractTitle(content) {
   const match = content.match(/^#\s+(.+)$/m);
   return match ? match[1].trim() : null;
 }
 
-function extractForwardLinks(content, sourcePath, repoRoot) {
-  const links = [];
-  const sourceDir = dirname(sourcePath);
-
-  // Wiki-style: [[target]]
-  const wikiRe = /\[\[([^\]]+)\]\]/g;
-  let m;
-  while ((m = wikiRe.exec(content)) !== null) {
-    let target = m[1].trim();
-    if (!target.endsWith('.md')) target += '.md';
-    links.push(normalize(target));
-  }
-
-  // Markdown-style: [text](relative/path.md)
-  const mdRe = /\[([^\]]*)\]\(([^)]+)\)/g;
-  while ((m = mdRe.exec(content)) !== null) {
-    const href = m[2].trim();
-    if (href.startsWith('http://') || href.startsWith('https://') || href.startsWith('#') || href.startsWith('mailto:')) continue;
-    const path = href.split('#')[0];
-    if (!path || !path.endsWith('.md')) continue;
-    const abs = resolve(sourceDir, path);
-    const rel = relative(repoRoot, abs);
-    links.push(normalize(rel));
-  }
-
-  return [...new Set(links)];
+function stubNode(id) {
+  return { id, title: id.split('/').pop().replace('.md', ''), domain: deriveDomain(id) };
 }
 
-function deriveDomain(relPath) {
-  const parts = relPath.split('/');
-  if (parts.length >= 3 && (parts[0] === 'operator' || parts[0] === 'core') && parts[1] === 'knowledge') {
-    return parts[2];
+function reportUnresolved(label, entries) {
+  if (entries.length === 0) return;
+  const byReason = new Map();
+  for (const e of entries) byReason.set(e.reason, (byReason.get(e.reason) ?? 0) + 1);
+  const summary = [...byReason.entries()].map(([r, n]) => `${r}=${n}`).join(' ');
+  console.warn(`${label}: ${entries.length} unresolved (${summary})`);
+  for (const e of entries) {
+    const extra = e.candidates ? ` [candidates: ${e.candidates.join(', ')}]` : '';
+    console.warn(`  ${e.reason}: ${e.source} -> ${e.target}${extra}`);
   }
-  return null;
 }
 
-// ── Main ────────────────────────────────────────────────────────────
-
-async function main(args) {
-  let root = DEFAULT_ROOT;
-
-  for (let i = 0; i < args.length; i++) {
-    const a = args[i];
-    if (a === '--help' || a === '-h') {
-      console.log(USAGE);
-      return 0;
-    }
-    if (a === '--root') {
-      root = resolve(args[++i]);
-    }
-  }
-
-  const scanDirs = [
-    join(root, 'operator', 'knowledge'),
-    join(root, 'core', 'knowledge'),
-  ];
+/**
+ * Build the knowledge graph over `root` and write
+ * `operator/knowledge/.graph.json`.
+ *
+ * Depends on `.backlinks.json` already being current -- run
+ * `buildBacklinkIndex()` first. Building the graph over a stale backlinks file
+ * silently produces a graph whose backlink edges describe the PREVIOUS tree.
+ *
+ * @param {object} args
+ * @param {string} args.root  workspace root the knowledge tree hangs off
+ * @param {object|null} [args.coverage]  optional provenance stamp merged into
+ *   the artefact as `coverage` (see backlink-indexer.mjs).
+ * @returns {Promise<object>}  outPath, counts, diagnostics and the unresolved
+ *   entries the CLI reports
+ */
+export async function buildKnowledgeGraph({ root, coverage = null }) {
+  const scanDirs = scanDirsFor(root);
 
+  // ── Pass 1: enumerate the tree ────────────────────────────────────────
+  // Resolution needs the whole file set before any link is interpreted —
+  // `[[slug]]` cannot be resolved without knowing which files exist.
   const allFiles = [];
+  const indexPaths = [];
   for (const dir of scanDirs) {
-    allFiles.push(...await walkDir(dir));
+    allFiles.push(...await walkFiles(dir));
+    indexPaths.push(...await walkFiles(dir, isIndexJson));
   }
+  const fileIds = allFiles.map((f) => toPosix(relative(root, f)));
+  const fileIndex = buildFileIndex(fileIds);
 
+  // ── Pass 2: nodes + forward edges ─────────────────────────────────────
   const nodesMap = new Map();
   const edges = [];
   const edgeSet = new Set();
+  const unresolvedLinks = [];
+
+  const addEdge = (from, to, type) => {
+    const edgeKey = `${type}:${from}->${to}`;
+    if (edgeSet.has(edgeKey)) return;
+    edgeSet.add(edgeKey);
+    edges.push({ from, to, type });
+  };
 
-  for (const filePath of allFiles) {
+  for (let i = 0; i < allFiles.length; i++) {
+    const filePath = allFiles[i];
+    const relPath = fileIds[i];
     const content = await readFile(filePath, 'utf8');
-    const relPath = relative(root, filePath);
     const title = extractTitle(content) || relPath.split('/').pop().replace('.md', '');
-    const domain = deriveDomain(relPath);
 
-    nodesMap.set(relPath, { id: relPath, title, domain });
+    nodesMap.set(relPath, { id: relPath, title, domain: deriveDomain(relPath) });
 
-    const forwardLinks = extractForwardLinks(content, filePath, root);
-    for (const target of forwardLinks) {
-      const edgeKey = `link:${relPath}->${target}`;
-      if (!edgeSet.has(edgeKey)) {
-        edgeSet.add(edgeKey);
-        edges.push({ from: relPath, to: target, type: 'link' });
-      }
+    const { links, unresolved } = extractLinks(content, relPath, fileIndex);
+    unresolvedLinks.push(...unresolved);
+    for (const target of links) addEdge(relPath, target, 'link');
+  }
+
+  // ── index.json hubs ───────────────────────────────────────────────────
+  const indexFiles = [];
+  for (const indexPath of indexPaths) {
+    const id = toPosix(relative(root, indexPath));
+    let data;
+    try {
+      data = JSON.parse(await readFile(indexPath, 'utf8'));
+    } catch (err) {
+      console.warn(`Skipping unparseable index: ${id} (${err.message})`);
+      continue;
     }
+    indexFiles.push({ id, data });
   }
+  const hubs = buildIndexHubs(indexFiles, fileIndex);
+  for (const node of hubs.nodes) nodesMap.set(node.id, node);
+  for (const edge of hubs.edges) addEdge(edge.from, edge.to, edge.type);
 
-  // Read backlinks and add backlink edges
+  // ── Backlink edges (reversed: target -> source) ────────────────────────
   const backlinksPath = join(root, 'operator', 'knowledge', '.backlinks.json');
+  let backlinksMissing = false;
   try {
     const raw = await readFile(backlinksPath, 'utf8');
     const backlinksData = JSON.parse(raw);
     const bl = backlinksData.backlinks || {};
     for (const [target, sources] of Object.entries(bl)) {
       for (const source of sources) {
-        const edgeKey = `backlink:${target}->${source}`;
-        if (!edgeSet.has(edgeKey)) {
-          edgeSet.add(edgeKey);
-          edges.push({ from: target, to: source, type: 'backlink' });
-        }
-        if (!nodesMap.has(target)) {
-          nodesMap.set(target, { id: target, title: target.split('/').pop().replace('.md', ''), domain: deriveDomain(target) });
-        }
-        if (!nodesMap.has(source)) {
-          nodesMap.set(source, { id: source, title: source.split('/').pop().replace('.md', ''), domain: deriveDomain(source) });
-        }
+        addEdge(target, source, 'backlink');
+        if (!nodesMap.has(target)) nodesMap.set(target, stubNode(target));
+        if (!nodesMap.has(source)) nodesMap.set(source, stubNode(source));
       }
     }
   } catch (err) {
     if (err.code !== 'ENOENT') throw err;
-    console.warn('No .backlinks.json found — run backlink-indexer.mjs first. Building graph from forward links only.');
+    backlinksMissing = true;
+  }
+
+  // Markdown links to files that do not exist on this host still become stub
+  // nodes — that is how a doc present in one entitlement tier and absent from
+  // the build host stays visible. Wiki links do not: an unresolved `[[slug]]`
+  // is a dangling reference, not a missing file.
+  for (const edge of edges) {
+    if (edge.type !== 'link') continue;
+    if (!nodesMap.has(edge.to)) nodesMap.set(edge.to, stubNode(edge.to));
   }
 
   const nodes = [...nodesMap.values()].sort((a, b) => a.id.localeCompare(b.id));
+  edges.sort((a, b) => a.type.localeCompare(b.type) || a.from.localeCompare(b.from) || a.to.localeCompare(b.to));
 
   const output = {
     version: '1.0',
     generatedAt: new Date().toISOString(),
+    diagnostics: {
+      markdownFiles: allFiles.length,
+      indexHubs: {
+        files: indexFiles.length,
+        entries: hubs.entryCount,
+        resolved: hubs.resolvedCount,
+        unresolved: hubs.unresolved.length,
+      },
+      wikiLinks: {
+        unresolved: unresolvedLinks.length,
+      },
+    },
     nodes,
     edges,
   };
+  if (coverage) output.coverage = coverage;
 
   const outPath = join(root, 'operator', 'knowledge', '.graph.json');
-  await writeFile(outPath, JSON.stringify(output, null, 2) + '\n', 'utf8');
-  console.log(`Wrote ${outPath} (${nodes.length} nodes, ${edges.length} edges)`);
+  await writeJsonAtomic(outPath, output);
+  return {
+    outPath,
+    nodeCount: nodes.length,
+    edgeCount: edges.length,
+    backlinksMissing,
+    indexFileCount: indexFiles.length,
+    hubEntryCount: hubs.entryCount,
+    hubResolvedCount: hubs.resolvedCount,
+    unresolvedLinks,
+    unresolvedIndexEntries: hubs.unresolved,
+    diagnostics: output.diagnostics,
+  };
+}
+
+async function main(args) {
+  let root = DEFAULT_ROOT;
+  let quiet = false;
+
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    if (a === '--help' || a === '-h') {
+      console.log(USAGE);
+      return 0;
+    }
+    if (a === '--quiet') quiet = true;
+    if (a === '--root') {
+      root = resolve(args[++i]);
+    }
+  }
+
+  const result = await buildKnowledgeGraph({ root });
+
+  if (result.backlinksMissing) {
+    console.warn('No .backlinks.json found — run backlink-indexer.mjs first. Building graph from forward links only.');
+  }
+  console.log(`Wrote ${result.outPath} (${result.nodeCount} nodes, ${result.edgeCount} edges)`);
+  console.log(
+    `Index hubs: ${result.indexFileCount} files, ${result.hubResolvedCount}/${result.hubEntryCount} entries resolved`,
+  );
+  if (!quiet) {
+    reportUnresolved('Wiki links', result.unresolvedLinks);
+    reportUnresolved('Index entries', result.unresolvedIndexEntries);
+  }
   return 0;
 }
 
-const exitCode = await main(process.argv.slice(2));
-process.exit(exitCode ?? 0);
+// See backlink-indexer.mjs for why both sides are realpath'd.
+function isDirectlyInvoked() {
+  if (!process.argv[1]) return false;
+  try {
+    return fileURLToPath(import.meta.url) === realpathSync(resolve(process.argv[1]));
+  } catch {
+    return false;
+  }
+}
+
+if (isDirectlyInvoked()) {
+  const exitCode = await main(process.argv.slice(2));
+  process.exit(exitCode ?? 0);
+}