@renzrollon/interlock 1.0.0

package/lib/graph/extract/walk.mjs

@@ -0,0 +1,272 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { rel } from '../paths.mjs';
+
+const DEFAULT_SKIP_DIRS = new Set([
+  'node_modules',
+  '.git',
+  '.next',
+  'dist',
+  'build',
+  'coverage',
+  '.turbo',
+  '.cache',
+  'vendor',
+  'interlock-graph-out',
+  '__pycache__',
+  '.venv',
+  'venv',
+  '.tox',
+  '.mypy_cache',
+  '.pytest_cache',
+  '.ruff_cache',
+  'site-packages',
+  '.eggs',
+  'eggs',
+  '.idea',
+  '.vscode',
+]);
+
+/** Code / scripts always indexed. */
+const SOURCE_EXTS = new Set([
+  '.ts',
+  '.tsx',
+  '.js',
+  '.jsx',
+  '.mjs',
+  '.cjs',
+  '.py',
+  '.pyi',
+  '.sh',
+  '.bash',
+]);
+
+/**
+ * Curated config JSON basenames (and simple globs via endsWith / includes).
+ * Lockfiles and large generated JSON are excluded separately.
+ */
+const CONFIG_JSON_NAMES = new Set([
+  'package.json',
+  'tsconfig.json',
+  'jsconfig.json',
+  'pyrightconfig.json',
+  'turbo.json',
+  'nx.json',
+  'components.json',
+  'nest-cli.json',
+  'openapi.json',
+  'swagger.json',
+  'settings.json',
+  'launch.json',
+  'extensions.json',
+  'manifest.json',
+  'appsscript.json',
+]);
+
+const CONFIG_JSON_MAX_BYTES = 64 * 1024;
+
+function loadIgnorePatterns(root) {
+  const patterns = [];
+  for (const name of ['.gitignore', '.interlockignore']) {
+    const p = path.join(root, name);
+    if (!fs.existsSync(p)) continue;
+    for (const line of fs.readFileSync(p, 'utf8').split('\n')) {
+      const t = line.trim();
+      if (!t || t.startsWith('#')) continue;
+      patterns.push(t.replace(/\/$/, ''));
+    }
+  }
+  return patterns;
+}
+
+function ignored(relPath, patterns) {
+  for (const pat of patterns) {
+    if (pat.startsWith('!')) continue;
+    if (pat.startsWith('*')) {
+      const suffix = pat.slice(1);
+      if (relPath.endsWith(suffix) || relPath.includes(`/${suffix}`)) return true;
+      continue;
+    }
+    if (relPath === pat || relPath.startsWith(`${pat}/`) || relPath.includes(`/${pat}/`)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+function isConfigJson(name, relPath, size) {
+  if (!name.endsWith('.json')) return false;
+  if (size > CONFIG_JSON_MAX_BYTES) return false;
+  const lower = name.toLowerCase();
+  if (
+    lower.includes('lock') ||
+    lower.endsWith('.min.json') ||
+    lower === 'package-lock.json' ||
+    lower === 'npm-shrinkwrap.json'
+  ) {
+    return false;
+  }
+  if (CONFIG_JSON_NAMES.has(name)) return true;
+  if (/^tsconfig\..+\.json$/.test(name)) return true;
+  if (name.endsWith('.eslintrc.json') || name === '.eslintrc.json') return true;
+  if (name.endsWith('mcp.json') || name.endsWith('mcp_config.json')) return true;
+  // Root-level small *.json only (avoid sweeping fixture dumps)
+  if (!relPath.includes('/') && size <= CONFIG_JSON_MAX_BYTES) return true;
+  return false;
+}
+
+function shouldSkipDir(root, dir, entName, relPath) {
+  // Skip the project's own `.claude` home (Interlock graph artifacts, memory) at repo root only —
+  // nested trees like harness/.claude/hooks/*.sh are still indexed.
+  if (entName === '.claude' && relPath === '.claude') return true;
+  if (DEFAULT_SKIP_DIRS.has(entName)) return true;
+  if (entName.endsWith('.egg-info')) return true;
+  return false;
+}
+
+/**
+ * Walk root for source + curated config files. Returns absolute paths.
+ */
+export function walkSourceFiles(root, { extraSkip = [] } = {}) {
+  const skipExtra = new Set(extraSkip);
+  const patterns = loadIgnorePatterns(root);
+  const out = [];
+
+  function visit(dir) {
+    let entries;
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const ent of entries) {
+      const full = path.join(dir, ent.name);
+      const r = rel(root, full);
+      if (ent.isDirectory()) {
+        if (skipExtra.has(ent.name) || shouldSkipDir(root, dir, ent.name, r) || ignored(r, patterns)) {
+          continue;
+        }
+        visit(full);
+        continue;
+      }
+      if (!ent.isFile()) continue;
+      if (ignored(r, patterns)) continue;
+      const ext = path.extname(ent.name);
+      if (SOURCE_EXTS.has(ext)) {
+        if (ent.name.endsWith('.d.ts')) continue;
+        out.push(full);
+        continue;
+      }
+      if (ext === '.json') {
+        let size = 0;
+        try {
+          size = fs.statSync(full).size;
+        } catch {
+          continue;
+        }
+        if (isConfigJson(ent.name, r, size)) out.push(full);
+      }
+    }
+  }
+
+  visit(root);
+  return out.sort();
+}
+
+// Accepts directories or individual markdown files, so callers can mix a docs
+// directory with root-level files such as README.md — on many repos those root
+// files are the only real documentation.
+export function walkMarkdown(root, subdirs) {
+  const out = [];
+  for (const sub of subdirs) {
+    const base = path.join(root, sub);
+    let stat;
+    try {
+      stat = fs.statSync(base);
+    } catch {
+      continue;
+    }
+    if (stat.isDirectory()) walkMd(base, out);
+    else if (/\.mdx?$/i.test(base)) out.push(base);
+  }
+  return [...new Set(out)].sort();
+}
+
+function walkMd(dir, out) {
+  let entries;
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch {
+    return;
+  }
+  for (const ent of entries) {
+    const full = path.join(dir, ent.name);
+    if (ent.isDirectory()) {
+      if (ent.name === 'node_modules' || ent.name === '.git' || ent.name === '__pycache__') continue;
+      walkMd(full, out);
+      continue;
+    }
+    if (ent.isFile() && ent.name.endsWith('.md')) out.push(full);
+  }
+}
+
+export { SOURCE_EXTS, CONFIG_JSON_NAMES, CONFIG_JSON_MAX_BYTES };
+
+/**
+ * Directory names projects actually use for prose, beyond `docs/`. Probed in
+ * order when the preferred roots turn up nothing — a repo that keeps its
+ * documentation in `doc/` or `site/` should not silently index zero files.
+ */
+const DOCS_DIR_CANDIDATES = [
+  'docs',
+  'doc',
+  'documentation',
+  'site/content',
+  'site',
+  'website/docs',
+  'guides',
+  'wiki',
+];
+
+/**
+ * Root-level markdown worth indexing on any repo. These are frequently the only
+ * real documentation a project has, and they live outside every docs directory.
+ */
+const ROOT_DOC_FILES = [
+  'README.md',
+  'ARCHITECTURE.md',
+  'CONTRIBUTING.md',
+  'DESIGN.md',
+  'ROADMAP.md',
+  'AGENTS.md',
+  'CLAUDE.md',
+];
+
+/**
+ * Resolve which markdown roots to index for a given repo.
+ *
+ * Returns the preferred roots when they contain anything, otherwise the first
+ * candidate directory that exists. `openspec/specs` is always included when
+ * present, and root-level docs are always appended, because on many repos
+ * (Go projects especially) `README.md` is the documentation.
+ */
+export function resolveDocsDirs(root, requested) {
+  if (requested && requested.length) return requested;
+
+  const has = (p) => {
+    try {
+      return fs.existsSync(`${root}/${p}`);
+    } catch {
+      return false;
+    }
+  };
+
+  const dirs = [];
+  const primary = DOCS_DIR_CANDIDATES.find(has);
+  if (primary) dirs.push(primary);
+  if (has('openspec/specs')) dirs.push('openspec/specs');
+
+  for (const f of ROOT_DOC_FILES) if (has(f)) dirs.push(f);
+
+  return dirs.length ? dirs : ['docs', 'openspec/specs'];
+}

package/lib/graph/format.mjs

@@ -0,0 +1,51 @@
+/**
+ * Format a subgraph as agent-facing plain text within a token budget.
+ * Rough token estimate: chars / 4.
+ */
+export function formatSubgraph({ mode, seeds, nodes, edges, budget = 1500 }) {
+  const lines = [];
+  lines.push(`Traversal: ${String(mode).toUpperCase()} | Start: [${seeds.join(', ')}] | ${nodes.length} nodes, ${edges.length} edges`);
+  lines.push('');
+
+  const nodeLines = [];
+  for (const n of nodes) {
+    nodeLines.push(
+      `  NODE ${n.id} [type=${n.type} src=${n.source_file || '?'} loc=${n.source_location || '?'}]`,
+    );
+  }
+  const edgeLines = [];
+  for (const e of edges) {
+    edgeLines.push(
+      `  EDGE ${short(e.source)} --${e.relation} [${e.confidence}]--> ${short(e.target)} (${e.source_file}:${e.source_location})`,
+    );
+  }
+
+  let body = [...nodeLines, '', ...edgeLines];
+  let text = `${lines.join('\n')}${body.join('\n')}\n`;
+  let tokens = estimateTokens(text);
+
+  if (tokens <= budget) return text;
+
+  // Trim edges first, then nodes
+  while (tokens > budget && edgeLines.length > 3) {
+    edgeLines.pop();
+    body = [...nodeLines, '', ...edgeLines, `  … truncated to fit budget ${budget}`];
+    text = `${lines.join('\n')}${body.join('\n')}\n`;
+    tokens = estimateTokens(text);
+  }
+  while (tokens > budget && nodeLines.length > 3) {
+    nodeLines.pop();
+    body = [...nodeLines, '', ...edgeLines, `  … truncated to fit budget ${budget}`];
+    text = `${lines.join('\n')}${body.join('\n')}\n`;
+    tokens = estimateTokens(text);
+  }
+  return text;
+}
+
+export function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+
+function short(id) {
+  return id.replace(/^(file|symbol|module|openspec|memory|external|unresolved):/, '');
+}

package/lib/graph/graph/build.mjs

@@ -0,0 +1,374 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { createExtractor } from '../extract/create-extractor.mjs';
+import { extractOpenspec } from '../extract/openspec.mjs';
+import { extractMemory } from '../extract/memory.mjs';
+import { walkSourceFiles } from '../extract/walk.mjs';
+import {
+  ensureGraphDir,
+  graphPath,
+  loadJson,
+  manifestPath,
+  rel,
+  reportPath,
+  sha256File,
+  writeJson,
+} from '../paths.mjs';
+import { writeReport } from './report.mjs';
+
+function langOf(filePath) {
+  const ext = path.posix.extname(filePath).toLowerCase();
+  if (ext === '.py' || ext === '.pyi') return 'python';
+  if (ext === '.sh' || ext === '.bash') return 'shell';
+  if (ext === '.json') return 'json';
+  return 'js';
+}
+
+function resolvePythonImport(fromFile, specifier, fileIndex) {
+  let modPath;
+  if (specifier.startsWith('.')) {
+    let dots = 0;
+    while (specifier[dots] === '.') dots += 1;
+    const rest = specifier.slice(dots);
+    let base = path.posix.dirname(fromFile);
+    for (let i = 1; i < dots; i++) {
+      const parent = path.posix.dirname(base);
+      if (parent === base) break;
+      base = parent;
+    }
+    modPath = rest ? path.posix.normalize(path.posix.join(base, rest.replace(/\./g, '/'))) : base;
+  } else {
+    modPath = specifier.replace(/\./g, '/');
+  }
+
+  const tryPaths = [`${modPath}.py`, `${modPath}.pyi`, `${modPath}/__init__.py`];
+  for (const p of tryPaths) {
+    if (fileIndex.has(p)) return { kind: 'file', id: `file:${p}`, path: p };
+  }
+
+  // Bare `import layer` → package __init__
+  if (!specifier.startsWith('.') && fileIndex.has(`${modPath}/__init__.py`)) {
+    return { kind: 'file', id: `file:${modPath}/__init__.py`, path: `${modPath}/__init__.py` };
+  }
+
+  if (specifier.startsWith('.')) {
+    return { kind: 'unresolved', id: `unresolved:${modPath}`, path: modPath };
+  }
+  return { kind: 'external', id: `external:${specifier}` };
+}
+
+function resolveShellImport(fromFile, specifier, fileIndex) {
+  const dir = path.posix.dirname(fromFile);
+  const cleaned = String(specifier).replace(/^\.\//, '');
+  const candidates = [
+    cleaned,
+    path.posix.normalize(path.posix.join(dir, cleaned)),
+    path.posix.basename(cleaned),
+  ];
+  // Also try basename match when path was variable-stripped (unique hit)
+  const basenames = [...fileIndex].filter((f) => f.endsWith(`/${path.posix.basename(cleaned)}`) || f === path.posix.basename(cleaned));
+
+  for (const p of candidates) {
+    if (fileIndex.has(p)) return { kind: 'file', id: `file:${p}`, path: p };
+  }
+  if (basenames.length === 1) {
+    const p = basenames[0];
+    return { kind: 'file', id: `file:${p}`, path: p };
+  }
+  return { kind: 'unresolved', id: `unresolved:${cleaned}`, path: cleaned };
+}
+
+function resolveJsImport(fromFile, specifier, fileIndex) {
+  if (!specifier.startsWith('.') && !specifier.startsWith('/')) {
+    return { kind: 'external', id: `external:${specifier}` };
+  }
+  const dir = path.posix.dirname(fromFile);
+  let candidate = path.posix.normalize(path.posix.join(dir, specifier));
+  if (candidate.startsWith('./')) candidate = candidate.slice(2);
+
+  const tryPaths = [
+    candidate,
+    `${candidate}.ts`,
+    `${candidate}.tsx`,
+    `${candidate}.js`,
+    `${candidate}.jsx`,
+    `${candidate}.mjs`,
+    `${candidate}.cjs`,
+    `${candidate}/index.ts`,
+    `${candidate}/index.tsx`,
+    `${candidate}/index.js`,
+    `${candidate}/index.mjs`,
+  ];
+
+  for (const p of tryPaths) {
+    if (fileIndex.has(p)) return { kind: 'file', id: `file:${p}`, path: p };
+  }
+  return { kind: 'unresolved', id: `unresolved:${candidate}`, path: candidate };
+}
+
+/**
+ * Resolve an import/require/source specifier against an importing file.
+ */
+export function resolveImport(fromFile, specifier, fileIndex) {
+  const lang = langOf(fromFile);
+  if (lang === 'python') return resolvePythonImport(fromFile, specifier, fileIndex);
+  if (lang === 'shell') return resolveShellImport(fromFile, specifier, fileIndex);
+  if (lang === 'json') {
+    // package.json main/bin paths are usually relative to package root
+    if (fileIndex.has(specifier)) return { kind: 'file', id: `file:${specifier}`, path: specifier };
+    const tryPaths = [
+      specifier,
+      specifier.replace(/^\.\//, ''),
+      `${specifier}.js`,
+      `${specifier}.mjs`,
+      `${specifier}.ts`,
+    ];
+    for (const p of tryPaths) {
+      if (fileIndex.has(p)) return { kind: 'file', id: `file:${p}`, path: p };
+    }
+    return { kind: 'unresolved', id: `unresolved:${specifier}`, path: specifier };
+  }
+  return resolveJsImport(fromFile, specifier, fileIndex);
+}
+
+function moduleIdFor(filePath) {
+  const parts = filePath.split('/');
+  if (parts.length <= 1) return 'module:.';
+  // Drop filename; use first dir, or first two dirs when nested (e.g. app/api)
+  const dirs = parts.slice(0, -1);
+  if (dirs.length === 1) return `module:${dirs[0]}`;
+  return `module:${dirs.slice(0, 2).join('/')}`;
+}
+
+/**
+ * Build a full graph for root. Returns { graph, stats, extractorMode }.
+ */
+export function buildGraph(root, { fileFilter = null } = {}) {
+  const extractor = createExtractor(root);
+  let files = walkSourceFiles(root);
+  if (fileFilter) {
+    const allow = new Set(fileFilter);
+    files = files.filter((f) => allow.has(rel(root, f)));
+  }
+
+  const fileIndex = new Set(files.map((f) => rel(root, f)));
+  const nodes = new Map();
+  const edges = [];
+  const symbolByName = new Map(); // name -> [symbol ids]
+  const hashes = {};
+
+  function addNode(node) {
+    if (!nodes.has(node.id)) nodes.set(node.id, node);
+  }
+
+  for (const absPath of files) {
+    const filePath = rel(root, absPath);
+    hashes[filePath] = sha256File(absPath);
+    const source = fs.readFileSync(absPath, 'utf8');
+    const extracted = extractor.extract(filePath, source);
+
+    addNode({
+      id: `file:${filePath}`,
+      type: 'file',
+      label: path.posix.basename(filePath),
+      source_file: filePath,
+      source_location: 'L1',
+    });
+
+    const mod = moduleIdFor(filePath);
+    addNode({
+      id: mod,
+      type: 'module',
+      label: mod.replace(/^module:/, ''),
+      source_file: filePath,
+      source_location: 'L1',
+    });
+    edges.push({
+      source: `file:${filePath}`,
+      target: mod,
+      relation: 'belongs_to',
+      confidence: 'EXTRACTED',
+      source_file: filePath,
+      source_location: 'L1',
+      weight: 1,
+    });
+
+    for (const exp of extracted.exports) {
+      const sid = `symbol:${filePath}#${exp.name}`;
+      addNode({
+        id: sid,
+        type: 'symbol',
+        label: exp.name,
+        source_file: filePath,
+        source_location: `L${exp.line}`,
+      });
+      edges.push({
+        source: `file:${filePath}`,
+        target: sid,
+        relation: 'exports',
+        confidence: exp.confidence,
+        source_file: filePath,
+        source_location: `L${exp.line}`,
+        weight: 1,
+      });
+      if (!symbolByName.has(exp.name)) symbolByName.set(exp.name, []);
+      symbolByName.get(exp.name).push(sid);
+    }
+
+    for (const imp of extracted.imports) {
+      const resolved = resolveImport(filePath, imp.from, fileIndex);
+      if (resolved.kind === 'external') {
+        addNode({
+          id: resolved.id,
+          type: 'external',
+          label: imp.from,
+          source_file: filePath,
+          source_location: `L${imp.line}`,
+        });
+      } else if (resolved.kind === 'unresolved') {
+        addNode({
+          id: resolved.id,
+          type: 'unresolved',
+          label: resolved.path,
+          source_file: filePath,
+          source_location: `L${imp.line}`,
+        });
+      }
+      edges.push({
+        source: `file:${filePath}`,
+        target: resolved.id,
+        relation: 'imports',
+        confidence: imp.confidence,
+        source_file: filePath,
+        source_location: `L${imp.line}`,
+        weight: 1,
+        imported: imp.imported,
+        local: imp.local,
+      });
+
+      if (resolved.kind === 'file' && imp.imported && imp.imported !== '*' && imp.imported !== 'default') {
+        const targetSym = `symbol:${resolved.path}#${imp.imported}`;
+        edges.push({
+          source: `file:${filePath}`,
+          target: targetSym,
+          relation: 'references',
+          confidence: 'INFERRED',
+          source_file: filePath,
+          source_location: `L${imp.line}`,
+          weight: 1,
+        });
+      }
+    }
+
+    for (const ref of extracted.references) {
+      const resolved = resolveImport(filePath, ref.from, fileIndex);
+      if (resolved.kind !== 'file') continue;
+      edges.push({
+        source: `file:${filePath}`,
+        target: resolved.id,
+        relation: 'references',
+        confidence: ref.confidence,
+        source_file: filePath,
+        source_location: `L${ref.line}`,
+        weight: 0.5,
+        name: ref.name,
+      });
+    }
+  }
+
+  const knownSymbols = new Set(symbolByName.keys());
+  const openspec = extractOpenspec(root, knownSymbols, fileIndex);
+  for (const n of openspec.nodes) addNode(n);
+  for (const e of openspec.edges) {
+    if (e.target.startsWith('symbol-name:')) {
+      const name = e.target.slice('symbol-name:'.length);
+      const ids = symbolByName.get(name) || [];
+      for (const sid of ids) {
+        edges.push({ ...e, target: sid });
+        // ensure symbol node exists (it should)
+      }
+    } else {
+      edges.push(e);
+    }
+  }
+
+  const memory = extractMemory(root, fileIndex);
+  for (const n of memory.nodes) addNode(n);
+  for (const e of memory.edges) {
+    if (nodes.has(e.source) && nodes.has(e.target)) edges.push(e);
+  }
+
+  // Ensure external/unresolved targets exist as nodes; drop dangling symbol refs
+  for (const e of edges) {
+    if ((e.target.startsWith('external:') || e.target.startsWith('unresolved:')) && !nodes.has(e.target)) {
+      addNode({
+        id: e.target,
+        type: e.target.startsWith('external:') ? 'external' : 'unresolved',
+        label: e.target.replace(/^(external|unresolved):/, ''),
+        source_file: e.source_file,
+        source_location: e.source_location,
+      });
+    }
+  }
+  const cleanEdges = edges.filter((e) => {
+    if (!nodes.has(e.source)) return false;
+    if (!nodes.has(e.target)) return false;
+    return true;
+  });
+
+  const graph = {
+    directed: true,
+    multigraph: true,
+    graph: {
+      built_at: new Date().toISOString(),
+      root: root,
+      extractor: extractor.mode,
+      schema_version: 1,
+    },
+    nodes: [...nodes.values()],
+    links: cleanEdges.map((e) => ({
+      source: e.source,
+      target: e.target,
+      relation: e.relation,
+      confidence: e.confidence,
+      source_file: e.source_file,
+      source_location: e.source_location,
+      weight: e.weight ?? 1,
+      ...(e.imported != null ? { imported: e.imported } : {}),
+      ...(e.local != null ? { local: e.local } : {}),
+      ...(e.name != null ? { name: e.name } : {}),
+    })),
+  };
+
+  const stats = {
+    files: fileIndex.size,
+    nodes: graph.nodes.length,
+    edges: graph.links.length,
+    symbols: graph.nodes.filter((n) => n.type === 'symbol').length,
+    modules: graph.nodes.filter((n) => n.type === 'module').length,
+    extractor: extractor.mode,
+  };
+
+  return { graph, stats, hashes, extractorMode: extractor.mode };
+}
+
+export function writeGraphArtifacts(root, { graph, stats, hashes }) {
+  ensureGraphDir(root);
+  writeJson(graphPath(root), graph);
+  writeJson(manifestPath(root), {
+    built_at: graph.graph.built_at,
+    files: hashes,
+    stats,
+  });
+  const report = writeReport(root, graph, stats);
+  fs.writeFileSync(reportPath(root), report);
+  return { graphPath: graphPath(root), reportPath: reportPath(root), stats };
+}
+
+export function loadGraph(root) {
+  const g = loadJson(graphPath(root), null);
+  if (!g) {
+    throw new Error(`No graph found at ${graphPath(root)}. Run: interlock-graph build`);
+  }
+  return g;
+}