genesis-compiler 1.10.5 → 1.10.7

package/src/index/code-indexers/npm-packages.js

@@ -9,7 +9,7 @@ const packageName = value => typeof value === 'string' && /^(?:@[a-zA-Z0-9._-]+\
 
 // npm owns manifest and export semantics. Read direct declarations and their
 // public entry files only; never walk dependency trees or enumerate internals.
-export async function inspectNpmPackages({ projectRoot, discovery, files, indexedFiles = [], signal }) {
+export async function inspectNpmPackages({ projectRoot, discovery, files, indexedFiles = [], cacheRoot = null, signal }) {
   const known = new Map(files.map(file => [file.path, file]));
   const packages = []; const inputs = []; const diagnostics = [];
   const manifests = new Map(); const workspaceScopes = []; const byRoot = new Map();
@@ -145,7 +145,8 @@ export async function inspectNpmPackages({ projectRoot, discovery, files, indexe
   }
   const projectIndexes = new Map(indexedFiles.map(file => [file.path, file]));
   const candidates = [...new Map(entries.filter(entry => projectIndexes.get(entry.file.path)?.hash !== entry.file.hash).map(entry => [entry.file.path, { path: entry.file.path, hash: entry.file.hash, mode: 0o100644, publicEntry: true }])).values()];
-  const result = candidates.length ? await extractCodeIndexes({ indexers: ['javascript'], files: candidates, projectRoot, signal }) : { contributions: [], diagnostics: [] };
+  // Entry files share the project's extraction cache, so unchanged packages are not parsed again.
+  const result = candidates.length ? await extractCodeIndexes({ indexers: ['javascript'], files: candidates, projectRoot, signal, cacheRoot }) : { contributions: [], diagnostics: [], cacheLocations: [] };
   diagnostics.push(...result.diagnostics);
   const indexed = new Map([...projectIndexes, ...result.contributions.flatMap(contribution => contribution.files.map(file => [file.path, file]))]);
   function publicSurface(filePath, visited = new Set()) {
@@ -184,7 +185,7 @@ export async function inspectNpmPackages({ projectRoot, discovery, files, indexe
     point.hash = file.hash; point.coverage = 'partial'; record.symbolCoverage = 'partial';
     point.symbols = publicSurface(file.path);
   }
-  return { packages, inputs: [...new Map(inputs.map(input => [input.path, input])).values()], diagnostics };
+  return { packages, inputs: [...new Map(inputs.map(input => [input.path, input])).values()], diagnostics, cacheLocations: result.cacheLocations };
 }
 
 // Metadata and entry files live outside Git's inventory. Recheck those exact

package/src/index/code-indexers/worker.js

@@ -1,4 +1,4 @@
-import { parseAsync, registerDynamicLanguage } from '@ast-grep/napi';
+import { parse, registerDynamicLanguage } from '@ast-grep/napi';
 import { astGrepCodeIndexers } from './ast-grep.js';
 
 try {
@@ -7,9 +7,12 @@ try {
   const input = JSON.parse(Buffer.concat(chunks).toString('utf8'));
   if (Object.keys(input.registrations).length) registerDynamicLanguage(input.registrations);
   const contributions = [];
+  const callSites = new Set(input.callSites || []);
   for (const id of input.indexers) {
     const result = await astGrepCodeIndexers.get(id).extract({
-      files: input.files, projectRoot: input.projectRoot, parseAsync, parserErrors: input.errors,
+      // Nothing else runs here while a file parses, so parsing stays on this
+      // thread instead of a round trip through the thread pool.
+      files: input.files, projectRoot: input.projectRoot, parse, parserErrors: input.errors, callSites,
     });
     contributions.push({ ...result, extractor: id });
   }

package/src/index/entry-points.js

@@ -0,0 +1,269 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+import { ENTRY_POINT_KINDS } from './architecture-observations.js';
+import { sha256 } from './utils.js';
+
+export const MAX_ENTRY_POINTS = 20_000;
+export const MAX_TEST_ENTRY_POINTS = 5_000;
+const MAX_MANIFEST_BYTES = 1024 * 1024;
+const TEST_FILE = [
+  /(?:^|\/)[^/]+\.(?:test|spec)\.[^/]+$/u, /(?:^|\/)test_[^/]+\.py$/u, /_test\.py$/u, /_test\.go$/u, /(?:^|\/)src\/test\//u,
+  /Test\.java$/u, /Test\.kt$/u, /Tests\.cs$/u, /(?:^|\/)spec\/(?:.+\/)?[^/]+_spec\.rb$/u, /(?:^|\/)tests\/(?:.+\/)?[^/]+\.rs$/u, /(?:^|\/)tests\/(?:.+\/)?[^/]+Test\.php$/u,
+];
+const KIND_ORDER = new Map(ENTRY_POINT_KINDS.map((kind, index) => [kind, index]));
+const isTestFile = filePath => TEST_FILE.some(pattern => pattern.test(filePath));
+
+const lineRange = (source, needle) => {
+  const lines = source.split('\n');
+  const index = needle ? lines.findIndex(line => line.includes(needle)) : -1;
+  const line = index < 0 ? 1 : index + 1;
+  const length = (lines[line - 1] || '').replace(/\r$/u, '').length;
+  return length ? { start: { line, column: 1 }, end: { line, column: length + 1 } } : { start: { line, column: 1 }, end: { line: line + 1, column: 1 } };
+};
+
+// Minimal TOML: `[section]` / `[[section]]` headers with `key = "value"` lines.
+function tomlSections(source) {
+  const sections = [];
+  let current = { name: '', values: {} };
+  sections.push(current);
+  for (const raw of source.split('\n')) {
+    const line = raw.replace(/#.*$/u, '').trim();
+    const header = line.match(/^\[\[?\s*([^\]]+?)\s*\]\]?$/u);
+    if (header) { current = { name: header[1], values: {} }; sections.push(current); continue; }
+    const pair = line.match(/^"?([^"=\s]+)"?\s*=\s*["']([^"']*)["']/u);
+    if (pair) current.values[pair[1]] = pair[2];
+  }
+  return sections;
+}
+
+/**
+ * Discovers language-level entry points from manifests, conventions, Program
+ * operations and adapter observations. Function identities are attached only
+ * when a call web exists (`attachEntryFunctions`).
+ */
+export async function discoverEntryPoints({ projectRoot, architecture, functions = [], program = null }) {
+  const files = new Map(architecture.files.map(file => [file.path, file]));
+  const sources = new Map();
+  async function read(filePath) {
+    if (sources.has(filePath)) return sources.get(filePath);
+    let source = null;
+    const file = files.get(filePath);
+    if (file) {
+      try {
+        const bytes = await readFile(path.join(projectRoot, filePath));
+        if (bytes.length <= MAX_MANIFEST_BYTES && sha256(bytes) === file.hash) source = bytes.toString('utf8');
+      } catch (error) { if (!['ENOENT', 'ENOTDIR', 'EISDIR'].includes(error.code)) throw error; }
+    }
+    sources.set(filePath, source);
+    return source;
+  }
+  const entries = new Map();
+  const evidence = (filePath, range) => [{ path: filePath, hash: files.get(filePath).hash, ...range }];
+  function add(record, target = null) {
+    if (!files.has(record.file) || entries.has(record.id)) return;
+    entries.set(record.id, { resolution: 'resolved', functionIds: [], pageId: null, componentId: null, operationKey: null, actionId: null, ...record, target });
+  }
+  const join = (directory, value) => path.posix.normalize(path.posix.join(directory, String(value).replace(/^\.\//u, '')));
+  const byFile = new Map();
+  for (const entry of functions) {
+    if (!byFile.has(entry.path)) byFile.set(entry.path, []);
+    byFile.get(entry.path).push(entry);
+  }
+  const paths = [...files.keys()];
+  const manifests = name => paths.filter(filePath => path.posix.basename(filePath) === name);
+  let libraryExports = 0;
+
+  for (const manifest of manifests('package.json')) {
+    const source = await read(manifest);
+    let document;
+    try { document = JSON.parse(source); } catch { continue; }
+    const bin = typeof document?.bin === 'string' ? { [String(document.name || '').split('/').at(-1) || path.posix.basename(document.bin)]: document.bin } : document?.bin;
+    for (const [name, value] of Object.entries(bin && typeof bin === 'object' ? bin : {})) {
+      if (typeof value !== 'string') continue;
+      const file = join(path.posix.dirname(manifest), value);
+      add({ id: `entry:cli-command:${manifest}#bin/${name}`, kind: 'cli-command', title: name, file, evidence: evidence(manifest, lineRange(source, `"${name}"`)) }, { module: file });
+    }
+  }
+  for (const manifest of manifests('pyproject.toml')) {
+    const source = await read(manifest);
+    if (source === null) continue;
+    const directory = path.posix.dirname(manifest);
+    const moduleFile = module => ['', 'src/'].flatMap(prefix => [`${prefix}${module.replaceAll('.', '/')}.py`, `${prefix}${module.replaceAll('.', '/')}/__init__.py`])
+      .map(candidate => join(directory, candidate)).find(candidate => files.has(candidate));
+    for (const section of tomlSections(source)) {
+      if (section.name === 'project.scripts') {
+        for (const [name, value] of Object.entries(section.values)) {
+          const [module, attribute] = value.split(':');
+          const file = moduleFile(module.trim());
+          if (file) add({ id: `entry:cli-command:${manifest}#scripts/${name}`, kind: 'cli-command', title: name, file, evidence: evidence(manifest, lineRange(source, name)) }, { name: attribute?.trim().split('.')[0], file });
+        }
+      }
+      if (section.name === 'project' && section.values.name) {
+        const file = moduleFile(section.values.name.toLowerCase().replace(/[-.]+/gu, '_'));
+        if (file?.endsWith('__init__.py')) {
+          for (const entry of (byFile.get(file) || []).filter(entry => entry.visibility === 'public' && !entry.container)) {
+            libraryExports++;
+            add({ id: `entry:library-export:${file}#${entry.name}`, kind: 'library-export', title: entry.name, file, evidence: evidence(manifest, lineRange(source, 'name')) }, { name: entry.name, file });
+          }
+        }
+      }
+    }
+  }
+  for (const manifest of manifests('Cargo.toml')) {
+    const source = await read(manifest);
+    if (source === null) continue;
+    const directory = path.posix.dirname(manifest);
+    const sections = tomlSections(source);
+    for (const section of sections.filter(entry => entry.name === 'bin' && entry.values.name)) {
+      const file = join(directory, section.values.path || `src/bin/${section.values.name}.rs`);
+      add({ id: `entry:cli-command:${manifest}#bin/${section.values.name}`, kind: 'cli-command', title: section.values.name, file, evidence: evidence(manifest, lineRange(source, section.values.name)) }, { name: 'main', file });
+    }
+    const main = join(directory, 'src/main.rs');
+    const crate = sections.find(entry => entry.name === 'package')?.values.name || path.posix.basename(directory === '.' ? projectRoot : directory);
+    add({ id: `entry:cli-command:${manifest}#main`, kind: 'cli-command', title: crate, file: main, evidence: evidence(manifest, lineRange(source, '[package]')) }, { name: 'main', file: main });
+    const library = join(directory, 'src/lib.rs');
+    for (const entry of (byFile.get(library) || []).filter(entry => entry.visibility === 'public' && !entry.container)) {
+      libraryExports++;
+      add({ id: `entry:library-export:${library}#${entry.name}`, kind: 'library-export', title: entry.name, file: library, evidence: evidence(manifest, lineRange(source, '[package]')) }, { name: entry.name, file: library });
+    }
+  }
+  for (const manifest of paths.filter(filePath => filePath.endsWith('.gemspec'))) {
+    const source = await read(manifest);
+    if (source === null) continue;
+    const list = source.match(/executables\s*=\s*(?:\[([^\]]*)\]|%w[[(]([^\])]*)[\])])/u);
+    const names = list ? (list[1] !== undefined ? [...list[1].matchAll(/['"]([^'"]+)['"]/gu)].map(match => match[1]) : list[2].trim().split(/\s+/u)) : [];
+    const bindir = source.match(/bindir\s*=\s*['"]([^'"]+)['"]/u)?.[1] || 'bin';
+    for (const name of names.filter(Boolean)) {
+      const file = join(path.posix.dirname(manifest), `${bindir}/${name}`);
+      add({ id: `entry:cli-command:${manifest}#executables/${name}`, kind: 'cli-command', title: name, file, evidence: evidence(manifest, lineRange(source, 'executables')) }, { module: file });
+    }
+  }
+  for (const manifest of manifests('composer.json')) {
+    const source = await read(manifest);
+    let document;
+    try { document = JSON.parse(source); } catch { continue; }
+    for (const value of [document?.bin].flat().filter(entry => typeof entry === 'string')) {
+      const file = join(path.posix.dirname(manifest), value);
+      add({ id: `entry:cli-command:${manifest}#bin/${value}`, kind: 'cli-command', title: path.posix.basename(value), file, evidence: evidence(manifest, lineRange(source, value)) }, { module: file });
+    }
+  }
+  // Language entry functions: Go `package main`, Java/Kotlin, C#, C++ `main`.
+  const goPackages = new Map();
+  for (const [filePath, file] of files) {
+    if (file.language !== 'go') continue;
+    const source = await read(filePath);
+    goPackages.set(filePath, source?.match(/^\s*package\s+(\w+)/mu)?.[1] || null);
+  }
+  for (const entry of functions) {
+    const file = files.get(entry.path);
+    const main = (entry.language === 'go' && entry.name === 'main' && !entry.container && goPackages.get(entry.path) === 'main')
+      || (entry.language === 'java' && entry.name === 'main' && entry.static)
+      || (entry.language === 'kotlin' && entry.name === 'main' && !entry.container)
+      || (entry.language === 'csharp' && entry.name === 'Main')
+      || (entry.language === 'cpp' && entry.name === 'main' && !entry.container);
+    if (main && file) add({ id: `entry:cli-command:${entry.path}#${entry.name}`, kind: 'cli-command', title: entry.container || path.posix.basename(entry.path).replace(/\.[^.]+$/u, ''), file: entry.path, evidence: evidence(entry.path, entry.start && entry.end ? { start: entry.start, end: entry.end } : lineRange('', null)) }, { name: entry.name, file: entry.path });
+    if (entry.language === 'go' && entry.visibility === 'public' && !entry.container && !isTestFile(entry.path) && goPackages.get(entry.path) && goPackages.get(entry.path) !== 'main') {
+      libraryExports++;
+      add({ id: `entry:library-export:${entry.path}#${entry.name}`, kind: 'library-export', title: entry.name, file: entry.path, evidence: evidence(entry.path, entry.start && entry.end ? { start: entry.start, end: entry.end } : lineRange('', null)) }, { name: entry.name, file: entry.path });
+    }
+  }
+  for (const filePath of paths.filter(value => path.posix.basename(value) === 'Program.cs' && !(byFile.get(value) || []).some(entry => entry.name === 'Main'))) {
+    add({ id: `entry:cli-command:${filePath}#top-level`, kind: 'cli-command', title: path.posix.basename(path.posix.dirname(filePath)) || 'Program', file: filePath, evidence: evidence(filePath, lineRange(await read(filePath) || '', null)) }, { module: filePath });
+  }
+  const commandFiles = new Set([...entries.values()].filter(entry => entry.kind === 'cli-command').map(entry => entry.file));
+  for (const [filePath, file] of files) {
+    if (file.mode !== 0o100755 || commandFiles.has(filePath)) continue;
+    const source = await read(filePath);
+    if (!source?.startsWith('#!')) continue;
+    add({ id: `entry:cli-command:${filePath}`, kind: 'cli-command', title: path.posix.basename(filePath), file: filePath, evidence: evidence(filePath, lineRange(source, '#!')) }, { module: filePath });
+    commandFiles.add(filePath);
+  }
+
+  for (const entry of architecture.packages || []) {
+    if (entry.opaque) continue;
+    for (const exported of entry.entryPoints || []) {
+      const specifier = exported.importPath === '.' ? entry.name : `${entry.name}${exported.importPath.slice(1)}`;
+      for (const symbol of exported.symbols || []) {
+        if (symbol.kind !== 'function' || !files.has(symbol.sourceFile)) continue;
+        libraryExports++;
+        add({ id: `entry:library-export:${symbol.sourceFile}#${symbol.sourceName || symbol.name}`, kind: 'library-export', title: `${symbol.name} (${specifier})`, file: symbol.sourceFile, evidence: evidence(symbol.sourceFile, { start: symbol.start, end: symbol.end }) }, { name: symbol.sourceName || symbol.name, file: symbol.sourceFile, within: { start: symbol.start, end: symbol.end } });
+      }
+    }
+  }
+  if (!libraryExports) {
+    // Without a declared package API, public functions outside commands and tests stand in for it.
+    for (const entry of functions.filter(value => value.visibility === 'public' && !value.container && value.role !== 'test' && !isTestFile(value.path) && !commandFiles.has(value.path) && value.start && value.end)) {
+      add({ id: `entry:library-export:${entry.path}#${entry.name}`, kind: 'library-export', title: entry.name, file: entry.path, evidence: evidence(entry.path, { start: entry.start, end: entry.end }) }, { name: entry.name, file: entry.path, within: { start: entry.start, end: entry.end } });
+    }
+  }
+  for (const operation of program?.buildings || []) {
+    const cited = operation.sources.filter(source => files.has(source));
+    if (!cited.length) continue;
+    const file = cited.includes(operation.mainSource) ? operation.mainSource : cited[0];
+    const names = [...new Set([...String(operation.publicContract || '').matchAll(/`(?:[A-Za-z_$][\w$]*[.:]+)*([A-Za-z_$][\w$]*)\s*\(/gu)].map(match => match[1]))];
+    add({ id: `entry:program-operation:${operation.path}`, kind: 'program-operation', title: operation.title, file, evidence: evidence(files.has(operation.path) ? operation.path : file, lineRange('', null)) }, { files: cited, names, module: file });
+  }
+  for (const [filePath, file] of files) {
+    if (file.language !== 'unknown' && isTestFile(filePath)) add({ id: `entry:test:${filePath}`, kind: 'test', title: path.posix.basename(filePath), file: filePath, evidence: evidence(filePath, lineRange('', null)) }, { all: filePath });
+  }
+  const ui = architecture.ui;
+  // A page entry is named by its first registration (for a web page, its URL
+  // path), which says more than a title taken from a file such as `[slug].vue`.
+  const registered = new Map();
+  for (const registration of ui?.registrations || []) if (!registered.has(registration.pageId)) registered.set(registration.pageId, registration.identity);
+  for (const page of ui?.pages || []) add({ id: `entry:page:${page.id}`, kind: 'page', title: registered.get(page.id) || page.title, file: page.file, pageId: page.id, evidence: page.evidence });
+  for (const component of ui?.components || []) if (component.file) add({ id: `entry:component:${component.id}`, kind: 'component', title: component.title, file: component.file, componentId: component.id, evidence: component.evidence });
+  for (const operation of ui?.entryOperations || []) {
+    if (operation.resolution !== 'resolved' || !operation.controllerFile) continue;
+    add({ id: `entry:route:${operation.id}`, kind: 'route', title: [operation.key.method, operation.key.operation].filter(Boolean).join(' '), file: operation.controllerFile, operationKey: operation.key, actionId: operation.actionId ?? null, evidence: operation.evidence }, { ids: operation.controllerFunctionId ? [operation.controllerFunctionId] : [] });
+  }
+  for (const record of ui?.entryPoints || []) {
+    add({ id: record.id.startsWith('entry:') ? record.id : `entry:${record.kind}:${record.id}`, kind: record.kind, title: record.title, file: record.file, pageId: record.pageId ?? null, componentId: record.componentId ?? null, operationKey: record.operationKey ?? null, actionId: record.actionId ?? null, evidence: record.evidence }, { ids: record.functionIds });
+  }
+  return capEntryPoints([...entries.values()]);
+}
+
+/** Applies the 20,000 total / 5,000 test caps in a stable order. */
+export function capEntryPoints(entries, { total = MAX_ENTRY_POINTS, tests = MAX_TEST_ENTRY_POINTS } = {}) {
+  const sorted = [...entries].sort((left, right) => KIND_ORDER.get(left.kind) - KIND_ORDER.get(right.kind) || left.file.localeCompare(right.file) || left.id.localeCompare(right.id));
+  let testCount = 0;
+  const kept = sorted.filter(entry => entry.kind !== 'test' || ++testCount <= tests).slice(0, total);
+  const truncated = kept.length < sorted.length;
+  return {
+    entries: kept,
+    capability: truncated ? 'partial' : 'complete',
+    diagnostics: truncated ? [{ code: 'ENTRY_POINTS_TRUNCATED', message: `Only ${kept.length} of ${sorted.length} entry points are listed (at most ${total}, of which ${tests} tests).` }] : [],
+  };
+}
+
+/** Fills `functionIds` from a call web; without one every list stays empty. */
+export function attachEntryFunctions(entries, callWeb) {
+  const byFile = new Map();
+  if (callWeb) {
+    for (const entry of callWeb.functions) {
+      const filePath = callWeb.files[entry.file].path;
+      if (!byFile.has(filePath)) byFile.set(filePath, []);
+      byFile.get(filePath).push(entry);
+    }
+  }
+  const known = new Set(callWeb ? callWeb.functions.map(entry => entry.id) : []);
+  const inside = (entry, range) => !range || ((entry.start.line > range.start.line || (entry.start.line === range.start.line && entry.start.column >= range.start.column))
+    && (entry.start.line < range.end.line || (entry.start.line === range.end.line && entry.start.column < range.end.column)));
+  const module = filePath => (byFile.get(filePath) || []).filter(entry => entry.kind === 'module').map(entry => entry.id);
+  return entries.map(({ target, ...entry }) => {
+    let ids = [];
+    if (callWeb && target) {
+      if (target.ids) ids = target.ids.filter(id => known.has(id));
+      else if (target.all) ids = (byFile.get(target.all) || []).map(fn => fn.id);
+      else if (target.files) {
+        ids = target.files.flatMap(filePath => (byFile.get(filePath) || []).filter(fn => fn.name && target.names.includes(fn.name)).map(fn => fn.id));
+        if (!ids.length) ids = module(target.module);
+      } else if (target.name) {
+        const candidates = (byFile.get(target.file) || []).filter(fn => fn.name === target.name && inside(fn, target.within));
+        ids = candidates.length ? [candidates[0].id] : module(target.file);
+      } else if (target.module) ids = module(target.module);
+    }
+    return { ...entry, functionIds: [...new Set(ids)] };
+  });
+}

package/src/index/extraction-cache.js

@@ -0,0 +1,81 @@
+import { createHash } from 'node:crypto';
+import { access, readFile, readdir, rm, rmdir, stat } from 'node:fs/promises';
+import path from 'node:path';
+
+import { runGitText } from './process.js';
+import { writeFileAtomic } from './utils.js';
+
+export const EXTRACTION_CACHE_RETENTION_MS = 14 * 24 * 60 * 60 * 1000;
+const STORE_BATCH = 64;
+
+// Git never lists its own directory, so worktrees of one repository share
+// these entries without an ignore line. Each kind of extraction has its own root.
+export async function extractionCacheRoot(projectRoot, kind) {
+  const common = await runGitText(projectRoot, ['rev-parse', '--path-format=absolute', '--git-common-dir']);
+  return path.join(common, 'genesis-cache', kind);
+}
+
+/** One entry per extractor version, language and parsed content. */
+export function extractionCachePath(root, { version, language, variant, hash }) {
+  const key = createHash('sha256').update(`${variant}\n${hash}`).digest('hex');
+  return path.join(root, String(version), language, `${key}.json`);
+}
+
+/** Reads cached extractions; unreadable or missing entries are extracted again. */
+export async function loadExtractions(root, entries) {
+  const values = new Map();
+  const misses = [];
+  for (let index = 0; index < entries.length; index += STORE_BATCH) {
+    await Promise.all(entries.slice(index, index + STORE_BATCH).map(async entry => {
+      try { values.set(entry.path, JSON.parse(await readFile(entry.location, 'utf8'))); }
+      catch (error) {
+        if (!(error instanceof SyntaxError) && !['ENOENT', 'ENOTDIR'].includes(error.code)) throw error;
+        misses.push(entry);
+      }
+    }));
+  }
+  return { values, misses: misses.sort((left, right) => left.path.localeCompare(right.path)) };
+}
+
+/** Entries without a cache file. Only presence is checked; nothing is read. */
+export async function absentExtractions(entries) {
+  const absent = [];
+  for (let index = 0; index < entries.length; index += STORE_BATCH) {
+    await Promise.all(entries.slice(index, index + STORE_BATCH).map(async entry => {
+      try { await access(entry.location); }
+      catch (error) {
+        if (!['ENOENT', 'ENOTDIR'].includes(error.code)) throw error;
+        absent.push(entry);
+      }
+    }));
+  }
+  return absent.sort((left, right) => left.path.localeCompare(right.path));
+}
+
+export async function storeExtractions(entries) {
+  for (let index = 0; index < entries.length; index += STORE_BATCH) {
+    await Promise.all(entries.slice(index, index + STORE_BATCH).map(({ location, value }) => writeFileAtomic(location, JSON.stringify(value))));
+  }
+}
+
+/** Deletes entries this snapshot does not use once they are older than the retention period. */
+export async function pruneExtractions(root, referenced, { now = Date.now(), retentionMs = EXTRACTION_CACHE_RETENTION_MS } = {}) {
+  const keep = new Set(referenced);
+  let removed = 0;
+  async function visit(directory, depth) {
+    let names;
+    try { names = await readdir(directory); }
+    catch (error) { if (['ENOENT', 'ENOTDIR'].includes(error.code)) return; throw error; }
+    for (const name of names) {
+      const location = path.join(directory, name);
+      if (depth < 2) { await visit(location, depth + 1); continue; }
+      if (keep.has(location)) continue;
+      try {
+        if (now - (await stat(location)).mtimeMs > retentionMs) { await rm(location, { force: true }); removed++; }
+      } catch (error) { if (!['ENOENT', 'ENOTDIR'].includes(error.code)) throw error; }
+    }
+    if (depth > 0) await rmdir(directory).catch(() => {});
+  }
+  await visit(root, 0);
+  return { removed };
+}

package/src/index/project-files.js

@@ -15,11 +15,15 @@ async function visiblePaths(projectRoot) {
     .sort();
 }
 
+const STATE_BATCH = 32;
+
 /** Hashable states for ordinary Git-visible files, excluding ignored output. */
 export async function gitVisibleFileStates(projectRoot, { includePath = () => true } = {}) {
-  const states = new Map();
-  for (const file of (await visiblePaths(projectRoot)).filter(includePath)) {
-    states.set(file, await pathState(path.join(projectRoot, file)));
+  const files = (await visiblePaths(projectRoot)).filter(includePath);
+  // Entries keep path order while a bounded batch of files is read at once.
+  const states = new Map(files.map(file => [file, null]));
+  for (let index = 0; index < files.length; index += STATE_BATCH) {
+    await Promise.all(files.slice(index, index + STATE_BATCH).map(async file => states.set(file, await pathState(path.join(projectRoot, file)))));
   }
   return states;
 }

package/src/index/project-inspection.js

@@ -24,6 +24,7 @@ export const GENESIS_BOOTSTRAP_PATHS = Object.freeze([
   'genesis/engineering.md',
   '.genesis/machine-city.json',
   '.genesis/program-city.json',
+  '.genesis/call-web.json',
   '.codex/hooks.json',
   '.claude/settings.json',
   '.opencode/plugins/genesis-project-guidance.js',
@@ -36,11 +37,13 @@ export const GENESIS_BOOTSTRAP_PATHS = Object.freeze([
 ]);
 const bootstrapPaths = new Set(GENESIS_BOOTSTRAP_PATHS);
 // Genesis creates .gitignore only to ignore its Cities; any other content is the project's.
+// Earlier releases wrote fewer derived-artifact lines; any such subset is still Genesis's.
 async function isBootstrapGitignore(projectRoot, paths) {
   if (!paths.includes('.gitignore')) return false;
   try {
-    return await readFile(path.join(projectRoot, '.gitignore'), 'utf8')
-      === `${GENESIS_DERIVED_ARTIFACTS_GITIGNORE_LINES.join('\n')}\n`;
+    const lines = (await readFile(path.join(projectRoot, '.gitignore'), 'utf8')).split('\n');
+    return lines.pop() === '' && lines.length > 0 && new Set(lines).size === lines.length
+      && lines.every(line => GENESIS_DERIVED_ARTIFACTS_GITIGNORE_LINES.includes(line));
   } catch (error) {
     if (['ENOENT', 'ENOTDIR', 'EISDIR'].includes(error?.code)) return false;
     throw error;

package/src/index/template-catalog.js

@@ -64,7 +64,7 @@ async function installedCatalogs(stackPackages, projectRoot) {
 }
 
 /** Catalogues are explicit data sources. A Stack or Skill never implicitly installs a template. */
-export async function listTemplates({ projectRoot = process.cwd(), stackPackages = [], templateSources = [] } = {}) {
+export async function listTemplates({ projectRoot = process.cwd(), stackPackages = [], templateSources = [], scratchRoot } = {}) {
   let recorded = [];
   try { recorded = (await readStack(projectRoot, { stackPackages })).stackPackages; } catch { /* Explicit sources also work before init. */ }
   stackPackages = [...new Set([...recorded, ...stackPackages])];
@@ -78,7 +78,7 @@ export async function listTemplates({ projectRoot = process.cwd(), stackPackages
     namespaces.add(source.namespace);
     let catalog = source.catalog;
     if (!catalog) {
-      const snapshot = await readGitSnapshot({ repository: source.repository, branch: source.branch || 'main' });
+      const snapshot = await readGitSnapshot({ repository: source.repository, branch: source.branch || 'main', scratchRoot });
       const file = snapshot.files.find(({ path: name }) => name === 'genesis.templates.json');
       if (!file) throw templateCatalogError(`${source.namespace} does not contain genesis.templates.json.`);
       try { catalog = JSON.parse(file.contents.toString('utf8')); }

package/src/index/template-project.js

@@ -1,5 +1,4 @@
-import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, rm, rmdir, symlink, writeFile } from 'node:fs/promises';
-import os from 'node:os';
+import { lstat, mkdir, mkdtemp, readFile, readdir, readlink, rm, symlink, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { parseBlueprintSource } from './blueprint.js';
 import { buildProjectIndex } from './code-index.js';
@@ -79,22 +78,23 @@ async function assertSeedable(root, stackPackages) {
 /** Apply one complete source tree without changing Git history or running application commands. */
 export async function applyTemplate({ projectRoot, templateId, stackPackages = [], templateSources = [] } = {}) {
   const root = (await gitContext(projectRoot)).repositoryRoot;
+  // The lock directory also holds the scratch repositories, so a host that
+  // confines Git to the project accepts them and they never outlive the lock.
   const lockPath = path.resolve(root, await runGitText(root, ['rev-parse', '--git-path', 'genesis-template.lock']));
   try { await mkdir(lockPath); }
   catch (error) {
     if (error.code === 'EEXIST') throw new GenesisError('TEMPLATE_PROJECT_BUSY', 'Another template operation holds this project lock.');
     throw error;
   }
-  let stagingRoot;
   try {
     await assertSeedable(root, stackPackages);
-    const { templates } = await listTemplates({ projectRoot: root, stackPackages, templateSources });
+    const { templates } = await listTemplates({ projectRoot: root, stackPackages, templateSources, scratchRoot: lockPath });
     const template = resolveTemplate(templates, String(templateId || ''));
     const preserved = new Map();
     // A seedable project's .gitignore can only be the one init wrote for the Cities.
     for (const name of [...GENESIS_BOOTSTRAP_PATHS, '.gitignore']) preserved.set(name, await existingFile(root, name));
-    const snapshot = await readGitSnapshot(template);
-    stagingRoot = await mkdtemp(path.join(os.tmpdir(), 'genesis-template-project-'));
+    const snapshot = await readGitSnapshot({ ...template, scratchRoot: lockPath });
+    const stagingRoot = await mkdtemp(path.join(lockPath, 'genesis-template-project-'));
     await runGit(stagingRoot, ['init', '--quiet']);
     for (const file of snapshot.files) {
       if (file.path.split('/').includes('node_modules') || file.path.split('/').some((part) => part === '.env' || part.startsWith('.env.') && part !== '.env.example')) {
@@ -168,7 +168,6 @@ export async function applyTemplate({ projectRoot, templateId, stackPackages = [
       guidance: 'The starting application is ready for its declared workspace setup. No application commands or Git commits were run.',
     };
   } finally {
-    if (stagingRoot) await rm(stagingRoot, { recursive: true, force: true });
-    await rmdir(lockPath);
+    await rm(lockPath, { recursive: true, force: true });
   }
 }

package/src/index/template-source.js

@@ -17,13 +17,16 @@ function validateRepository(repository) {
   }
 }
 
-/** Fetch a branch once and read ordinary blobs; never execute or check out remote hooks/filters. */
-export async function readGitSnapshot({ repository, branch }) {
+/**
+ * Fetch a branch once and read ordinary blobs; never execute or check out remote hooks/filters.
+ * The scratch repository is created below scratchRoot (the system temporary directory by default).
+ */
+export async function readGitSnapshot({ repository, branch, scratchRoot = os.tmpdir() }) {
   validateRepository(repository);
   if (typeof branch !== 'string' || !branch || branch.startsWith('-')) {
     throw new GenesisError('TEMPLATE_SOURCE_INVALID', 'A template branch is required.');
   }
-  const temporaryRoot = await mkdtemp(path.join(os.tmpdir(), 'genesis-template-source-'));
+  const temporaryRoot = await mkdtemp(path.join(scratchRoot, 'genesis-template-source-'));
   try {
     await runGit(temporaryRoot, ['init', '--bare', '--quiet']);
     await runGit(temporaryRoot, ['check-ref-format', `refs/heads/${branch}`]);

package/src/index/utils.js

@@ -25,6 +25,40 @@ export function stableJson(value, { space = 2 } = {}) {
   return `${JSON.stringify(stable(value), null, space)}\n`;
 }
 
+const omittedJson = value => value === undefined || typeof value === 'function' || typeof value === 'symbol';
+
+/**
+ * The text of stableJson(value, { space: 0 }) as consecutive parts. Members of
+ * the top two object levels and elements of the top three array levels are
+ * separate parts, so a large document is never copied or held whole.
+ */
+export function* stableJsonParts(value, depth = 0) {
+  if (Array.isArray(value) && depth < 3) {
+    yield '[';
+    for (let index = 0; index < value.length; index++) {
+      if (index) yield ',';
+      if (omittedJson(value[index])) yield 'null';
+      else yield* stableJsonParts(value[index], depth + 1);
+    }
+    yield ']';
+  } else if (value && typeof value === 'object' && !Array.isArray(value) && depth < 2) {
+    // The member order of the copy stable() builds, including integer-like keys.
+    const sorted = Object.fromEntries(Object.keys(value).sort().map(key => [key, value[key]]));
+    let separator = '';
+    yield '{';
+    for (const [key, item] of Object.entries(sorted)) {
+      if (omittedJson(item)) continue;
+      yield `${separator}${JSON.stringify(key)}:`;
+      separator = ',';
+      yield* stableJsonParts(item, depth + 1);
+    }
+    yield '}';
+  } else {
+    yield JSON.stringify(stable(value));
+  }
+  if (depth === 0) yield '\n';
+}
+
 export function uniqueSorted(values = []) {
   return [...new Set(values)].sort();
 }

package/src/index.js

@@ -202,9 +202,10 @@ export function indexCodebase({
   stackPackages = [],
   write = true,
   observations = null,
+  callWeb = false,
   signal,
 } = {}) {
-  return buildProjectIndex({ projectRoot, queries, stackPackages, write, observations, signal });
+  return buildProjectIndex({ projectRoot, queries, stackPackages, write, observations, callWeb, signal });
 }
 
 export function inspectDerivedArtifacts() {
@@ -246,6 +247,7 @@ export { inspectSubsystems } from './index/subsystems.js';
 export { inspectOpenCodePlugin, installOpenCodePlugin as syncOpenCodePlugin } from './index/opencode-plugin.js';
 
 export { ARCHITECTURE_OBSERVATIONS_SCHEMA, validateArchitectureObservations, connectArchitectureObservations, mergeArchitectureObservations } from './index/architecture-observations.js';
+export { CALL_WEB_SCHEMA, validateCallWeb, readCallWeb } from './index/call-web.js';
 
 export { validateArchitectureFacts } from './index/architecture.js';
 export { CITY_SCHEMA_VERSION, MACHINE_CITY_SCHEMA, PROGRAM_CITY_SCHEMA, validateCityDocument } from './index/city-documents.js';