genesis-compiler 1.10.6 → 1.10.8

package/src/index/code-index.js

@@ -1,7 +1,9 @@
-import { readFile, lstat } from 'node:fs/promises';
+import { createHash, randomUUID } from 'node:crypto';
+import { createReadStream } from 'node:fs';
+import { lstat, mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 
-import { extractCodeIndexes } from './code-indexers/ast-grep.js';
+import { extractCodeIndexes, extractionCandidates, extractorIdentity } from './code-indexers/ast-grep.js';
 import { inspectNpmPackages, recheckNpmInputs } from './code-indexers/npm-packages.js';
 import { asDiagnostic, GenesisError } from './errors.js';
 import { buildCityPresentation } from './city-presentation.js';
@@ -11,13 +13,19 @@ import { isProjectContentPath } from './paths.js';
 import { inspectProgram } from './program.js';
 import { inspectSubsystems } from './subsystems.js';
 import { buildArchitectureFacts } from './architecture.js';
+import { CALL_WEB_PATH, buildCallWeb } from './call-web.js';
+import { absentExtractions, extractionCachePath, extractionCacheRoot, loadExtractions, pruneExtractions, storeExtractions } from './extraction-cache.js';
+import { attachEntryFunctions, discoverEntryPoints } from './entry-points.js';
+import { runGit } from './process.js';
 import { CITY_SCHEMA_VERSION, MACHINE_CITY_SCHEMA, PROGRAM_CITY_SCHEMA } from './city-documents.js';
 import { gitVisibleFileStates } from './project-files.js';
 import { readStack } from './stack.js';
-import { sha256, stableJson, writeFileAtomic } from './utils.js';
+import { sha256, stableJson, stableJsonParts } from './utils.js';
 
 export const MACHINE_CITY_PATH = '.genesis/machine-city.json';
 export const PROGRAM_CITY_PATH = '.genesis/program-city.json';
+// Small write batches are reclaimed young instead of piling up as large objects.
+const WRITE_BATCH_CHARACTERS = 64 * 1024;
 
 export const GENESIS_DERIVED_ARTIFACTS = Object.freeze([
   Object.freeze({
@@ -30,6 +38,11 @@ export const GENESIS_DERIVED_ARTIFACTS = Object.freeze([
     path: PROGRAM_CITY_PATH,
     recreator: 'index-codebase',
   }),
+  Object.freeze({
+    id: 'call-web',
+    path: CALL_WEB_PATH,
+    recreator: 'index-codebase',
+  }),
 ]);
 
 // Both Cities are rebuilt from the working tree, so projects ignore them.
@@ -234,16 +247,93 @@ async function programCity(projectRoot) {
   }
 }
 
+// Only these facts carry forward, and returning releases the rest of the
+// previous city at once.
+async function previousArchitecture(root) {
+  try {
+    const { callWeb, capabilities, entryPoints, sourceHash, stackHash, ui } = JSON.parse(await readFile(path.join(root, MACHINE_CITY_PATH), 'utf8')).architecture || {};
+    return { callWeb, capabilities, entryPoints, sourceHash, stackHash, ui };
+  } catch (error) {
+    if (!(error instanceof SyntaxError) && !['ENOENT', 'ENOTDIR'].includes(error.code)) throw error;
+    return undefined;
+  }
+}
+
+async function fileDigest(location) {
+  const hash = createHash('sha256');
+  for await (const chunk of createReadStream(location)) hash.update(chunk);
+  return hash.digest('hex');
+}
+
+// A document is written beside its destination and compared there, so
+// neither it nor the previous version is ever held whole in memory.
 async function writeIfChanged(projectRoot, relativePath, value) {
   const location = path.join(projectRoot, relativePath);
-  const source = stableJson(value, { space: 0 });
+  const temporary = `${location}.${randomUUID()}.tmp`;
+  const written = createHash('sha256');
+  function* batches() {
+    let pending = '';
+    for (const part of typeof value === 'string' ? [value] : stableJsonParts(value)) {
+      pending += part;
+      if (pending.length < WRITE_BATCH_CHARACTERS) continue;
+      written.update(pending);
+      yield pending;
+      pending = '';
+    }
+    written.update(pending);
+    yield pending;
+  }
+  await mkdir(path.dirname(location), { recursive: true });
   try {
-    if (await readFile(location, 'utf8') === source) return false;
+    await writeFile(temporary, batches(), { flag: 'wx' });
+    let existing = null;
+    try { existing = await stat(location); }
+    catch (error) { if (!['ENOENT', 'ENOTDIR'].includes(error?.code)) throw error; }
+    if (existing?.size === (await stat(temporary)).size && await fileDigest(location) === written.digest('hex')) return false;
+    await rename(temporary, location);
+    return true;
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+
+// The call web is never written where Git would offer it for commit.
+async function callWebIgnored(root) {
+  try {
+    await runGit(root, ['check-ignore', '--quiet', CALL_WEB_PATH]);
+    return true;
   } catch (error) {
-    if (!['ENOENT', 'ENOTDIR'].includes(error?.code)) throw error;
+    if (error.details?.status === 1) return false;
+    throw error;
+  }
+}
+
+// Call sites are only checked for presence up front. Their values are read
+// when the call web is built, after the observation reader, so a host's
+// inspector never runs beside them.
+async function prepareCallSites(root, indexers, files) {
+  const cacheRoot = await extractionCacheRoot(root, 'call-sites');
+  const version = await extractorIdentity();
+  const candidates = extractionCandidates(indexers, files)
+    .map(entry => ({ ...entry, location: extractionCachePath(cacheRoot, { version, ...entry }) }));
+  return { cacheRoot, candidates, misses: await absentExtractions(candidates), extracted: 0 };
+}
+
+// An entry that was present when checked but is unreadable now is extracted
+// again; one the extraction could not produce is not retried.
+async function readCallSites(root, indexers, files, callSites, signal) {
+  const { values, misses } = await loadExtractions(callSites.cacheRoot, callSites.candidates);
+  const late = misses.filter(entry => !callSites.misses.includes(entry));
+  if (late.length) {
+    const locations = new Map(late.map(entry => [entry.path, entry.location]));
+    const { contributions } = await extractCodeIndexes({ indexers, files: files.filter(file => locations.has(file.path)), projectRoot: root, signal, callSites: [...locations.keys()] });
+    const extracted = contributions.flatMap(contribution => contribution.files).filter(file => file.callSites);
+    for (const file of extracted) values.set(file.path, file.callSites);
+    await storeExtractions(extracted.map(file => ({ location: locations.get(file.path), value: file.callSites })));
+    callSites.misses.push(...late);
+    callSites.extracted += extracted.length;
   }
-  await writeFileAtomic(location, source);
-  return true;
+  return values;
 }
 
 function functionMatches(entry, queries) {
@@ -265,6 +355,7 @@ export async function buildProjectIndex({
   stackPackages = [],
   write = true,
   observations = null,
+  callWeb = false,
   signal,
 } = {}) {
   signal?.throwIfAborted();
@@ -285,7 +376,22 @@ export async function buildProjectIndex({
     ))
     .map(([filePath, state]) => ({ path: filePath, hash: state.hash, mode: state.mode }));
   const indexers = [...new Set(stack.components.flatMap((component) => component.indexers || []))].sort();
-  const { contributions, diagnostics } = await extractCodeIndexes({ indexers, files, projectRoot: root, signal });
+  const callWebDiagnostics = [];
+  let callSites = null;
+  if (callWeb) {
+    if (await callWebIgnored(root)) callSites = await prepareCallSites(root, indexers, files);
+    else callWebDiagnostics.push({ code: 'CALL_WEB_NOT_IGNORED', path: CALL_WEB_PATH, message: 'Add `/.genesis/call-web.json` to .gitignore' });
+  }
+  const extractionRoot = await extractionCacheRoot(root, 'extractions');
+  const { contributions, diagnostics, cacheLocations } = await extractCodeIndexes({ indexers, files, projectRoot: root, signal, callSites: callSites?.misses.map(entry => entry.path) || [], cacheRoot: extractionRoot });
+  if (callSites) {
+    // Extraction is content-addressed, so it is cached even if this build later fails.
+    const locations = new Map(callSites.misses.map(entry => [entry.path, entry.location]));
+    const extracted = contributions.flatMap(contribution => contribution.files).filter(file => file.callSites && locations.has(file.path));
+    await storeExtractions(extracted.map(file => ({ location: locations.get(file.path), value: file.callSites })));
+    for (const file of extracted) delete file.callSites;
+    callSites.extracted = extracted.length;
+  }
   const machine = machineCity({
     cityRegions: stack.cityRegions,
     components: stack.components.map(({ id }) => id),
@@ -309,14 +415,11 @@ export async function buildProjectIndex({
   for (const discovery of stack.packageDiscovery.discoveries) {
     signal?.throwIfAborted();
     if (discovery.adapter !== 'npm') { architectureDiagnostics.push({ code: 'PACKAGE_ADAPTER_UNSUPPORTED', path: discovery.manifests[0], message: `Package adapter ${discovery.adapter} has no installed inspector; public boundaries are unavailable.` }); continue; }
-    const inspected = await inspectNpmPackages({ projectRoot: root, discovery, files, indexedFiles: contributions.flatMap(contribution => contribution.files), signal });
+    const inspected = await inspectNpmPackages({ projectRoot: root, discovery, files, indexedFiles: contributions.flatMap(contribution => contribution.files), cacheRoot: extractionRoot, signal });
     packages.push(...inspected.packages); packageInputs.push(...inspected.inputs); architectureDiagnostics.push(...inspected.diagnostics);
+    cacheLocations.push(...inspected.cacheLocations);
   }
-  let previous;
-  if (!observations && !observationReader) {
-    try { previous = JSON.parse(await readFile(path.join(root, MACHINE_CITY_PATH), 'utf8')).architecture; }
-    catch (error) { if (!(error instanceof SyntaxError) && !['ENOENT', 'ENOTDIR'].includes(error.code)) throw error; }
-  }
+  const previous = observations || observationReader ? undefined : await previousArchitecture(root);
   machine.architecture = buildArchitectureFacts({ machine, subsystemMap, program, uiDiscovery: stack.uiDiscovery, packageDiscovery: stack.packageDiscovery, packages, stackHash: stack.identityHash, diagnostics: architectureDiagnostics, observations, previous, inventory: files, cityRegions: stack.cityRegions });
   if (observationReader) {
     signal?.throwIfAborted();
@@ -325,6 +428,35 @@ export async function buildProjectIndex({
     if (!observations) throw new GenesisError('ARCHITECTURE_OBSERVATIONS_INVALID', 'An architecture inspector must return its observation document.');
     machine.architecture = buildArchitectureFacts({ machine, subsystemMap, program, uiDiscovery: stack.uiDiscovery, packageDiscovery: stack.packageDiscovery, packages, stackHash: stack.identityHash, diagnostics: architectureDiagnostics, observations, inventory: files, cityRegions: stack.cityRegions });
   }
+  const architecture = machine.architecture;
+  const discovered = await discoverEntryPoints({ projectRoot: root, architecture, functions: machine.functions, program });
+  architecture.diagnostics.push(...discovered.diagnostics, ...callWebDiagnostics);
+  architecture.capabilities.entryPoints = discovered.capability;
+  let callWebDocument = null;
+  let callWebSource = null;
+  architecture.callWeb = null;
+  if (callSites) {
+    const extractions = await readCallSites(root, indexers, files, callSites, signal);
+    const built = buildCallWeb({ architecture, extractions, machineFunctions: machine.functions, observations, sourceIdentity: architecture.sourceHash });
+    callWebDocument = built.document;
+    callWebSource = stableJson(callWebDocument, { space: 0 });
+    architecture.callWeb = { path: CALL_WEB_PATH, identity: sha256(callWebSource), sourceIdentity: architecture.sourceHash, counts: built.counts, coverage: built.coverage };
+    const adapterCoverage = name => observations?.capabilities?.[name];
+    architecture.capabilities.functions = adapterCoverage('functions') === 'partial' ? 'partial' : 'complete';
+    architecture.capabilities.calls = built.truncated.length || adapterCoverage('calls') === 'partial' ? 'partial' : 'complete';
+    architecture.entryPoints = attachEntryFunctions(discovered.entries, callWebDocument);
+  } else if (callWeb) {
+    architecture.capabilities.functions = 'unconfigured';
+    architecture.capabilities.calls = 'unconfigured';
+    architecture.entryPoints = attachEntryFunctions(discovered.entries, null);
+  } else {
+    // An unchanged snapshot keeps its call web, as it keeps its observations.
+    const current = previous?.callWeb && previous.sourceHash === architecture.sourceHash && previous.stackHash === architecture.stackHash ? previous : null;
+    const previousIds = new Map((current?.entryPoints || []).map(entry => [entry.id, entry.functionIds]));
+    architecture.callWeb = current?.callWeb || null;
+    if (current) for (const name of ['functions', 'calls']) if (current.capabilities?.[name]) architecture.capabilities[name] = current.capabilities[name];
+    architecture.entryPoints = attachEntryFunctions(discovered.entries, null).map(entry => ({ ...entry, functionIds: previousIds.get(entry.id) || [] }));
+  }
   // Do not publish a join from different revisions or destroy the previous
   // valid artifact after an inspector/parser read racing a source change.
   const finalStates = await gitVisibleFileStates(root, { includePath });
@@ -338,10 +470,15 @@ export async function buildProjectIndex({
   await recheckNpmInputs(root, packageInputs, signal);
   const changedFiles = [];
   if (write) {
+    // Machine City names the call web's identity, so it is published second.
+    if (callWebSource !== null && await writeIfChanged(root, CALL_WEB_PATH, callWebSource)) changedFiles.push(CALL_WEB_PATH);
     if (await writeIfChanged(root, MACHINE_CITY_PATH, machine)) changedFiles.push(MACHINE_CITY_PATH);
     if (await writeIfChanged(root, PROGRAM_CITY_PATH, program)) changedFiles.push(PROGRAM_CITY_PATH);
+    await pruneExtractions(extractionRoot, cacheLocations);
+    if (callSites) await pruneExtractions(callSites.cacheRoot, callSites.candidates.map(entry => entry.location));
   }
   const allMatches = machine.functions.filter((entry) => functionMatches(entry, queries));
+  // Entry-point and call-web diagnostics are already part of the architecture's.
   const allDiagnostics = [...diagnostics, ...program.diagnostics, ...architectureDiagnostics];
   return {
     status: allDiagnostics.length > 0 ? 'completed-with-warning' : 'ready',
@@ -350,7 +487,7 @@ export async function buildProjectIndex({
       `in ${formatCount(machine.buildings.length, 'file')}`,
       `and ${formatCount(program.buildings.length, 'Program operation')}.`,
     ].join(' '),
-    paths: { machine: MACHINE_CITY_PATH, program: PROGRAM_CITY_PATH },
+    paths: { machine: MACHINE_CITY_PATH, program: PROGRAM_CITY_PATH, ...(callWebDocument ? { callWeb: CALL_WEB_PATH } : {}) },
     changedFiles,
     diagnostics: allDiagnostics,
     functionCount: machine.functions.length,
@@ -360,5 +497,6 @@ export async function buildProjectIndex({
     matches: allMatches.slice(0, 200),
     machine,
     program,
+    ...(callWebDocument ? { callWeb: callWebDocument, callSites: { extracted: callSites.extracted, cached: callSites.candidates.length - callSites.misses.length } } : {}),
   };
 }