@timber-js/app 0.2.0-alpha.191 → 0.2.0-alpha.194

package/src/analyze/graph-cache.ts

@@ -0,0 +1,332 @@
+/**
+ * graph-cache — the mtime-keyed output cache for `timber graph`.
+ *
+ * The cold crawl costs seconds (design/47 §6) and its cost is setup +
+ * dependency compilation, not project-file count — so the cacheable
+ * unit is the classification OUTPUT, never Vite's transforms. One
+ * crawl per project state: the first question after a clone pays for
+ * the crawl, every subsequent question is a cache hit until the
+ * project changes.
+ *
+ * Validity is a fingerprint over (path, mtime, size) of every project
+ * source file — the same disk walk the crawl itself uses (scan.ts), so
+ * cache coverage and crawl coverage cannot drift — plus the config
+ * files and the nearest lockfile, since dependency changes change
+ * resolution (export conditions can flip a file's environment without
+ * touching project code).
+ *
+ * Deliberately vite-free: a cache hit answers without importing Vite.
+ *
+ * Design docs: 47-module-environment-tooling.md §"Cold-path policy".
+ */
+
+import { createHash } from 'node:crypto';
+import { createRequire } from 'node:module';
+import { mkdir, readFile, rename, stat, writeFile } from 'node:fs/promises';
+import { dirname, isAbsolute, join } from 'node:path';
+import { buildSourceExtensions, scanProjectSourceFiles } from './scan.ts';
+import type { CrawlReport } from './crawl.ts';
+
+/**
+ * JSON contract version (design/47 §9) — one constant for every
+ * frontend (CLI and dev-server endpoint), so the two cannot drift.
+ * Lives here rather than graph-command.ts because this module is
+ * vite-free and both frontends already depend on the analyze core.
+ */
+export const GRAPH_JSON_VERSION = 1;
+
+/** Bump when the report shape or fingerprint inputs change. */
+export const GRAPH_CACHE_VERSION = 2;
+
+/** Cache location under the app root. */
+const CACHE_RELATIVE_PATH = join('.timber', 'graph-cache.json');
+
+/**
+ * Config/manifest files at the root that participate in the
+ * fingerprint. The `.env` family is included because a Vite config
+ * using `loadEnv(mode, …)` can select aliases, plugins, the app root,
+ * or pageExtensions from it — the disk scan skips dot-prefixed files,
+ * so without these entries an `.env.production` edit would never miss
+ * the cache (codex review, PR #1056). The build resolves under mode
+ * `production`; the development variants are tracked anyway — they are
+ * loadable through `loadEnv` with any mode argument, and an absent
+ * file contributes nothing.
+ */
+const ROOT_MANIFEST_FILES = [
+  'package.json',
+  'tsconfig.json',
+  'timber.config.ts',
+  'timber.config.js',
+  'timber.config.mjs',
+  '.env',
+  '.env.local',
+  '.env.production',
+  '.env.production.local',
+  '.env.development',
+  '.env.development.local',
+];
+
+/** Lockfile names, searched upward from the root (monorepo workspaces). */
+const LOCKFILE_NAMES = [
+  'pnpm-lock.yaml',
+  'package-lock.json',
+  'yarn.lock',
+  'bun.lock',
+  'bun.lockb',
+];
+
+export interface GraphCacheFile {
+  version: number;
+  fingerprint: string;
+  /** The Vite config file the cached crawl booted with. */
+  configFile: string;
+  /**
+   * The resolved pageExtensions from the timber config at crawl time.
+   * Stored so the vite-free hit path can reproduce the same scan
+   * extension set (custom extensions like 'foo' must be admitted on
+   * both the write and the hit path, or the fingerprint disagrees
+   * with what was cached).
+   */
+  pageExtensions?: string[];
+  /**
+   * Files the fingerprint tracks BEYOND the disk scan, discovered at
+   * crawl time: Vite's `configFileDependencies` (a config helper
+   * imported from outside the app root can reshape the whole app) and
+   * every file the report itself classifies (the graph can reach files
+   * the scan skips — `app/.well-known/page.tsx`, a route under a
+   * directory named `dist`). The hit path re-stats exactly this list,
+   * so an edit to any of them misses the cache.
+   */
+  trackedExtras: string[];
+  report: CrawlReport;
+}
+
+/**
+ * Follow a tsconfig/jsconfig `extends` chain, returning every file on
+ * it (the start file included). A base config outside the app —
+ * `"extends": "../tsconfig.base.json"` with a `paths` mapping — can
+ * reshape Vite's resolution without touching anything else the
+ * fingerprint tracks. Parsing is a targeted regex, not JSONC: tsconfig
+ * allows comments, and all we need is the `extends` value. Package
+ * specifiers resolve through Node resolution (an approximation of
+ * TypeScript's — the `.json` append covers the common shapes): a
+ * WORKSPACE package like `@repo/tsconfig/base.json` is a link, so its
+ * edits change neither the lockfile nor any scanned file and must be
+ * fingerprinted directly; registry packages resolve too, harmlessly.
+ * Cycles and unresolvable targets just end the walk (bounded at 16).
+ */
+/**
+ * Strip JSONC comments so a commented-out `"extends"` line cannot
+ * shadow the real one. String-aware: comment markers inside string
+ * literals (a path containing `//`) survive.
+ */
+export function stripJsonComments(raw: string): string {
+  let out = '';
+  let inString = false;
+  for (let i = 0; i < raw.length; i++) {
+    const ch = raw[i];
+    if (inString) {
+      out += ch;
+      if (ch === '\\') {
+        out += raw[i + 1] ?? '';
+        i++;
+      } else if (ch === '"') {
+        inString = false;
+      }
+    } else if (ch === '"') {
+      inString = true;
+      out += ch;
+    } else if (ch === '/' && raw[i + 1] === '/') {
+      while (i < raw.length && raw[i] !== '\n') i++;
+      out += '\n';
+    } else if (ch === '/' && raw[i + 1] === '*') {
+      i += 2;
+      while (i < raw.length && !(raw[i] === '*' && raw[i + 1] === '/')) i++;
+      i++;
+    } else {
+      out += ch;
+    }
+  }
+  return out;
+}
+
+async function collectTsconfigChain(startFile: string): Promise<string[]> {
+  const chain: string[] = [];
+  const queue = [startFile];
+  const seen = new Set(queue);
+  while (queue.length > 0 && chain.length < 16) {
+    const current = queue.shift()!;
+    let raw: string;
+    try {
+      raw = stripJsonComments(await readFile(current, 'utf8'));
+    } catch {
+      continue;
+    }
+    chain.push(current);
+    const match = raw.match(/"extends"\s*:\s*("([^"]+)"|\[([^\]]*)\])/);
+    if (!match) continue;
+    const targets = match[2]
+      ? [match[2]]
+      : (match[3]?.match(/"([^"]+)"/g) ?? []).map((t) => t.slice(1, -1));
+    for (const target of targets) {
+      let next: string | null = null;
+      if (isAbsolute(target)) {
+        // An absolute extends target names the base directly — joining
+        // it under the current config's directory would fabricate a
+        // path that does not exist and drop the real base.
+        next = target;
+        if (!/\.json$/.test(next)) next += '.json';
+      } else if (target.startsWith('.')) {
+        next = join(dirname(current), target);
+        if (!/\.json$/.test(next)) next += '.json';
+      } else {
+        const req = createRequire(current);
+        for (const candidate of [target, `${target}.json`, `${target}/tsconfig.json`]) {
+          try {
+            next = req.resolve(candidate);
+            break;
+          } catch {
+            // try the next shape
+          }
+        }
+      }
+      if (next && !seen.has(next)) {
+        seen.add(next);
+        queue.push(next);
+      }
+    }
+  }
+  return chain;
+}
+
+/** Nearest lockfile at or above `root`, or null. */
+async function findLockfile(root: string): Promise<string | null> {
+  let dir = root;
+  for (;;) {
+    for (const name of LOCKFILE_NAMES) {
+      const candidate = join(dir, name);
+      try {
+        await stat(candidate);
+        return candidate;
+      } catch {
+        // keep looking
+      }
+    }
+    const parent = dirname(dir);
+    if (parent === dir) return null;
+    dir = parent;
+  }
+}
+
+/** `path\0mtime\0size` entries for a file list; absent files contribute nothing. */
+export async function fingerprintEntriesForFiles(files: Iterable<string>): Promise<string[]> {
+  const entries: string[] = [];
+  for (const file of new Set(files)) {
+    try {
+      const stats = await stat(file);
+      entries.push(`${file}\0${stats.mtimeMs}\0${stats.size}`);
+    } catch {
+      // Absent files simply contribute nothing; when one appears
+      // later, its entry appears and the fingerprint changes.
+    }
+  }
+  return entries;
+}
+
+/**
+ * Entries for the scan-derived base set: project source files, the
+ * config, root manifests, and the nearest lockfile. Scan failures are
+ * folded into the entry list rather than thrown: an unreadable
+ * directory changes the answer's completeness, so it must change the
+ * key too.
+ */
+export async function collectBaseFingerprintEntries(
+  root: string,
+  configFile: string,
+  pageExtensions?: readonly string[]
+): Promise<string[]> {
+  const extensions = buildSourceExtensions(pageExtensions);
+  const sourceFiles: string[] = [];
+  const failures: string[] = [];
+  await scanProjectSourceFiles(
+    root,
+    sourceFiles,
+    (failure) => failures.push(`${failure.dir}:${failure.message}`),
+    0,
+    extensions
+  );
+  const tracked = [...sourceFiles, configFile, ...ROOT_MANIFEST_FILES.map((f) => join(root, f))];
+  // Manifest/.env files beside the CONFIG too, when it lives outside
+  // the app root — `loadEnv(mode, process.cwd())` in a monorepo config
+  // reads the .env next to the config, not the one under root.
+  const configDir = dirname(configFile);
+  if (configDir !== root) {
+    tracked.push(...ROOT_MANIFEST_FILES.map((f) => join(configDir, f)));
+  }
+  // ts/jsconfig extends chains: a base config outside the app can carry
+  // the `paths` mappings resolution depends on.
+  for (const name of ['tsconfig.json', 'jsconfig.json']) {
+    tracked.push(...(await collectTsconfigChain(join(root, name))));
+    if (configDir !== root) tracked.push(...(await collectTsconfigChain(join(configDir, name))));
+  }
+  const lockfile = await findLockfile(root);
+  if (lockfile) tracked.push(lockfile);
+  return [...failures, ...(await fingerprintEntriesForFiles(tracked))];
+}
+
+/** Hash an entry list order-independently. */
+export function hashFingerprintEntries(entries: string[]): string {
+  const unique = [...new Set(entries)].sort();
+  return createHash('sha256').update(unique.join('\n')).digest('hex');
+}
+
+/**
+ * Fingerprint the project state `timber graph`'s answers depend on:
+ * the scan-derived base set plus `extraFiles` (the cache's
+ * trackedExtras on the hit path; config dependencies + report files on
+ * the write path).
+ */
+export async function computeProjectFingerprint(
+  root: string,
+  configFile: string,
+  extraFiles: readonly string[] = [],
+  pageExtensions?: readonly string[]
+): Promise<string> {
+  const entries = await collectBaseFingerprintEntries(root, configFile, pageExtensions);
+  entries.push(...(await fingerprintEntriesForFiles(extraFiles)));
+  return hashFingerprintEntries(entries);
+}
+
+/** Read the cache file; anything unreadable or version-mismatched is a miss. */
+export async function readGraphCache(root: string): Promise<GraphCacheFile | null> {
+  try {
+    const raw = await readFile(join(root, CACHE_RELATIVE_PATH), 'utf8');
+    const parsed = JSON.parse(raw) as Partial<GraphCacheFile>;
+    if (
+      parsed.version !== GRAPH_CACHE_VERSION ||
+      typeof parsed.fingerprint !== 'string' ||
+      typeof parsed.configFile !== 'string' ||
+      !Array.isArray(parsed.trackedExtras) ||
+      typeof parsed.report !== 'object' ||
+      parsed.report === null
+    ) {
+      return null;
+    }
+    return parsed as GraphCacheFile;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Write the cache atomically (temp file + rename) so a concurrent
+ * reader — parallel agent invocations are the expected workload —
+ * never sees a torn JSON file.
+ */
+export async function writeGraphCache(root: string, cache: GraphCacheFile): Promise<void> {
+  const target = join(root, CACHE_RELATIVE_PATH);
+  await mkdir(dirname(target), { recursive: true });
+  const temp = `${target}.${process.pid}.tmp`;
+  await writeFile(temp, JSON.stringify(cache));
+  await rename(temp, target);
+}