@timber-js/app 0.2.0-alpha.21 → 0.2.0-alpha.210

package/src/analyze/crawl-entry.ts

@@ -0,0 +1,68 @@
+/**
+ * crawl-entry — child-process entry point for the cold crawl.
+ *
+ * The crawl's memory footprint is transient but large (~1.3 GB RSS on
+ * plugin-heavy apps — design/47 §6), so consumers spawn this module as a
+ * short-lived child process instead of crawling in-process:
+ *
+ *   node --import tsx/esm crawl-entry.ts --root <dir> [--config <file>]
+ *        [--out <file>] [--timeout <ms>]
+ *
+ * With `--out`, the JSON report is written to that file. Otherwise it is
+ * printed to stdout after a marker line — plugins in the booted app may
+ * write their own stdout noise, so consumers parse everything after the
+ * marker, not the whole stream.
+ */
+
+import { writeFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { crawlProject, CRAWL_RESULT_MARKER } from './crawl.ts';
+
+function readFlag(argv: string[], name: string): string | undefined {
+  const index = argv.indexOf(name);
+  return index !== -1 ? argv[index + 1] : undefined;
+}
+
+async function main(): Promise<void> {
+  const argv = process.argv.slice(2);
+  const rootArg = readFlag(argv, '--root');
+  if (!rootArg) {
+    process.stderr.write('usage: crawl-entry --root <dir> [--config <file>] [--out <file>]\n');
+    process.exitCode = 2;
+    return;
+  }
+
+  const timeoutArg = readFlag(argv, '--timeout');
+  const report = await crawlProject({
+    root: resolve(rootArg),
+    configFile: readFlag(argv, '--config'),
+    timeoutMs: timeoutArg ? Number(timeoutArg) : undefined,
+  });
+
+  const json = JSON.stringify(report);
+  const outFile = readFlag(argv, '--out');
+  if (outFile) {
+    await writeFile(resolve(outFile), json);
+  } else {
+    // Await the flush: stdout to a pipe is async, and process.exit()
+    // below does NOT flush pending pipe writes — a large report would
+    // be truncated right after the marker.
+    await new Promise<void>((resolveWrite, rejectWrite) => {
+      process.stdout.write(`\n${CRAWL_RESULT_MARKER}\n${json}\n`, (error) =>
+        error ? rejectWrite(error) : resolveWrite()
+      );
+    });
+  }
+}
+
+main().then(
+  () => {
+    // The booted app's plugins may hold handles (watchers, sockets) past
+    // server.close(); the crawl is a read-only child, so exit explicitly.
+    process.exit(process.exitCode ?? 0);
+  },
+  (error) => {
+    process.stderr.write(`[timber] crawl failed: ${error?.stack ?? String(error)}\n`);
+    process.exit(1);
+  }
+);

package/src/analyze/crawl.ts

@@ -0,0 +1,500 @@
+/**
+ * crawl — programmatic middleware-mode crawl (the cold path).
+ *
+ * Boots `createServer({ middlewareMode: true })` — no HTTP port, no route
+ * rendering — and BFS-walks `env.transformRequest(url)` → importedModules
+ * edges from `virtual:timber-rsc-entry`, then from every discovered
+ * `'use client'` boundary through the SSR environment. This reuses Vite's
+ * actual resolution (aliases, tsconfig paths, react-server export
+ * conditions, virtual modules), so it cannot drift from what dev or the
+ * build does; hand-rolled resolution was evaluated and rejected
+ * (design/47 §"Verified Prototype Findings").
+ *
+ * Cost model (design/47 §6): 2.5 s on small apps to ~10 s on dep-heavy
+ * plugin stacks, with a transient ~1.3 GB RSS — callers MUST run this in
+ * a short-lived child process (see crawl-entry.ts) and cache the
+ * classification output, not the Vite transforms. The crawl always
+ * covers the full project: one crawl per project state, never per
+ * question — marginal per-file cost is ~zero once booted.
+ *
+ * `TIMBER_ANALYZE=1` is set before the server boots so platform dev
+ * plugins (wrangler bindings — a measured ~6.8 s remote-bindings
+ * handshake on website) skip work the crawl never needs: classification
+ * wants import edges, not live Cloudflare connections.
+ *
+ * Design docs: 47-module-environment-tooling.md §6, §8.
+ */
+
+import { readFile, realpath } from 'node:fs/promises';
+import { join } from 'node:path';
+import type { ViteDevServer } from 'vite';
+import { normalizePath } from 'vite';
+import {
+  classifyModules,
+  detectDirective,
+  type Directive,
+  type ModuleClassification,
+} from './classify.ts';
+import { buildSourceExtensions, scanProjectSourceFiles, isScannableSourceFile } from './scan.ts';
+import { stitchImportChains, type ChainGraph, type ImportChain } from './chains.ts';
+import {
+  collectImportSpecifiers,
+  importSignals,
+  poisoningsFor,
+  type Poisoning,
+} from './poison-scan.ts';
+import { isTimberPluginApi, resolveAppDir } from '../plugin-context.ts';
+
+const RSC_ENTRY_URL = 'virtual:timber-rsc-entry';
+
+/**
+ * Marker preceding the JSON report on crawl-entry's stdout. Lives here
+ * (not in crawl-entry.ts) so consumers can import it without executing
+ * the child entry's main().
+ */
+export const CRAWL_RESULT_MARKER = '__TIMBER_CRAWL_RESULT__';
+
+export interface CrawlOptions {
+  /** Project root (the app directory containing vite.config). */
+  root: string;
+  /** Explicit Vite config file; discovered from root when omitted. */
+  configFile?: string;
+  /** Overall crawl deadline. Default 120s. */
+  timeoutMs?: number;
+}
+
+export interface CrawlError {
+  /** 'scan' covers disk-scan and directive-read failures. */
+  environment: 'rsc' | 'ssr' | 'scan';
+  id: string;
+  message: string;
+}
+
+/**
+ * One project file's full graph answer — the classification plus the
+ * remaining JSON-contract fields (`boundary`, `chains`, `poisonings`)
+ * that need the open graphs / original sources to compute. This is the
+ * per-module unit `timber graph` caches and serves.
+ */
+export interface GraphModuleReport extends ModuleClassification {
+  /** True when the file itself carries `'use client'`. */
+  boundary: boolean;
+  /** Boundary-aware import chains (design/47 §7); [] when in no graph. */
+  chains: ImportChain[];
+  /** Report-only poisoning conflicts (design/47 §3); [] when clean. */
+  poisonings: Poisoning[];
+}
+
+export interface CrawlReport {
+  root: string;
+  /**
+   * True when the crawl deadline expired mid-walk — the report is
+   * truncated, not wrong, and the condition is transient (a slow
+   * machine, a cold dep-optimizer). Consumers must not cache an
+   * incomplete report: with unchanged files it would be served forever,
+   * never retrying after the transient cause cleared. Deterministic
+   * per-module transform failures do NOT set this — they recur on
+   * retry, so caching them is correct.
+   */
+  incomplete: boolean;
+  /** Classification of every project file, sorted by path. */
+  modules: GraphModuleReport[];
+  /**
+   * Transform failures encountered during the BFS. These are classified,
+   * not swallowed — CSS/asset URLs and non-transformable modules land
+   * here (44 on website in the §6 measurement) and the crawl continues.
+   */
+  errors: CrawlError[];
+  stats: {
+    rscGraphModules: number;
+    ssrGraphModules: number;
+    projectFiles: number;
+    durationMs: number;
+  };
+}
+
+/** Marker error for deadline expiry — structurally distinguishable
+ * from a deterministic transform failure (consumers must not cache a
+ * deadline-truncated report; a transform failure recurs on retry and
+ * caches fine). */
+class CrawlDeadlineError extends Error {}
+
+/** Race a promise against the crawl deadline. */
+function withDeadline<T>(promise: Promise<T>, remainingMs: number, label: string): Promise<T> {
+  return new Promise<T>((resolvePromise, rejectPromise) => {
+    const timer = setTimeout(
+      () =>
+        rejectPromise(
+          new CrawlDeadlineError(`crawl deadline exceeded while transforming ${label}`)
+        ),
+      Math.max(remainingMs, 1)
+    );
+    promise.then(
+      (value) => {
+        clearTimeout(timer);
+        resolvePromise(value);
+      },
+      (error) => {
+        clearTimeout(timer);
+        rejectPromise(error);
+      }
+    );
+  });
+}
+
+/** Duck-typed view of the Vite dev environment surface the crawl uses. */
+export interface CrawlEnvironment {
+  transformRequest(url: string): Promise<unknown>;
+  moduleGraph: {
+    getModuleByUrl(url: string): Promise<CrawlModuleNode | undefined>;
+  };
+}
+
+interface CrawlModuleNode {
+  id: string | null;
+  url: string;
+  file: string | null;
+  importedModules: Set<CrawlModuleNode>;
+  transformResult: unknown;
+}
+
+interface EnvCrawlResult {
+  /** Absolute paths of project files reached in this environment. */
+  projectFiles: Set<string>;
+  /** Project files whose own transform failed (membership unconfirmed subtree). */
+  failedFiles: Set<string>;
+  /** Total graph modules visited (project + deps + virtual). */
+  visited: number;
+  /** The deadline expired mid-walk — the result is truncated. */
+  deadlineHit: boolean;
+}
+
+/**
+ * Whether a graph node's file belongs in the report. `root` must be
+ * slash-normalized; module-node paths already are (Vite normalizes
+ * them, even on Windows — comparing against a backslashed root would
+ * reject every physical module there and misreport the whole app as
+ * pending). The rule set is the scan's own (isScannableSourceFile), so
+ * report membership, disk-scan coverage, and the cache fingerprint are
+ * one set — a graph-reachable file the fingerprint never tracked would
+ * let its edits serve a stale cache (codex review, PR #1056).
+ */
+function isProjectFile(file: string | null, root: string, extensions?: RegExp): file is string {
+  if (!file) return false;
+  return isScannableSourceFile(normalizePath(file), root, extensions);
+}
+
+/**
+ * BFS one environment from the given seed URLs, transforming every
+ * discovered module unconditionally. Transforming (rather than reading
+ * live graphs) is what closes the §8 blind spot: modules behind
+ * unexecuted dynamic imports are transformed here like any other edge.
+ *
+ * Exported for tests (deadline behavior is exercised with a fake env).
+ */
+export async function crawlEnvironment(
+  env: CrawlEnvironment,
+  envName: 'rsc' | 'ssr',
+  seeds: string[],
+  root: string,
+  deadlineAt: number,
+  errors: CrawlError[],
+  extensions?: RegExp
+): Promise<EnvCrawlResult> {
+  const queue = [...seeds];
+  const enqueued = new Set(queue);
+  const projectFiles = new Set<string>();
+  const failedFiles = new Set<string>();
+  let visited = 0;
+  let deadlineHit = false;
+
+  // Index-cursor BFS — the queue only grows, so this walks each entry once.
+  for (let cursor = 0; cursor < queue.length; cursor++) {
+    const url = queue[cursor];
+    const remaining = deadlineAt - Date.now();
+    if (remaining <= 0) {
+      errors.push({ environment: envName, id: url, message: 'crawl deadline exceeded' });
+      deadlineHit = true;
+      break;
+    }
+
+    let node: CrawlModuleNode | undefined;
+    try {
+      await withDeadline(env.transformRequest(url), remaining, url);
+      node = await env.moduleGraph.getModuleByUrl(url);
+    } catch (error) {
+      if (error instanceof CrawlDeadlineError) deadlineHit = true;
+      errors.push({
+        environment: envName,
+        id: url,
+        message: error instanceof Error ? error.message : String(error),
+      });
+      // The URL may still have a graph node with a file — it IS a graph
+      // member (reachable), its transform just failed. Record membership
+      // AND the failure so classification reports it as reachable with
+      // subtreeUnknown, not as pending/unreferenced.
+      node = await env.moduleGraph.getModuleByUrl(url).catch(() => undefined);
+      if (node && isProjectFile(node.file, root, extensions)) {
+        projectFiles.add(node.file);
+        failedFiles.add(node.file);
+      }
+      continue;
+    }
+
+    if (!node) continue;
+    visited++;
+    if (isProjectFile(node.file, root, extensions)) projectFiles.add(node.file);
+
+    for (const child of node.importedModules) {
+      const childUrl = child.url ?? child.id;
+      if (!childUrl || enqueued.has(childUrl)) continue;
+      enqueued.add(childUrl);
+      queue.push(childUrl);
+    }
+  }
+
+  return { projectFiles, failedFiles, visited, deadlineHit };
+}
+
+/** Root-relative Vite URL for an absolute project file path.
+ * Both arguments must be slash-normalized (the crawl normalizes at
+ * the boundary, so callers always satisfy this). */
+function fileToUrl(file: string, root: string): string {
+  return file.slice(root.length);
+}
+
+/** What one read of a module's original source yields. */
+interface SourceFacts {
+  directive: Directive;
+  /** Import-specifier exclusivity signals (poison-scan). */
+  signals: Poisoning[];
+}
+
+/**
+ * Read a file's first directive and poison signals from its original
+ * source on disk, in one read. A failure is recorded, not swallowed —
+ * a lost 'use client' here would silently drop the whole client
+ * subtree from the SSR crawl.
+ */
+async function sourceFactsOf(file: string, errors: CrawlError[]): Promise<SourceFacts> {
+  try {
+    const code = await readFile(file, 'utf8');
+    return {
+      directive: detectDirective(code, file),
+      signals: importSignals(collectImportSpecifiers(code, file)),
+    };
+  } catch (error) {
+    errors.push({
+      environment: 'scan',
+      id: file,
+      message: `directive read failed: ${error instanceof Error ? error.message : String(error)}`,
+    });
+    return { directive: null, signals: [] };
+  }
+}
+
+/**
+ * Crawl a project cold and classify every project file.
+ *
+ * Boots a middleware-mode Vite server with the app's own config (all
+ * timber plugins, real resolution), crawls the RSC graph from the entry,
+ * detects directives from original sources, crawls the SSR graph from
+ * every `'use client'` boundary, and intersects the memberships.
+ */
+export async function crawlProject(options: CrawlOptions): Promise<CrawlReport> {
+  const startedAt = Date.now();
+  const deadlineAt = startedAt + (options.timeoutMs ?? 120_000);
+  // Signal platform dev plugins (wrangler bindings) to skip boot work the
+  // crawl never uses. Must be set before createServer runs plugin hooks.
+  process.env.TIMBER_ANALYZE = '1';
+
+  const { createServer } = await import('vite');
+  const server: ViteDevServer = await createServer({
+    root: options.root,
+    configFile: options.configFile,
+    server: { middlewareMode: true, watch: null },
+    logLevel: 'error',
+  });
+
+  try {
+    // The root must live in the same path space as the graph nodes.
+    // By default Vite resolves through symlinks, so nodes carry
+    // realpaths and a symlinked root (pnpm workspaces, ~ links) would
+    // string-mismatch every file and misclassify the whole project as
+    // unreferenced (design/43 open question 3). Under
+    // resolve.preserveSymlinks the graphs keep the SYMLINK paths, and
+    // realpathing would create exactly the mismatch it normally
+    // prevents — honor the resolved setting. Slash-normalize either
+    // way: module-node paths are slash-normalized even on Windows.
+    const root = normalizePath(
+      server.config.resolve.preserveSymlinks
+        ? server.config.root
+        : await realpath(server.config.root)
+    );
+    // Structural assignment, no cast: CrawlEnvironment/CrawlModuleNode
+    // are supertypes of Vite's DevEnvironment/EnvironmentModuleNode for
+    // the members the BFS reads, so an upstream shape change fails here
+    // at compile time instead of inside the walk.
+    const rscEnv: CrawlEnvironment | undefined = server.environments.rsc;
+    const ssrEnv: CrawlEnvironment | undefined = server.environments.ssr;
+    if (!rscEnv?.transformRequest || !ssrEnv?.transformRequest) {
+      throw new Error('[timber] crawl requires the rsc and ssr Vite environments');
+    }
+
+    const errors: CrawlError[] = [];
+
+    // appDir and pageExtensions come from the booted server's own
+    // resolved timber config (the root-sync plugin api — the single
+    // supported way to read it; cf. cli-check.ts). Resolved before
+    // Phase 1 so the scan extension set is available to isProjectFile
+    // and scanProjectSourceFiles — custom pageExtensions must be
+    // admitted throughout, or graph membership disagrees with the
+    // disk scan and fingerprint (TIM-1392).
+    const rootSyncPlugin = server.config.plugins.find((p) => p.name === 'timber-root-sync');
+    let appDir: string;
+    let pageExtensions: readonly string[] | undefined;
+    if (rootSyncPlugin && isTimberPluginApi(rootSyncPlugin.api)) {
+      const pluginCtx = rootSyncPlugin.api.getContext();
+      appDir = pluginCtx.appDir;
+      pageExtensions = pluginCtx.config.pageExtensions;
+    } else {
+      try {
+        appDir = resolveAppDir(root);
+      } catch {
+        // No app directory at all — provisional convention hints just
+        // never fire; graph-membership classification is unaffected.
+        appDir = join(root, 'app');
+      }
+    }
+    // Normalize once at the boundary: appDir comes from node:path
+    // join() (backslashes on Windows) but all downstream comparisons
+    // — classify, chains, isRouteRoot — expect slash-normalized paths.
+    appDir = normalizePath(appDir);
+    const extensions = buildSourceExtensions(pageExtensions);
+
+    // Phase 1 — RSC graph from the entry.
+    const rsc = await crawlEnvironment(
+      rscEnv,
+      'rsc',
+      [RSC_ENTRY_URL],
+      root,
+      deadlineAt,
+      errors,
+      extensions
+    );
+
+    // Phase 2 — directives + poison signals for everything discovered so
+    // far plus files on disk that no graph reached (the provisional/
+    // pending set). The disk walk is the same one the graph cache
+    // fingerprints (scan.ts) so cache validity covers exactly this set.
+    const diskFiles: string[] = [];
+    await scanProjectSourceFiles(
+      root,
+      diskFiles,
+      (failure) => errors.push({ environment: 'scan', id: failure.dir, message: failure.message }),
+      0,
+      extensions
+    );
+    // One path space throughout: graph paths are already slash-
+    // normalized, disk-walk paths are node-joined (backslashed on
+    // Windows) — mixing them in one Set would duplicate every file.
+    const allFiles = new Set([...rsc.projectFiles, ...diskFiles.map(normalizePath)]);
+    const facts = new Map<string, SourceFacts>();
+    for (const file of allFiles) {
+      facts.set(file, await sourceFactsOf(file, errors));
+    }
+
+    // Phase 3 — SSR graph from every RSC-REACHABLE 'use client'
+    // boundary. SSR renders exactly the tree the browser hydrates, so
+    // this yields the client tree (boundaries plus everything they
+    // transitively import). Disk-only boundaries are deliberately NOT
+    // seeded: an unreferenced 'use client' file must keep its
+    // provisional pending classification, not get promoted to a
+    // confirmed boundary (and drag its imports in as client-internal)
+    // by the act of analyzing it.
+    const boundarySeeds = [...rsc.projectFiles]
+      .filter((file) => facts.get(file)?.directive === 'use client')
+      .map((file) => fileToUrl(file, root));
+    const ssr = await crawlEnvironment(
+      ssrEnv,
+      'ssr',
+      boundarySeeds,
+      root,
+      deadlineAt,
+      errors,
+      extensions
+    );
+
+    // SSR-only discoveries (client-internal files) need facts too.
+    for (const file of ssr.projectFiles) {
+      if (!facts.has(file)) {
+        facts.set(file, await sourceFactsOf(file, errors));
+        allFiles.add(file);
+      }
+    }
+
+    const classifications = classifyModules(
+      [...allFiles].map((file) => ({
+        file,
+        inRsc: rsc.projectFiles.has(file),
+        inSsr: ssr.projectFiles.has(file),
+        // The crawl transforms unconditionally, so the untransformed
+        // frontier only arises where the transform itself failed.
+        ssrTransformed: ssr.projectFiles.has(file) ? !ssr.failedFiles.has(file) : undefined,
+        directive: facts.get(file)?.directive ?? null,
+      })),
+      { appDir, pageExtensions }
+    );
+
+    // Phase 4 — chains, from the live graphs of the still-open server.
+    // ChainGraph is a structural supertype of Vite's module graphs
+    // (TIM-1382), so the assignment is cast-free and an upstream shape
+    // change fails at compile time.
+    const rscChainGraph: ChainGraph = server.environments.rsc.moduleGraph;
+    const ssrChainGraph: ChainGraph = server.environments.ssr.moduleGraph;
+    const chainCtx = {
+      rsc: rscChainGraph,
+      ssr: ssrChainGraph,
+      appDir,
+      pageExtensions,
+      // facts keys are already slash-normalized (allFiles is built from
+      // graph membership + diskFiles.map(normalizePath)), so no
+      // per-lookup normalization is needed.
+      getDirective: (file: string) => facts.get(file)?.directive ?? null,
+    };
+
+    const modules: GraphModuleReport[] = classifications.map((classification) => ({
+      ...classification,
+      boundary: facts.get(classification.file)?.directive === 'use client',
+      chains: stitchImportChains(classification.file, chainCtx),
+      poisonings: poisoningsFor(
+        classification.kind,
+        classification.pending,
+        facts.get(classification.file)?.signals ?? []
+      ),
+    }));
+
+    return {
+      root,
+      incomplete: rsc.deadlineHit || ssr.deadlineHit,
+      modules,
+      errors,
+      stats: {
+        rscGraphModules: rsc.visited,
+        ssrGraphModules: ssr.visited,
+        projectFiles: allFiles.size,
+        durationMs: Date.now() - startedAt,
+      },
+    };
+  } finally {
+    // Bound the cleanup: after a deadline expiry the very transforms
+    // that hung are what server.close() waits on — an unbounded await
+    // here would hang the child past its own --timeout and the report
+    // would never be emitted. The child's process.exit reaps whatever
+    // close left behind.
+    await Promise.race([
+      server.close().catch(() => {}),
+      new Promise<void>((resolveTimer) => setTimeout(resolveTimer, 5_000)),
+    ]);
+  }
+}