@timber-js/app 0.2.0-alpha.21 → 0.2.0-alpha.211

package/src/server/static-generator.ts

@@ -0,0 +1,680 @@
+/**
+ * Static site generator — enumerates routes and renders them to
+ * HTML + RSC flight payloads for `output: 'static'` mode.
+ *
+ * Runs INSIDE the built RSC bundle (same pattern as prebuilt-builder.ts)
+ * so it has access to the route manifest, page module loaders, and the
+ * request pipeline handler.
+ *
+ * For each page route:
+ *   - Renders HTML (Accept: text/html) for initial load
+ *   - Renders RSC flight (Accept: text/x-component) for client SPA nav
+ *
+ * For each API route (route.ts):
+ *   - Calls the GET handler and captures the response
+ *
+ * The caller (plugins/static-build.ts) handles file I/O — keeping
+ * node:fs out of the server bundle so it runs on Cloudflare Workers.
+ *
+ * Design docs: design/11-platform.md §"static output mode"
+ */
+
+import type { ManifestRoot, ManifestSegmentNode, ManifestFile } from './route-matcher.ts';
+import { hasDynamicSegments, hasUserSitemap, resolvePathWithParams } from './sitemap-generator.ts';
+import { loadModule } from './safe-load.ts';
+import { readAllBytesWithDeadline } from './stream-utils.ts';
+import { DEFAULT_RENDER_TIMEOUT_MS, resolveRenderTimeoutMs } from './render-timeout.ts';
+import { METADATA_ROUTE_CONVENTIONS, resolveServePathForFile } from './metadata-routes.ts';
+import { RSC_CONTENT_TYPE } from '../shared/rsc-media-type.ts';
+import {
+  NOT_FOUND_ENTRY_PATH,
+  NOT_FOUND_PROBE_PATH,
+  resolveNotFoundPage,
+  type NotFoundProbe,
+} from './static-not-found.ts';
+
+// ─── Types ───────────────────────────────────────────────────────────────
+
+/** A single rendered static page ready for the caller to write to disk. */
+export interface StaticPageEntry {
+  /** The URL path (e.g., "/about", "/blog/first-post"). */
+  urlPath: string;
+  /** HTTP status code from the response. */
+  status: number;
+  /** Response headers. */
+  headers: Record<string, string>;
+  /** The response body as bytes. */
+  body: Uint8Array;
+  /** Content type for choosing file extension. */
+  contentType: string;
+  /**
+   * Whether this is an RSC flight payload (for client navigation)
+   * vs an HTML page (for initial load).
+   */
+  kind: 'html' | 'rsc' | 'api';
+}
+
+/** An error encountered during static generation. */
+export interface StaticGenerationError {
+  urlPath: string;
+  message: string;
+  status?: number;
+}
+
+/** A redirect discovered during static generation. */
+export interface StaticRedirectEntry {
+  /** The source URL path (e.g., "/old-page"). */
+  from: string;
+  /** The redirect target (Location header value). */
+  to: string;
+  /** HTTP status code (301, 302, 307, 308). */
+  status: number;
+}
+
+/** Result of the static generation pass. */
+export interface StaticGenerationSummary {
+  /** Total pages rendered (HTML + RSC pairs count as 2). */
+  rendered: number;
+  /** Pages that failed to render. */
+  errors: StaticGenerationError[];
+  /** Redirects from middleware/proxy that need platform redirect rules. */
+  redirects: StaticRedirectEntry[];
+  /**
+   * Non-fatal problems the build should surface. Unlike `errors`, these do not
+   * fail the build — they name a degraded outcome the developer can act on
+   * (e.g. the not-found probe answering with something other than a 404).
+   */
+  warnings: string[];
+}
+
+export interface StaticGenerationOptions {
+  /** Called for each rendered page/asset. File I/O is the caller's job. */
+  onEntry: (entry: StaticPageEntry) => Promise<void>;
+  /** Progress/warning logger. */
+  log?: (message: string) => void;
+  /** Timeout per page render in milliseconds. Default: 30000. */
+  renderTimeoutMs?: number;
+  /**
+   * When true and no user-authored sitemap exists, the generator will
+   * render `/sitemap.xml` (and any paginated sub-sitemaps) through the
+   * handler to capture auto-generated sitemaps in the static output.
+   */
+  autoSitemapEnabled?: boolean;
+}
+
+// ─── Route enumeration ───────────────────────────────────────────────────
+
+interface StaticUrl {
+  urlPath: string;
+  kind: 'page' | 'api';
+}
+
+/**
+ * Walk the manifest tree and collect all concrete URLs to render.
+ *
+ * Static routes produce their URL directly. Dynamic routes call
+ * `generateStaticSegmentParams()` from the page module. Dynamic routes
+ * without the export are a build error in static mode.
+ */
+export async function enumerateStaticUrls(
+  root: ManifestSegmentNode,
+  log: (message: string) => void,
+  timeoutMs: number = DEFAULT_RENDER_TIMEOUT_MS
+): Promise<{ urls: StaticUrl[]; errors: StaticGenerationError[] }> {
+  const urls: StaticUrl[] = [];
+  const errors: StaticGenerationError[] = [];
+
+  async function walk(node: ManifestSegmentNode): Promise<void> {
+    if (node.segmentType === 'intercepting' || node.segmentType === 'private') return;
+
+    // Page routes
+    if (node.page) {
+      const isDynamic = hasDynamicSegments(node.urlPath);
+      if (isDynamic) {
+        const expanded = await expandDynamicRoute(node.urlPath, node.page, log, timeoutMs);
+        if (expanded === null) {
+          errors.push({
+            urlPath: node.urlPath,
+            message:
+              `Dynamic route ${node.urlPath} must export generateStaticSegmentParams ` +
+              `in static mode — there is no server to render it at request time.`,
+          });
+        } else {
+          for (const path of expanded) {
+            urls.push({ urlPath: path, kind: 'page' });
+          }
+        }
+      } else {
+        urls.push({ urlPath: node.urlPath || '/', kind: 'page' });
+      }
+    }
+
+    // API routes (route.ts)
+    if (node.route) {
+      // Validate that the route exports GET — static mode only sends GET
+      // requests, so POST/PUT/DELETE-only routes would return 405.
+      const hasGet = await routeExportsGet(node.route, timeoutMs);
+      if (!hasGet) {
+        errors.push({
+          urlPath: node.urlPath || '/',
+          message:
+            `API route ${node.urlPath || '/'} does not export a GET handler. ` +
+            `Static mode can only render GET routes — add a GET export or ` +
+            `remove this route from the static build. If this route relies ` +
+            `on middleware redirects, use a _redirects file instead.`,
+        });
+      } else {
+        const isDynamic = hasDynamicSegments(node.urlPath);
+        if (isDynamic) {
+          const expanded = await expandDynamicApiRoute(node.urlPath, node.route, log, timeoutMs);
+          if (expanded === null) {
+            errors.push({
+              urlPath: node.urlPath,
+              message:
+                `Dynamic API route ${node.urlPath} must export generateStaticSegmentParams ` +
+                `in static mode.`,
+            });
+          } else {
+            for (const path of expanded) {
+              urls.push({ urlPath: path, kind: 'api' });
+            }
+          }
+        } else {
+          urls.push({ urlPath: node.urlPath || '/', kind: 'api' });
+        }
+      }
+    }
+
+    // Metadata routes (robots.ts, sitemap.ts, icon.png, etc.)
+    if (node.metadataRoutes) {
+      for (const [baseName, file] of Object.entries(node.metadataRoutes)) {
+        const convention = METADATA_ROUTE_CONVENTIONS[baseName];
+        if (!convention) continue;
+        if (!convention.nestable && node.urlPath !== '/') continue;
+        const servePath = resolveServePathForFile(baseName, file.filePath);
+        const segmentPrefix = node.urlPath === '/' ? '' : node.urlPath;
+
+        if (hasDynamicSegments(node.urlPath)) {
+          // Metadata route under a dynamic segment — expand using params
+          // from the page, route, or the metadata file itself (metadata-only
+          // dynamic segments are valid per design/16-metadata.md).
+          const paramSource = node.page ?? node.route ?? file;
+          const expanded = await expandDynamicRoute(node.urlPath, paramSource, log, timeoutMs);
+          if (expanded === null) {
+            errors.push({
+              urlPath: `${segmentPrefix}/${servePath}`,
+              message:
+                `Metadata route ${baseName} in dynamic segment ${node.urlPath} ` +
+                `requires generateStaticSegmentParams in static mode.`,
+            });
+          } else {
+            for (const expandedPath of expanded) {
+              const prefix = expandedPath === '/' ? '' : expandedPath;
+              urls.push({ urlPath: `${prefix}/${servePath}`, kind: 'api' });
+            }
+          }
+        } else {
+          urls.push({ urlPath: `${segmentPrefix}/${servePath}`, kind: 'api' });
+        }
+      }
+    }
+
+    for (const child of node.children) {
+      await walk(child);
+    }
+  }
+
+  await walk(root);
+  urls.sort((a, b) => a.urlPath.localeCompare(b.urlPath));
+
+  // Drop repeats of the same (path, kind). `generateStaticSegmentParams` is
+  // an application function returning whatever a CMS query gave it, and a
+  // repeated combo asks for the same page twice — identical bytes to the same
+  // file, so it is waste rather than ambiguity. Removing it here keeps the
+  // output-path collision check (TIM-1266) meaning what it says: two
+  // *different* routes competing for one file.
+  const seen = new Set<string>();
+  const unique = urls.filter(({ urlPath, kind }) => {
+    const key = `${kind}:${urlPath}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+
+  return { urls: unique, errors };
+}
+
+/**
+ * Load a page module and call generateStaticSegmentParams to get
+ * all param combos for a dynamic route. Returns null if the export
+ * is missing (build error in static mode).
+ */
+async function expandDynamicRoute(
+  urlPath: string,
+  page: ManifestFile,
+  log: (message: string) => void,
+  timeoutMs: number = DEFAULT_RENDER_TIMEOUT_MS
+): Promise<string[] | null> {
+  const mod = await loadModuleWithTimeout(page, timeoutMs);
+
+  const generateFn = mod.generateStaticSegmentParams;
+  if (typeof generateFn !== 'function') return null;
+
+  const combos = (await (generateFn as () => Promise<unknown>)()) as Record<
+    string,
+    string | string[]
+  >[];
+  if (!Array.isArray(combos)) {
+    throw new Error(
+      `generateStaticSegmentParams for ${urlPath} must return an array, got ${typeof combos}`
+    );
+  }
+
+  const paths: string[] = [];
+  for (const combo of combos) {
+    const resolved = resolvePathWithParams(urlPath, combo);
+    if (resolved !== null) paths.push(resolved);
+  }
+
+  if (paths.length === 0) {
+    log(
+      `[timber] Static generation: generateStaticSegmentParams for ${urlPath} ` +
+        `returned an empty array — this route will have no pages in the static output`
+    );
+  }
+
+  return paths;
+}
+
+/**
+ * Check whether a route module exports a GET handler.
+ * Static mode sends GET requests — routes without GET return 405.
+ */
+async function routeExportsGet(
+  route: ManifestFile,
+  timeoutMs: number = DEFAULT_RENDER_TIMEOUT_MS
+): Promise<boolean> {
+  const mod = await loadModuleWithTimeout(route, timeoutMs);
+  return typeof mod.GET === 'function';
+}
+
+async function expandDynamicApiRoute(
+  urlPath: string,
+  route: ManifestFile,
+  log: (message: string) => void,
+  timeoutMs: number = DEFAULT_RENDER_TIMEOUT_MS
+): Promise<string[] | null> {
+  const mod = await loadModuleWithTimeout(route, timeoutMs);
+
+  const generateFn = mod.generateStaticSegmentParams;
+  if (typeof generateFn !== 'function') return null;
+
+  const combos = (await (generateFn as () => Promise<unknown>)()) as Record<
+    string,
+    string | string[]
+  >[];
+  if (!Array.isArray(combos)) {
+    throw new Error(
+      `generateStaticSegmentParams for ${urlPath} must return an array, got ${typeof combos}`
+    );
+  }
+
+  const paths: string[] = [];
+  for (const combo of combos) {
+    const resolved = resolvePathWithParams(urlPath, combo);
+    if (resolved !== null) paths.push(resolved);
+  }
+  return paths;
+}
+
+// ─── Redirect normalization ─────────────────────────────────────────────
+
+/**
+ * Normalize a Location header value for _redirects output.
+ * Strips the synthetic http://localhost origin from same-origin
+ * redirects so _redirects contains pathname-only targets.
+ */
+function normalizeRedirectTarget(rawLocation: string): string {
+  try {
+    const parsed = new URL(rawLocation);
+    if (parsed.hostname === 'localhost') {
+      return parsed.pathname + parsed.search + parsed.hash;
+    }
+  } catch {
+    // Relative URL — use as-is
+  }
+  return rawLocation;
+}
+
+// ─── Rendering ───────────────────────────────────────────────────────────
+
+async function loadModuleWithTimeout(
+  file: ManifestFile,
+  timeoutMs: number
+): Promise<Record<string, unknown>> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      reject(new Error(`module load timed out after ${timeoutMs}ms for ${file.filePath}`));
+    }, timeoutMs);
+  });
+  try {
+    return (await Promise.race([loadModule(file), deadline])) as Record<string, unknown>;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function headersToRecord(headers: Headers): Record<string, string> {
+  const record: Record<string, string> = {};
+  headers.forEach((value, key) => {
+    record[key] = value;
+  });
+  return record;
+}
+
+// ─── Static generation orchestration ─────────────────────────────────────
+
+/**
+ * Generate all static pages from the route manifest.
+ *
+ * For each page: renders HTML (for initial load) and RSC flight (for SPA nav).
+ * For each API route: calls the GET handler.
+ *
+ * Must run inside the built RSC bundle — the handler and manifest are
+ * module instances from this bundle's graph.
+ */
+export async function generateStaticSite(
+  manifest: ManifestRoot,
+  handler: (req: Request) => Promise<Response>,
+  options: StaticGenerationOptions
+): Promise<StaticGenerationSummary> {
+  const log = options.log ?? ((msg: string) => console.log(msg));
+  const timeoutMs = resolveRenderTimeoutMs(options.renderTimeoutMs);
+  const summary: StaticGenerationSummary = {
+    rendered: 0,
+    errors: [],
+    redirects: [],
+    warnings: [],
+  };
+  // Warnings are recorded and logged from one place so the summary the build
+  // reports and the line the developer reads can never describe different sets.
+  const warn = (message: string) => {
+    summary.warnings.push(message);
+    log(`[timber] Static generation: ${message}`);
+  };
+
+  log('[timber] Static generation: enumerating routes…');
+
+  const { urls, errors: enumErrors } = await enumerateStaticUrls(manifest.root, log, timeoutMs);
+  summary.errors.push(...enumErrors);
+
+  if (enumErrors.length > 0) {
+    const details = enumErrors.map((e) => `  - ${e.urlPath}: ${e.message}`).join('\n');
+    throw new Error(
+      `[timber] Static generation failed: ${enumErrors.length} route(s) cannot be rendered:\n${details}`
+    );
+  }
+
+  // Auto-sitemap: if enabled and no user sitemap exists, render the
+  // auto-generated /sitemap.xml (and paginated sub-sitemaps) through
+  // the handler. Must come after enumeration so user-authored sitemaps
+  // are discovered first.
+  if (options.autoSitemapEnabled && !hasUserSitemap(manifest.root)) {
+    urls.push({ urlPath: '/sitemap.xml', kind: 'api' });
+  }
+
+  log(`[timber] Static generation: rendering ${urls.length} routes…`);
+
+  // Track rendered API bodies so we can discover sub-sitemaps from the index.
+  const renderedEntries = new Map<string, Uint8Array>();
+
+  for (const { urlPath, kind } of urls) {
+    if (kind === 'page') {
+      // Render HTML for initial load
+      let htmlOk = false;
+      try {
+        const htmlResponse = await renderUrl(handler, urlPath, 'text/html', timeoutMs);
+
+        if (htmlResponse.status >= 300 && htmlResponse.status < 400) {
+          const rawLocation = htmlResponse.headers.get('location');
+          if (rawLocation) {
+            const location = normalizeRedirectTarget(rawLocation);
+            summary.redirects.push({ from: urlPath, to: location, status: htmlResponse.status });
+            log(`[timber] Static generation: ${urlPath} → ${location} (${htmlResponse.status})`);
+          } else {
+            summary.errors.push({
+              urlPath,
+              message: `HTML render returned ${htmlResponse.status} with no Location header`,
+              status: htmlResponse.status,
+            });
+          }
+        } else if (htmlResponse.status >= 400) {
+          summary.errors.push({
+            urlPath,
+            message: `HTML render returned status ${htmlResponse.status}`,
+            status: htmlResponse.status,
+          });
+        } else {
+          await options.onEntry({
+            urlPath,
+            status: htmlResponse.status,
+            headers: headersToRecord(htmlResponse.headers),
+            body: htmlResponse.body,
+            contentType: htmlResponse.headers.get('content-type') ?? 'text/html',
+            kind: 'html',
+          });
+          summary.rendered++;
+          htmlOk = true;
+        }
+      } catch (error) {
+        summary.errors.push({
+          urlPath,
+          message: `HTML render failed: ${error instanceof Error ? error.message : String(error)}`,
+        });
+      }
+
+      // Render RSC flight for client navigation — independent of HTML
+      // success so a transient write error doesn't orphan the flight file.
+      if (htmlOk) {
+        try {
+          const rscResponse = await renderUrl(handler, urlPath, RSC_CONTENT_TYPE, timeoutMs);
+
+          if (rscResponse.status < 400) {
+            await options.onEntry({
+              urlPath,
+              status: rscResponse.status,
+              headers: headersToRecord(rscResponse.headers),
+              body: rscResponse.body,
+              contentType: rscResponse.headers.get('content-type') ?? RSC_CONTENT_TYPE,
+              kind: 'rsc',
+            });
+            summary.rendered++;
+          }
+        } catch {
+          // RSC flight failure is non-fatal — the page will work with full
+          // page loads instead of SPA navigation for this route.
+          log(
+            `[timber] Static generation: RSC flight for ${urlPath} failed, SPA nav will fallback`
+          );
+        }
+      }
+    } else {
+      // API route — call the handler and capture the response
+      try {
+        const apiResponse = await renderUrl(handler, urlPath, '*/*', timeoutMs);
+
+        if (apiResponse.status >= 300 && apiResponse.status < 400) {
+          const rawLocation = apiResponse.headers.get('location');
+          if (rawLocation) {
+            const location = normalizeRedirectTarget(rawLocation);
+            summary.redirects.push({ from: urlPath, to: location, status: apiResponse.status });
+            log(`[timber] Static generation: ${urlPath} → ${location} (${apiResponse.status})`);
+          } else {
+            summary.errors.push({
+              urlPath,
+              message: `API route returned ${apiResponse.status} with no Location header`,
+              status: apiResponse.status,
+            });
+          }
+          continue;
+        }
+
+        if (apiResponse.status >= 400) {
+          summary.errors.push({
+            urlPath,
+            message: `API route render returned status ${apiResponse.status}`,
+            status: apiResponse.status,
+          });
+          continue;
+        }
+
+        const apiBody = apiResponse.body;
+        if (urlPath === '/sitemap.xml') renderedEntries.set(urlPath, apiBody);
+        await options.onEntry({
+          urlPath,
+          status: apiResponse.status,
+          headers: headersToRecord(apiResponse.headers),
+          body: apiBody,
+          contentType: apiResponse.headers.get('content-type') ?? 'application/octet-stream',
+          kind: 'api',
+        });
+        summary.rendered++;
+      } catch (error) {
+        summary.errors.push({
+          urlPath,
+          message: `API route failed: ${error instanceof Error ? error.message : String(error)}`,
+        });
+      }
+    }
+  }
+
+  // Write 404.html. Always — the classification of what to write, and what
+  // to tell the developer about it, lives in static-not-found.ts.
+  let probe: NotFoundProbe;
+  try {
+    const response = await renderUrl(handler, NOT_FOUND_PROBE_PATH, 'text/html', timeoutMs);
+    probe = {
+      kind: 'response',
+      status: response.status,
+      body: response.body,
+      headers: headersToRecord(response.headers),
+      contentType: response.headers.get('content-type') ?? 'text/html',
+    };
+  } catch (error) {
+    probe = { kind: 'threw', error };
+  }
+
+  const notFound = resolveNotFoundPage(probe, manifest.root);
+  if (notFound.warning) warn(notFound.warning);
+  await options.onEntry({
+    urlPath: NOT_FOUND_ENTRY_PATH,
+    status: 404,
+    headers: notFound.headers,
+    body: notFound.body,
+    contentType: notFound.contentType,
+    kind: 'html',
+  });
+  log(notFound.logLine);
+
+  // Discover and render paginated sub-sitemaps. If /sitemap.xml was an
+  // index (contains /sitemap/N.xml references), render each sub-sitemap.
+  if (options.autoSitemapEnabled && !hasUserSitemap(manifest.root)) {
+    const sitemapEntry = renderedEntries.get('/sitemap.xml');
+    if (sitemapEntry) {
+      const xml = new TextDecoder().decode(sitemapEntry);
+      const subUrls = discoverSubSitemapUrls(xml);
+      for (const subUrl of subUrls) {
+        try {
+          const resp = await renderUrl(handler, subUrl, '*/*', timeoutMs);
+          if (resp.status >= 400) {
+            summary.errors.push({
+              urlPath: subUrl,
+              message: `Sub-sitemap render returned status ${resp.status}`,
+              status: resp.status,
+            });
+          } else {
+            await options.onEntry({
+              urlPath: subUrl,
+              status: resp.status,
+              headers: headersToRecord(resp.headers),
+              body: resp.body,
+              contentType: resp.headers.get('content-type') ?? 'application/xml',
+              kind: 'api',
+            });
+            summary.rendered++;
+          }
+        } catch (error) {
+          summary.errors.push({
+            urlPath: subUrl,
+            message: `Sub-sitemap failed: ${error instanceof Error ? error.message : String(error)}`,
+          });
+        }
+      }
+    }
+  }
+
+  return summary;
+}
+
+/** Extract /sitemap/N.xml URLs from a sitemap index XML. */
+export function discoverSubSitemapUrls(xml: string): string[] {
+  const urls: string[] = [];
+  const locPattern = /<loc>([^<]*\/sitemap\/\d+\.xml)<\/loc>/g;
+  let match;
+  while ((match = locPattern.exec(xml)) !== null) {
+    try {
+      const url = new URL(match[1]);
+      urls.push(url.pathname);
+    } catch {
+      // If it's already a pathname (no origin), use it directly
+      const loc = match[1];
+      if (loc.startsWith('/')) urls.push(loc);
+    }
+  }
+  return urls;
+}
+
+async function renderUrl(
+  handler: (req: Request) => Promise<Response>,
+  urlPath: string,
+  accept: string,
+  timeoutMs: number
+): Promise<{ status: number; headers: Headers; body: Uint8Array }> {
+  const url = `http://localhost${urlPath}`;
+  const controller = new AbortController();
+  const request = new Request(url, {
+    method: 'GET',
+    headers: { Accept: accept },
+    signal: controller.signal,
+  });
+
+  // Race the handler call itself against the timeout — a stalled
+  // middleware or proxy could hang the build if only the body drain
+  // is guarded. See design/13-security.md A2.
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      controller.abort();
+      reject(new Error(`handler did not respond within ${timeoutMs}ms for ${urlPath}`));
+    }, timeoutMs);
+  });
+
+  let response: Response;
+  try {
+    response = await Promise.race([handler(request), deadline]);
+  } finally {
+    clearTimeout(timer);
+  }
+
+  // The reader is cancelled on every exit — design/13-security.md A3.
+  const body = response.body
+    ? await readAllBytesWithDeadline(response.body, timeoutMs, 'response')
+    : new Uint8Array(0);
+
+  return {
+    status: response.status,
+    headers: response.headers,
+    body,
+  };
+}