@timber-js/app 0.2.0-alpha.182 → 0.2.0-alpha.183

package/src/server/static-generator.ts

@@ -0,0 +1,692 @@
+/**
+ * Static site generator — enumerates routes and renders them to
+ * HTML + RSC flight payloads for `output: 'static'` mode.
+ *
+ * Runs INSIDE the built RSC bundle (same pattern as prebuilt-builder.ts)
+ * so it has access to the route manifest, page module loaders, and the
+ * request pipeline handler.
+ *
+ * For each page route:
+ *   - Renders HTML (Accept: text/html) for initial load
+ *   - Renders RSC flight (Accept: text/x-component) for client SPA nav
+ *
+ * For each API route (route.ts):
+ *   - Calls the GET handler and captures the response
+ *
+ * The caller (plugins/static-build.ts) handles file I/O — keeping
+ * node:fs out of the server bundle so it runs on Cloudflare Workers.
+ *
+ * Design docs: design/11-platform.md §"static output mode"
+ */
+
+import type { ManifestRoot, ManifestSegmentNode, ManifestFile } from './route-matcher.js';
+import { hasDynamicSegments, hasUserSitemap, resolvePathWithParams } from './sitemap-generator.js';
+import { loadModule } from './safe-load.js';
+import { METADATA_ROUTE_CONVENTIONS, resolveServePathForFile } from './metadata-routes.js';
+
+// ─── Types ───────────────────────────────────────────────────────────────
+
+/** A single rendered static page ready for the caller to write to disk. */
+export interface StaticPageEntry {
+  /** The URL path (e.g., "/about", "/blog/first-post"). */
+  urlPath: string;
+  /** HTTP status code from the response. */
+  status: number;
+  /** Response headers. */
+  headers: Record<string, string>;
+  /** The response body as bytes. */
+  body: Uint8Array;
+  /** Content type for choosing file extension. */
+  contentType: string;
+  /**
+   * Whether this is an RSC flight payload (for client navigation)
+   * vs an HTML page (for initial load).
+   */
+  kind: 'html' | 'rsc' | 'api';
+}
+
+/** An error encountered during static generation. */
+export interface StaticGenerationError {
+  urlPath: string;
+  message: string;
+  status?: number;
+}
+
+/** A redirect discovered during static generation. */
+export interface StaticRedirectEntry {
+  /** The source URL path (e.g., "/old-page"). */
+  from: string;
+  /** The redirect target (Location header value). */
+  to: string;
+  /** HTTP status code (301, 302, 307, 308). */
+  status: number;
+}
+
+/** Result of the static generation pass. */
+export interface StaticGenerationSummary {
+  /** Total pages rendered (HTML + RSC pairs count as 2). */
+  rendered: number;
+  /** Pages that failed to render. */
+  errors: StaticGenerationError[];
+  /** Redirects from middleware/proxy that need platform redirect rules. */
+  redirects: StaticRedirectEntry[];
+}
+
+export interface StaticGenerationOptions {
+  /** Called for each rendered page/asset. File I/O is the caller's job. */
+  onEntry: (entry: StaticPageEntry) => Promise<void>;
+  /** Progress/warning logger. */
+  log?: (message: string) => void;
+  /** Timeout per page render in milliseconds. Default: 30000. */
+  renderTimeoutMs?: number;
+  /**
+   * When true and no user-authored sitemap exists, the generator will
+   * render `/sitemap.xml` (and any paginated sub-sitemaps) through the
+   * handler to capture auto-generated sitemaps in the static output.
+   */
+  autoSitemapEnabled?: boolean;
+}
+
+// ─── Route enumeration ───────────────────────────────────────────────────
+
+interface StaticUrl {
+  urlPath: string;
+  kind: 'page' | 'api';
+}
+
+/**
+ * Walk the manifest tree and collect all concrete URLs to render.
+ *
+ * Static routes produce their URL directly. Dynamic routes call
+ * `generateStaticSegmentParams()` from the page module. Dynamic routes
+ * without the export are a build error in static mode.
+ */
+export async function enumerateStaticUrls(
+  root: ManifestSegmentNode,
+  log: (message: string) => void,
+  timeoutMs: number = DEFAULT_RENDER_TIMEOUT_MS
+): Promise<{ urls: StaticUrl[]; errors: StaticGenerationError[] }> {
+  const urls: StaticUrl[] = [];
+  const errors: StaticGenerationError[] = [];
+
+  async function walk(node: ManifestSegmentNode): Promise<void> {
+    if (node.segmentType === 'intercepting' || node.segmentType === 'private') return;
+
+    // Page routes
+    if (node.page) {
+      const isDynamic = hasDynamicSegments(node.urlPath);
+      if (isDynamic) {
+        const expanded = await expandDynamicRoute(node.urlPath, node.page, log, timeoutMs);
+        if (expanded === null) {
+          errors.push({
+            urlPath: node.urlPath,
+            message:
+              `Dynamic route ${node.urlPath} must export generateStaticSegmentParams ` +
+              `in static mode — there is no server to render it at request time.`,
+          });
+        } else {
+          for (const path of expanded) {
+            urls.push({ urlPath: path, kind: 'page' });
+          }
+        }
+      } else {
+        urls.push({ urlPath: node.urlPath || '/', kind: 'page' });
+      }
+    }
+
+    // API routes (route.ts)
+    if (node.route) {
+      // Validate that the route exports GET — static mode only sends GET
+      // requests, so POST/PUT/DELETE-only routes would return 405.
+      const hasGet = await routeExportsGet(node.route, timeoutMs);
+      if (!hasGet) {
+        errors.push({
+          urlPath: node.urlPath || '/',
+          message:
+            `API route ${node.urlPath || '/'} does not export a GET handler. ` +
+            `Static mode can only render GET routes — add a GET export or ` +
+            `remove this route from the static build. If this route relies ` +
+            `on middleware redirects, use a _redirects file instead.`,
+        });
+      } else {
+        const isDynamic = hasDynamicSegments(node.urlPath);
+        if (isDynamic) {
+          const expanded = await expandDynamicApiRoute(node.urlPath, node.route, log, timeoutMs);
+          if (expanded === null) {
+            errors.push({
+              urlPath: node.urlPath,
+              message:
+                `Dynamic API route ${node.urlPath} must export generateStaticSegmentParams ` +
+                `in static mode.`,
+            });
+          } else {
+            for (const path of expanded) {
+              urls.push({ urlPath: path, kind: 'api' });
+            }
+          }
+        } else {
+          urls.push({ urlPath: node.urlPath || '/', kind: 'api' });
+        }
+      }
+    }
+
+    // Metadata routes (robots.ts, sitemap.ts, icon.png, etc.)
+    if (node.metadataRoutes) {
+      for (const [baseName, file] of Object.entries(node.metadataRoutes)) {
+        const convention = METADATA_ROUTE_CONVENTIONS[baseName];
+        if (!convention) continue;
+        if (!convention.nestable && node.urlPath !== '/') continue;
+        const servePath = resolveServePathForFile(baseName, file.filePath);
+        const segmentPrefix = node.urlPath === '/' ? '' : node.urlPath;
+
+        if (hasDynamicSegments(node.urlPath)) {
+          // Metadata route under a dynamic segment — expand using params
+          // from the page, route, or the metadata file itself (metadata-only
+          // dynamic segments are valid per design/16-metadata.md).
+          const paramSource = node.page ?? node.route ?? file;
+          const expanded = await expandDynamicRoute(node.urlPath, paramSource, log, timeoutMs);
+          if (expanded === null) {
+            errors.push({
+              urlPath: `${segmentPrefix}/${servePath}`,
+              message:
+                `Metadata route ${baseName} in dynamic segment ${node.urlPath} ` +
+                `requires generateStaticSegmentParams in static mode.`,
+            });
+          } else {
+            for (const expandedPath of expanded) {
+              const prefix = expandedPath === '/' ? '' : expandedPath;
+              urls.push({ urlPath: `${prefix}/${servePath}`, kind: 'api' });
+            }
+          }
+        } else {
+          urls.push({ urlPath: `${segmentPrefix}/${servePath}`, kind: 'api' });
+        }
+      }
+    }
+
+    for (const child of node.children) {
+      await walk(child);
+    }
+  }
+
+  await walk(root);
+  urls.sort((a, b) => a.urlPath.localeCompare(b.urlPath));
+  return { urls, errors };
+}
+
+/**
+ * Load a page module and call generateStaticSegmentParams to get
+ * all param combos for a dynamic route. Returns null if the export
+ * is missing (build error in static mode).
+ */
+async function expandDynamicRoute(
+  urlPath: string,
+  page: ManifestFile,
+  log: (message: string) => void,
+  timeoutMs: number = DEFAULT_RENDER_TIMEOUT_MS
+): Promise<string[] | null> {
+  const mod = await loadModuleWithTimeout(page, timeoutMs);
+
+  let generateFn = mod.generateStaticSegmentParams;
+  if (typeof generateFn !== 'function' && typeof mod.generateStaticParams === 'function') {
+    generateFn = mod.generateStaticParams;
+    log(
+      `[timber] ${page.filePath}: generateStaticParams is deprecated — ` +
+        `rename to generateStaticSegmentParams`
+    );
+  }
+
+  if (typeof generateFn !== 'function') return null;
+
+  const combos = (await (generateFn as () => Promise<unknown>)()) as Record<
+    string,
+    string | string[]
+  >[];
+  if (!Array.isArray(combos)) {
+    throw new Error(
+      `generateStaticSegmentParams for ${urlPath} must return an array, got ${typeof combos}`
+    );
+  }
+
+  const paths: string[] = [];
+  for (const combo of combos) {
+    const resolved = resolvePathWithParams(urlPath, combo);
+    if (resolved !== null) paths.push(resolved);
+  }
+
+  if (paths.length === 0) {
+    log(
+      `[timber] Static generation: generateStaticSegmentParams for ${urlPath} ` +
+        `returned an empty array — this route will have no pages in the static output`
+    );
+  }
+
+  return paths;
+}
+
+/**
+ * Check whether a route module exports a GET handler.
+ * Static mode sends GET requests — routes without GET return 405.
+ */
+async function routeExportsGet(
+  route: ManifestFile,
+  timeoutMs: number = DEFAULT_RENDER_TIMEOUT_MS
+): Promise<boolean> {
+  const mod = await loadModuleWithTimeout(route, timeoutMs);
+  return typeof mod.GET === 'function';
+}
+
+async function expandDynamicApiRoute(
+  urlPath: string,
+  route: ManifestFile,
+  log: (message: string) => void,
+  timeoutMs: number = DEFAULT_RENDER_TIMEOUT_MS
+): Promise<string[] | null> {
+  const mod = await loadModuleWithTimeout(route, timeoutMs);
+
+  let generateFn = mod.generateStaticSegmentParams;
+  if (typeof generateFn !== 'function' && typeof mod.generateStaticParams === 'function') {
+    generateFn = mod.generateStaticParams;
+    log(
+      `[timber] ${route.filePath}: generateStaticParams is deprecated — ` +
+        `rename to generateStaticSegmentParams`
+    );
+  }
+
+  if (typeof generateFn !== 'function') return null;
+
+  const combos = (await (generateFn as () => Promise<unknown>)()) as Record<
+    string,
+    string | string[]
+  >[];
+  if (!Array.isArray(combos)) {
+    throw new Error(
+      `generateStaticSegmentParams for ${urlPath} must return an array, got ${typeof combos}`
+    );
+  }
+
+  const paths: string[] = [];
+  for (const combo of combos) {
+    const resolved = resolvePathWithParams(urlPath, combo);
+    if (resolved !== null) paths.push(resolved);
+  }
+  return paths;
+}
+
+// ─── Redirect normalization ─────────────────────────────────────────────
+
+/**
+ * Normalize a Location header value for _redirects output.
+ * Strips the synthetic http://localhost origin from same-origin
+ * redirects so _redirects contains pathname-only targets.
+ */
+function normalizeRedirectTarget(rawLocation: string): string {
+  try {
+    const parsed = new URL(rawLocation);
+    if (parsed.hostname === 'localhost') {
+      return parsed.pathname + parsed.search + parsed.hash;
+    }
+  } catch {
+    // Relative URL — use as-is
+  }
+  return rawLocation;
+}
+
+// ─── Rendering ───────────────────────────────────────────────────────────
+
+const DEFAULT_RENDER_TIMEOUT_MS = 30_000;
+
+async function loadModuleWithTimeout(
+  file: ManifestFile,
+  timeoutMs: number
+): Promise<Record<string, unknown>> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      reject(new Error(`module load timed out after ${timeoutMs}ms for ${file.filePath}`));
+    }, timeoutMs);
+  });
+  try {
+    return (await Promise.race([loadModule(file), deadline])) as Record<string, unknown>;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function drainResponse(response: Response, timeoutMs: number): Promise<Uint8Array> {
+  if (!response.body) return new Uint8Array(0);
+
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let timedOut = false;
+
+  const deadline = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      timedOut = true;
+      reject(new Error(`response did not complete within ${timeoutMs}ms`));
+    }, timeoutMs);
+  });
+
+  try {
+    for (;;) {
+      const { done, value } = await Promise.race([reader.read(), deadline]);
+      if (done) break;
+      chunks.push(value as Uint8Array);
+    }
+  } finally {
+    clearTimeout(timer);
+    // Always cancel — design/13-security.md A3: never leave a reader open.
+    await reader.cancel().catch(() => {});
+  }
+
+  let total = 0;
+  for (const c of chunks) total += c.byteLength;
+  const out = new Uint8Array(total);
+  let off = 0;
+  for (const c of chunks) {
+    out.set(c, off);
+    off += c.byteLength;
+  }
+  return out;
+}
+
+function headersToRecord(headers: Headers): Record<string, string> {
+  const record: Record<string, string> = {};
+  headers.forEach((value, key) => {
+    record[key] = value;
+  });
+  return record;
+}
+
+// ─── Static generation orchestration ─────────────────────────────────────
+
+/**
+ * Generate all static pages from the route manifest.
+ *
+ * For each page: renders HTML (for initial load) and RSC flight (for SPA nav).
+ * For each API route: calls the GET handler.
+ *
+ * Must run inside the built RSC bundle — the handler and manifest are
+ * module instances from this bundle's graph.
+ */
+export async function generateStaticSite(
+  manifest: ManifestRoot,
+  handler: (req: Request) => Promise<Response>,
+  options: StaticGenerationOptions
+): Promise<StaticGenerationSummary> {
+  const log = options.log ?? ((msg: string) => console.log(msg));
+  const timeoutMs = options.renderTimeoutMs ?? DEFAULT_RENDER_TIMEOUT_MS;
+  const summary: StaticGenerationSummary = { rendered: 0, errors: [], redirects: [] };
+
+  log('[timber] Static generation: enumerating routes…');
+
+  const { urls, errors: enumErrors } = await enumerateStaticUrls(manifest.root, log, timeoutMs);
+  summary.errors.push(...enumErrors);
+
+  if (enumErrors.length > 0) {
+    const details = enumErrors.map((e) => `  - ${e.urlPath}: ${e.message}`).join('\n');
+    throw new Error(
+      `[timber] Static generation failed: ${enumErrors.length} route(s) cannot be rendered:\n${details}`
+    );
+  }
+
+  // Auto-sitemap: if enabled and no user sitemap exists, render the
+  // auto-generated /sitemap.xml (and paginated sub-sitemaps) through
+  // the handler. Must come after enumeration so user-authored sitemaps
+  // are discovered first.
+  if (options.autoSitemapEnabled && !hasUserSitemap(manifest.root)) {
+    urls.push({ urlPath: '/sitemap.xml', kind: 'api' });
+  }
+
+  log(`[timber] Static generation: rendering ${urls.length} routes…`);
+
+  // Track rendered API bodies so we can discover sub-sitemaps from the index.
+  const renderedEntries = new Map<string, Uint8Array>();
+
+  for (const { urlPath, kind } of urls) {
+    if (kind === 'page') {
+      // Render HTML for initial load
+      let htmlOk = false;
+      try {
+        const htmlResponse = await renderUrl(handler, urlPath, 'text/html', timeoutMs);
+
+        if (htmlResponse.status >= 300 && htmlResponse.status < 400) {
+          const rawLocation = htmlResponse.headers.get('location');
+          if (rawLocation) {
+            const location = normalizeRedirectTarget(rawLocation);
+            summary.redirects.push({ from: urlPath, to: location, status: htmlResponse.status });
+            log(`[timber] Static generation: ${urlPath} → ${location} (${htmlResponse.status})`);
+          } else {
+            summary.errors.push({
+              urlPath,
+              message: `HTML render returned ${htmlResponse.status} with no Location header`,
+              status: htmlResponse.status,
+            });
+          }
+        } else if (htmlResponse.status >= 400) {
+          summary.errors.push({
+            urlPath,
+            message: `HTML render returned status ${htmlResponse.status}`,
+            status: htmlResponse.status,
+          });
+        } else {
+          await options.onEntry({
+            urlPath,
+            status: htmlResponse.status,
+            headers: headersToRecord(htmlResponse.headers),
+            body: htmlResponse.body,
+            contentType: htmlResponse.headers.get('content-type') ?? 'text/html',
+            kind: 'html',
+          });
+          summary.rendered++;
+          htmlOk = true;
+        }
+      } catch (error) {
+        summary.errors.push({
+          urlPath,
+          message: `HTML render failed: ${error instanceof Error ? error.message : String(error)}`,
+        });
+      }
+
+      // Render RSC flight for client navigation — independent of HTML
+      // success so a transient write error doesn't orphan the flight file.
+      if (htmlOk) {
+        try {
+          const rscResponse = await renderUrl(handler, urlPath, 'text/x-component', timeoutMs);
+
+          if (rscResponse.status < 400) {
+            await options.onEntry({
+              urlPath,
+              status: rscResponse.status,
+              headers: headersToRecord(rscResponse.headers),
+              body: rscResponse.body,
+              contentType: rscResponse.headers.get('content-type') ?? 'text/x-component',
+              kind: 'rsc',
+            });
+            summary.rendered++;
+          }
+        } catch {
+          // RSC flight failure is non-fatal — the page will work with full
+          // page loads instead of SPA navigation for this route.
+          log(
+            `[timber] Static generation: RSC flight for ${urlPath} failed, SPA nav will fallback`
+          );
+        }
+      }
+    } else {
+      // API route — call the handler and capture the response
+      try {
+        const apiResponse = await renderUrl(handler, urlPath, '*/*', timeoutMs);
+
+        if (apiResponse.status >= 300 && apiResponse.status < 400) {
+          const rawLocation = apiResponse.headers.get('location');
+          if (rawLocation) {
+            const location = normalizeRedirectTarget(rawLocation);
+            summary.redirects.push({ from: urlPath, to: location, status: apiResponse.status });
+            log(`[timber] Static generation: ${urlPath} → ${location} (${apiResponse.status})`);
+          } else {
+            summary.errors.push({
+              urlPath,
+              message: `API route returned ${apiResponse.status} with no Location header`,
+              status: apiResponse.status,
+            });
+          }
+          continue;
+        }
+
+        if (apiResponse.status >= 400) {
+          summary.errors.push({
+            urlPath,
+            message: `API route render returned status ${apiResponse.status}`,
+            status: apiResponse.status,
+          });
+          continue;
+        }
+
+        const apiBody = apiResponse.body;
+        if (urlPath === '/sitemap.xml') renderedEntries.set(urlPath, apiBody);
+        await options.onEntry({
+          urlPath,
+          status: apiResponse.status,
+          headers: headersToRecord(apiResponse.headers),
+          body: apiBody,
+          contentType: apiResponse.headers.get('content-type') ?? 'application/octet-stream',
+          kind: 'api',
+        });
+        summary.rendered++;
+      } catch (error) {
+        summary.errors.push({
+          urlPath,
+          message: `API route failed: ${error instanceof Error ? error.message : String(error)}`,
+        });
+      }
+    }
+  }
+
+  // Render 404.html — the app's not-found page for unmatched URLs.
+  // Static hosts (GitHub Pages, Cloudflare Pages, Netlify) serve this
+  // automatically when no matching file exists.
+  // Uses /_timber/__not_found as the probe path: _timber/ is the framework's
+  // reserved prefix and __not_found is a private segment (double underscore),
+  // so user routes (including catch-all [..slug]) cannot claim this path.
+  try {
+    const notFoundResponse = await renderUrl(
+      handler,
+      '/_timber/__not_found',
+      'text/html',
+      timeoutMs
+    );
+    if (notFoundResponse.status === 404 && notFoundResponse.body.byteLength > 0) {
+      await options.onEntry({
+        urlPath: '/_timber_not_found',
+        status: notFoundResponse.status,
+        headers: headersToRecord(notFoundResponse.headers),
+        body: notFoundResponse.body,
+        contentType: notFoundResponse.headers.get('content-type') ?? 'text/html',
+        kind: 'html',
+      });
+      log('[timber] Static generation: rendered 404.html');
+    }
+  } catch {
+    log('[timber] Static generation: 404 page render failed — skipping 404.html');
+  }
+
+  // Discover and render paginated sub-sitemaps. If /sitemap.xml was an
+  // index (contains /sitemap/N.xml references), render each sub-sitemap.
+  if (options.autoSitemapEnabled && !hasUserSitemap(manifest.root)) {
+    const sitemapEntry = renderedEntries.get('/sitemap.xml');
+    if (sitemapEntry) {
+      const xml = new TextDecoder().decode(sitemapEntry);
+      const subUrls = discoverSubSitemapUrls(xml);
+      for (const subUrl of subUrls) {
+        try {
+          const resp = await renderUrl(handler, subUrl, '*/*', timeoutMs);
+          if (resp.status >= 400) {
+            summary.errors.push({
+              urlPath: subUrl,
+              message: `Sub-sitemap render returned status ${resp.status}`,
+              status: resp.status,
+            });
+          } else {
+            await options.onEntry({
+              urlPath: subUrl,
+              status: resp.status,
+              headers: headersToRecord(resp.headers),
+              body: resp.body,
+              contentType: resp.headers.get('content-type') ?? 'application/xml',
+              kind: 'api',
+            });
+            summary.rendered++;
+          }
+        } catch (error) {
+          summary.errors.push({
+            urlPath: subUrl,
+            message: `Sub-sitemap failed: ${error instanceof Error ? error.message : String(error)}`,
+          });
+        }
+      }
+    }
+  }
+
+  return summary;
+}
+
+/** Extract /sitemap/N.xml URLs from a sitemap index XML. */
+export function discoverSubSitemapUrls(xml: string): string[] {
+  const urls: string[] = [];
+  const locPattern = /<loc>([^<]*\/sitemap\/\d+\.xml)<\/loc>/g;
+  let match;
+  while ((match = locPattern.exec(xml)) !== null) {
+    try {
+      const url = new URL(match[1]);
+      urls.push(url.pathname);
+    } catch {
+      // If it's already a pathname (no origin), use it directly
+      const loc = match[1];
+      if (loc.startsWith('/')) urls.push(loc);
+    }
+  }
+  return urls;
+}
+
+async function renderUrl(
+  handler: (req: Request) => Promise<Response>,
+  urlPath: string,
+  accept: string,
+  timeoutMs: number
+): Promise<{ status: number; headers: Headers; body: Uint8Array }> {
+  const url = `http://localhost${urlPath}`;
+  const controller = new AbortController();
+  const request = new Request(url, {
+    method: 'GET',
+    headers: { Accept: accept },
+    signal: controller.signal,
+  });
+
+  // Race the handler call itself against the timeout — a stalled
+  // middleware or proxy could hang the build if only the body drain
+  // is guarded. See design/13-security.md A2.
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const deadline = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      controller.abort();
+      reject(new Error(`handler did not respond within ${timeoutMs}ms for ${urlPath}`));
+    }, timeoutMs);
+  });
+
+  let response: Response;
+  try {
+    response = await Promise.race([handler(request), deadline]);
+  } finally {
+    clearTimeout(timer);
+  }
+
+  const body = await drainResponse(response, timeoutMs);
+
+  return {
+    status: response.status,
+    headers: response.headers,
+    body,
+  };
+}