@timber-js/app 0.2.0-alpha.19 → 0.2.0-alpha.190

package/src/server/sitemap-generator.ts

@@ -0,0 +1,339 @@
+/**
+ * Auto-generate sitemap.xml from the route tree.
+ *
+ * When enabled via `sitemap: { enabled: true, baseUrl: '...' }` in timber.config.ts,
+ * this module walks the route tree and produces sitemap XML for all discoverable pages.
+ *
+ * Features:
+ * - Collects all static page routes automatically
+ * - Calls `generateStaticSegmentParams()` on dynamic routes to enumerate URLs
+ * - Skips API routes (route.ts), layout-only segments, slots, intercepting routes
+ * - Skips auth-protected routes (access.ts in segment chain) by default
+ * - Automatic pagination: sitemap index when > 50,000 URLs
+ *
+ * See design/16-metadata.md §"Auto-generated Sitemap"
+ */
+
+import { serializeSitemap, serializeSitemapIndex } from './pipeline-metadata.js';
+import type { SegmentNode } from '../routing/types.js';
+
+// ─── Types ───────────────────────────────────────────────────────────────────
+
+/** Configuration for sitemap generation from timber.config.ts. */
+export interface SitemapConfig {
+  /** Must be explicitly true to enable auto-generation. */
+  enabled: boolean;
+  /** Base URL for absolute URLs in the sitemap. Required. e.g., 'https://example.com' */
+  baseUrl: string;
+  /** Default changefreq for all entries. Optional. */
+  defaultChangeFrequency?: string;
+  /** Default priority for all entries (0.0–1.0). Optional. */
+  defaultPriority?: number;
+  /** Include routes protected by access.ts. Default: false. */
+  includeProtected?: boolean;
+}
+
+/** A single sitemap entry ready for serialization. */
+export interface SitemapEntry {
+  url: string;
+  changeFrequency?: string;
+  priority?: number;
+}
+
+/**
+ * A route discovered during tree walk that may need async resolution
+ * (dynamic routes with generateStaticSegmentParams).
+ */
+export interface SitemapRoute {
+  /** The URL path pattern (e.g., '/products/[id]'). */
+  urlPath: string;
+  /** Whether this route has dynamic segments that need generateStaticSegmentParams. */
+  isDynamic: boolean;
+  /** Loader for the page module (from route manifest). */
+  pageLoader?: { load: () => Promise<unknown>; filePath: string };
+}
+
+/**
+ * Function signature for loading a page module.
+ * Takes the manifest loader and returns the module exports.
+ */
+export type ModuleLoader = (loader: {
+  load: () => Promise<unknown>;
+  filePath: string;
+}) => Promise<Record<string, unknown>>;
+
+/** Result of sitemap generation. */
+export type SitemapResult =
+  | { type: 'single'; xml: string }
+  | { type: 'index'; indexXml: string; pages: Map<number, string> };
+
+// ─── Constants ───────────────────────────────────────────────────────────────
+
+/** Maximum URLs per sitemap file per the sitemap protocol. */
+export const URLS_PER_SITEMAP = 50_000;
+
+// ─── Tree Walking ────────────────────────────────────────────────────────────
+
+/**
+ * Walk the route tree and collect all routes eligible for the sitemap.
+ *
+ * Pure synchronous function — no module loading. Returns routes that may
+ * need async resolution (dynamic routes with generateStaticSegmentParams).
+ *
+ * @param root - The root segment node from the route tree
+ * @param config - Sitemap configuration
+ * @returns Array of sitemap routes to resolve
+ */
+export function collectSitemapRoutes(root: SegmentNode, config: SitemapConfig): SitemapRoute[] {
+  const routes: SitemapRoute[] = [];
+  walkNode(root, routes, config.includeProtected ?? false, false);
+  return routes;
+}
+
+/**
+ * Recursive tree walker. Collects page routes, skipping ineligible segments.
+ *
+ * @param node - Current segment node
+ * @param routes - Accumulator for discovered routes
+ * @param includeProtected - Whether to include routes with access.ts
+ * @param hasAccessInChain - Whether any ancestor has access.ts
+ */
+function walkNode(
+  node: SegmentNode,
+  routes: SitemapRoute[],
+  includeProtected: boolean,
+  hasAccessInChain: boolean
+): void {
+  // Track whether this node or any ancestor has access.ts
+  const protectedChain = hasAccessInChain || !!node.access;
+
+  // Skip intercepting routes — they're conditional rewrites, not real pages
+  if (node.segmentType === 'intercepting') return;
+
+  // Skip private folders
+  if (node.segmentType === 'private') return;
+
+  // Skip slots — parallel routes don't contribute independent URLs
+  // (slots are handled via the parent's slots map, but we skip them here)
+
+  // Check if this node has a page (not a route.ts API endpoint)
+  if (node.page && !node.route) {
+    // Skip protected routes unless configured to include them
+    if (!protectedChain || includeProtected) {
+      const isDynamic = hasDynamicSegments(node.urlPath);
+      // At runtime (manifest), page has a `load` function. At scan time (RouteFile),
+      // it only has filePath + extension. We check for `load` at runtime.
+      const page = node.page as {
+        filePath: string;
+        extension: string;
+        load?: () => Promise<unknown>;
+      };
+      routes.push({
+        urlPath: node.urlPath,
+        isDynamic,
+        pageLoader: page.load ? { load: page.load, filePath: page.filePath } : undefined,
+      });
+    }
+  }
+
+  // Recurse into children (not slots — slots don't contribute sitemap URLs)
+  for (const child of node.children) {
+    walkNode(child, routes, includeProtected, protectedChain);
+  }
+
+  // Do NOT recurse into slots — they are parallel routes at the same URL
+}
+
+/**
+ * Check if a URL path contains dynamic segments.
+ *
+ * Dynamic segments are enclosed in brackets: [param], [...param], [[...param]]
+ */
+export function hasDynamicSegments(urlPath: string): boolean {
+  return /\[/.test(urlPath);
+}
+
+// ─── URL Resolution ──────────────────────────────────────────────────────────
+
+/**
+ * Resolve sitemap routes into concrete URLs.
+ *
+ * For static routes, the URL is derived directly from the path.
+ * For dynamic routes, calls `generateStaticSegmentParams()` from the page module.
+ *
+ * @param routes - Routes collected by collectSitemapRoutes
+ * @param config - Sitemap configuration
+ * @param loadModule - Async function to load a page module by file path
+ * @returns Array of sitemap entries with absolute URLs
+ */
+export async function resolveSitemapUrls(
+  routes: SitemapRoute[],
+  config: SitemapConfig,
+  loadModule?: ModuleLoader
+): Promise<SitemapEntry[]> {
+  const baseUrl = config.baseUrl.replace(/\/+$/, ''); // strip trailing slash
+  const entries: SitemapEntry[] = [];
+
+  for (const route of routes) {
+    if (!route.isDynamic) {
+      // Static route — direct URL
+      const url = route.urlPath === '/' ? baseUrl + '/' : baseUrl + route.urlPath;
+      entries.push({
+        url,
+        changeFrequency: config.defaultChangeFrequency,
+        priority: config.defaultPriority,
+      });
+    } else if (loadModule && route.pageLoader) {
+      // Dynamic route — try to call generateStaticSegmentParams
+      try {
+        const mod = await loadModule(route.pageLoader);
+        const generateFn = mod.generateStaticSegmentParams;
+        if (typeof generateFn !== 'function') {
+          // No generateStaticSegmentParams export — skip this dynamic route
+          continue;
+        }
+
+        const paramSets: Record<string, string>[] = await generateFn();
+        for (const params of paramSets) {
+          const resolvedPath = resolvePathWithParams(route.urlPath, params);
+          if (resolvedPath) {
+            entries.push({
+              url: baseUrl + resolvedPath,
+              changeFrequency: config.defaultChangeFrequency,
+              priority: config.defaultPriority,
+            });
+          }
+        }
+      } catch (error) {
+        // Log and skip — don't fail the entire sitemap for one bad route
+        const filePath = route.pageLoader.filePath;
+        console.warn(
+          `[timber] sitemap: failed to call generateStaticSegmentParams for ${filePath}:`,
+          error instanceof Error ? error.message : error
+        );
+      }
+    }
+    // If no loadModule provided and route is dynamic, skip it
+  }
+
+  return entries;
+}
+
+/**
+ * Replace dynamic segments in a URL path with concrete param values.
+ *
+ * Handles:
+ * - `[param]` → single value
+ * - `[...param]` → slash-joined values
+ * - `[[...param]]` → slash-joined values (or empty string if not provided)
+ *
+ * @returns The resolved path, or null if params are missing
+ */
+export function resolvePathWithParams(
+  urlPath: string,
+  params: Record<string, string | string[]>
+): string | null {
+  let resolved = urlPath;
+
+  // Optional catch-all: [[...param]]
+  resolved = resolved.replace(/\[\[\.\.\.([^\]]+)\]\]/g, (_, name) => {
+    const value = params[name];
+    if (value === undefined || value === '') return '';
+    if (Array.isArray(value)) return value.join('/');
+    return value;
+  });
+
+  // Catch-all: [...param]
+  resolved = resolved.replace(/\[\.\.\.([^\]]+)\]/g, (_, name) => {
+    const value = params[name];
+    if (value === undefined) return '';
+    if (Array.isArray(value)) return value.join('/');
+    return value;
+  });
+
+  // Single dynamic: [param] (possibly with affixes like img-[id].png)
+  // Match optional prefix before [ and optional suffix after ]
+  let missingRequired = false;
+  resolved = resolved.replace(/([^/[]*)\[([^\]]+)\]([^/[]*)/g, (match, prefix, name, suffix) => {
+    const value = params[name];
+    if (value === undefined) {
+      missingRequired = true;
+      return match; // keep original to avoid malformed URL
+    }
+    const v = Array.isArray(value) ? (value[0] ?? '') : value;
+    return prefix + v + suffix;
+  });
+
+  if (missingRequired) return null;
+
+  // Clean up double slashes from empty optional catch-alls
+  resolved = resolved.replace(/\/+/g, '/');
+
+  // Remove trailing slash (except for root)
+  if (resolved.length > 1 && resolved.endsWith('/')) {
+    resolved = resolved.slice(0, -1);
+  }
+
+  return resolved;
+}
+
+// ─── Sitemap Generation ──────────────────────────────────────────────────────
+
+/**
+ * Generate the complete sitemap result from a route tree.
+ *
+ * Orchestrates tree walking, URL resolution, and XML serialization.
+ * Returns either a single sitemap or a paginated sitemap index.
+ *
+ * @param root - The root segment node from the route tree
+ * @param config - Sitemap configuration
+ * @param loadModule - Optional async function to load page modules (for generateStaticSegmentParams)
+ * @returns The sitemap result (single or index with pages)
+ */
+export async function generateSitemap(
+  root: SegmentNode,
+  config: SitemapConfig,
+  loadModule?: ModuleLoader
+): Promise<SitemapResult> {
+  const routes = collectSitemapRoutes(root, config);
+  const entries = await resolveSitemapUrls(routes, config, loadModule);
+
+  if (entries.length <= URLS_PER_SITEMAP) {
+    // Single sitemap
+    return {
+      type: 'single',
+      xml: serializeSitemap(entries),
+    };
+  }
+
+  // Paginated — split into chunks
+  const baseUrl = config.baseUrl.replace(/\/+$/, '');
+  const pageCount = Math.ceil(entries.length / URLS_PER_SITEMAP);
+  const pages = new Map<number, string>();
+  const sitemapUrls: string[] = [];
+
+  for (let i = 0; i < pageCount; i++) {
+    const start = i * URLS_PER_SITEMAP;
+    const end = Math.min(start + URLS_PER_SITEMAP, entries.length);
+    const chunk = entries.slice(start, end);
+    pages.set(i, serializeSitemap(chunk));
+    sitemapUrls.push(`${baseUrl}/sitemap/${i}.xml`);
+  }
+
+  return {
+    type: 'index',
+    indexXml: serializeSitemapIndex(sitemapUrls),
+    pages,
+  };
+}
+
+/**
+ * Check if the route tree root has a user-authored sitemap metadata route.
+ *
+ * When a user-authored sitemap exists at the root, auto-generation is disabled
+ * to avoid conflicts — the user takes full control.
+ */
+export function hasUserSitemap(root: SegmentNode<unknown>): boolean {
+  if (!root.metadataRoutes) return false;
+  return 'sitemap' in root.metadataRoutes;
+}

package/src/server/sitemap-handler.ts

@@ -0,0 +1,157 @@
+/**
+ * Auto-sitemap request handler for the pipeline.
+ *
+ * Creates a handler function that intercepts `/sitemap.xml` and `/sitemap/N.xml`
+ * requests, generating sitemap XML from the route tree on-the-fly.
+ *
+ * This module bridges the pipeline (which doesn't know about route trees)
+ * with the sitemap generator (which doesn't know about HTTP).
+ *
+ * See design/16-metadata.md §"Auto-generated Sitemap"
+ */
+
+import {
+  generateSitemap,
+  hasUserSitemap,
+  type SitemapConfig,
+  type SitemapResult,
+} from './sitemap-generator.js';
+import { loadModule } from './safe-load.js';
+import type { SegmentNode } from '../routing/types.js';
+
+/** Regex to match /sitemap/N.xml where N is a non-negative integer. */
+const SITEMAP_PAGE_PATTERN = /^\/sitemap\/(\d+)\.xml$/;
+
+/**
+ * Create the auto-sitemap request handler for the pipeline.
+ *
+ * Returns a function that handles `/sitemap.xml` and `/sitemap/N.xml`,
+ * or returns `null` to pass through to regular routing.
+ *
+ * Returns `undefined` if auto-sitemap is not enabled (no overhead in pipeline).
+ */
+export function createAutoSitemapHandler(
+  root: SegmentNode,
+  runtimeConfig: Record<string, unknown>
+): ((pathname: string) => Promise<Response | null>) | undefined {
+  // Check if sitemap generation is enabled
+  const sitemapConfig = runtimeConfig.sitemap as
+    | {
+        enabled?: boolean;
+        baseUrl?: string;
+        defaultChangeFrequency?: string;
+        defaultPriority?: number;
+        includeProtected?: boolean;
+      }
+    | undefined;
+
+  if (!sitemapConfig?.enabled) return undefined;
+
+  if (!sitemapConfig.baseUrl) {
+    console.warn(
+      '[timber] sitemap.enabled is true but sitemap.baseUrl is missing. Sitemap generation disabled.'
+    );
+    return undefined;
+  }
+
+  // Don't generate if user has their own sitemap at the root
+  if (hasUserSitemap(root)) return undefined;
+
+  const config: SitemapConfig = {
+    enabled: true,
+    baseUrl: sitemapConfig.baseUrl,
+    defaultChangeFrequency: sitemapConfig.defaultChangeFrequency,
+    defaultPriority: sitemapConfig.defaultPriority,
+    includeProtected: sitemapConfig.includeProtected ?? false,
+  };
+
+  // Module loader adapter — uses the safe-load helper which wraps
+  // the manifest's lazy import functions.
+  const moduleLoader = async (loader: { load: () => Promise<unknown>; filePath: string }) => {
+    return loadModule<Record<string, unknown>>(loader);
+  };
+
+  // Cache the generated result to avoid regenerating on every request.
+  // In dev, this cache is busted by file changes (the handler is recreated
+  // when the route tree changes). In production, it's generated once.
+  let cachedResult: SitemapResult | null = null;
+  let inflight: Promise<SitemapResult> | null = null;
+
+  async function getOrGenerateSitemap(): Promise<SitemapResult> {
+    if (cachedResult) return cachedResult;
+    if (!inflight) {
+      inflight = generateSitemap(root, config, moduleLoader).then(
+        (result) => {
+          cachedResult = result;
+          inflight = null;
+          return result;
+        },
+        (err) => {
+          inflight = null;
+          throw err;
+        }
+      );
+    }
+    // Every caller races against its own timeout — not just the first.
+    // Per design/06-caching.md — singleflight maps must have a timeout
+    // to prevent deadlock if generateSitemap hangs.
+    const timeout = new Promise<never>((_, reject) =>
+      setTimeout(() => {
+        inflight = null;
+        reject(new Error('[timber] sitemap generation timed out'));
+      }, 30_000)
+    );
+    return Promise.race([inflight, timeout]);
+  }
+
+  return async (pathname: string): Promise<Response | null> => {
+    // Handle /sitemap.xml
+    if (pathname === '/sitemap.xml') {
+      const result = await getOrGenerateSitemap();
+      const xml = result.type === 'single' ? result.xml : result.indexXml;
+      return new Response(xml, {
+        status: 200,
+        headers: {
+          'Content-Type': 'application/xml; charset=utf-8',
+          'Cache-Control': 'public, max-age=14400, must-revalidate',
+        },
+      });
+    }
+
+    // Handle /sitemap/N.xml (paginated sub-sitemaps)
+    const pageMatch = SITEMAP_PAGE_PATTERN.exec(pathname);
+    if (pageMatch) {
+      const result = await getOrGenerateSitemap();
+      if (result.type === 'single') {
+        // No pagination — only page 0 is valid for a single sitemap
+        const pageIndex = parseInt(pageMatch[1], 10);
+        if (pageIndex === 0) {
+          return new Response(result.xml, {
+            status: 200,
+            headers: {
+              'Content-Type': 'application/xml; charset=utf-8',
+              'Cache-Control': 'public, max-age=14400, must-revalidate',
+            },
+          });
+        }
+        return new Response(null, { status: 404 });
+      }
+
+      const pageIndex = parseInt(pageMatch[1], 10);
+      const pageXml = result.pages.get(pageIndex);
+      if (!pageXml) {
+        return new Response(null, { status: 404 });
+      }
+      return new Response(pageXml, {
+        status: 200,
+        headers: {
+          'Content-Type': 'application/xml; charset=utf-8',
+          'Cache-Control': 'public, max-age=14400, must-revalidate',
+        },
+      });
+    }
+
+    // Not a sitemap URL — pass through
+    return null;
+  };
+}

package/src/server/skippable-prefix.ts

@@ -0,0 +1,74 @@
+/**
+ * Skippable-prefix truncation.
+ *
+ * Extracted from `route-element-builder.ts`, which was over the 500-line limit
+ * (design/18-build-system.md). This is a self-contained question — "may this
+ * segment's layout still be skipped, given what its slots would render?" — with
+ * no dependency on element construction.
+ */
+
+import type { ManifestSegmentNode } from './route-matcher.js';
+import type { RouteMatch } from './pipeline.js';
+import { shouldSkipSlot, type ClientStateTree } from './state-tree-diff.js';
+import { consumedUrlParts } from './chain-url-parts.js';
+import { computeSlotContentKey, computeSlotKey } from '../routing/segment-keys.js';
+import { matchUrlParts } from './tree-match.js';
+import { slotUrlParts } from './slot-resolver.js';
+
+/**
+ * Truncate the skippable prefix when a skippable layout has slots whose
+ * content changed between the departing and destination URLs (TIM-1232).
+ *
+ * Without this, changed slots at skipped layouts are never sent to the
+ * client — the `continue` in the main loop skips slot resolution entirely.
+ * Removing the segment from the skippable set forces it to render with
+ * its slots. The prefix must stay contiguous, so all segments after the
+ * first changed-slot segment are also removed.
+ *
+ * Uses key-based diffing (TIM-1370): computes the destination content key
+ * per slot and checks if the client's advertised set contains it.
+ */
+export function truncateSkippableForChangedSlots(
+  skippableSet: Set<number>,
+  segments: ManifestSegmentNode[],
+  segmentKeys: string[],
+  match: RouteMatch,
+  clientStateTree: ClientStateTree,
+  req: Request
+): void {
+  const clientSlotKeys = clientStateTree.slots ?? null;
+  const destUrl = req.url;
+
+  for (let i = 0; i < segments.length; i++) {
+    if (!skippableSet.has(i)) continue;
+    const segment = segments[i];
+    const slotEntries = Object.entries(segment.slots ?? {});
+    if (slotEntries.length === 0) continue;
+
+    const segIdx = match.segments.indexOf(segment);
+    const ownerParts = segIdx >= 0 ? consumedUrlParts(match.segments, segIdx, match) : [];
+    const destParts = slotUrlParts(segment, match, destUrl);
+
+    for (const [slotName, slotNode] of slotEntries) {
+      const m = slotNode as ManifestSegmentNode;
+      const k = computeSlotKey(segmentKeys[i], `@${slotName}`);
+
+      // Slots with access.ts are never skip candidates (TIM-1364).
+      const destMatch = matchUrlParts(m, destParts);
+      const hasAccessInChain = !!m.access || (destMatch?.chain.some((seg) => seg.access) ?? false);
+
+      const destLeaf = destMatch?.chain[destMatch.chain.length - 1];
+      const entryFile = destLeaf?.page?.filePath ?? null;
+      const contentKey = computeSlotContentKey(k, ownerParts, entryFile, destMatch?.params ?? {});
+
+      if (
+        hasAccessInChain ||
+        m.children.some((c) => c.segmentType === 'intercepting') ||
+        !shouldSkipSlot(contentKey, clientSlotKeys)
+      ) {
+        for (let j = i; j < segments.length; j++) skippableSet.delete(j);
+        return;
+      }
+    }
+  }
+}