@timber-js/app 0.2.0-alpha.9 → 0.2.0-alpha.90

package/src/server/node-stream-transforms.ts

@@ -0,0 +1,509 @@
+/**
+ * Node.js native stream transforms for SSR HTML post-processing.
+ *
+ * These are Node.js Transform stream equivalents of the Web Stream
+ * transforms in html-injectors.ts. Used on Node.js/Bun where native
+ * streams (C++ backed) are faster than Web Streams (JS reimplementation).
+ *
+ * The transforms are pure string operations on HTML chunks — the same
+ * logic as the Web Stream versions, just wrapped in Node.js Transform
+ * instead of Web TransformStream.
+ *
+ * Architecture:
+ *   renderToPipeableStream → pipe(errorHandler) → pipe(headInjector)
+ *     → pipe(flightInjector) → Readable.toWeb() → Response
+ *
+ * All chunks stay in C++ Node.js stream buffers until the final
+ * Readable.toWeb() conversion for the Response body.
+ */
+
+import { Transform } from 'node:stream';
+import { createGzip, constants } from 'node:zlib';
+
+// ─── Buffered Transform ──────────────────────────────────────────────────────
+
+/**
+ * Options for the Node.js buffered transform.
+ */
+export interface NodeBufferedTransformOptions {
+  /**
+   * Flush synchronously once the buffer reaches this many bytes.
+   * Prevents unbounded memory growth for very large Fizz flushes.
+   * Default: Infinity (no byte limit — flush only on tick boundary).
+   */
+  readonly maxBufferByteLength?: number;
+}
+
+/**
+ * Node.js Transform that buffers incoming chunks and coalesces them
+ * within a single event loop tick.
+ *
+ * Equivalent to createBufferedTransformStream() in html-injectors.ts.
+ * React Fizz may emit multiple micro-chunks within a single flush.
+ * Without buffering, downstream transforms (especially flight injection)
+ * could see chunk boundaries in the middle of HTML tags or attributes.
+ *
+ * This transform collects all chunks that arrive in the same tick and
+ * emits them as a single concatenated Buffer on the next `setImmediate`.
+ *
+ * **Not a polling loop.** Uses a single-shot `setImmediate` per flush
+ * cycle — no recursive scheduling, no busy-wait. See design/02 §"No Polling".
+ *
+ * Inspired by Next.js `createBufferedTransformStream`.
+ */
+export function createNodeBufferedTransform(options: NodeBufferedTransformOptions = {}): Transform {
+  const { maxBufferByteLength = Infinity } = options;
+
+  let bufferedChunks: Buffer[] = [];
+  let bufferByteLength = 0;
+  let pendingImmediate: ReturnType<typeof setImmediate> | null = null;
+
+  const transform = new Transform({
+    transform(chunk: Buffer, _encoding, callback) {
+      bufferedChunks.push(chunk);
+      bufferByteLength += chunk.byteLength;
+
+      if (bufferByteLength >= maxBufferByteLength) {
+        // Synchronous flush — buffer is too large to hold
+        flushBuffer();
+      } else if (!pendingImmediate) {
+        // Schedule a deferred flush for end of this tick.
+        // Single-shot setImmediate — NOT a recursive loop.
+        // See design/02 §"No Polling".
+        pendingImmediate = setImmediate(() => {
+          pendingImmediate = null;
+          flushBuffer();
+        });
+      }
+
+      callback();
+    },
+    flush(callback) {
+      // Cancel any pending deferred flush and flush synchronously
+      if (pendingImmediate) {
+        clearImmediate(pendingImmediate);
+        pendingImmediate = null;
+      }
+      flushBuffer();
+      callback();
+    },
+  });
+
+  function flushBuffer() {
+    if (bufferedChunks.length === 0) return;
+
+    const merged = Buffer.concat(bufferedChunks, bufferByteLength);
+    bufferedChunks = [];
+    bufferByteLength = 0;
+    transform.push(merged);
+  }
+
+  return transform;
+}
+
+// ─── Injection Transforms ────────────────────────────────────────────────────
+
+import { createMachine } from '../utils/state-machine.js';
+import { flightChunkScript } from './flight-scripts.js';
+import {
+  flightInjectionTransitions,
+  isHtmlDone,
+  isPullDone,
+  type FlightInjectionState,
+  type FlightInjectionEvent,
+} from './flight-injection-state.js';
+import { withTimeout, RenderTimeoutError } from './render-timeout.js';
+import { logStreamingError } from './logger.js';
+
+// ─── Move Suffix Transform ───────────────────────────────────────────────────
+
+const SUFFIX = '</body></html>';
+const SUFFIX_BUF = Buffer.from(SUFFIX, 'utf-8');
+
+/**
+ * Node.js Transform that moves `</body></html>` to the end of the stream.
+ *
+ * Equivalent to createMoveSuffixStream() in html-injectors.ts.
+ * Strips the suffix when first encountered and re-emits it in flush().
+ * If no suffix is found, it's appended anyway for well-formed HTML.
+ */
+export function createNodeMoveSuffixTransform(): Transform {
+  let foundSuffix = false;
+
+  return new Transform({
+    transform(chunk: Buffer, _encoding, callback) {
+      if (foundSuffix) {
+        this.push(chunk);
+        callback();
+        return;
+      }
+
+      const text = chunk.toString('utf-8');
+      const idx = text.indexOf(SUFFIX);
+      if (idx === -1) {
+        this.push(chunk);
+        callback();
+        return;
+      }
+
+      foundSuffix = true;
+
+      // If the entire chunk is exactly the suffix, skip it
+      if (chunk.byteLength === SUFFIX_BUF.byteLength) {
+        callback();
+        return;
+      }
+
+      // Emit content before the suffix
+      const before = text.slice(0, idx);
+      const after = text.slice(idx + SUFFIX.length);
+      if (before) this.push(Buffer.from(before, 'utf-8'));
+      if (after) this.push(Buffer.from(after, 'utf-8'));
+      callback();
+    },
+    flush(callback) {
+      // Always emit the suffix at the very end
+      this.push(SUFFIX_BUF);
+      callback();
+    },
+  });
+}
+
+// ─── Head Injection ──────────────────────────────────────────────────────────
+
+/**
+ * Node.js Transform that injects HTML content before </head>.
+ *
+ * Equivalent to injectHead() in html-injectors.ts. Streams chunks
+ * through immediately, keeping only a small trailing buffer to handle
+ * </head> split across chunk boundaries.
+ */
+export function createNodeHeadInjector(headHtml: string): Transform {
+  if (!headHtml) {
+    return new Transform({
+      transform(chunk, _enc, cb) {
+        cb(null, chunk);
+      },
+    });
+  }
+
+  const target = '</head>';
+  const tailLen = target.length - 1;
+  let injected = false;
+  let tail = '';
+
+  return new Transform({
+    transform(chunk: Buffer, _encoding, callback) {
+      if (injected) {
+        callback(null, chunk);
+        return;
+      }
+
+      const text = tail + chunk.toString('utf-8');
+      const tagIndex = text.indexOf(target);
+
+      if (tagIndex !== -1) {
+        const before = text.slice(0, tagIndex);
+        const after = text.slice(tagIndex);
+        this.push(Buffer.from(before + headHtml + after, 'utf-8'));
+        injected = true;
+        tail = '';
+        callback();
+      } else {
+        const safeEnd = Math.max(0, text.length - tailLen);
+        if (safeEnd > 0) {
+          this.push(Buffer.from(text.slice(0, safeEnd), 'utf-8'));
+        }
+        tail = text.slice(safeEnd);
+        callback();
+      }
+    },
+    flush(callback) {
+      if (!injected && tail) {
+        this.push(Buffer.from(tail, 'utf-8'));
+      }
+      callback();
+    },
+  });
+}
+
+// ─── RSC Flight Injection ────────────────────────────────────────────────────
+
+/**
+ * Node.js Transform that merges RSC script tags into the HTML stream.
+ *
+ * Reads RSC chunks from the provided ReadableStream and injects them
+ * as `<script>` tags between HTML chunks. Scripts are buffered in
+ * pending[] and only drained from transform() (after a complete HTML
+ * chunk) or flush() — never pushed directly from the pull loop.
+ *
+ * Suffix stripping (</body></html>) is handled upstream by
+ * createNodeMoveSuffixTransform. This transform only interleaves
+ * RSC scripts at safe chunk boundaries.
+ *
+ * The RSC stream is a Web ReadableStream (from the tee'd RSC Flight
+ * stream). We read from it using the Web API — this is the one bridge
+ * point between Web Streams and Node.js streams in the pipeline.
+ */
+/**
+ * Options for the Node.js flight injector.
+ */
+export interface NodeFlightInjectorOptions {
+  /**
+   * Timeout in milliseconds for individual RSC stream reads.
+   * If a single `rscReader.read()` call does not resolve within
+   * this duration, the read is aborted and the stream errors with
+   * a RenderTimeoutError. Default: 30000 (30s).
+   */
+  renderTimeoutMs?: number;
+}
+
+export function createNodeFlightInjector(
+  rscStream: ReadableStream<Uint8Array> | undefined,
+  options?: NodeFlightInjectorOptions
+): Transform {
+  if (!rscStream) {
+    return new Transform({
+      transform(chunk, _enc, cb) {
+        cb(null, chunk);
+      },
+    });
+  }
+
+  const timeoutMs = options?.renderTimeoutMs ?? 30_000;
+  const rscReader = rscStream.getReader();
+  const decoder = new TextDecoder('utf-8', { fatal: true });
+
+  const machine = createMachine<FlightInjectionState, FlightInjectionEvent>({
+    initial: { phase: 'init' },
+    transitions: flightInjectionTransitions,
+  });
+
+  // Stored promise from pullLoop — awaited in flush() via .then()
+  // instead of polling. See design/02 §"No Polling".
+  let pullPromise: Promise<void> | null = null;
+
+  // RSC script chunks waiting to be drained at a safe boundary.
+  // pullLoop buffers here; transform() and flush() drain.
+  // RSC script chunks waiting to be drained at a safe boundary.
+  // pullLoop buffers here; transform(), flush(), and pullLoop's
+  // post-yield drain all consume from this buffer.
+  const pending: Buffer[] = [];
+  async function pullLoop(): Promise<void> {
+    // Yield once so the first transform() call can process the shell
+    // HTML chunk before we start reading RSC data.
+    await new Promise<void>((r) => setImmediate(r));
+    try {
+      for (;;) {
+        // Guard each RSC read with a timeout so a permanently hung
+        // RSC stream eventually aborts instead of blocking forever.
+        // See design/02-rendering-pipeline.md §"Streaming Constraints".
+        const readPromise = rscReader.read();
+        const { done, value } =
+          timeoutMs > 0
+            ? await withTimeout(readPromise, timeoutMs, 'RSC stream read timed out')
+            : await readPromise;
+        if (done) {
+          machine.send({ type: 'PULL_DONE' });
+          return;
+        }
+        const decoded = decoder.decode(value, { stream: true });
+        const scriptBuf = Buffer.from(flightChunkScript(decoded), 'utf-8');
+        // Buffer the script — drained by the next transform() call,
+        // flush(), or by the scheduled drain below.
+        pending.push(scriptBuf);
+        // Yield between reads so HTML chunks get priority in the event
+        // loop — but only while HTML is still streaming. Once flush()
+        // fires, read without yielding to drain remaining RSC data.
+        if (!isHtmlDone(machine.state)) {
+          await new Promise<void>((r) => setImmediate(r));
+          // After yielding, if no transform() call drained the buffer
+          // (i.e., no new HTML chunk arrived), drain now. This ensures
+          // RSC flight data reaches the client at shell-flush time —
+          // without this, hydration blocks on createFromReadableStream
+          // waiting for data that's stuck in pending[].
+          // This is safe: we're between event loop ticks, so no
+          // transform() call is mid-execution (no mid-tag risk).
+          if (pending.length > 0) {
+            drainPending();
+          }
+        }
+      }
+    } catch (err) {
+      // On timeout, cancel the RSC reader to release resources.
+      if (err instanceof RenderTimeoutError) {
+        rscReader.cancel(err).catch(() => {});
+      }
+      machine.send({ type: 'PULL_ERROR', error: err });
+    }
+  }
+
+  /** Drain all buffered RSC script chunks to the transform output. */
+  function drainPending(): void {
+    while (pending.length > 0) {
+      transform.push(pending.shift()!);
+    }
+  }
+
+  // No bootstrap script here — the init script is in <head> via
+  // flightInitScript() (see flight-scripts.ts). This ensures __timber_f
+  // exists before any chunk scripts execute.
+
+  const transform = new Transform({
+    transform(chunk: Buffer, _encoding, callback) {
+      const isFirst = machine.state.phase === 'init';
+      if (isFirst) {
+        machine.send({ type: 'FIRST_CHUNK' });
+      }
+
+      // Emit the HTML chunk, then drain any buffered RSC scripts.
+      // Scripts always come AFTER a complete HTML chunk — never mid-tag.
+      // The buffered transform upstream (TIM-528) ensures each chunk is
+      // a coherent HTML fragment. Suffix stripping is handled upstream
+      // by createNodeMoveSuffixTransform (TIM-530).
+      transform.push(chunk);
+      drainPending();
+
+      // Start the pull loop on the first HTML chunk.
+      if (isFirst) {
+        pullPromise = pullLoop();
+      }
+      callback();
+    },
+    flush(callback) {
+      // All HTML chunks have been emitted. Transition to flushing —
+      // the pull loop will stop yielding between RSC reads since
+      // isHtmlDone() now returns true.
+      machine.send({ type: 'HTML_DONE' });
+
+      const finish = () => {
+        // Drain any remaining buffered RSC scripts
+        drainPending();
+        if (machine.state.phase === 'error') {
+          const err = machine.state.error;
+          transform.destroy(err instanceof Error ? err : new Error(String(err)));
+          return;
+        }
+        callback();
+      };
+
+      if (isPullDone(machine.state)) {
+        finish();
+        return;
+      }
+      // Wait for the RSC pull loop promise to resolve instead of
+      // polling with setImmediate. No CPU spin, no busy-poll —
+      // just a Promise chain. See design/02 §"No Polling".
+      if (!pullPromise) {
+        pullPromise = pullLoop();
+      }
+      pullPromise.then(finish, (err) => {
+        machine.send({ type: 'PULL_ERROR', error: err });
+        finish();
+      });
+    },
+  });
+
+  return transform;
+}
+
+// ─── Error Handling ──────────────────────────────────────────────────────────
+
+const NOINDEX_SCRIPT =
+  '<script>document.head.appendChild(Object.assign(document.createElement("meta"),{name:"robots",content:"noindex"}))</script>';
+
+/**
+ * Node.js Transform that catches post-shell streaming errors.
+ *
+ * Equivalent to wrapStreamWithErrorHandling() in ssr-render.ts.
+ * Catches errors from React's streaming phase (deny/throw inside Suspense
+ * after the shell has flushed) and closes the stream cleanly.
+ */
+export function createNodeErrorHandler(signal?: AbortSignal): Transform {
+  const transform = new Transform({
+    transform(chunk, _encoding, callback) {
+      callback(null, chunk);
+    },
+  });
+
+  transform.on('error', (error) => {
+    const isAbort =
+      (error instanceof DOMException && error.name === 'AbortError') ||
+      (error instanceof Error && error.name === 'AbortError') ||
+      signal?.aborted;
+
+    if (isAbort) {
+      transform.end();
+      return;
+    }
+
+    logStreamingError({ error });
+    transform.push(Buffer.from(NOINDEX_SCRIPT, 'utf-8'));
+    transform.end();
+  });
+
+  return transform;
+}
+
+// ─── Compression ─────────────────────────────────────────────────────────────
+
+const COMPRESSIBLE_TYPES = new Set([
+  'text/html',
+  'text/css',
+  'text/plain',
+  'text/xml',
+  'text/javascript',
+  'text/x-component',
+  'application/json',
+  'application/javascript',
+  'application/xml',
+  'application/xhtml+xml',
+  'application/rss+xml',
+  'application/atom+xml',
+  'image/svg+xml',
+]);
+
+/**
+ * Create a Node.js gzip Transform using native node:zlib.
+ *
+ * Uses `createGzip()` which is backed by C++ zlib — significantly faster
+ * than the Web Streams `CompressionStream` API (which is a JS wrapper
+ * around the same zlib but with per-chunk Promise overhead).
+ *
+ * Returns null if the response shouldn't be compressed (wrong content type,
+ * client doesn't accept gzip, already encoded, etc.).
+ */
+export function createNodeGzipCompressor(
+  requestHeaders: Headers,
+  responseHeaders: Headers
+): Transform | null {
+  // Check Accept-Encoding
+  const acceptEncoding = requestHeaders.get('accept-encoding') || '';
+  if (!acceptEncoding.includes('gzip')) return null;
+
+  // Check content type is compressible
+  const contentType = responseHeaders.get('content-type') || '';
+  const mimeType = contentType.split(';')[0].trim().toLowerCase();
+  if (!COMPRESSIBLE_TYPES.has(mimeType)) return null;
+
+  // Don't double-compress
+  if (responseHeaders.has('content-encoding')) return null;
+
+  // Set response headers for gzip
+  responseHeaders.set('content-encoding', 'gzip');
+  responseHeaders.delete('content-length');
+  const existingVary = responseHeaders.get('vary');
+  if (existingVary) {
+    if (!existingVary.toLowerCase().includes('accept-encoding')) {
+      responseHeaders.set('vary', existingVary + ', Accept-Encoding');
+    }
+  } else {
+    responseHeaders.set('vary', 'Accept-Encoding');
+  }
+
+  // Z_SYNC_FLUSH ensures each chunk is flushed to the output immediately.
+  // Without it, gzip buffers internally and the browser doesn't receive
+  // the HTML shell until the gzip stream closes — breaking streaming.
+  // ~2–5% size overhead vs Z_NO_FLUSH but preserves correct streaming.
+  return createGzip({ flush: constants.Z_SYNC_FLUSH });
+}

package/src/server/page-deny-boundary.tsx

@@ -0,0 +1,56 @@
+/**
+ * PageDenyBoundary — catches DenySignal from page components server-side.
+ *
+ * Wraps a server component page by calling it as an async function (the same
+ * thing React Flight does internally). If the page throws DenySignal, the
+ * boundary catches it and renders the matching deny page as a normal element.
+ * No error reaches React Flight — the Flight stream is clean, SSR succeeds,
+ * and the entire request uses a single renderToReadableStream call.
+ *
+ * Client component pages ('use client') are NOT wrapped — they can't call
+ * deny() (server-only API) and must go through createElement normally.
+ *
+ * See design/04-authorization.md, design/10-error-handling.md, TIM-666.
+ */
+
+import { DenySignal } from './primitives.js';
+import type { DenyPageEntry } from './deny-page-resolver.js';
+import { renderMatchingDenyPage, setDenyStatus } from './deny-page-resolver.js';
+import { withSpan } from './tracing.js';
+
+/**
+ * Async server component that wraps a page call with DenySignal catching.
+ *
+ * Calls the page component as an async function, awaits it, and catches
+ * DenySignal. On catch, renders the deny page in-tree. On success, returns
+ * the page's rendered output normally.
+ */
+export async function PageDenyBoundary({
+  Page,
+  route,
+  denyPages,
+}: {
+  /** The page server component function. */
+  Page: (...args: unknown[]) => unknown;
+  /** Route path for OTEL tracing. */
+  route: string;
+  /** Deny page fallback chain from the segment chain. */
+  denyPages: DenyPageEntry[];
+}): Promise<React.ReactElement> {
+  try {
+    // Call the page as an async function — same as React Flight does.
+    // Wrap in OTEL span for tracing (replaces the TracedPage wrapper).
+    const result = await withSpan('timber.page', { 'timber.route': route }, () => Page({}));
+    return result as React.ReactElement;
+  } catch (error: unknown) {
+    if (error instanceof DenySignal) {
+      const denyElement = renderMatchingDenyPage(denyPages, error.status, error.data);
+      if (denyElement) {
+        setDenyStatus(error.status);
+        return denyElement;
+      }
+    }
+    // Non-deny errors (RedirectSignal, runtime errors) propagate normally.
+    throw error;
+  }
+}

package/src/server/pipeline-interception.ts

@@ -9,6 +9,8 @@
  * See design/07-routing.md §"Intercepting Routes"
  */
 
+import { classifyUrlSegment } from '../routing/segment-classify.js';
+
 /** Result of a successful interception match. */
 export interface InterceptionMatchResult {
   /** The pathname to re-match (the source/intercepting route's parent). */
@@ -26,7 +28,7 @@ export interface InterceptionMatchResult {
 export function findInterceptionMatch(
   targetPathname: string,
   sourceUrl: string,
-  rewrites: import('#/routing/interception.js').InterceptionRewrite[]
+  rewrites: import('../routing/interception.js').InterceptionRewrite[]
 ): InterceptionMatchResult | null {
   for (const rewrite of rewrites) {
     // Check if the source URL starts with the intercepting prefix
@@ -53,23 +55,22 @@ export function pathnameMatchesPattern(pathname: string, pattern: string): boole
 
   let pi = 0;
   for (let i = 0; i < patternParts.length; i++) {
-    const segment = patternParts[i];
-
-    // Catch-all: [...param] or [[...param]] — matches rest of URL
-    if (segment.startsWith('[...') || segment.startsWith('[[...')) {
-      return pi < pathParts.length || segment.startsWith('[[...');
-    }
+    const seg = classifyUrlSegment(patternParts[i]);
 
-    // Dynamic: [param] — matches any single segment
-    if (segment.startsWith('[') && segment.endsWith(']')) {
-      if (pi >= pathParts.length) return false;
-      pi++;
-      continue;
+    switch (seg.kind) {
+      case 'catch-all':
+        return pi < pathParts.length;
+      case 'optional-catch-all':
+        return true;
+      case 'dynamic':
+        if (pi >= pathParts.length) return false;
+        pi++;
+        continue;
+      case 'static':
+        if (pi >= pathParts.length || pathParts[pi] !== seg.value) return false;
+        pi++;
+        continue;
     }
-
-    // Static — must match exactly
-    if (pi >= pathParts.length || pathParts[pi] !== segment) return false;
-    pi++;
   }
 
   return pi === pathParts.length;

package/src/server/pipeline-metadata.ts

@@ -79,6 +79,19 @@ export function serializeSitemap(
   return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>`;
 }
 
+/**
+ * Serialize a sitemap index (list of sub-sitemap URLs) to XML.
+ * Used for pagination when the total URL count exceeds 50,000.
+ * Follows the sitemap.org protocol: https://www.sitemaps.org/protocol.html
+ */
+export function serializeSitemapIndex(sitemapUrls: string[]): string {
+  const sitemaps = sitemapUrls
+    .map((url) => `  <sitemap>\n    <loc>${escapeXml(url)}</loc>\n  </sitemap>`)
+    .join('\n');
+
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${sitemaps}\n</sitemapindex>`;
+}
+
 /** Escape special XML characters. */
 export function escapeXml(str: string): string {
   return str