@rangojs/router 0.0.0-experimental.bd6e11bc → 0.0.0-experimental.bdaf10aa

package/src/build/route-types/per-module-writer.ts

@@ -3,7 +3,10 @@ import ts from "typescript";
 import { extractParamsFromPattern } from "./param-extraction.js";
 import { extractRoutesFromSource } from "./ast-route-extraction.js";
 import { generatePerModuleTypesSource } from "./codegen.js";
-import { buildCombinedRouteMapWithSearch } from "./include-resolution.js";
+import {
+  buildCombinedRouteMapWithSearch,
+  createScanMemo,
+} from "./include-resolution.js";
 import type { ScanFilter } from "./scan-filter.js";
 import { findTsFiles } from "./scan-filter.js";
 
@@ -78,10 +81,20 @@ export function writePerModuleRouteTypesForFile(filePath: string): void {
     if (varNames.length > 0) {
       // Follow includes recursively via the combined route map builder.
       // The visited set in buildCombinedRouteMapWithSearch prevents infinite loops.
+      // Share one per-file scan memo across all urls() variables so a shared
+      // include target is read+parsed once for this file, not once per variable.
       routes = [];
+      const memo = createScanMemo();
       for (const varName of varNames) {
         const { routes: routeMap, searchSchemas } =
-          buildCombinedRouteMapWithSearch(filePath, varName);
+          buildCombinedRouteMapWithSearch(
+            filePath,
+            varName,
+            undefined,
+            undefined,
+            undefined,
+            memo,
+          );
         for (const [name, pattern] of Object.entries(routeMap)) {
           const params = extractParamsFromPattern(pattern);
           routes.push({

package/src/build/route-types/router-processing.ts

@@ -54,8 +54,33 @@ function isRoutableSourceFile(name: string): boolean {
   );
 }
 
-function findRouterFilesRecursive(
+function isExcludedScanDir(name: string): boolean {
+  return (
+    name === "node_modules" ||
+    name === "dist" ||
+    name === "coverage" ||
+    name === "__tests__" ||
+    name === "__mocks__" ||
+    name.startsWith(".")
+  );
+}
+
+/**
+ * Recursively collect source files whose code contains `pattern` (a comment- or
+ * string-only mention is ignored via firstCodeMatchIndex). Shared by createRouter
+ * and createHostRouter discovery, which differ only in the call pattern and
+ * `stopAtMatchDir`: createRouter treats a directory containing a match as a router
+ * root and stops descending it; createHostRouter descends the whole tree (the host
+ * entry sits above the sub-app router roots).
+ *
+ * `pattern` is the non-global tester (no lastIndex state); `patternG` is its global
+ * twin for the code-region scan.
+ */
+function findCallSiteFilesRecursive(
   dir: string,
+  pattern: RegExp,
+  patternG: RegExp,
+  stopAtMatchDir: boolean,
   filter: ScanFilter | undefined,
   results: string[],
 ): void {
@@ -70,21 +95,12 @@ function findRouterFilesRecursive(
   }
 
   const childDirs: string[] = [];
-  const routerFilesInDir: string[] = [];
+  const matchesInDir: string[] = [];
 
   for (const entry of entries) {
     const fullPath = join(dir, entry.name);
     if (entry.isDirectory()) {
-      if (
-        entry.name === "node_modules" ||
-        entry.name === "dist" ||
-        entry.name === "coverage" ||
-        entry.name === "__tests__" ||
-        entry.name === "__mocks__" ||
-        entry.name.startsWith(".")
-      )
-        continue;
-      childDirs.push(fullPath);
+      if (!isExcludedScanDir(entry.name)) childDirs.push(fullPath);
       continue;
     }
 
@@ -100,26 +116,29 @@ function findRouterFilesRecursive(
       // so a mention inside a comment or string is not mistaken for a real
       // router file (which previously triggered a spurious "Multiple routers
       // found" error).
-      if (
-        ROUTER_CALL_PATTERN.test(source) &&
-        firstCodeMatchIndex(source, ROUTER_CALL_PATTERN_G) >= 0
-      ) {
-        routerFilesInDir.push(fullPath);
+      if (pattern.test(source) && firstCodeMatchIndex(source, patternG) >= 0) {
+        matchesInDir.push(fullPath);
       }
     } catch {
       continue;
     }
   }
 
-  // A directory that contains a router file is treated as a router root.
-  // Once found, deeper directories are skipped to avoid redundant scans.
-  if (routerFilesInDir.length > 0) {
-    results.push(...routerFilesInDir);
-    return;
-  }
-
-  for (const childDir of childDirs) {
-    findRouterFilesRecursive(childDir, filter, results);
+  results.push(...matchesInDir);
+
+  // createRouter (stopAtMatchDir): a directory that contains a match is a router
+  // root, so deeper directories are skipped. createHostRouter: always descend.
+  if (!stopAtMatchDir || matchesInDir.length === 0) {
+    for (const childDir of childDirs) {
+      findCallSiteFilesRecursive(
+        childDir,
+        pattern,
+        patternG,
+        stopAtMatchDir,
+        filter,
+        results,
+      );
+    }
   }
 }
 
@@ -322,12 +341,6 @@ export function extractBasenameFromRouter(code: string): string | undefined {
   return result;
 }
 
-/** @deprecated Use extractUrlsFromRouter instead */
-export function extractUrlsVariableFromRouter(code: string): string | null {
-  const result = extractUrlsFromRouter(code);
-  return result?.kind === "variable" ? result.name : null;
-}
-
 /** Apply a basename prefix to all route patterns in a result set. */
 function applyBasenameToRoutes(
   result: {
@@ -558,7 +571,38 @@ export function detectUnresolvableIncludesForUrlsFile(
  */
 export function findRouterFiles(root: string, filter?: ScanFilter): string[] {
   const result: string[] = [];
-  findRouterFilesRecursive(root, filter, result);
+  findCallSiteFilesRecursive(
+    root,
+    ROUTER_CALL_PATTERN,
+    ROUTER_CALL_PATTERN_G,
+    true,
+    filter,
+    result,
+  );
+  return result;
+}
+
+const HOST_ROUTER_CALL_PATTERN = /\bcreateHostRouter\s*[<(]/;
+const HOST_ROUTER_CALL_PATTERN_G = /\bcreateHostRouter\s*[<(]/g;
+
+/**
+ * Scan for files containing createHostRouter() and return their paths. Unlike
+ * findRouterFiles, this does NOT stop at the first router-root directory -- a host
+ * entry typically sits above the sub-app router roots, so the whole tree is scanned.
+ */
+export function findHostRouterFiles(
+  root: string,
+  filter?: ScanFilter,
+): string[] {
+  const result: string[] = [];
+  findCallSiteFilesRecursive(
+    root,
+    HOST_ROUTER_CALL_PATTERN,
+    HOST_ROUTER_CALL_PATTERN_G,
+    false,
+    filter,
+    result,
+  );
   return result;
 }
 
@@ -617,9 +661,6 @@ export function writeCombinedRouteTypes(
       ? readFileSync(outPath, "utf-8")
       : null;
 
-    // When the static parser can't extract routes (e.g. callback-style urls()),
-    // write an empty placeholder so the build-time transform's injected import
-    // resolves. Runtime discovery will overwrite this with the real routes.
     if (Object.keys(result.routes).length === 0) {
       if (!existing) {
         const emptySource = generateRouteTypesSource({});
@@ -635,11 +676,6 @@ export function writeCombinedRouteTypes(
       hasSearchSchemas ? result.searchSchemas : undefined,
     );
     if (existing !== source) {
-      // On initial dev startup, don't overwrite a file from runtime discovery
-      // (which has all dynamic routes) with a smaller set from the static
-      // parser. The static parser can't see routes generated by Array.from()
-      // or other dynamic code. During HMR (file watcher), always write so
-      // newly added routes appear immediately.
       if (opts?.preserveIfLarger && existing) {
         const existingCount = countPublicRouteEntries(existing);
         const newCount = Object.keys(result.routes).filter(

package/src/build/route-types/source-scan.ts

@@ -13,12 +13,19 @@
 // Memory: O(1) for the boolean check; O(#matches) for the index list. No
 // stripped copy and no per-char array are ever materialized.
 //
-// Pragmatic scanner, not a full tokenizer: regex literals are not special-cased
-// (a target token inside one is implausible) and template interpolations are
-// treated as opaque string content. One intentional consequence: a token whose
-// match would only complete by treating an interleaved comment as whitespace
-// (e.g. `createRouter /* x */ (`) is not detected — real calls never interleave
-// a comment between the callee and its arguments.
+// Pragmatic scanner, not a full tokenizer: regex literals ARE coarsely skipped
+// (see below) and template interpolations are treated as opaque string content.
+// One intentional consequence: a token whose match would only complete by
+// treating an interleaved comment as whitespace (e.g. `createRouter /* x */ (`)
+// is not detected — real calls never interleave a comment between the callee
+// and its arguments.
+//
+// Regex literals are skipped because a literal containing a quote or comment
+// char (e.g. `const re = /it's a "x"/g;`) would otherwise open a phantom string
+// at the inner quote and swallow the following REAL code — dropping a router
+// file from discovery. We only treat a `/` as a regex start when it is in
+// "regex position" (the previous significant code char is not value-producing),
+// so genuine division (`a / b`) is left untouched.
 
 // JS line terminators end a `//` comment: LF, CR, LS (U+2028), PS (U+2029).
 function isLineTerminator(ch: string): boolean {
@@ -27,6 +34,56 @@ function isLineTerminator(ch: string): boolean {
   return c === 10 || c === 13 || c === 0x2028 || c === 0x2029;
 }
 
+// Identifier-position keywords after which a `/` begins a regex literal, not
+// division: the keyword cannot be the left operand of a division, so `return
+// /re/`, `typeof /re/`, `case /re/`, etc. are regexes. After any OTHER identifier
+// or number (a value), `/` is division.
+const REGEX_PRECEDING_KEYWORDS = new Set([
+  "return",
+  "typeof",
+  "instanceof",
+  "in",
+  "of",
+  "new",
+  "delete",
+  "void",
+  "do",
+  "else",
+  "yield",
+  "await",
+  "case",
+  "throw",
+]);
+
+// A `/` at `slashPos` is a regex-literal start (not division) when the previous
+// significant code char cannot end an expression. A closing `)`/`]`/`}` and an
+// identifier/digit/`$`/`_` are value-producing (division); everything else
+// (operators, `(`, `,`, `=`, `:`, `{`, `;`, `<`, `>`, ...) and the start-of-file
+// put `/` in regex position. The one subtlety: an identifier that is actually a
+// regex-preceding KEYWORD (`return /re/`) ends in a word char, so the
+// previous-char-only test misread it as division and then let the regex body's
+// inner quotes open a phantom string — dropping a later real `createRouter()`.
+// So when the previous char is a word char we walk back over any whitespace and
+// the identifier and treat `/` as a regex iff that identifier is such a keyword.
+// `}` stays value-producing to avoid swallowing an object/block followed by
+// division; the cost is only that a regex right after a block isn't skipped.
+function isRegexPositionAt(
+  code: string,
+  slashPos: number,
+  prevChar: string | undefined,
+): boolean {
+  if (prevChar === undefined) return true; // start of file
+  if (prevChar === ")" || prevChar === "]" || prevChar === "}") return false;
+  if (!/[\w$]/.test(prevChar)) return true; // operator / `(` / `,` / `=` / ...
+  // Previous char ends an identifier or number: regex only after a keyword that
+  // expects an expression. Walk back over whitespace + the identifier run.
+  let k = slashPos - 1;
+  while (k >= 0 && /\s/.test(code[k])) k--;
+  const wordEnd = k + 1;
+  while (k >= 0 && /[\w$]/.test(code[k])) k--;
+  return REGEX_PRECEDING_KEYWORDS.has(code.slice(k + 1, wordEnd));
+}
+
 /**
  * Build a classifier that answers "is offset `q` in code (not a comment or
  * string)?" for STRICTLY INCREASING `q`. The internal cursor only moves forward,
@@ -37,6 +94,9 @@ function makeCodeClassifier(code: string): (q: number) => boolean {
   let i = 0; // forward cursor: everything before `i` is already classified
   let skipStart = -1; // last detected comment/string region (cache)
   let skipEnd = -1;
+  // Last significant code char, used to disambiguate `/` (regex vs division).
+  // Comments are transparent (don't update it); strings/regex are value-producing.
+  let lastSig: string | undefined;
 
   return (q: number): boolean => {
     if (q >= skipStart && q < skipEnd) return false; // q in the cached region
@@ -44,14 +104,17 @@ function makeCodeClassifier(code: string): (q: number) => boolean {
       const c = code[i];
       const d = i + 1 < n ? code[i + 1] : "";
       let end = -1;
+      let transparent = false; // comment: skipped but does not set lastSig
       if (c === "/" && d === "/") {
         let j = i + 2;
         while (j < n && !isLineTerminator(code[j])) j++;
         end = j;
+        transparent = true;
       } else if (c === "/" && d === "*") {
         let j = i + 2;
         while (j < n && !(code[j] === "*" && code[j + 1] === "/")) j++;
         end = Math.min(n, j + 2);
+        transparent = true;
       } else if (c === '"' || c === "'" || c === "`") {
         let j = i + 1;
         while (j < n) {
@@ -66,16 +129,51 @@ function makeCodeClassifier(code: string): (q: number) => boolean {
           j++;
         }
         end = j;
+      } else if (
+        c === "/" &&
+        d !== "/" &&
+        d !== "*" &&
+        isRegexPositionAt(code, i, lastSig)
+      ) {
+        // Coarse regex-literal skip. A regex literal cannot span a raw newline;
+        // `/` inside a `[...]` character class is literal (not a terminator).
+        // Bail (treat the `/` as a normal char) if no closing `/` on the line
+        // so a stray division-looking `/` never swallows the rest of the line.
+        let j = i + 1;
+        let inClass = false;
+        let closed = false;
+        while (j < n && !isLineTerminator(code[j])) {
+          const r = code[j];
+          if (r === "\\") {
+            j += 2;
+            continue;
+          }
+          if (r === "[") inClass = true;
+          else if (r === "]") inClass = false;
+          else if (r === "/" && !inClass) {
+            j++;
+            closed = true;
+            break;
+          }
+          j++;
+        }
+        if (closed) {
+          while (j < n && /[a-z]/.test(code[j])) j++; // flags
+          end = j;
+        }
       }
       if (end >= 0) {
-        // Comment/string region [i, end). `q >= i` here (loop condition).
+        // Comment/string/regex region [i, end). `q >= i` here (loop condition).
         if (q < end) {
           skipStart = i;
           skipEnd = end;
           return false;
         }
         i = end;
+        // Strings and regex literals are value-producing; comments are not.
+        if (!transparent) lastSig = "x";
       } else {
+        if (!/\s/.test(c)) lastSig = c;
         i++;
       }
     }

package/src/build/runtime-discovery.ts

@@ -139,7 +139,10 @@ export async function discoverAndWriteRouteTypes(
     for (const [id, router] of registry) {
       if (!router.urlpatterns) continue;
 
-      const manifest = generateManifest(router.urlpatterns, routerMountIndex);
+      const manifest = await generateManifest(
+        router.urlpatterns,
+        routerMountIndex,
+      );
       routerMountIndex++;
 
       // Filter out auto-generated route names that the runtime creates for

package/src/cache/cache-error.ts

@@ -0,0 +1,104 @@
+/**
+ * Cache error reporting.
+ *
+ * Caches are best-effort: a read failure degrades to a miss (render fresh) and a
+ * write failure degrades to a no-op - they MUST NOT throw up and fail the
+ * request. But the failure must still be LOUD: it is logged to the console (so
+ * it is visible even in a background waitUntil task or when no hook is wired)
+ * AND routed through the router's onError callback (via the request context's
+ * deduped _reportBackgroundError) so consumers can observe cache degradation in
+ * their own telemetry.
+ *
+ * The one deliberate exception is the invalidation WRITE verb (updateTag ->
+ * store.invalidateTags): a failed durable marker write is rejected so an awaited
+ * updateTag() surfaces it (read-your-own-writes honesty). That is not a data
+ * read/write and does not go through this helper's swallow-and-degrade path.
+ */
+
+import { _getRequestContext } from "../server/request-context.js";
+
+/**
+ * Minimal shape of a request context for error reporting. Passed explicitly by
+ * background tasks (waitUntil) where the ALS context is already gone, so the
+ * error can still reach the router's onError. Structural to avoid importing the
+ * full RequestContext type (request-context.ts imports CacheErrorCategory from
+ * here - a mutual type-only reference).
+ */
+export interface CacheErrorReporter {
+  _reportBackgroundError?: (
+    error: unknown,
+    category: CacheErrorCategory,
+  ) => void;
+}
+
+export type CacheErrorCategory =
+  /** A read failed (transient infra: KV/Cache API error). Degrade to a miss. */
+  | "cache-read"
+  /** A write failed. Degrade to a no-op (entry simply not cached). */
+  | "cache-write"
+  /** A delete/eviction failed. Best-effort. */
+  | "cache-delete"
+  /**
+   * A STORED entry could not be parsed/deserialized (partial KV read, truncated
+   * Cache API body, malformed envelope/RSC payload). The entry is faulty and is
+   * evicted so subsequent reads do not keep failing on it. Distinct from
+   * cache-read so consumers can tell corruption from a transient outage.
+   */
+  | "cache-corrupt"
+  /** A tag-invalidation side effect failed (e.g. the eager CDN purge hook). */
+  | "cache-invalidate"
+  /**
+   * A background stale-while-revalidate refresh failed (the `"use cache"`
+   * read-through path). The stale value was already served; the refresh that
+   * would have replaced it errored.
+   */
+  | "stale-revalidation";
+
+/**
+ * Report a non-fatal cache error loudly without failing the request: always logs
+ * (label + error) and, when a request context is available, routes the error
+ * through the router's onError callback. Never throws.
+ *
+ * `ctx` is for callers running in a detached background task (waitUntil), where
+ * the ALS request context is already gone (so `_getRequestContext()` is null):
+ * they capture the context up front and pass it here so onError still fires.
+ * Foreground callers omit it and fall back to the ALS context.
+ */
+export function reportCacheError(
+  error: unknown,
+  category: CacheErrorCategory,
+  label: string,
+  ctx?: CacheErrorReporter,
+): void {
+  console.error(`${label}:`, error);
+  try {
+    const target = ctx ?? _getRequestContext();
+    target?._reportBackgroundError?.(error, category);
+  } catch {
+    // Reporting must never itself break the cache path.
+  }
+}
+
+/**
+ * Run a best-effort async cache task (typically scheduled via waitUntil), catching
+ * any rejection and routing it through reportCacheError so background cache work
+ * (non-blocking L1 writes, KV persistence, L1 promotion) reports failures via
+ * onError instead of throwing or silently swallowing. Never rejects.
+ *
+ * Pass `ctx` when the task runs detached (the ALS context is gone) and the
+ * failure should still reach onError; omit it to fall back to the ALS context.
+ *
+ * @example this.waitUntil(() => reportingAsync(() => cache.put(req, res), "cache-write", "[CFCacheStore] L1 write"))
+ */
+export async function reportingAsync(
+  task: () => Promise<unknown>,
+  category: CacheErrorCategory,
+  label: string,
+  ctx?: CacheErrorReporter,
+): Promise<void> {
+  try {
+    await task();
+  } catch (error) {
+    reportCacheError(error, category, label, ctx);
+  }
+}

package/src/cache/cache-key-utils.ts

@@ -6,24 +6,45 @@
  * document-cache, and loader-cache.
  */
 
+import { encodeKV } from "../encode-kv.js";
+
+/**
+ * Reserved URL query params that the router owns and must never key the cache
+ * on. `_rsc*` is the router's internal navigation/action/loader prefix (matched
+ * by prefix). `__no_cache` is the single `__`-prefixed param the router reads
+ * (handler.ts / testing dispatch.ts use it to bypass the store); it and the
+ * other router-internal `__`-prefixed request params are matched by an EXACT
+ * allowlist, not a blanket `__` prefix. A blanket `__` filter would silently
+ * collapse consumer params like `__variant=a` vs `__variant=b` onto one cache
+ * slot; an allowlist keeps the router's own params out of the key while leaving
+ * consumer `__` params intact.
+ */
+const RESERVED_SEARCH_PARAMS = new Set([
+  "__no_cache",
+  "__rsc",
+  "__html",
+  "__debug_manifest",
+  "__prerender_collect",
+]);
+
+function isReservedSearchParam(key: string): boolean {
+  return key.startsWith("_rsc") || RESERVED_SEARCH_PARAMS.has(key);
+}
+
 /**
  * Build a sorted, deterministic query string from URLSearchParams,
- * excluding internal _rsc* and __* params.
+ * excluding the router's reserved params (see isReservedSearchParam).
  *
  * Returns empty string when no user-facing params exist.
  */
 export function sortedSearchString(searchParams: URLSearchParams): string {
   const pairs: [string, string][] = [];
   for (const [k, v] of searchParams) {
-    if (!k.startsWith("_rsc") && !k.startsWith("__")) {
+    if (!isReservedSearchParam(k)) {
       pairs.push([k, v]);
     }
   }
-  if (pairs.length === 0) return "";
-  pairs.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
-  return pairs
-    .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
-    .join("&");
+  return encodeKV(pairs, { sort: true });
 }
 
 /**
@@ -35,10 +56,34 @@ export function sortedRouteParams(
   params: Record<string, string> | undefined,
 ): string {
   if (!params) return "";
-  const entries = Object.entries(params);
-  if (entries.length === 0) return "";
-  return entries
-    .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
-    .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`)
-    .join("&");
+  return encodeKV(Object.entries(params), { sort: true });
+}
+
+/**
+ * Host-namespaced cache key base: `${host}${pathname}[:params][?search]`.
+ *
+ * The ONE composition of the host-namespacing rule, shared by the segment tier
+ * (cache-scope.ts) and the document tier (document-cache.ts) so the rule cannot
+ * drift between them. Host prefixing matters because VercelCacheStore /
+ * MemorySegmentCacheStore key by the raw string (only CFCacheStore adds host
+ * internally) -- on a single function serving multiple domains an
+ * un-namespaced key bleeds tenant A's cached response to tenant B.
+ *
+ * Output is BYTE-STABLE by contract: changing the composition silently
+ * invalidates every persisted cache entry on upgrade. Callers append their own
+ * tier-specific suffixes (`:rsc`/`:html`, segment hash) after this base.
+ */
+export function cacheKeyBase(
+  host: string,
+  pathname: string,
+  searchParams?: URLSearchParams,
+  params?: Record<string, string>,
+): string {
+  const paramStr = sortedRouteParams(params);
+  const searchStr = searchParams ? sortedSearchString(searchParams) : "";
+
+  let key = `${host}${pathname}`;
+  if (paramStr) key += `:${paramStr}`;
+  if (searchStr) key += `?${searchStr}`;
+  return key;
 }