mnfst-render 0.5.51 → 0.5.53

package/manifest.render.mjs

@@ -411,6 +411,8 @@ function parseArgs() {
     if (args[i] === '--retries' && args[i + 1]) { out.retries = parseInt(args[++i], 10); continue; }
     if (args[i] === '--incremental') { out.incremental = true; continue; }
     if (args[i] === '--no-incremental') { out.noIncremental = true; continue; }
+    if (args[i] === '--run-cache') { out.runCache = true; continue; }
+    if (args[i] === '--no-run-cache') { out.runCache = false; continue; }
     if (args[i] === '--dry-run') { out.dryRun = true; continue; }
     if (args[i] === '--force-lock') { out.forceLock = true; continue; }
     if (args[i] === '--debug-prerender') { out.debugPrerender = true; continue; }
@@ -524,6 +526,25 @@ function expandLocaleRoutePaths({ routeSegments, locales, defaultLocale, localeR
   return paths;
 }
 
+/**
+ * Incremental participation policy.  Recording is on whenever the project
+ * PARTICIPATES: config `render.incremental: true`, the --incremental flag, or
+ * a valid ledger already on disk (so one opted-in render upgrades every later
+ * plain run).  --no-incremental disables skipping only — a participating run
+ * still records and refreshes the ledger.  Explicit config `incremental:
+ * false` is fully off, ledger or not.  Never-opted-in stays fully inert.
+ */
+function resolveIncrementalPolicy({ configValue, flagIncremental, flagNoIncremental, ledgerExists }) {
+  const participates = flagIncremental === true
+    ? true
+    : configValue === false ? false : (configValue === true || !!ledgerExists);
+  return {
+    record: participates,
+    skip: participates && flagNoIncremental !== true,
+    explicit: flagIncremental === true || configValue === true,
+  };
+}
+
 function resolveConfig() {
   const cli = parseArgs();
   const cwd = process.cwd();
@@ -580,9 +601,23 @@ function resolveConfig() {
     paths: Array.isArray(pre.paths)
       ? pre.paths.map((p) => String(p).replace(/^\/+|\/+$/g, '')).filter(Boolean)
       : [],
-    // Incremental renders (opt-in): skip Puppeteer routes whose fingerprinted
-    // inputs are unchanged since the last successful run. --no-incremental wins.
-    incremental: cli.noIncremental ? false : !!(cli.incremental ?? pre.incremental ?? false),
+    // Incremental renders: see resolveIncrementalPolicy.  `incremental` gates
+    // skipping; `incrementalRecord` gates capture + ledger writes.
+    ...(() => {
+      const p = resolveIncrementalPolicy({
+        configValue: pre.incremental,
+        flagIncremental: cli.incremental,
+        flagNoIncremental: cli.noIncremental,
+        ledgerExists: loadRouteLedger(root) !== null,
+      });
+      return { incremental: p.skip, incrementalRecord: p.record, incrementalExplicit: p.explicit };
+    })(),
+    // Requests ABORTED during prerender (substring match, `*` glob within a
+    // pattern).  blockTelemetry (default on) adds TELEMETRY_BLOCK_PATTERNS.
+    blockRequests: Array.isArray(pre.blockRequests) ? pre.blockRequests.map((p) => String(p)).filter(Boolean) : [],
+    blockTelemetry: pre.blockTelemetry !== false,
+    // Run-scoped cross-origin response cache (render.runCache / --no-run-cache).
+    runCache: cli.runCache ?? pre.runCache !== false,
     dryRun: !!cli.dryRun,
     forceLock: !!cli.forceLock,
     debugPrerender: !!cli.debugPrerender,
@@ -3492,6 +3527,249 @@ const PINNED_VERSION_RE = /@\d+\.\d+\.\d+/;
 // Bodies above this are never fingerprinted — the route renders instead.
 const EXTERNAL_BODY_HASH_LIMIT = 8 * 1024 * 1024;
 
+// A prerender is not a visitor (Playcom measured 465 fake GA page_views per
+// render), and error/RUM reporters phone home from headless too (a widget
+// crash in headless spams the site's production error tracker every render).
+// Analytics + error/RUM collection endpoints only — never script loads.
+const TELEMETRY_BLOCK_PATTERNS = [
+  'google-analytics.com/g/collect',
+  '*.google-analytics.com/mp/collect',
+  'stats.g.doubleclick.net',
+  '*.analytics.google.com/g/collect',
+  'plausible.io/api/event',
+  'api.segment.io/v1',
+  'api.mixpanel.com/track',
+  '.facebook.com/tr?',
+  '.facebook.com/tr/',
+  'ingest.sentry.io',
+  'notify.bugsnag.com',
+  'api.rollbar.com',
+  'cloudflareinsights.com/cdn-cgi/rum',
+  'bam.nr-data.net',
+  'browser-intake-datadoghq.com',
+];
+
+/** URL predicate from block patterns: substring match, `*` globs within a
+ *  pattern.  null when there is nothing to block. */
+function compileBlockMatcher(patterns) {
+  const regs = [];
+  for (const p of patterns || []) {
+    const src = String(p).split('*').map((s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('.*');
+    if (src) regs.push(new RegExp(src));
+  }
+  if (!regs.length) return null;
+  return (url) => regs.some((r) => r.test(String(url)));
+}
+
+// --- Run-scoped cross-origin response cache ------------------------------------
+// Within one run every cross-origin GET is fetched AT MOST ONCE: the first 2xx
+// body+headers are replayed to every later page.  A prerender is a point-in-
+// time snapshot, so run-level consistency is a feature — and N pages
+// re-fetching identical framework assets manufactures the burst that fails.
+const RUN_CACHE_MAX_ENTRY_BYTES = EXTERNAL_BODY_HASH_LIMIT;
+const RUN_CACHE_MAX_TOTAL_BYTES = 64 * 1024 * 1024;
+const RUN_CACHE_MAX_ENTRIES = 500;
+
+/** Interception verdict: blocked URLs abort FIRST (never cached/served); only
+ *  cross-origin http(s) GETs are cache-eligible; everything else passes. */
+function classifyRunCacheRequest({ url, method, localOrigin, isBlocked }) {
+  if (isBlocked && isBlocked(url)) return 'block';
+  let u;
+  try { u = new URL(url); } catch { return 'bypass'; }
+  if (u.protocol !== 'http:' && u.protocol !== 'https:') return 'bypass';
+  if (method !== 'GET' || u.origin === localOrigin) return 'bypass';
+  return 'cache';
+}
+
+/** Headers safe to replay via request.respond: the stored body is the decoded
+ *  bytes, so the original transfer's encoding/length no longer apply.  ETag,
+ *  Last-Modified and Content-Type pass through verbatim — capture must observe
+ *  identical metadata on served and fetched responses. */
+function runCacheReplayHeaders(headers) {
+  const out = {};
+  for (const [k, v] of Object.entries(headers || {})) {
+    const key = k.toLowerCase();
+    if (key === 'content-encoding' || key === 'content-length'
+      || key === 'transfer-encoding' || key === 'set-cookie') continue;
+    out[k] = v;
+  }
+  return out;
+}
+
+function createRunCache({
+  maxEntryBytes = RUN_CACHE_MAX_ENTRY_BYTES,
+  maxTotalBytes = RUN_CACHE_MAX_TOTAL_BYTES,
+  maxEntries = RUN_CACHE_MAX_ENTRIES,
+} = {}) {
+  const entries = new Map(); // url → { status, headers, body }; insertion = recency
+  const inflight = new Map(); // url → { promise, settle }
+  let totalBytes = 0;
+  const stats = { served: 0, servedUrls: new Set(), stored: 0 };
+  return {
+    stats,
+    get size() { return entries.size; },
+    get totalBytes() { return totalBytes; },
+    /** Hit refreshes LRU recency. */
+    get(url) {
+      const e = entries.get(url);
+      if (!e) return null;
+      entries.delete(url); entries.set(url, e);
+      return e;
+    },
+    peek: (url) => entries.get(url) || null,
+    /** First 2xx wins; failures and over-cap bodies are never cached. */
+    set(url, entry) {
+      const { status, body } = entry || {};
+      if (!(status >= 200 && status < 300)) return false;
+      if (!body || typeof body.length !== 'number' || body.length > maxEntryBytes) return false;
+      if (entries.has(url)) return true;
+      entries.set(url, entry);
+      totalBytes += body.length;
+      stats.stored++;
+      while (entries.size > maxEntries || totalBytes > maxTotalBytes) {
+        const oldest = entries.keys().next().value;
+        totalBytes -= entries.get(oldest).body.length;
+        entries.delete(oldest);
+      }
+      return true;
+    },
+    noteServed(url) { stats.served++; stats.servedUrls.add(url); },
+    flight: (url) => inflight.get(url) || null,
+    /** One network fetch per URL at a time: concurrent pages await the first.
+     *  The timer settles null if nothing ever answers (page died mid-flight). */
+    beginFlight(url, timeoutMs = 30000) {
+      let resolve;
+      const promise = new Promise((r) => { resolve = r; });
+      const f = {
+        promise,
+        settle(entry) {
+          if (inflight.get(url) === f) inflight.delete(url);
+          clearTimeout(timer);
+          resolve(entry || null);
+        },
+      };
+      const timer = setTimeout(() => f.settle(null), timeoutMs);
+      timer.unref?.();
+      inflight.set(url, f);
+      return f;
+    },
+    settleFlight(url, entry) { inflight.get(url)?.settle(entry); },
+  };
+}
+
+/** Navigation budget: min(pageTimeout, 60s), keeping pageTimeout's 0=disabled. */
+const deriveNavTimeoutMs = (pageTimeoutMs) =>
+  (pageTimeoutMs > 0 ? Math.min(pageTimeoutMs, 60000) : 0);
+
+/** Cross-origin response → { validator } | { flag } | {} (2xx without
+ *  validator: content-address the body).  A 304 is the page's OWN conditional
+ *  revalidation answered "unchanged" — a hit, not a failure: the validator
+ *  comes from the 304's headers, else the request's conditional values.
+ *  A 304 has no body, so it never falls through to buffer(). */
+function classifyExternalResponse(status, responseHeaders = {}, requestHeaders = {}) {
+  if (status === 304) {
+    const etag = responseHeaders['etag'] || requestHeaders['if-none-match'];
+    if (etag) return { validator: { etag } };
+    const lastModified = responseHeaders['last-modified'] || requestHeaders['if-modified-since'];
+    if (lastModified) return { validator: { lastModified } };
+    return { flag: 'external 304 without validator' };
+  }
+  if (status < 200 || status >= 300) return { flag: `external ${status}` };
+  const etag = responseHeaders['etag'];
+  const lastModified = responseHeaders['last-modified'];
+  if (etag || lastModified) return { validator: etag ? { etag } : { lastModified } };
+  return {};
+}
+
+/**
+ * Incremental input-capture hooks over plain request/response shapes
+ * (Puppeteer-independent for unit tests).  Same-origin GETs map to disk files
+ * via the static server's own resolution; anything un-fingerprintable marks
+ * the route non-cacheable.  A blocked request never happened: not an input,
+ * not a failure, never awaited.
+ */
+function createCaptureHooks({ capture, localOrigin, resolveRequest, isBlocked }) {
+  const flag = (reason) => flagNonCacheable(capture, reason);
+  return {
+    request({ url, method }) {
+      try {
+        const u = new URL(url);
+        if (u.protocol !== 'http:' && u.protocol !== 'https:') return;
+        if (isBlocked && isBlocked(u.href)) return;
+        if (method !== 'GET') { flag(`non-GET ${method} ${u.href}`); return; }
+        if (u.origin !== localOrigin) {
+          // Pinned-version URL: the URL is its own identity.  Anything else
+          // must get a PROCESSED response (validator or content hash) by the
+          // end of the route or it stays in `awaiting` → route renders.
+          if (PINNED_VERSION_RE.test(u.pathname)) capture.externalPinned.add(u.href);
+          else capture.awaiting.add(u.href);
+          return;
+        }
+        const rel = resolveRequest(u.pathname);
+        if (rel) { capture.requested.set(u.pathname, rel); capture.files.add(rel); }
+        else flag(`unresolvable request ${u.pathname}`);
+      } catch { flag('unparseable request URL'); }
+    },
+    requestFailed({ url }) {
+      if (isBlocked && isBlocked(url)) return;
+      try { capture.awaiting.delete(new URL(url).href); } catch { /* flagged below */ }
+      flag(`request failed: ${url}`);
+    },
+    response({ url, status, headers, requestHeaders, buffer }) {
+      try {
+        const u = new URL(url);
+        if (u.protocol !== 'http:' && u.protocol !== 'https:') return;
+        if (isBlocked && isBlocked(u.href)) return;
+        if (u.origin === localOrigin || PINNED_VERSION_RE.test(u.pathname)) {
+          if (status >= 400) flag(`${status} ${u.href}`);
+          return;
+        }
+        const verdict = classifyExternalResponse(status, headers || {}, requestHeaders || {});
+        if (verdict.flag) { capture.awaiting.delete(u.href); flag(`${verdict.flag} ${u.href}`); return; }
+        if (verdict.validator) {
+          capture.externalValidated.set(u.href, verdict.validator);
+          capture.awaiting.delete(u.href);
+          return;
+        }
+        // No validator (e.g. cdn.manifestx.dev): content-address the bytes;
+        // revalidation re-downloads and compares.  Unreadable or oversized
+        // body → render.  `awaiting` clears only once the hash is recorded.
+        capture.pending.push(
+          Promise.resolve().then(buffer)
+            .then((body) => {
+              if (body.length > EXTERNAL_BODY_HASH_LIMIT) { flag(`external response too large to fingerprint: ${u.href}`); return; }
+              capture.externalValidated.set(u.href, { contentHash: sha256Full(body) });
+              capture.awaiting.delete(u.href);
+            })
+            .catch(() => flag(`external response unreadable: ${u.href}`))
+        );
+      } catch { flag('unparseable response URL'); }
+    },
+  };
+}
+
+// ALL distinct non-cacheable causes, not just the first: one render surfaces
+// every third party to block/fix instead of one per 65-minute run.
+const NONCACHEABLE_REASONS_CAP = 20;
+
+/** URLs in a cause reduced to host+path (no scheme/query) so per-page query
+ *  strings collapse into one actionable line. */
+function normalizeNonCacheableCause(reason) {
+  return String(reason).replace(/https?:\/\/[^\s]+/g, (m) => {
+    try { const u = new URL(m); return u.host + u.pathname; } catch { return m; }
+  });
+}
+
+/** `.reason` stays the first cause (ledger back-compat); `.reasons` collects
+ *  every distinct cause, ordered, deduped on the normalized form, capped. */
+function flagNonCacheable(capture, reason) {
+  if (!capture.nonCacheable) { capture.nonCacheable = true; capture.reason = reason; }
+  const list = capture.reasons || (capture.reasons = []);
+  if (list.length >= NONCACHEABLE_REASONS_CAP) return;
+  const key = normalizeNonCacheableCause(reason);
+  if (!list.some((r) => normalizeNonCacheableCause(r) === key)) list.push(reason);
+}
+
 const externalValidatorToken = (url, v) =>
   `${url}#etag:${v?.etag || ''}|lm:${v?.lastModified || ''}|ch:${v?.contentHash || ''}`;
 
@@ -3500,8 +3778,7 @@ const externalValidatorToken = (url, v) =>
  *  UNVERIFIED external dependency — the route must render. */
 function settleCaptureAwaiting(capture) {
   if (capture.awaiting && capture.awaiting.size > 0 && !capture.nonCacheable) {
-    capture.nonCacheable = true;
-    capture.reason = `unverified external: ${[...capture.awaiting][0]}`;
+    flagNonCacheable(capture, `unverified external: ${[...capture.awaiting][0]}`);
   }
   return capture;
 }
@@ -3656,7 +3933,7 @@ async function evaluateRouteSkip({ entry, hashInput, resolveRequest, revalidateE
  * Run-scoped incremental state: ledger, memoized hashing/resolution, the skip
  * pass, per-route capture records, and the end-of-run ledger write.
  */
-function createIncrementalRun({ rootResolved, previousOutput, stagingOutput, globalKey, deps }) {
+function createIncrementalRun({ rootResolved, previousOutput, stagingOutput, globalKey, deps, skipEnabled = true }) {
   const ledger = loadRouteLedger(rootResolved);
   const revalidate = createExternalRevalidator();
   const hashFileAt = (base, relPath) => {
@@ -3671,7 +3948,7 @@ function createIncrementalRun({ rootResolved, previousOutput, stagingOutput, glo
   };
   const captures = new Map(); // routeKey → capture record of a route rendered this run
   const carried = new Map();  // routeKey → previous entry revalidated and reused this run
-  const state = { disabled: false, skipped: 0, reasons: {} };
+  const state = { disabled: false, skipped: 0, reasons: {}, skipEnabled };
   const disable = (reason) => {
     if (state.disabled) return;
     state.disabled = true;
@@ -3702,6 +3979,7 @@ function createIncrementalRun({ rootResolved, previousOutput, stagingOutput, glo
     /** Skip pass: returns the subset of `paths` that must render; copies the
      *  previous outputs of the rest into staging and carries their entries. */
     async partition(paths) {
+      if (!skipEnabled) return paths; // recording-only run: render all, refresh the ledger
       if (!ledger) {
         for (let i = 0; i < paths.length; i++) noteRender('no ledger entry');
         return paths;
@@ -3768,7 +4046,13 @@ function createIncrementalRun({ rootResolved, previousOutput, stagingOutput, glo
       const routes = {};
       for (const [key, entry] of carried) routes[key] = entry;
       for (const [key, cap] of captures) {
-        if (cap.nonCacheable) { routes[key] = { nonCacheable: true, reason: cap.reason }; continue; }
+        if (cap.nonCacheable) {
+          routes[key] = { nonCacheable: true, reason: cap.reason };
+          // Additive: tolerated by the malformed-entry check (non-cacheable
+          // entries short-circuit before the shape test), so no schema bump.
+          if (Array.isArray(cap.reasons) && cap.reasons.length) routes[key].reasons = cap.reasons;
+          continue;
+        }
         const requested = {};
         const requestedPairs = [];
         for (const pathname of [...cap.requested.keys()].sort()) {
@@ -4275,6 +4559,9 @@ async function runPrerender(config) {
   // a wedged page.evaluate() may have poisoned the shared browser process.
   // pageTimeout is configurable via manifest.prerender.pageTimeout; 0 disables.
   const pageTimeoutMs = Math.max(0, pre.pageTimeout ?? Math.max(120000, (config.wait ?? 30000) * 4));
+  // Navigation shares the page budget up to 60s (was a hard 30s cap that
+  // failed runs while a 120s page budget sat unused); 0 stays disabled.
+  const navTimeoutMs = deriveNavTimeoutMs(pageTimeoutMs);
   // The recycle gate must never swap the browser out from under a live page,
   // so its drain wait has to outlast the per-page timeout above (every page
   // is guaranteed to release its slot by then). Configurable via
@@ -4437,15 +4724,22 @@ async function runPrerender(config) {
   // baseHtmlCache: base path segment → raw DOM HTML captured before any Node.js transforms
   const baseHtmlCache = new Map();
 
-  // --- Incremental skip pass (opt-in; see the incremental-rendering section) --
+  // --- Incremental skip pass (see the incremental-rendering section) ----------
   // Skips reuse files from the PREVIOUS final output, which still exists while
-  // this run builds into staging.  Off → no capture, no ledger, no new work.
+  // this run builds into staging.  Recording runs whenever the project
+  // participates (incrementalRecord); skipping only when `incremental` is on.
+  // Never-opted-in → no capture, no ledger, no new work.
   let inc = null;
-  if (config.incremental && !config.serve) {
+  if (config.incrementalRecord && !config.serve) {
     // Input capture maps request pathnames to disk via the built-in server's
     // resolution; an external --local server voids that mapping.
-    process.stdout.write('prerender: incremental needs the built-in static server — ignoring it for this run\n');
-  } else if (config.incremental) {
+    if (config.incrementalExplicit) {
+      process.stdout.write('prerender: incremental needs the built-in static server — ignoring it for this run\n');
+    }
+  } else if (config.incrementalRecord) {
+    if (!config.incrementalExplicit) {
+      process.stdout.write('prerender: incremental active from existing ledger (.mnfst-cache/routes.json; --no-incremental to disable)\n');
+    }
     let manifestContentHash = 'missing';
     try { manifestContentHash = sha256Full(readFileSync(join(config.root, 'manifest.json'))); } catch { /* missing */ }
     // Output-relevant resolved options (CLI included).  pathList is in here
@@ -4466,6 +4760,9 @@ async function runPrerender(config) {
       liveUrl: config.liveUrl,
       utilitiesBundle: bundleUtilities,
       tailwindBuilt,
+      // Blocking changes what a page loads; entries from another regime must not skip.
+      blockRequests: [...(config.blockRequests || [])].sort(),
+      blockTelemetry: config.blockTelemetry,
       pathList: [...pathList].sort(),
     }));
     inc = createIncrementalRun({
@@ -4481,8 +4778,29 @@ async function runPrerender(config) {
         getRawHtml: (seg) => baseHtmlCache.get(seg),
         pushUtilityBlock: (b) => utilityBlocks.push(b),
       },
+      skipEnabled: config.incremental,
     });
   }
+
+  // --- Request blocking during prerender ---------------------------------------
+  // Telemetry collection endpoints are blocked by default (blockTelemetry:
+  // false opts out); render.blockRequests adds user patterns.  Blocked
+  // requests are aborted via CDP and are invisible to incremental capture.
+  const blockPatterns = [
+    ...(config.blockTelemetry ? TELEMETRY_BLOCK_PATTERNS : []),
+    ...(config.blockRequests || []),
+  ];
+  const blockMatcher = compileBlockMatcher(blockPatterns);
+  const telemetryMatcher = config.blockTelemetry ? compileBlockMatcher(TELEMETRY_BLOCK_PATTERNS) : null;
+  const cdpBlockedUrls = blockPatterns.map((p) => `*${p}*`);
+  let telemetryBlockedCount = 0;
+
+  // Run cache (render.runCache, default on): one cross-origin fetch per URL per
+  // run, replayed to every later page.  With it active, blocking folds into the
+  // same interception path (Network.setBlockedURLs does not layer under
+  // request interception).
+  const runCache = config.runCache ? createRunCache() : null;
+
   const puppeteerRenderPaths = inc ? await inc.partition(puppeteerPaths) : puppeteerPaths;
   let incRendered = puppeteerRenderPaths.length;
   const puppeteerTotal = puppeteerRenderPaths.length;
@@ -4535,66 +4853,24 @@ async function runPrerender(config) {
       capture = {
         requested: new Map(), files: new Set(),
         externalPinned: new Set(), externalValidated: new Map(),
-        awaiting: new Set(), pending: [], nonCacheable: false, reason: null,
-      };
-      const localOrigin = new URL(config.localUrl).origin;
-      const flag = (reason) => {
-        if (!capture.nonCacheable) { capture.nonCacheable = true; capture.reason = reason; }
+        awaiting: new Set(), pending: [], nonCacheable: false, reason: null, reasons: [],
       };
-      page.on('request', (req) => {
-        try {
-          const u = new URL(req.url());
-          if (u.protocol !== 'http:' && u.protocol !== 'https:') return;
-          if (req.method() !== 'GET') { flag(`non-GET ${req.method()} ${u.href}`); return; }
-          if (u.origin !== localOrigin) {
-            // Pinned-version URL: the URL is its own identity.  Anything else
-            // must get a PROCESSED response (validator or content hash) by the
-            // end of the route or it stays in `awaiting` → route renders.
-            if (PINNED_VERSION_RE.test(u.pathname)) capture.externalPinned.add(u.href);
-            else capture.awaiting.add(u.href);
-            return;
-          }
-          const rel = resolveStaticRequestFile(rootResolved, u.pathname);
-          if (rel) { capture.requested.set(u.pathname, rel); capture.files.add(rel); }
-          else flag(`unresolvable request ${u.pathname}`);
-        } catch { flag('unparseable request URL'); }
-      });
-      page.on('requestfailed', (req) => {
-        try { capture.awaiting.delete(new URL(req.url()).href); } catch { /* flagged below */ }
-        flag(`request failed: ${req.url()}`);
-      });
-      page.on('response', (res) => {
-        try {
-          const u = new URL(res.url());
-          if (u.protocol !== 'http:' && u.protocol !== 'https:') return;
-          const status = res.status();
-          if (u.origin === localOrigin || PINNED_VERSION_RE.test(u.pathname)) {
-            if (status >= 400) flag(`${status} ${u.href}`);
-            return;
-          }
-          if (status < 200 || status >= 300) { flag(`external ${status} ${u.href}`); capture.awaiting.delete(u.href); return; }
-          const headers = res.headers();
-          const etag = headers['etag'];
-          const lastModified = headers['last-modified'];
-          if (etag || lastModified) {
-            capture.externalValidated.set(u.href, etag ? { etag } : { lastModified });
-            capture.awaiting.delete(u.href);
-            return;
-          }
-          // No validator (e.g. cdn.manifestx.dev): content-address the bytes;
-          // revalidation re-downloads and compares.  Unreadable or oversized
-          // body → render.  `awaiting` clears only once the hash is recorded.
-          capture.pending.push(
-            res.buffer()
-              .then((body) => {
-                if (body.length > EXTERNAL_BODY_HASH_LIMIT) { flag(`external response too large to fingerprint: ${u.href}`); return; }
-                capture.externalValidated.set(u.href, { contentHash: sha256Full(body) });
-                capture.awaiting.delete(u.href);
-              })
-              .catch(() => flag(`external response unreadable: ${u.href}`))
-          );
-        } catch { flag('unparseable response URL'); }
+      const hooks = createCaptureHooks({
+        capture,
+        localOrigin: new URL(config.localUrl).origin,
+        resolveRequest: (pathname) => resolveStaticRequestFile(rootResolved, pathname),
+        isBlocked: blockMatcher,
       });
+      page.on('request', (req) => hooks.request({ url: req.url(), method: req.method() }));
+      page.on('requestfailed', (req) => hooks.requestFailed({ url: req.url() }));
+      page.on('response', (res) => hooks.response({
+        url: res.url(), status: res.status(), headers: res.headers(),
+        requestHeaders: res.request().headers(),
+        buffer: () => res.buffer(),
+      }));
+    }
+    if (telemetryMatcher) {
+      page.on('requestfailed', (req) => { if (telemetryMatcher(req.url())) telemetryBlockedCount++; });
     }
     // Render at a typical desktop viewport so layouts dependent on viewport
     // width (responsive flex/grid, container queries, media queries) settle
@@ -4604,6 +4880,64 @@ async function runPrerender(config) {
     // viewport-driven flex distribution.
     await page.setViewport({ width: 1200, height: 800, deviceScaleFactor: 1 });
     try {
+      if (runCache) {
+        // Run cache via request interception.  Blocked URLs abort FIRST and are
+        // never cached or served; same-origin and non-GET traffic passes
+        // through untouched.  A served response is a real fulfilled response:
+        // the page.on capture listeners above observe it like a network one.
+        const pageLocalOrigin = new URL(config.localUrl).origin;
+        const done = (p) => p.catch(() => { /* page/request gone */ });
+        await page.setRequestInterception(true);
+        page.on('request', (req) => {
+          try {
+            const verdict = classifyRunCacheRequest({
+              url: req.url(), method: req.method(),
+              localOrigin: pageLocalOrigin, isBlocked: blockMatcher,
+            });
+            if (verdict === 'block') return done(req.abort('blockedbyclient'));
+            if (verdict === 'bypass') return done(req.continue());
+            const href = new URL(req.url()).href;
+            const hit = runCache.get(href);
+            if (hit) { runCache.noteServed(href); return done(req.respond(hit)); }
+            const flight = runCache.flight(href);
+            if (flight) {
+              return flight.promise.then((entry) => {
+                if (entry) { runCache.noteServed(href); return done(req.respond(entry)); }
+                return done(req.continue()); // first fetch failed: retry the network
+              });
+            }
+            runCache.beginFlight(href);
+            return done(req.continue());
+          } catch { return done(req.continue()); }
+        });
+        page.on('response', (res) => {
+          try {
+            if (res.request().method() !== 'GET') return;
+            const u = new URL(res.url());
+            if ((u.protocol !== 'http:' && u.protocol !== 'https:') || u.origin === pageLocalOrigin) return;
+            const prior = runCache.peek(u.href);
+            if (prior) return runCache.settleFlight(u.href, prior);
+            const status = res.status();
+            if (status < 200 || status >= 300) return runCache.settleFlight(u.href, null);
+            const headers = runCacheReplayHeaders(res.headers());
+            res.buffer()
+              .then((body) => {
+                const entry = { status, headers, body };
+                runCache.settleFlight(u.href, runCache.set(u.href, entry) ? entry : null);
+              })
+              .catch(() => runCache.settleFlight(u.href, null));
+          } catch { /* not a cacheable response */ }
+        });
+        page.on('requestfailed', (req) => {
+          try { runCache.settleFlight(new URL(req.url()).href, null); } catch { /* unparseable */ }
+        });
+      } else if (cdpBlockedUrls.length > 0) {
+        // CDP URL blocking (telemetry + render.blockRequests): requests abort at
+        // the network layer; coexists with the page.on capture listeners above.
+        const cdp = await page.createCDPSession();
+        await cdp.send('Network.enable');
+        await cdp.send('Network.setBlockedURLs', { urls: cdpBlockedUrls });
+      }
       // Align <html lang> with the URL being prerendered before any app script runs.
       // initializeDataSourcesPlugin picks locale from document.documentElement.lang first; a mismatch
       // (e.g. headless default vs /en/...) leaves $x.* empty while x-route sections still render.
@@ -4781,7 +5115,7 @@ async function runPrerender(config) {
       pushDebug({ path: displayPath, stage: 'start' });
       await page.goto(url, {
         waitUntil: 'domcontentloaded',
-        timeout: Math.min(timeout, 30000),
+        timeout: navTimeoutMs,
       });
 
       // Settle waits.  These give Manifest plugins (especially the components
@@ -5947,6 +6281,7 @@ async function runPrerender(config) {
           externalValidated: capture.externalValidated,
           nonCacheable: capture.nonCacheable,
           reason: capture.reason,
+          reasons: capture.reasons,
           outputs,
           utility: routeUtilityBlocks,
         });
@@ -6059,8 +6394,36 @@ async function runPrerender(config) {
   }
 
   if (inc) {
-    const why = Object.entries(inc.state.reasons).sort((a, b) => b[1] - a[1]).map(([r, c]) => `${c} ${r}`);
-    process.stdout.write(`prerender: skipped ${inc.state.skipped} unchanged, rendered ${incRendered}${why.length ? ` (${why.join(', ')})` : ''}\n`);
+    if (!inc.state.skipEnabled) {
+      process.stdout.write(`prerender: rendered ${incRendered} (ledger refreshed, skipping off)\n`);
+    } else {
+      const why = Object.entries(inc.state.reasons).sort((a, b) => b[1] - a[1]).map(([r, c]) => `${c} ${r}`);
+      process.stdout.write(`prerender: skipped ${inc.state.skipped} unchanged, rendered ${incRendered}${why.length ? ` (${why.join(', ')})` : ''}\n`);
+    }
+    // Every distinct non-cacheable cause across the run, host+path visible:
+    // what to block or fix, found in ONE render instead of serially.
+    const causes = new Map();
+    for (const cap of inc.captures.values()) {
+      if (!cap.nonCacheable) continue;
+      const list = Array.isArray(cap.reasons) && cap.reasons.length ? cap.reasons : [cap.reason];
+      for (const r of list) {
+        if (!r) continue;
+        const key = normalizeNonCacheableCause(r);
+        causes.set(key, (causes.get(key) || 0) + 1);
+      }
+    }
+    if (causes.size > 0) {
+      const rows = [...causes.entries()].sort((a, b) => b[1] - a[1]);
+      process.stdout.write('prerender: non-cacheable causes —\n');
+      for (const [cause, n] of rows.slice(0, 10)) process.stdout.write(`  ${n}× ${cause}\n`);
+      if (rows.length > 10) process.stdout.write(`  … +${rows.length - 10} more cause(s)\n`);
+    }
+  }
+  if (runCache && runCache.stats.served > 0) {
+    process.stdout.write(`prerender: run cache served ${runCache.stats.served} cross-origin requests (${runCache.stats.servedUrls.size} unique)\n`);
+  }
+  if (telemetryBlockedCount > 0) {
+    process.stdout.write(`prerender: blocked ${telemetryBlockedCount} telemetry requests (render.blockTelemetry: false to allow)\n`);
   }
 
   if (config.debugPrerender) {
@@ -6187,4 +6550,16 @@ export {
   isSafeRelPath,
   settleCaptureAwaiting,
   EXTERNAL_BODY_HASH_LIMIT,
+  resolveIncrementalPolicy,
+  classifyExternalResponse,
+  createCaptureHooks,
+  compileBlockMatcher,
+  TELEMETRY_BLOCK_PATTERNS,
+  createRunCache,
+  classifyRunCacheRequest,
+  runCacheReplayHeaders,
+  deriveNavTimeoutMs,
+  flagNonCacheable,
+  normalizeNonCacheableCause,
+  NONCACHEABLE_REASONS_CAP,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mnfst-render",
-  "version": "0.5.51",
+  "version": "0.5.53",
   "description": "Render Manifest sites to static HTML for SEO",
   "type": "module",
   "bin": {