@pagepocket/lib 0.13.0 → 0.14.5

package/dist/replay/templates/match-api-source.js

@@ -107,12 +107,25 @@ export const matchAPISource = `function matchAPI(options) {
         const recordPathSearch = stripTrailingSlash(toPathSearch(recordUrl));
         if (inputPathSearch === recordPathSearch)
             return true;
-        const inputPath = stripTrailingSlash(toPathname(inputUrl));
-        const recordPath = stripTrailingSlash(toPathname(recordUrl));
-        if (inputPath === recordPath)
-            return true;
         return false;
     };
+    const normalizePathLoose = (input) => {
+        const parsed = safeUrl(input);
+        if (parsed) {
+            return stripTrailingSlash(parsed.pathname);
+        }
+        const rawValue = String(input ?? "");
+        if (!rawValue) {
+            return "";
+        }
+        const valueWithoutHash = rawValue.split("#")[0] ?? "";
+        const valueWithoutQuery = valueWithoutHash.split("?")[0] ?? "";
+        const withoutOrigin = valueWithoutQuery.replace(/^([a-zA-Z][a-zA-Z\\d+\\-.]*:)?\\/\\/[^/]+/, "");
+        if (!withoutOrigin) {
+            return "/";
+        }
+        return stripTrailingSlash(withoutOrigin.startsWith("/") ? withoutOrigin : "/" + withoutOrigin);
+    };
     const scanRecords = (keyMethod, keyBody) => {
         for (const record of records || []) {
             if (!record || !record.url || !record.method)
@@ -128,10 +141,79 @@ export const matchAPISource = `function matchAPI(options) {
         }
         return undefined;
     };
+    const scanRecordsIgnoreQueryAndProtocol = (keyMethod, keyBody) => {
+        const inputPathname = normalizePathLoose(url);
+        if (!inputPathname) {
+            return undefined;
+        }
+        const candidates = [];
+        for (const record of records || []) {
+            if (!record || !record.url || !record.method)
+                continue;
+            if (record.method.toUpperCase() !== keyMethod)
+                continue;
+            const recordPathname = normalizePathLoose(record.url);
+            if (!recordPathname || recordPathname !== inputPathname)
+                continue;
+            const recordBody = record.requestBody || record.requestBodyBase64 || "";
+            if (keyBody && recordBody !== keyBody)
+                continue;
+            candidates.push(record);
+        }
+        if (candidates.length === 0)
+            return undefined;
+        if (candidates.length === 1) {
+            const candidate = candidates[0];
+            const inputParsed = safeUrl(url);
+            const candidateParsed = safeUrl(candidate.url);
+            if (inputParsed && candidateParsed) {
+                const inputKeys = Array.from(inputParsed.searchParams.keys());
+                const candidateKeys = new Set(candidateParsed.searchParams.keys());
+                const inputAllContained = inputKeys.length > 0 && inputKeys.every(k => candidateKeys.has(k));
+                const candidateHasExtra = candidateKeys.size > inputKeys.length;
+                if (inputAllContained && candidateHasExtra)
+                    return undefined;
+            }
+            return candidate;
+        }
+        const inputParsed = safeUrl(url);
+        const inputSearch = inputParsed ? inputParsed.search : "";
+        let bestRecord = undefined;
+        let bestScore = -Infinity;
+        for (const candidate of candidates) {
+            const candidateParsed = safeUrl(candidate.url);
+            const candidateSearch = candidateParsed ? candidateParsed.search : "";
+            if (candidateSearch === inputSearch)
+                return candidate;
+            const inputParams = inputParsed ? inputParsed.searchParams : new URLSearchParams();
+            const candidateParams = candidateParsed ? candidateParsed.searchParams : new URLSearchParams();
+            let shared = 0;
+            let recordOnly = 0;
+            for (const [key, value] of candidateParams.entries()) {
+                if (inputParams.get(key) === value) {
+                    shared++;
+                }
+                else {
+                    recordOnly++;
+                }
+            }
+            const score = shared - recordOnly * 2;
+            if (score > bestScore) {
+                bestScore = score;
+                bestRecord = candidate;
+            }
+        }
+        return bestRecord;
+    };
     for (const [keyMethod, keyBody] of matchOrder) {
         const record = scanRecords(keyMethod, keyBody);
         if (record)
             return record;
     }
+    for (const [keyMethod, keyBody] of matchOrder) {
+        const record = scanRecordsIgnoreQueryAndProtocol(keyMethod, keyBody);
+        if (record)
+            return record;
+    }
     return undefined;
 }`;

package/dist/replay/templates/replay-script-template.part-2.js

@@ -37,6 +37,25 @@ export const buildReplayInjectedScriptPart2 = (options) => {
     return snapshot;
   })();
 
+
+  // Synchronous fast-path: when the loader page has already set
+  // window.__pagepocketResourcesPath / window.__pagepocketApiSnapshot,
+  // build resourceIndex immediately so document.write URL rewriting
+  // works before the async ready promise resolves.
+  try {
+    var preloadedResources = window.__pagepocketResourcesPath;
+    if (preloadedResources && preloadedResources.version === "1.0" && Array.isArray(preloadedResources.items)) {
+      resourceIndex = __pagepocketResourceProxy.buildIndex(preloadedResources);
+    }
+  } catch (syncInitError) {}
+
+  try {
+    var preloadedApi = window.__pagepocketApiSnapshot;
+    if (preloadedApi && preloadedApi.version === "1.0" && Array.isArray(preloadedApi.records)) {
+      primeLookups(preloadedApi);
+    }
+  } catch (syncInitError) {}
+
   const isLocalResource = (value) => {
     if (!value) return false;
     if (value.startsWith("data:") || value.startsWith("blob:")) return true;
@@ -202,7 +221,11 @@ export const buildReplayInjectedScriptPart2 = (options) => {
   };
 
   const responseFromRecord = (record) => {
-    const headers = new Headers(record.responseHeaders || {});
+    var headers = new Headers();
+    var rawHeaders = record.responseHeaders || {};
+    for (var key in rawHeaders) {
+      try { headers.append(key, rawHeaders[key]); } catch (e) {}
+    }
     if (record.responseEncoding === "base64" && record.responseBodyBase64) {
       const bytes = decodeBase64(record.responseBodyBase64);
       return new Response(bytes, {

package/dist/resource-filter.js

@@ -4,17 +4,43 @@ const isSkippableUrl = (url) => url.startsWith("data:") ||
     url.startsWith("mailto:") ||
     url.startsWith("tel:") ||
     url.startsWith("javascript:");
+/**
+ * Checks whether a response mimeType represents a saveable binary asset.
+ *
+ * Some SPAs (e.g. Feishu, Notion) load images and fonts via `fetch()` or
+ * `XMLHttpRequest` instead of `<img>` tags.  Puppeteer marks those requests
+ * as `resourceType: "fetch"` / `"xhr"`, but the response is still an image
+ * or font that must be persisted for offline replay.
+ *
+ * We detect this by inspecting the response `mimeType`.
+ */
+const isSaveableMimeType = (mimeType) => {
+    const lower = mimeType.toLowerCase().split(";")[0].trim();
+    if (lower.startsWith("image/")) {
+        return true;
+    }
+    if (lower.startsWith("font/")) {
+        return true;
+    }
+    if (lower.startsWith("audio/") || lower.startsWith("video/")) {
+        return true;
+    }
+    if (lower === "application/font-woff" || lower === "application/font-woff2" || lower === "application/octet-stream") {
+        return true;
+    }
+    return false;
+};
 export const createDefaultResourceFilter = () => ({
     shouldSave(req, res) {
         if (isSkippableUrl(req.url)) {
             return false;
         }
-        if (req.resourceType && (req.resourceType === "fetch" || req.resourceType === "xhr")) {
-            return false;
-        }
         if (res && res.status >= 400) {
             return false;
         }
+        if (req.resourceType && (req.resourceType === "fetch" || req.resourceType === "xhr")) {
+            return res?.mimeType ? isSaveableMimeType(res.mimeType) : false;
+        }
         if (req.resourceType) {
             return DEFAULT_ALLOWED.has(req.resourceType);
         }

package/dist/snapshot-builder/build-snapshot.js

@@ -8,6 +8,7 @@ import { responseMimeType } from "./http.js";
 import { docDirFromUrl, resolveSnapshotPath } from "./path-map.js";
 import { buildResourcesPathSnapshot } from "./resources-path.js";
 import { maybeRewriteScript, maybeRewriteStylesheet } from "./rewrite-resource.js";
+import { buildLoaderHtml } from "../replay/templates/loader-template.js";
 export const buildSnapshot = async (input) => {
     const warnings = input.warnings;
     const contentStore = input.capture.contentStore;
@@ -46,10 +47,11 @@ export const buildSnapshot = async (input) => {
         const apiPath = ensureLeadingSlash(multiDoc ? `${sanitizePosixPath(docDir)}/api.json` : "/api.json");
         for (const resource of group.resources) {
             if (resource.request.resourceType === "document") {
-                const path = urlToPath.get(resource.request.url) ?? "/index.html";
+                const originalPath = urlToPath.get(resource.request.url) ?? "/index.html";
+                const pagePath = toPagePath(originalPath);
                 const { file } = await emitDocumentFile({
                     resource,
-                    path,
+                    path: pagePath,
                     entryUrl: input.entryUrl,
                     groupUrl: group.url,
                     apiPath,
@@ -60,8 +62,23 @@ export const buildSnapshot = async (input) => {
                     snapshotEntryUrl: input.entryUrl
                 });
                 files.push(file);
+                const loaderHtml = buildLoaderHtml({
+                    pageUrl: pagePath,
+                    apiPath,
+                    resourcesPathUrl: "/resources_path.json"
+                });
+                const loaderBytes = new TextEncoder().encode(loaderHtml);
+                const loaderRef = await contentStore.put({ kind: "buffer", data: loaderBytes }, { url: originalPath, mimeType: "text/html", sizeHint: loaderBytes.byteLength });
+                files.push({
+                    path: originalPath,
+                    mimeType: "text/html",
+                    size: loaderBytes.byteLength,
+                    source: loaderRef,
+                    originalUrl: resource.request.url,
+                    resourceType: "document"
+                });
                 if (resource.request.url === input.entryUrl || !entryPath) {
-                    entryPath = path;
+                    entryPath = originalPath;
                 }
                 continue;
             }
@@ -135,3 +152,16 @@ export const buildSnapshot = async (input) => {
     };
 };
 const resourceMimeType = (resource) => resource.mimeType || responseMimeType(resource.response) || undefined;
+/**
+ * Derive the "page" path from the original document path.
+ * `/index.html` → `/_page.html`, `/foo/bar.html` → `/foo/_bar.html`.
+ */
+const toPagePath = (originalPath) => {
+    const lastSlash = originalPath.lastIndexOf("/");
+    if (lastSlash === -1) {
+        return `_${originalPath}`;
+    }
+    const dir = originalPath.slice(0, lastSlash + 1);
+    const filename = originalPath.slice(lastSlash + 1);
+    return `${dir}_${filename}`;
+};

package/dist/units/runner.js

@@ -39,6 +39,7 @@ export const runCapture = async (input) => {
     const mergePatch = (_ctx, patch) => mergePatchIntoFreshContext(patch);
     let ctx = { value: {} };
     let result;
+    const unitErrors = [];
     const totalUnits = input.units.length;
     try {
         for (let i = 0; i < input.units.length; i++) {
@@ -102,6 +103,7 @@ export const runCapture = async (input) => {
             }
             catch (err) {
                 debugLog(`[runner] unit "${unit.id}" threw:`, err);
+                unitErrors.push({ unitId: unit.id, error: err });
                 rt.publish(PROGRESS, {
                     type: "unit:end",
                     unitId: unit.id,
@@ -136,6 +138,19 @@ export const runCapture = async (input) => {
         await rt._awaitDeferred();
     }
     if (!result) {
+        if (unitErrors.length > 0) {
+            const first = unitErrors[0];
+            const rootCause = first.error instanceof Error ? first.error : new Error(String(first.error));
+            if (unitErrors.length > 1) {
+                const summary = unitErrors
+                    .map((e) => `${e.unitId}: ${e.error instanceof Error ? e.error.message : String(e.error)}`)
+                    .join("; ");
+                const wrapped = new Error(`Pipeline failed: ${unitErrors.length} unit(s) threw errors. ${summary}`);
+                wrapped.cause = rootCause;
+                throw wrapped;
+            }
+            throw rootCause;
+        }
         throw new Error("No terminal result. Ensure a terminal unit returns a CaptureResult or calls finish().");
     }
     return result;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pagepocket/lib",
-  "version": "0.13.0",
+  "version": "0.14.5",
   "description": "Library for rewriting HTML snapshots and inlining local resources.",
   "type": "module",
   "main": "dist/index.js",
@@ -20,9 +20,9 @@
   "dependencies": {
     "cheerio": "^1.0.0-rc.12",
     "domhandler": "^5.0.3",
-    "@pagepocket/contracts": "0.13.0",
-    "@pagepocket/shared": "0.13.0",
-    "@pagepocket/uni-fs": "0.13.0"
+    "@pagepocket/contracts": "0.14.5",
+    "@pagepocket/uni-fs": "0.14.5",
+    "@pagepocket/shared": "0.14.5"
   },
   "devDependencies": {
     "@playwright/test": "^1.50.1",