@pagepocket/lib 0.6.3 → 0.7.0

package/dist/hackers/replay-dom-rewrite.js

@@ -5,7 +5,9 @@ exports.replayDomRewriter = {
     id: "replay-dom-rewriter",
     stage: "replay",
     build: () => `
-  // Placeholder data URLs for missing resources.
+  // NOTE: When a resource is not found in the snapshot, we keep the original URL.
+  // (User choice) We still preserve a limited placeholder behavior for some tags,
+  // but only when we have a recorded response body to convert to a data URL.
   const transparentGif = "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==";
   const emptyScript = "data:text/javascript,/*pagepocket-missing*/";
   const emptyStyle = "data:text/css,/*pagepocket-missing*/";
@@ -93,7 +95,7 @@ exports.replayDomRewriter = {
       return;
     }
     const tag = (element.tagName || "").toLowerCase();
-    if (tag === "img" || tag === "source" || tag === "video" || tag === "audio" || tag === "script" || tag === "iframe") {
+    if (tag === "img" || tag === "source" || tag === "video" || tag === "audio" || tag === "script" || tag === "iframe" || tag === "object" || tag === "embed") {
       const src = currentSrc;
       if (src && !isLocalResource(src) && !src.startsWith("data:") && !src.startsWith("blob:")) {
         const localPath = findLocalPath(src);
@@ -102,8 +104,13 @@ exports.replayDomRewriter = {
           return;
         }
         const record = findByUrl(src);
-        const fallback = tag === "script" ? emptyScript : transparentGif;
-        element.setAttribute("src", record ? toDataUrl(record) : fallback);
+        if (record) {
+          element.setAttribute("src", toDataUrl(record));
+          return;
+        }
+
+        // Keep original URL when we have no snapshot match.
+        return;
       }
     }
 
@@ -117,8 +124,12 @@ exports.replayDomRewriter = {
           return;
         }
         const record = findByUrl(href);
-        const fallback = rel === "stylesheet" ? emptyStyle : emptyStyle;
-        element.setAttribute("href", record ? toDataUrl(record, "text/css") : fallback);
+        if (record) {
+          element.setAttribute("href", toDataUrl(record, "text/css"));
+          return;
+        }
+
+        return;
       }
     }
 
@@ -166,14 +177,9 @@ exports.replayDomRewriter = {
         const dataUrl = toDataUrl(record, fallbackType);
         return originalSetAttribute.call(this, name, dataUrl);
       }
-      if (attr === "src") {
-        const fallback = tag === "script" ? emptyScript : transparentGif;
-        return originalSetAttribute.call(this, name, fallback);
-      }
-      if (attr === "href") {
-        const fallback = relLower === "stylesheet" ? emptyStyle : emptyStyle;
-        return originalSetAttribute.call(this, name, fallback);
-      }
+
+      // Keep original URL on miss.
+      return originalSetAttribute.call(this, name, value);
     }
     return originalSetAttribute.call(this, name, value);
   };
@@ -207,8 +213,12 @@ exports.replayDomRewriter = {
           return;
         }
         const record = findByUrl(rawValue);
-        const next = record ? toDataUrl(record) : transparentGif;
-        setter.call(this, next);
+        if (record) {
+          setter.call(this, toDataUrl(record));
+          return;
+        }
+
+        setter.call(this, rawValue);
       });
       return;
     }
@@ -222,8 +232,12 @@ exports.replayDomRewriter = {
       return;
     }
     const record = findByUrl(rawValue);
-    const next = record ? toDataUrl(record) : transparentGif;
-    setter.call(this, next);
+    if (record) {
+      setter.call(this, toDataUrl(record));
+      return;
+    }
+
+    setter.call(this, rawValue);
   });
 
   patchProperty(HTMLScriptElement.prototype, "src", function(value, setter) {
@@ -240,8 +254,12 @@ exports.replayDomRewriter = {
           return;
         }
         const record = findByUrl(rawValue);
-        const next = record ? toDataUrl(record) : emptyScript;
-        setter.call(this, next);
+        if (record) {
+          setter.call(this, toDataUrl(record));
+          return;
+        }
+
+        setter.call(this, rawValue);
       });
       return;
     }
@@ -255,8 +273,13 @@ exports.replayDomRewriter = {
       return;
     }
     const record = findByUrl(rawValue);
-    const next = record ? toDataUrl(record) : emptyScript;
-    setter.call(this, next);
+    if (record) {
+      setter.call(this, toDataUrl(record));
+      return;
+    }
+
+    // Keep original URL on miss.
+    setter.call(this, rawValue);
   });
 
   patchProperty(HTMLLinkElement.prototype, "href", function(value, setter) {
@@ -275,8 +298,12 @@ exports.replayDomRewriter = {
           return;
         }
         const record = findByUrl(rawValue);
-        const next = record ? toDataUrl(record, relLower.includes("stylesheet") ? "text/css" : undefined) : emptyStyle;
-        setter.call(this, next);
+        if (record) {
+          setter.call(this, toDataUrl(record, relLower.includes("stylesheet") ? "text/css" : undefined));
+          return;
+        }
+
+        setter.call(this, rawValue);
       });
       return;
     }
@@ -290,8 +317,12 @@ exports.replayDomRewriter = {
       return;
     }
     const record = findByUrl(rawValue);
-    const next = record ? toDataUrl(record, relLower.includes("stylesheet") ? "text/css" : undefined) : emptyStyle;
-    setter.call(this, next);
+    if (record) {
+      setter.call(this, toDataUrl(record, relLower.includes("stylesheet") ? "text/css" : undefined));
+      return;
+    }
+
+    setter.call(this, rawValue);
   });
 
   patchProperty(HTMLImageElement.prototype, "srcset", function(value, setter) {

package/dist/index.d.ts

@@ -1,11 +1,13 @@
 export { PagePocket } from "./pagepocket";
-export type { ApiRecord, ApiSnapshot, CaptureOptions, CompletionContext, CompletionStrategy, ContentRef, ContentStore, ContentStoreHandle, NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent, PagePocketOptions, PageSnapshot, PathResolver, ResourceFilter, SnapshotFile, WriteFSOptions, WriteResult, ZipOptions } from "./types";
+export type { ApiRecord, ApiSnapshot, ApplyOptions, CaptureOptions, CompletionContext, CompletionStrategy, ContentRef, ContentStore, ContentStoreHandle, MatchQuery, NetworkEventStream, NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent, PagePocketOptions, PageSnapshot, PathResolver, ReplaceAction, ReplaceElementContext, ReplaceElementFn, ReplaceElementFnWithQuery, ReplaceElementRule, ReplaceElementsConfig, ResourceFilter, SnapshotFile, WriteFSOptions, WriteResult, ZipWriteResult, ZipResult, ZipOptions } from "./types";
 export { HybridContentStore } from "./content-store";
 export { createDefaultPathResolver, withPrefixPathResolver } from "./path-resolver";
 export { createDefaultResourceFilter } from "./resource-filter";
+export * from "./builtin-blacklist";
 export { networkIdle, timeout } from "./completion";
 export { buildReplayScript } from "./replay-script";
 export { buildPreloadScript } from "./preload";
 export { rewriteEntryHtml, rewriteJsText } from "./rewrite-links";
 export { rewriteCssText } from "./css-rewrite";
 export { writeToFS, toZip } from "./writers";
+export { applyReplaceElements } from "./replace-elements";

package/dist/index.js

@@ -1,6 +1,20 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.toZip = exports.writeToFS = exports.rewriteCssText = exports.rewriteJsText = exports.rewriteEntryHtml = exports.buildPreloadScript = exports.buildReplayScript = exports.timeout = exports.networkIdle = exports.createDefaultResourceFilter = exports.withPrefixPathResolver = exports.createDefaultPathResolver = exports.HybridContentStore = exports.PagePocket = void 0;
+exports.applyReplaceElements = exports.toZip = exports.writeToFS = exports.rewriteCssText = exports.rewriteJsText = exports.rewriteEntryHtml = exports.buildPreloadScript = exports.buildReplayScript = exports.timeout = exports.networkIdle = exports.createDefaultResourceFilter = exports.withPrefixPathResolver = exports.createDefaultPathResolver = exports.HybridContentStore = exports.PagePocket = void 0;
 var pagepocket_1 = require("./pagepocket");
 Object.defineProperty(exports, "PagePocket", { enumerable: true, get: function () { return pagepocket_1.PagePocket; } });
 var content_store_1 = require("./content-store");
@@ -10,6 +24,7 @@ Object.defineProperty(exports, "createDefaultPathResolver", { enumerable: true,
 Object.defineProperty(exports, "withPrefixPathResolver", { enumerable: true, get: function () { return path_resolver_1.withPrefixPathResolver; } });
 var resource_filter_1 = require("./resource-filter");
 Object.defineProperty(exports, "createDefaultResourceFilter", { enumerable: true, get: function () { return resource_filter_1.createDefaultResourceFilter; } });
+__exportStar(require("./builtin-blacklist"), exports);
 var completion_1 = require("./completion");
 Object.defineProperty(exports, "networkIdle", { enumerable: true, get: function () { return completion_1.networkIdle; } });
 Object.defineProperty(exports, "timeout", { enumerable: true, get: function () { return completion_1.timeout; } });
@@ -25,3 +40,5 @@ Object.defineProperty(exports, "rewriteCssText", { enumerable: true, get: functi
 var writers_1 = require("./writers");
 Object.defineProperty(exports, "writeToFS", { enumerable: true, get: function () { return writers_1.writeToFS; } });
 Object.defineProperty(exports, "toZip", { enumerable: true, get: function () { return writers_1.toZip; } });
+var replace_elements_1 = require("./replace-elements");
+Object.defineProperty(exports, "applyReplaceElements", { enumerable: true, get: function () { return replace_elements_1.applyReplaceElements; } });

package/dist/inflight-tracker.d.ts

@@ -0,0 +1,19 @@
+import type { NetworkEvent } from "@pagepocket/interceptor";
+type InflightStats = {
+    inflightRequests: number;
+    lastNetworkTs: number;
+    totalRequests: number;
+};
+export declare class InflightTracker {
+    private inflight;
+    private inflightRequests;
+    private lastNetworkTs;
+    private totalRequests;
+    private shouldIgnore;
+    constructor(options?: {
+        shouldIgnore?: (event: NetworkEvent) => boolean;
+    });
+    handleEvent(event: NetworkEvent): InflightStats;
+    getStats(): InflightStats;
+}
+export {};

package/dist/inflight-tracker.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InflightTracker = void 0;
+class InflightTracker {
+    constructor(options) {
+        this.inflight = new Set();
+        this.inflightRequests = 0;
+        this.lastNetworkTs = Date.now();
+        this.totalRequests = 0;
+        this.shouldIgnore = options?.shouldIgnore ?? (() => false);
+    }
+    handleEvent(event) {
+        if (this.shouldIgnore(event)) {
+            return this.getStats();
+        }
+        if (event?.timestamp) {
+            this.lastNetworkTs = event.timestamp;
+        }
+        else {
+            this.lastNetworkTs = Date.now();
+        }
+        if (event?.type === "request") {
+            this.totalRequests += 1;
+            if (!this.inflight.has(event.requestId)) {
+                this.inflight.add(event.requestId);
+                this.inflightRequests += 1;
+            }
+        }
+        if (event?.type === "response" || event?.type === "failed") {
+            if (this.inflight.delete(event.requestId)) {
+                this.inflightRequests = Math.max(0, this.inflightRequests - 1);
+            }
+        }
+        return {
+            inflightRequests: this.inflightRequests,
+            lastNetworkTs: this.lastNetworkTs,
+            totalRequests: this.totalRequests
+        };
+    }
+    getStats() {
+        return {
+            inflightRequests: this.inflightRequests,
+            lastNetworkTs: this.lastNetworkTs,
+            totalRequests: this.totalRequests
+        };
+    }
+}
+exports.InflightTracker = InflightTracker;

package/dist/pagepocket.d.ts

@@ -1,10 +1,12 @@
 import type { InterceptTarget } from "@pagepocket/interceptor";
-import type { CaptureOptions, PagePocketOptions, PageSnapshot } from "./types";
+import type { CaptureOptions, NetworkEventStream, PagePocketOptions, PageSnapshot } from "./types";
 export declare class PagePocket {
     private target;
     private options;
+    private eventStream;
     private constructor();
     static fromURL(url: string, options?: PagePocketOptions): PagePocket;
     static fromTarget(target: InterceptTarget, options?: PagePocketOptions): PagePocket;
+    interceptedRequestEvents(): NetworkEventStream;
     capture(options?: CaptureOptions): Promise<PageSnapshot>;
 }

package/dist/pagepocket.js

@@ -3,12 +3,111 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.PagePocket = void 0;
 const completion_1 = require("./completion");
 const content_store_1 = require("./content-store");
+const debug_1 = require("./debug");
+const inflight_tracker_1 = require("./inflight-tracker");
 const network_store_1 = require("./network-store");
 const path_resolver_1 = require("./path-resolver");
 const resource_filter_1 = require("./resource-filter");
 const snapshot_builder_1 = require("./snapshot-builder");
+class AsyncEventQueue {
+    constructor() {
+        this.values = [];
+        this.pending = [];
+        this.closed = false;
+    }
+    push(value) {
+        if (this.closed) {
+            return;
+        }
+        if (this.pending.length > 0) {
+            const resolve = this.pending.shift();
+            resolve?.({ kind: "value", value });
+            return;
+        }
+        this.values.push({ kind: "value", value });
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        if (this.pending.length > 0) {
+            for (const resolve of this.pending) {
+                resolve({ kind: "close" });
+            }
+            this.pending = [];
+            return;
+        }
+        this.values.push({ kind: "close" });
+    }
+    async *iterate() {
+        while (true) {
+            if (this.values.length > 0) {
+                const item = this.values.shift();
+                if (!item || item.kind === "close") {
+                    return;
+                }
+                yield item.value;
+                continue;
+            }
+            if (this.closed) {
+                return;
+            }
+            const item = await new Promise((resolve) => {
+                this.pending.push(resolve);
+            });
+            if (item.kind === "close") {
+                return;
+            }
+            yield item.value;
+        }
+    }
+}
+class BroadcastAsyncIterable {
+    constructor() {
+        this.queues = new Set();
+        this.closed = false;
+    }
+    subscribe() {
+        const queue = new AsyncEventQueue();
+        if (this.closed) {
+            queue.close();
+        }
+        this.queues.add(queue);
+        return (async function* (owner) {
+            try {
+                for await (const value of queue.iterate()) {
+                    yield value;
+                }
+            }
+            finally {
+                owner.queues.delete(queue);
+                queue.close();
+            }
+        })(this);
+    }
+    emit(value) {
+        if (this.queues.size === 0) {
+            return;
+        }
+        for (const queue of this.queues) {
+            queue.push(value);
+        }
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        for (const queue of this.queues) {
+            queue.close();
+        }
+        this.queues.clear();
+    }
+}
 class PagePocket {
     constructor(target, options) {
+        this.eventStream = new BroadcastAsyncIterable();
         this.target = target;
         this.options = options ?? {};
     }
@@ -18,77 +117,92 @@ class PagePocket {
     static fromTarget(target, options) {
         return new PagePocket(target, options);
     }
+    interceptedRequestEvents() {
+        return this.eventStream.subscribe();
+    }
     async capture(options) {
         if (!options?.interceptor) {
             throw new Error("CaptureOptions.interceptor is required.");
         }
         const contentStore = options?.contentStore ?? new content_store_1.HybridContentStore();
-        const filter = options?.filter ?? (0, resource_filter_1.createDefaultResourceFilter)();
+        const baseFilter = options?.filter ?? (0, resource_filter_1.createDefaultResourceFilter)();
+        const blacklist = options?.blacklist ?? [];
+        const isBlacklisted = (url) => typeof url === "string" && blacklist.some((pattern) => pattern.test(url));
+        const filter = blacklist.length === 0
+            ? baseFilter
+            : {
+                shouldSave(req, res) {
+                    if (isBlacklisted(req.url)) {
+                        (0, debug_1.debug_log)("[pagepocket] blacklist match (skip save)", req.url);
+                        return false;
+                    }
+                    return baseFilter.shouldSave(req, res);
+                }
+            };
         const pathResolver = options?.pathResolver ?? (0, path_resolver_1.createDefaultPathResolver)();
         const rewriteEntry = options?.rewriteEntry ?? true;
         const rewriteCSS = options?.rewriteCSS ?? true;
+        const replaceElements = options?.replaceElements;
         const limits = options?.limits;
         const completionStrategies = (0, completion_1.normalizeCompletion)(options?.completion);
-        const completion = completionStrategies.length > 0 ? completionStrategies : [(0, completion_1.networkIdle)(1000), (0, completion_1.timeout)(5000)];
+        const idleMs = options?.timeoutMs ?? 5000;
+        const maxDurationMs = options?.maxDurationMs;
+        const completion = completionStrategies.length > 0
+            ? completionStrategies
+            : [(0, completion_1.networkIdle)(idleMs), ...(maxDurationMs !== undefined ? [(0, completion_1.timeout)(maxDurationMs)] : [])];
         const store = new network_store_1.NetworkStore({
             contentStore,
             filter,
             limits
         });
-        const inflight = new Set();
-        let inflightRequests = 0;
-        let lastNetworkTs = Date.now();
-        let totalRequests = 0;
-        const pendingEvents = new Set();
-        const onEvent = (event) => {
-            if (event?.timestamp) {
-                lastNetworkTs = event.timestamp;
-            }
-            else {
-                lastNetworkTs = Date.now();
-            }
-            if (event?.type === "request") {
-                totalRequests += 1;
-                if (!inflight.has(event.requestId)) {
-                    inflight.add(event.requestId);
-                    inflightRequests += 1;
-                }
-            }
-            if (event?.type === "response" || event?.type === "failed") {
-                if (inflight.delete(event.requestId)) {
-                    inflightRequests = Math.max(0, inflightRequests - 1);
+        const inflightTracker = new inflight_tracker_1.InflightTracker({
+            shouldIgnore: (event) => {
+                if (!isBlacklisted(event.url)) {
+                    return false;
                 }
+                (0, debug_1.debug_log)("[pagepocket] blacklist match (ignore inflight)", event.url);
+                return true;
             }
+        });
+        const getStats = () => {
+            const stats = inflightTracker.getStats();
+            (0, debug_1.debug_log)("[pagepocket] stats", stats);
+            return stats;
+        };
+        const pendingEvents = new Set();
+        const onEvent = (event) => {
+            this.eventStream.emit(event);
+            const stats = inflightTracker.handleEvent(event);
+            (0, debug_1.debug_log)("[pagepocket] inflight", stats.inflightRequests, "event", event.type, event.requestId, event.url);
             const task = store.handleEvent(event);
             pendingEvents.add(task);
             task.finally(() => pendingEvents.delete(task));
         };
-        const session = await options.interceptor.start(this.target, { onEvent });
+        (0, debug_1.debug_log)("[pagepocket] session start");
+        const session = await options.interceptor.start(this.target, { onEvent }, { timeoutMs: options?.timeoutMs, maxDurationMs: options?.maxDurationMs });
+        (0, debug_1.debug_log)("[pagepocket] session started");
         if (this.target.kind === "url" && session?.navigate) {
+            (0, debug_1.debug_log)("[pagepocket] navigate", this.target.url);
             await session.navigate(this.target.url);
+            (0, debug_1.debug_log)("[pagepocket] navigated");
         }
         if (completion.length === 1) {
             await completion[0].wait({
                 now: () => Date.now(),
-                getStats: () => ({
-                    inflightRequests,
-                    lastNetworkTs,
-                    totalRequests
-                })
+                getStats
             });
         }
         else if (completion.length > 1) {
             await Promise.race(completion.map((strategy) => strategy.wait({
                 now: () => Date.now(),
-                getStats: () => ({
-                    inflightRequests,
-                    lastNetworkTs,
-                    totalRequests
-                })
+                getStats
             })));
         }
+        (0, debug_1.debug_log)("[pagepocket] session stop");
         await session.stop();
+        (0, debug_1.debug_log)("[pagepocket] session stopped");
         await Promise.all(pendingEvents);
+        this.eventStream.close();
         const entryUrl = this.target.kind === "url" ? this.target.url : "";
         return (0, snapshot_builder_1.buildSnapshot)({
             entryUrl,
@@ -99,6 +213,7 @@ class PagePocket {
             pathResolver,
             rewriteEntry,
             rewriteCSS,
+            replaceElements,
             warnings: store.getWarnings()
         });
     }

package/dist/replace-elements.d.ts

@@ -0,0 +1,9 @@
+import type { CheerioAPI } from "cheerio";
+import type { ReplaceElementsConfig } from "./types";
+export declare const applyReplaceElements: (input: {
+    $: CheerioAPI;
+    entryUrl: string;
+    url: string;
+    replaceElements?: ReplaceElementsConfig;
+    isEntryDocument: boolean;
+}) => Promise<void>;