@pagepocket/lib 0.6.3 → 0.7.0

package/dist/resource-proxy.js

@@ -0,0 +1,284 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveToLocalPath = exports.buildResourceProxyIndex = void 0;
+const addMulti = (map, key, value) => {
+    const existing = map.get(key);
+    if (!existing) {
+        map.set(key, value);
+        return;
+    }
+    if (Array.isArray(existing)) {
+        existing.push(value);
+        return;
+    }
+    map.set(key, [existing, value]);
+};
+const toArray = (value) => {
+    if (!value) {
+        return [];
+    }
+    return Array.isArray(value) ? value : [value];
+};
+const stripHash = (value) => {
+    const index = value.indexOf("#");
+    return index === -1 ? value : value.slice(0, index);
+};
+const stripTrailingSlash = (value) => {
+    if (!value || value === "/") {
+        return value;
+    }
+    return value.endsWith("/") ? value.slice(0, -1) : value;
+};
+const looksAlreadyEscapedForStaticServers = (value) => {
+    // Heuristic: if the path contains "%25XX" patterns, it was likely already
+    // escaped once ("%" -> "%25") to survive static-server decoding.
+    //
+    // This is intentionally conservative; double-escaping breaks lookups.
+    return /%25[0-9a-fA-F]{2}/.test(value);
+};
+const escapePercentForStaticServersOnce = (value) => {
+    if (!value) {
+        return value;
+    }
+    if (looksAlreadyEscapedForStaticServers(value)) {
+        return value;
+    }
+    return value.split("%").join("%25");
+};
+const isLikelyHostname = (value) => {
+    // Keep this loose; we only use it as a guard for embedded-URL detection.
+    if (!value)
+        return false;
+    if (value === "localhost")
+        return true;
+    return value.includes(".");
+};
+const encodeEmbeddedUrlTailIfPresent = (pathname) => {
+    // Some CDNs embed a full absolute URL into a single path segment using
+    // encodeURIComponent (e.g. ".../https%3A%2F%2Fexample.com%2Fa.png").
+    //
+    // Other runtimes may request the *decoded* form in-path (e.g.
+    // ".../https://example.com/a.png"), which changes path segments.
+    //
+    // To be resilient, detect an embedded absolute URL tail (http(s)://...) and
+    // produce an alternate pathname with that tail collapsed into one encoded
+    // segment.
+    const raw = String(pathname || "");
+    if (!raw.includes("/http")) {
+        return null;
+    }
+    const parts = raw.split("/");
+    for (let i = 0; i < parts.length; i += 1) {
+        const scheme = parts[i];
+        if (scheme !== "http:" && scheme !== "https:") {
+            continue;
+        }
+        // A real absolute URL in-path is typically split like:
+        //   ["...", "https:", "", "example.com", "a", "b.png"]
+        const hasDoubleSlash = parts[i + 1] === "";
+        const host = parts[i + 2] || "";
+        if (!hasDoubleSlash || !isLikelyHostname(host)) {
+            continue;
+        }
+        const embedded = scheme + "//" + parts.slice(i + 2).join("/");
+        const encoded = encodeURIComponent(embedded);
+        const nextParts = parts.slice(0, i).concat(encoded);
+        const rebuilt = nextParts.join("/") || "/";
+        return rebuilt.startsWith("/") ? rebuilt : "/" + rebuilt;
+    }
+    return null;
+};
+const makePathnameVariants = (pathname) => {
+    const variants = new Set();
+    const push = (value) => {
+        if (!value)
+            return;
+        variants.add(value);
+    };
+    push(pathname);
+    push(stripTrailingSlash(pathname));
+    const encodedTail = encodeEmbeddedUrlTailIfPresent(pathname);
+    if (encodedTail && encodedTail !== pathname) {
+        push(encodedTail);
+        push(stripTrailingSlash(encodedTail));
+    }
+    return Array.from(variants);
+};
+const getBasename = (pathname) => {
+    const clean = pathname.split("?")[0] || "";
+    const parts = clean.split("/").filter(Boolean);
+    return parts[parts.length - 1] || "";
+};
+const toUrlOrNull = (value) => {
+    try {
+        return new URL(value);
+    }
+    catch {
+        return null;
+    }
+};
+const buildResourceProxyIndex = (snapshot) => {
+    const byExactUrl = new Map();
+    const byPathnameWithSearch = new Map();
+    const byPathname = new Map();
+    const byBasename = new Map();
+    for (const item of snapshot.items || []) {
+        if (!item || !item.url || !item.path) {
+            continue;
+        }
+        const parsed = toUrlOrNull(item.url);
+        if (!parsed) {
+            continue;
+        }
+        const pathname = parsed.pathname || "/";
+        const pathnameWithSearch = pathname + (parsed.search || "");
+        const basename = getBasename(pathname);
+        const indexed = {
+            ...item,
+            parsed,
+            pathname,
+            pathnameWithSearch,
+            basename
+        };
+        // Prefer first-seen item for exact URL.
+        if (!byExactUrl.has(parsed.toString())) {
+            byExactUrl.set(parsed.toString(), indexed);
+        }
+        addMulti(byPathnameWithSearch, pathnameWithSearch, indexed);
+        addMulti(byPathname, pathname, indexed);
+        if (basename) {
+            addMulti(byBasename, basename, indexed);
+        }
+    }
+    return {
+        byExactUrl,
+        byPathnameWithSearch,
+        byPathname,
+        byBasename
+    };
+};
+exports.buildResourceProxyIndex = buildResourceProxyIndex;
+const uniqByPath = (items) => {
+    const seen = new Set();
+    const out = [];
+    for (const item of items) {
+        if (seen.has(item.path))
+            continue;
+        seen.add(item.path);
+        out.push(item);
+    }
+    return out;
+};
+const preferSingle = (items, baseUrl, suffixLength) => {
+    if (items.length <= 1) {
+        return items[0] ?? null;
+    }
+    const baseParsed = (() => {
+        try {
+            return new URL(baseUrl);
+        }
+        catch {
+            return null;
+        }
+    })();
+    if (baseParsed) {
+        const sameOrigin = items.filter((i) => i.parsed.origin === baseParsed.origin);
+        if (sameOrigin.length === 1) {
+            return sameOrigin[0];
+        }
+        if (sameOrigin.length > 1) {
+            items = sameOrigin;
+        }
+    }
+    // If still ambiguous, only accept when the match key is strong.
+    // We treat very short suffix matches (or basename-only) as too risky.
+    if (suffixLength < 2) {
+        return null;
+    }
+    return null;
+};
+const tryCandidates = (items, baseUrl, suffixLength) => {
+    const unique = uniqByPath(items);
+    if (unique.length === 0) {
+        return null;
+    }
+    if (unique.length === 1) {
+        return unique[0];
+    }
+    return preferSingle(unique, baseUrl, suffixLength);
+};
+const makeSuffixes = (pathname) => {
+    const parts = pathname.split("/").filter(Boolean);
+    const out = [];
+    for (let i = 0; i < parts.length; i += 1) {
+        const suffix = "/" + parts.slice(i).join("/");
+        out.push({ key: suffix, depth: parts.length - i });
+    }
+    return out;
+};
+const resolveToLocalPath = (options) => {
+    const { requestUrl, baseUrl, index } = options;
+    if (!requestUrl) {
+        return undefined;
+    }
+    let abs = null;
+    try {
+        abs = new URL(requestUrl, baseUrl);
+    }
+    catch {
+        abs = null;
+    }
+    if (!abs) {
+        return undefined;
+    }
+    const absString = abs.toString();
+    const exact = index.byExactUrl.get(absString);
+    if (exact) {
+        return escapePercentForStaticServersOnce(exact.path);
+    }
+    const withoutHash = stripHash(absString);
+    if (withoutHash !== absString) {
+        const found = index.byExactUrl.get(withoutHash);
+        if (found) {
+            return escapePercentForStaticServersOnce(found.path);
+        }
+    }
+    const pathname = abs.pathname || "/";
+    const pathnameVariants = makePathnameVariants(pathname);
+    const search = abs.search || "";
+    const pathnameWithSearchVariants = pathnameVariants.map((p) => p + search);
+    for (const key of pathnameWithSearchVariants) {
+        const items = toArray(index.byPathnameWithSearch.get(key));
+        const match = tryCandidates(items, baseUrl, 99);
+        if (match) {
+            return escapePercentForStaticServersOnce(match.path);
+        }
+    }
+    for (const key of pathnameVariants) {
+        const items = toArray(index.byPathname.get(key));
+        const match = tryCandidates(items, baseUrl, 99);
+        if (match) {
+            return escapePercentForStaticServersOnce(match.path);
+        }
+    }
+    // Suffix fallback: progressively remove leading segments.
+    for (const variant of pathnameVariants) {
+        for (const suffix of makeSuffixes(variant)) {
+            const items = toArray(index.byPathname.get(suffix.key));
+            const match = tryCandidates(items, baseUrl, suffix.depth);
+            if (match) {
+                return escapePercentForStaticServersOnce(match.path);
+            }
+        }
+    }
+    const basename = getBasename(pathname);
+    if (basename) {
+        const items = toArray(index.byBasename.get(basename));
+        const match = tryCandidates(items, baseUrl, 1);
+        if (match) {
+            return escapePercentForStaticServersOnce(match.path);
+        }
+    }
+    return undefined;
+};
+exports.resolveToLocalPath = resolveToLocalPath;

package/dist/rewrite-links.d.ts

@@ -1,3 +1,4 @@
+import type { ReplaceElementsConfig } from "./types";
 type UrlResolver = (absoluteUrl: string) => string | null;
 export declare const rewriteJsText: (source: string, resolve: UrlResolver, baseUrl: string) => Promise<string>;
 export declare const rewriteEntryHtml: (input: {
@@ -6,6 +7,13 @@ export declare const rewriteEntryHtml: (input: {
     apiPath: string;
     resolve: UrlResolver;
     rewriteLinks?: boolean;
+    replaceElements?: ReplaceElementsConfig;
+    isEntryDocument?: boolean;
+    /**
+     * The top-level entry URL for this snapshot (may differ from entryUrl when
+     * multiple documents/frames are captured).
+     */
+    snapshotEntryUrl?: string;
 }) => Promise<{
     html: string;
     title?: string;

package/dist/rewrite-links.js

@@ -37,6 +37,7 @@ exports.rewriteEntryHtml = exports.rewriteJsText = void 0;
 const cheerio = __importStar(require("cheerio"));
 const css_rewrite_1 = require("./css-rewrite");
 const hack_html_1 = require("./hack-html");
+const replace_elements_1 = require("./replace-elements");
 const shouldSkipValue = (value) => {
     const trimmed = value.trim();
     return (!trimmed ||
@@ -347,6 +348,13 @@ const rewriteEntryHtml = async (input) => {
         baseUrl: baseUrl,
         apiPath: input.apiPath
     });
+    await (0, replace_elements_1.applyReplaceElements)({
+        $,
+        entryUrl: input.snapshotEntryUrl ?? baseUrl,
+        url: baseUrl,
+        replaceElements: input.replaceElements,
+        isEntryDocument: input.isEntryDocument ?? true
+    });
     const title = $("title").first().text() || undefined;
     return { html: $.html(), title };
 };

package/dist/snapshot-builder.d.ts

@@ -1,5 +1,5 @@
 import type { ApiEntry, StoredResource } from "./network-store";
-import type { ContentStore, PageSnapshot, PathResolver } from "./types";
+import type { ContentStore, PageSnapshot, PathResolver, ReplaceElementsConfig } from "./types";
 type BuildOptions = {
     entryUrl: string;
     createdAt: number;
@@ -9,6 +9,7 @@ type BuildOptions = {
     pathResolver?: PathResolver;
     rewriteEntry: boolean;
     rewriteCSS: boolean;
+    replaceElements?: ReplaceElementsConfig;
     warnings: string[];
 };
 export declare const buildSnapshot: (input: BuildOptions) => Promise<PageSnapshot>;

package/dist/snapshot-builder.js

@@ -131,6 +131,42 @@ const buildApiSnapshot = (url, createdAt, entries) => ({
     createdAt,
     records: entries.map((entry) => entry.record)
 });
+const buildResourcesPathSnapshot = (createdAt, files) => {
+    const items = [];
+    for (const file of files) {
+        const originalUrl = file.originalUrl;
+        if (!originalUrl) {
+            continue;
+        }
+        // Exclude HTML documents. We only want static resources.
+        if (file.resourceType === "document") {
+            continue;
+        }
+        // Skip snapshot-local pseudo URLs.
+        if (originalUrl.startsWith("/")) {
+            continue;
+        }
+        // Be defensive: only include valid absolute URLs.
+        try {
+            new URL(originalUrl);
+        }
+        catch {
+            continue;
+        }
+        items.push({
+            url: originalUrl,
+            path: file.path,
+            resourceType: file.resourceType,
+            mimeType: file.mimeType,
+            size: file.size
+        });
+    }
+    return {
+        version: "1.0",
+        createdAt,
+        items
+    };
+};
 const buildSnapshot = async (input) => {
     const warnings = input.warnings;
     const groups = groupResources({
@@ -184,7 +220,10 @@ const buildSnapshot = async (input) => {
                     entryUrl: group.url,
                     apiPath,
                     resolve,
-                    rewriteLinks: input.rewriteEntry
+                    rewriteLinks: input.rewriteEntry,
+                    replaceElements: input.replaceElements,
+                    isEntryDocument: group.url === input.entryUrl,
+                    snapshotEntryUrl: input.entryUrl
                 });
                 html = rewritten.html;
                 if (!title) {
@@ -276,6 +315,17 @@ const buildSnapshot = async (input) => {
             originalUrl: apiPath
         });
     }
+    {
+        const resourcesPath = buildResourcesPathSnapshot(input.createdAt, files);
+        const bytes = new TextEncoder().encode(JSON.stringify(resourcesPath, null, 2));
+        // Snapshot-local artifact. It intentionally has no originalUrl.
+        files.push({
+            path: "/resources_path.json",
+            mimeType: "application/json",
+            size: bytes.byteLength,
+            source: { kind: "memory", data: bytes }
+        });
+    }
     const totalBytes = files.reduce((sum, file) => sum + (file.size ?? 0), 0);
     const totalFiles = files.length;
     const snapshotUrl = input.entryUrl || groups[0]?.url || "";

package/dist/types.d.ts

@@ -1,5 +1,7 @@
 import type { BodySource, NetworkInterceptorAdapter, NetworkRequestEvent, NetworkResponseEvent, ResourceType } from "@pagepocket/interceptor";
+import type { Cheerio, CheerioAPI } from "cheerio";
 export type { BodySource, InterceptOptions, InterceptSession, InterceptTarget, InterceptorActions, InterceptorCapabilities, NavigateOptions, NetworkEvent, NetworkEventHandlers, NetworkInterceptorAdapter, NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent, ResourceType, TriggerAction } from "@pagepocket/interceptor";
+import type { NetworkEvent } from "@pagepocket/interceptor";
 export interface PathResolver {
     resolve(input: {
         url: string;
@@ -47,20 +49,94 @@ export interface CompletionStrategy {
 }
 export interface PagePocketOptions {
 }
+export type NetworkEventStream = AsyncIterable<NetworkEvent>;
 export interface CaptureOptions {
     interceptor: NetworkInterceptorAdapter;
     completion?: CompletionStrategy | CompletionStrategy[];
+    /**
+     * Network idle duration (ms) used to determine capture completion.
+     *
+     * If `completion` is not provided, PagePocket will wait until the network has
+     * been idle (no inflight requests) for this duration.
+     *
+     * Note: this is NOT a wall-clock timeout from capture start.
+     */
+    timeoutMs?: number;
+    /**
+     * Hard wall-clock limit (ms) for the overall capture session.
+     *
+     * When `completion` is not provided, PagePocket will stop after either:
+     * - network has been idle for `timeoutMs`, OR
+     * - `maxDurationMs` has elapsed.
+     */
+    maxDurationMs?: number;
     filter?: ResourceFilter;
     pathResolver?: PathResolver;
     contentStore?: ContentStore;
     rewriteEntry?: boolean;
     rewriteCSS?: boolean;
+    blacklist?: RegExp[];
+    /**
+     * Replace parts of the captured HTML (Document response body) during the HTML
+     * rewrite stage (Cheerio).
+     */
+    replaceElements?: ReplaceElementsConfig;
     limits?: {
         maxTotalBytes?: number;
         maxSingleResourceBytes?: number;
         maxResources?: number;
     };
 }
+export type ReplaceElementsConfig = Array<ReplaceElementRule | ReplaceElementFn | ReplaceElementFnWithQuery>;
+export type MatchQuery = string | {
+    selector?: string;
+    tagName?: string;
+    id?: string;
+    attrs?: Record<string, string | RegExp | true>;
+};
+export type ReplaceAction = {
+    type: "replaceWithHtml";
+    html: string;
+} | {
+    type: "replaceWithElement";
+    tagName: string;
+    textContent?: string;
+    html?: string;
+    attrs?: Record<string, string | null>;
+} | {
+    type: "renameTag";
+    to: string;
+    keepAttributes?: boolean;
+    keepChildren?: boolean;
+} | {
+    type: "remove";
+};
+export interface ApplyOptions {
+    scope?: "document" | "allFrames";
+    limit?: number | "all";
+    onReplaced?: "stop" | "continue";
+}
+export interface ReplaceElementRule {
+    name?: string;
+    match: MatchQuery;
+    replace: ReplaceAction;
+    apply?: ApplyOptions;
+}
+export interface ReplaceElementContext {
+    $: CheerioAPI;
+    $el: Cheerio<any>;
+    url: string;
+    entryUrl: string;
+    ruleIndex: number;
+    matchIndex: number;
+}
+export type ReplaceElementFn = (ctx: ReplaceElementContext) => void | ReplaceAction | ReplaceAction[] | Promise<void | ReplaceAction | ReplaceAction[]>;
+export interface ReplaceElementFnWithQuery {
+    name?: string;
+    query: string;
+    run: ReplaceElementFn;
+    apply?: ApplyOptions;
+}
 export interface SnapshotFile {
     path: string;
     mimeType?: string;
@@ -84,19 +160,30 @@ export interface PageSnapshot {
     };
     content: ContentStoreHandle;
     toDirectory(outDir: string, options?: WriteFSOptions): Promise<WriteResult>;
-    toZip(options?: ZipOptions): Promise<Uint8Array | Blob>;
+    toZip(options?: ZipOptions): Promise<ZipResult>;
 }
 export interface WriteFSOptions {
     clearCache?: boolean;
+    overwrite?: boolean;
+    suffix?: string;
 }
 export interface WriteResult {
     filesWritten: number;
     totalBytes: number;
+    outputDir?: string;
 }
 export interface ZipOptions {
     asBlob?: boolean;
     clearCache?: boolean;
+    overwrite?: boolean;
+    suffix?: string;
+    outputPath?: string;
+}
+export interface ZipWriteResult {
+    data: Uint8Array | Blob;
+    outputPath: string;
 }
+export type ZipResult = Uint8Array | Blob | ZipWriteResult;
 export interface ApiRecord {
     url: string;
     method: string;

package/dist/writers.d.ts

@@ -1,3 +1,3 @@
-import type { PageSnapshot, WriteFSOptions, WriteResult, ZipOptions } from "./types";
+import type { PageSnapshot, WriteFSOptions, WriteResult, ZipOptions, ZipResult } from "./types";
 export declare const writeToFS: (snapshot: PageSnapshot, outDir: string, options?: WriteFSOptions) => Promise<WriteResult>;
-export declare const toZip: (snapshot: PageSnapshot, options?: ZipOptions) => Promise<Uint8Array | Blob>;
+export declare const toZip: (snapshot: PageSnapshot, options?: ZipOptions) => Promise<ZipResult>;

package/dist/writers.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.toZip = exports.writeToFS = void 0;
 const uni_fs_1 = require("@pagepocket/uni-fs");
 const utils_1 = require("./utils");
+const DEFAULT_SUFFIX_PATTERN = "_{num}";
 const normalizePath = (value) => value.replace(/\\/g, "/");
 const joinPath = (base, relative) => {
     const cleanBase = normalizePath(base).replace(/\/+$/, "");
@@ -24,6 +25,39 @@ const splitPathExtension = (value) => {
     }
     return { filename: clean, extension: "" };
 };
+const trimTrailingSlash = (value) => {
+    const normalized = normalizePath(value);
+    if (normalized === "/") {
+        return normalized;
+    }
+    return normalized.replace(/\/+$/, "");
+};
+const buildSuffix = (pattern, index) => {
+    const template = pattern ?? DEFAULT_SUFFIX_PATTERN;
+    return template.includes("{num}") ? template.replace("{num}", String(index)) : `${template}${index}`;
+};
+const appendDirectorySuffix = (basePath, suffix) => {
+    return `${trimTrailingSlash(basePath)}${suffix}`;
+};
+const appendFileSuffix = (basePath, suffix) => {
+    const { filename, extension } = splitPathExtension(basePath);
+    if (!extension) {
+        return `${filename}${suffix}`;
+    }
+    return `${filename}${suffix}.${extension}`;
+};
+const resolveUniquePath = async (basePath, options) => {
+    if (options.overwrite) {
+        return basePath;
+    }
+    const applySuffix = options.kind === "directory" ? appendDirectorySuffix : appendFileSuffix;
+    for (let index = 0;; index += 1) {
+        const candidate = index === 0 ? basePath : applySuffix(basePath, buildSuffix(options.suffix, index));
+        if (!(await (0, uni_fs_1.existsPath)(candidate, ""))) {
+            return candidate;
+        }
+    }
+};
 const streamToUint8Array = async (stream) => {
     const reader = stream.getReader();
     const chunks = [];
@@ -48,9 +82,14 @@ const streamToUint8Array = async (stream) => {
 const writeToFS = async (snapshot, outDir, options) => {
     let filesWritten = 0;
     let totalBytes = 0;
+    const outputDir = await resolveUniquePath(outDir, {
+        overwrite: options?.overwrite ?? false,
+        suffix: options?.suffix,
+        kind: "directory"
+    });
     for (const file of snapshot.files) {
         const relative = (0, utils_1.stripLeadingSlash)(file.path);
-        const outputPath = joinPath(outDir, relative);
+        const outputPath = joinPath(outputDir, relative);
         const { filename, extension } = splitPathExtension(outputPath);
         const stream = await snapshot.content.open(file.source);
         const data = await streamToUint8Array(stream);
@@ -61,7 +100,7 @@ const writeToFS = async (snapshot, outDir, options) => {
     if (options?.clearCache ?? true) {
         await snapshot.content.dispose?.();
     }
-    return { filesWritten, totalBytes };
+    return { filesWritten, totalBytes, outputDir };
 };
 exports.writeToFS = writeToFS;
 const crc32Table = (() => {
@@ -164,12 +203,25 @@ const toZip = async (snapshot, options) => {
         writeUint16(0)
     ]);
     const zipBytes = concatBytes([...localChunks, centralDirectory, endRecord]);
-    const output = options?.asBlob && typeof Blob !== "undefined"
+    const outputData = options?.asBlob && typeof Blob !== "undefined"
         ? new Blob([zipBytes], { type: "application/zip" })
         : zipBytes;
+    if (options?.outputPath) {
+        const outputPath = await resolveUniquePath(options.outputPath, {
+            overwrite: options?.overwrite ?? false,
+            suffix: options?.suffix,
+            kind: "file"
+        });
+        const { filename, extension } = splitPathExtension(outputPath);
+        await (0, uni_fs_1.write)(filename, extension, outputData);
+        if (options?.clearCache ?? true) {
+            await snapshot.content.dispose?.();
+        }
+        return { data: outputData, outputPath };
+    }
     if (options?.clearCache ?? true) {
         await snapshot.content.dispose?.();
     }
-    return output;
+    return outputData;
 };
 exports.toZip = toZip;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pagepocket/lib",
-  "version": "0.6.3",
+  "version": "0.7.0",
   "description": "Library for rewriting HTML snapshots and inlining local resources.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -12,8 +12,8 @@
   "license": "ISC",
   "dependencies": {
     "cheerio": "^1.0.0-rc.12",
-    "@pagepocket/interceptor": "0.6.3",
-    "@pagepocket/uni-fs": "0.6.3"
+    "@pagepocket/interceptor": "0.7.0",
+    "@pagepocket/uni-fs": "0.7.0"
   },
   "devDependencies": {
     "@playwright/test": "^1.50.1",