@pagepocket/lib 0.6.2 → 0.7.0

package/dist/types.d.ts

@@ -1,5 +1,7 @@
 import type { BodySource, NetworkInterceptorAdapter, NetworkRequestEvent, NetworkResponseEvent, ResourceType } from "@pagepocket/interceptor";
+import type { Cheerio, CheerioAPI } from "cheerio";
 export type { BodySource, InterceptOptions, InterceptSession, InterceptTarget, InterceptorActions, InterceptorCapabilities, NavigateOptions, NetworkEvent, NetworkEventHandlers, NetworkInterceptorAdapter, NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent, ResourceType, TriggerAction } from "@pagepocket/interceptor";
+import type { NetworkEvent } from "@pagepocket/interceptor";
 export interface PathResolver {
     resolve(input: {
         url: string;
@@ -47,20 +49,94 @@ export interface CompletionStrategy {
 }
 export interface PagePocketOptions {
 }
+export type NetworkEventStream = AsyncIterable<NetworkEvent>;
 export interface CaptureOptions {
     interceptor: NetworkInterceptorAdapter;
     completion?: CompletionStrategy | CompletionStrategy[];
+    /**
+     * Network idle duration (ms) used to determine capture completion.
+     *
+     * If `completion` is not provided, PagePocket will wait until the network has
+     * been idle (no inflight requests) for this duration.
+     *
+     * Note: this is NOT a wall-clock timeout from capture start.
+     */
+    timeoutMs?: number;
+    /**
+     * Hard wall-clock limit (ms) for the overall capture session.
+     *
+     * When `completion` is not provided, PagePocket will stop after either:
+     * - network has been idle for `timeoutMs`, OR
+     * - `maxDurationMs` has elapsed.
+     */
+    maxDurationMs?: number;
     filter?: ResourceFilter;
     pathResolver?: PathResolver;
     contentStore?: ContentStore;
     rewriteEntry?: boolean;
     rewriteCSS?: boolean;
+    blacklist?: RegExp[];
+    /**
+     * Replace parts of the captured HTML (Document response body) during the HTML
+     * rewrite stage (Cheerio).
+     */
+    replaceElements?: ReplaceElementsConfig;
     limits?: {
         maxTotalBytes?: number;
         maxSingleResourceBytes?: number;
         maxResources?: number;
     };
 }
+export type ReplaceElementsConfig = Array<ReplaceElementRule | ReplaceElementFn | ReplaceElementFnWithQuery>;
+export type MatchQuery = string | {
+    selector?: string;
+    tagName?: string;
+    id?: string;
+    attrs?: Record<string, string | RegExp | true>;
+};
+export type ReplaceAction = {
+    type: "replaceWithHtml";
+    html: string;
+} | {
+    type: "replaceWithElement";
+    tagName: string;
+    textContent?: string;
+    html?: string;
+    attrs?: Record<string, string | null>;
+} | {
+    type: "renameTag";
+    to: string;
+    keepAttributes?: boolean;
+    keepChildren?: boolean;
+} | {
+    type: "remove";
+};
+export interface ApplyOptions {
+    scope?: "document" | "allFrames";
+    limit?: number | "all";
+    onReplaced?: "stop" | "continue";
+}
+export interface ReplaceElementRule {
+    name?: string;
+    match: MatchQuery;
+    replace: ReplaceAction;
+    apply?: ApplyOptions;
+}
+export interface ReplaceElementContext {
+    $: CheerioAPI;
+    $el: Cheerio<any>;
+    url: string;
+    entryUrl: string;
+    ruleIndex: number;
+    matchIndex: number;
+}
+export type ReplaceElementFn = (ctx: ReplaceElementContext) => void | ReplaceAction | ReplaceAction[] | Promise<void | ReplaceAction | ReplaceAction[]>;
+export interface ReplaceElementFnWithQuery {
+    name?: string;
+    query: string;
+    run: ReplaceElementFn;
+    apply?: ApplyOptions;
+}
 export interface SnapshotFile {
     path: string;
     mimeType?: string;
@@ -84,19 +160,30 @@ export interface PageSnapshot {
     };
     content: ContentStoreHandle;
     toDirectory(outDir: string, options?: WriteFSOptions): Promise<WriteResult>;
-    toZip(options?: ZipOptions): Promise<Uint8Array | Blob>;
+    toZip(options?: ZipOptions): Promise<ZipResult>;
 }
 export interface WriteFSOptions {
     clearCache?: boolean;
+    overwrite?: boolean;
+    suffix?: string;
 }
 export interface WriteResult {
     filesWritten: number;
     totalBytes: number;
+    outputDir?: string;
 }
 export interface ZipOptions {
     asBlob?: boolean;
     clearCache?: boolean;
+    overwrite?: boolean;
+    suffix?: string;
+    outputPath?: string;
+}
+export interface ZipWriteResult {
+    data: Uint8Array | Blob;
+    outputPath: string;
 }
+export type ZipResult = Uint8Array | Blob | ZipWriteResult;
 export interface ApiRecord {
     url: string;
     method: string;

package/dist/writers.d.ts

@@ -1,3 +1,3 @@
-import type { PageSnapshot, WriteFSOptions, WriteResult, ZipOptions } from "./types";
+import type { PageSnapshot, WriteFSOptions, WriteResult, ZipOptions, ZipResult } from "./types";
 export declare const writeToFS: (snapshot: PageSnapshot, outDir: string, options?: WriteFSOptions) => Promise<WriteResult>;
-export declare const toZip: (snapshot: PageSnapshot, options?: ZipOptions) => Promise<Uint8Array | Blob>;
+export declare const toZip: (snapshot: PageSnapshot, options?: ZipOptions) => Promise<ZipResult>;

package/dist/writers.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.toZip = exports.writeToFS = void 0;
 const uni_fs_1 = require("@pagepocket/uni-fs");
 const utils_1 = require("./utils");
+const DEFAULT_SUFFIX_PATTERN = "_{num}";
 const normalizePath = (value) => value.replace(/\\/g, "/");
 const joinPath = (base, relative) => {
     const cleanBase = normalizePath(base).replace(/\/+$/, "");
@@ -24,6 +25,39 @@ const splitPathExtension = (value) => {
     }
     return { filename: clean, extension: "" };
 };
+const trimTrailingSlash = (value) => {
+    const normalized = normalizePath(value);
+    if (normalized === "/") {
+        return normalized;
+    }
+    return normalized.replace(/\/+$/, "");
+};
+const buildSuffix = (pattern, index) => {
+    const template = pattern ?? DEFAULT_SUFFIX_PATTERN;
+    return template.includes("{num}") ? template.replace("{num}", String(index)) : `${template}${index}`;
+};
+const appendDirectorySuffix = (basePath, suffix) => {
+    return `${trimTrailingSlash(basePath)}${suffix}`;
+};
+const appendFileSuffix = (basePath, suffix) => {
+    const { filename, extension } = splitPathExtension(basePath);
+    if (!extension) {
+        return `${filename}${suffix}`;
+    }
+    return `${filename}${suffix}.${extension}`;
+};
+const resolveUniquePath = async (basePath, options) => {
+    if (options.overwrite) {
+        return basePath;
+    }
+    const applySuffix = options.kind === "directory" ? appendDirectorySuffix : appendFileSuffix;
+    for (let index = 0;; index += 1) {
+        const candidate = index === 0 ? basePath : applySuffix(basePath, buildSuffix(options.suffix, index));
+        if (!(await (0, uni_fs_1.existsPath)(candidate, ""))) {
+            return candidate;
+        }
+    }
+};
 const streamToUint8Array = async (stream) => {
     const reader = stream.getReader();
     const chunks = [];
@@ -48,9 +82,14 @@ const streamToUint8Array = async (stream) => {
 const writeToFS = async (snapshot, outDir, options) => {
     let filesWritten = 0;
     let totalBytes = 0;
+    const outputDir = await resolveUniquePath(outDir, {
+        overwrite: options?.overwrite ?? false,
+        suffix: options?.suffix,
+        kind: "directory"
+    });
     for (const file of snapshot.files) {
         const relative = (0, utils_1.stripLeadingSlash)(file.path);
-        const outputPath = joinPath(outDir, relative);
+        const outputPath = joinPath(outputDir, relative);
         const { filename, extension } = splitPathExtension(outputPath);
         const stream = await snapshot.content.open(file.source);
         const data = await streamToUint8Array(stream);
@@ -61,7 +100,7 @@ const writeToFS = async (snapshot, outDir, options) => {
     if (options?.clearCache ?? true) {
         await snapshot.content.dispose?.();
     }
-    return { filesWritten, totalBytes };
+    return { filesWritten, totalBytes, outputDir };
 };
 exports.writeToFS = writeToFS;
 const crc32Table = (() => {
@@ -164,12 +203,25 @@ const toZip = async (snapshot, options) => {
         writeUint16(0)
     ]);
     const zipBytes = concatBytes([...localChunks, centralDirectory, endRecord]);
-    const output = options?.asBlob && typeof Blob !== "undefined"
+    const outputData = options?.asBlob && typeof Blob !== "undefined"
         ? new Blob([zipBytes], { type: "application/zip" })
         : zipBytes;
+    if (options?.outputPath) {
+        const outputPath = await resolveUniquePath(options.outputPath, {
+            overwrite: options?.overwrite ?? false,
+            suffix: options?.suffix,
+            kind: "file"
+        });
+        const { filename, extension } = splitPathExtension(outputPath);
+        await (0, uni_fs_1.write)(filename, extension, outputData);
+        if (options?.clearCache ?? true) {
+            await snapshot.content.dispose?.();
+        }
+        return { data: outputData, outputPath };
+    }
     if (options?.clearCache ?? true) {
         await snapshot.content.dispose?.();
     }
-    return output;
+    return outputData;
 };
 exports.toZip = toZip;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pagepocket/lib",
-  "version": "0.6.2",
+  "version": "0.7.0",
   "description": "Library for rewriting HTML snapshots and inlining local resources.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -12,8 +12,8 @@
   "license": "ISC",
   "dependencies": {
     "cheerio": "^1.0.0-rc.12",
-    "@pagepocket/uni-fs": "0.6.2",
-    "@pagepocket/interceptor": "0.6.2"
+    "@pagepocket/interceptor": "0.7.0",
+    "@pagepocket/uni-fs": "0.7.0"
   },
   "devDependencies": {
     "@playwright/test": "^1.50.1",