@pagepocket/lib 0.7.1 → 0.8.1

package/dist/core/_impl/inflight-tracker.d.ts

@@ -0,0 +1,19 @@
+import type { NetworkEvent } from "@pagepocket/interceptor";
+type InflightStats = {
+    inflightRequests: number;
+    lastNetworkTs: number;
+    totalRequests: number;
+};
+export declare class InflightTracker {
+    private inflight;
+    private inflightRequests;
+    private lastNetworkTs;
+    private totalRequests;
+    private shouldIgnore;
+    constructor(options?: {
+        shouldIgnore?: (event: NetworkEvent) => boolean;
+    });
+    handleEvent(event: NetworkEvent): InflightStats;
+    getStats(): InflightStats;
+}
+export {};

package/dist/core/_impl/inflight-tracker.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InflightTracker = void 0;
+class InflightTracker {
+    constructor(options) {
+        this.inflight = new Set();
+        this.inflightRequests = 0;
+        this.lastNetworkTs = Date.now();
+        this.totalRequests = 0;
+        this.shouldIgnore = options?.shouldIgnore ?? (() => false);
+    }
+    handleEvent(event) {
+        if (this.shouldIgnore(event)) {
+            return this.getStats();
+        }
+        if (event?.timestamp) {
+            this.lastNetworkTs = event.timestamp;
+        }
+        else {
+            this.lastNetworkTs = Date.now();
+        }
+        if (event?.type === "request") {
+            this.totalRequests += 1;
+            if (!this.inflight.has(event.requestId)) {
+                this.inflight.add(event.requestId);
+                this.inflightRequests += 1;
+            }
+        }
+        if (event?.type === "response" || event?.type === "failed") {
+            if (this.inflight.delete(event.requestId)) {
+                this.inflightRequests = Math.max(0, this.inflightRequests - 1);
+            }
+        }
+        return {
+            inflightRequests: this.inflightRequests,
+            lastNetworkTs: this.lastNetworkTs,
+            totalRequests: this.totalRequests
+        };
+    }
+    getStats() {
+        return {
+            inflightRequests: this.inflightRequests,
+            lastNetworkTs: this.lastNetworkTs,
+            totalRequests: this.totalRequests
+        };
+    }
+}
+exports.InflightTracker = InflightTracker;

package/dist/core/_impl/pagepocket.d.ts

@@ -0,0 +1,27 @@
+import type { NetworkEvent } from "@pagepocket/interceptor";
+import type { NetworkEventStream, PagePocketOptions } from "../../types";
+import type { CaptureWithPluginsOptions, PagePocketCaptureResult } from "../plugin/types";
+export declare class PagePocket {
+    private target;
+    private options;
+    private eventStream;
+    private constructor();
+    static fromURL(url: string, options?: PagePocketOptions): PagePocket;
+    static fromPuppeteerPage(page: unknown, options?: PagePocketOptions): PagePocket;
+    static fromCDPTab(tabId: number, options?: PagePocketOptions): PagePocket;
+    static fromHTMLString(htmlString: string, options: {
+        baseUrl: string;
+        url?: string;
+    } & PagePocketOptions): PagePocket;
+    interceptedRequestEvents(): NetworkEventStream;
+    /** @internal */
+    _emitNetworkEvent(event: NetworkEvent): void;
+    /** @internal */
+    _closeNetworkEventStream(): void;
+    static fromDocument(doc: unknown, options: {
+        baseUrl: string;
+        url?: string;
+        serialize?: (doc: unknown) => string;
+    } & PagePocketOptions): PagePocket;
+    capture(options: CaptureWithPluginsOptions): Promise<PagePocketCaptureResult>;
+}

package/dist/core/_impl/pagepocket.js

@@ -0,0 +1,155 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PagePocket = void 0;
+const runner_1 = require("../plugin/runner");
+class AsyncEventQueue {
+    constructor() {
+        this.values = [];
+        this.pending = [];
+        this.closed = false;
+    }
+    push(value) {
+        if (this.closed) {
+            return;
+        }
+        if (this.pending.length > 0) {
+            const resolve = this.pending.shift();
+            resolve?.({ kind: "value", value });
+            return;
+        }
+        this.values.push({ kind: "value", value });
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        if (this.pending.length > 0) {
+            for (const resolve of this.pending) {
+                resolve({ kind: "close" });
+            }
+            this.pending = [];
+            return;
+        }
+        this.values.push({ kind: "close" });
+    }
+    async *iterate() {
+        while (true) {
+            if (this.values.length > 0) {
+                const item = this.values.shift();
+                if (!item || item.kind === "close") {
+                    return;
+                }
+                yield item.value;
+                continue;
+            }
+            if (this.closed) {
+                return;
+            }
+            const item = await new Promise((resolve) => {
+                this.pending.push(resolve);
+            });
+            if (item.kind === "close") {
+                return;
+            }
+            yield item.value;
+        }
+    }
+}
+class BroadcastAsyncIterable {
+    constructor() {
+        this.queues = new Set();
+        this.closed = false;
+    }
+    subscribe() {
+        const queue = new AsyncEventQueue();
+        if (this.closed) {
+            queue.close();
+        }
+        this.queues.add(queue);
+        return (async function* (owner) {
+            try {
+                for await (const value of queue.iterate()) {
+                    yield value;
+                }
+            }
+            finally {
+                owner.queues.delete(queue);
+                queue.close();
+            }
+        })(this);
+    }
+    emit(value) {
+        if (this.queues.size === 0) {
+            return;
+        }
+        for (const queue of this.queues) {
+            queue.push(value);
+        }
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        for (const queue of this.queues) {
+            queue.close();
+        }
+        this.queues.clear();
+    }
+}
+class PagePocket {
+    constructor(target, options) {
+        this.eventStream = new BroadcastAsyncIterable();
+        this.target = target;
+        this.options = options ?? {};
+    }
+    static fromURL(url, options) {
+        return new PagePocket({ kind: "url", url }, options);
+    }
+    static fromPuppeteerPage(page, options) {
+        return new PagePocket({ kind: "puppeteer-page", page }, options);
+    }
+    static fromCDPTab(tabId, options) {
+        return new PagePocket({ kind: "cdp-tab", tabId }, options);
+    }
+    static fromHTMLString(htmlString, options) {
+        const { baseUrl, url, ...rest } = options;
+        return new PagePocket({ kind: "html", htmlString, baseUrl, ...(url ? { url } : {}) }, rest);
+    }
+    interceptedRequestEvents() {
+        return this.eventStream.subscribe();
+    }
+    /** @internal */
+    _emitNetworkEvent(event) {
+        this.eventStream.emit(event);
+    }
+    /** @internal */
+    _closeNetworkEventStream() {
+        this.eventStream.close();
+    }
+    static fromDocument(doc, options) {
+        const serialize = options.serialize ??
+            ((value) => {
+                if (value && typeof value === "object" && "documentElement" in value) {
+                    const el = value.documentElement;
+                    if (el && typeof el.outerHTML === "string") {
+                        return el.outerHTML;
+                    }
+                }
+                throw new Error("fromDocument requires options.serialize(doc) or doc.documentElement.outerHTML");
+            });
+        const htmlString = serialize(doc);
+        const { baseUrl, url, serialize: _serialize, ...rest } = options;
+        return new PagePocket({ kind: "html", htmlString, baseUrl, ...(url ? { url } : {}) }, rest);
+    }
+    async capture(options) {
+        return (0, runner_1.captureWithPlugins)({
+            pocket: this,
+            pocketOptions: this.options,
+            target: this.target,
+            options
+        });
+    }
+}
+exports.PagePocket = PagePocket;

package/dist/core/capture/_impl/memory-content-store.d.ts

@@ -0,0 +1,4 @@
+import type { ContentStore } from "../../../types";
+export declare const createMemoryContentStore: (name?: string) => (ContentStore & {
+    _bytes: Map<string, Uint8Array>;
+}) | ContentStore;

package/dist/core/capture/_impl/memory-content-store.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createMemoryContentStore = void 0;
+const utils_1 = require("../../../utils");
+const streams_1 = require("../../../utils/streams");
+const createMemoryContentStore = (name = "memory") => {
+    const bytesById = new Map();
+    let nextId = 1;
+    const storeBytes = (bytes) => {
+        const id = String(nextId);
+        nextId += 1;
+        bytesById.set(id, bytes);
+        return id;
+    };
+    const getStoredBytes = (id) => {
+        const bytes = bytesById.get(id);
+        if (!bytes) {
+            throw new Error(`Missing memory content for ref id=${id}`);
+        }
+        return bytes;
+    };
+    return {
+        name,
+        _bytes: bytesById,
+        async put(body, _meta) {
+            const bytes = await (0, utils_1.toUint8Array)(body);
+            const id = storeBytes(bytes);
+            return { kind: "store-ref", id };
+        },
+        async open(ref) {
+            if (ref.kind === "memory") {
+                return (0, streams_1.streamFromUint8Array)(ref.data);
+            }
+            const bytes = getStoredBytes(ref.id);
+            return (0, streams_1.streamFromUint8Array)(bytes);
+        },
+        async dispose() {
+            bytesById.clear();
+        }
+    };
+};
+exports.createMemoryContentStore = createMemoryContentStore;

package/dist/core/capture/_impl/types.d.ts

@@ -0,0 +1,61 @@
+import type { ContentRef, ContentStore, ResourceType } from "../../../types";
+export type Header = {
+    name: string;
+    value: string;
+};
+export type CaptureBodyRef = ContentRef;
+export type CaptureCapabilities = {
+    requestHeaders: "sent" | "approx" | "none";
+    responseHeaders: "raw" | "approx" | "none";
+    requestBodies: boolean;
+    responseBodies: "decoded" | "wire" | "none";
+    httpVersion: boolean;
+    remoteIp: boolean;
+    headerOrderPreserved: boolean;
+};
+export type CaptureRequestEvent = {
+    type: "http.request";
+    requestId: string;
+    url: string;
+    method: string;
+    headers: Header[];
+    timestamp: number;
+    frameId?: string;
+    resourceType?: ResourceType;
+    initiator?: {
+        type?: string;
+        url?: string;
+    };
+};
+export type CaptureResponseEvent = {
+    type: "http.response";
+    requestId: string;
+    url: string;
+    status: number;
+    statusText?: string;
+    headers: Header[];
+    timestamp: number;
+    mimeType?: string;
+    fromDiskCache?: boolean;
+    fromServiceWorker?: boolean;
+    bodyRef?: CaptureBodyRef;
+    bodySize?: number;
+};
+export type CaptureFailedEvent = {
+    type: "http.failed";
+    requestId: string;
+    url: string;
+    errorText: string;
+    timestamp: number;
+};
+export type CaptureEvent = CaptureRequestEvent | CaptureResponseEvent | CaptureFailedEvent;
+export type CaptureArtifacts = {
+    /** Ordered list of captured network events. */
+    events: CaptureEvent[];
+    /** Content store containing captured bodies referenced by events. */
+    contentStore: ContentStore;
+    capabilities: CaptureCapabilities;
+};
+export type CaptureContext = {
+    capture?: CaptureArtifacts;
+};

package/dist/core/capture/_impl/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/capture/internal/memory-content-store.d.ts

@@ -0,0 +1,4 @@
+import type { ContentStore } from "../../../types";
+export declare const createMemoryContentStore: (name?: string) => (ContentStore & {
+    _bytes: Map<string, Uint8Array>;
+}) | ContentStore;

package/dist/core/capture/internal/memory-content-store.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createMemoryContentStore = void 0;
+const utils_1 = require("../../../utils");
+const streams_1 = require("../../../utils/streams");
+const createMemoryContentStore = (name = "memory") => {
+    const bytesById = new Map();
+    let nextId = 1;
+    const storeBytes = (bytes) => {
+        const id = String(nextId);
+        nextId += 1;
+        bytesById.set(id, bytes);
+        return id;
+    };
+    const getStoredBytes = (id) => {
+        const bytes = bytesById.get(id);
+        if (!bytes) {
+            throw new Error(`Missing memory content for ref id=${id}`);
+        }
+        return bytes;
+    };
+    return {
+        name,
+        _bytes: bytesById,
+        async put(body, _meta) {
+            const bytes = await (0, utils_1.toUint8Array)(body);
+            const id = storeBytes(bytes);
+            return { kind: "store-ref", id };
+        },
+        async open(ref) {
+            if (ref.kind === "memory") {
+                return (0, streams_1.streamFromUint8Array)(ref.data);
+            }
+            const bytes = getStoredBytes(ref.id);
+            return (0, streams_1.streamFromUint8Array)(bytes);
+        },
+        async dispose() {
+            bytesById.clear();
+        }
+    };
+};
+exports.createMemoryContentStore = createMemoryContentStore;

package/dist/core/capture/internal/types.d.ts

@@ -0,0 +1,61 @@
+import type { ContentRef, ContentStore, ResourceType } from "../../../types";
+export type Header = {
+    name: string;
+    value: string;
+};
+export type CaptureBodyRef = ContentRef;
+export type CaptureCapabilities = {
+    requestHeaders: "sent" | "approx" | "none";
+    responseHeaders: "raw" | "approx" | "none";
+    requestBodies: boolean;
+    responseBodies: "decoded" | "wire" | "none";
+    httpVersion: boolean;
+    remoteIp: boolean;
+    headerOrderPreserved: boolean;
+};
+export type CaptureRequestEvent = {
+    type: "http.request";
+    requestId: string;
+    url: string;
+    method: string;
+    headers: Header[];
+    timestamp: number;
+    frameId?: string;
+    resourceType?: ResourceType;
+    initiator?: {
+        type?: string;
+        url?: string;
+    };
+};
+export type CaptureResponseEvent = {
+    type: "http.response";
+    requestId: string;
+    url: string;
+    status: number;
+    statusText?: string;
+    headers: Header[];
+    timestamp: number;
+    mimeType?: string;
+    fromDiskCache?: boolean;
+    fromServiceWorker?: boolean;
+    bodyRef?: CaptureBodyRef;
+    bodySize?: number;
+};
+export type CaptureFailedEvent = {
+    type: "http.failed";
+    requestId: string;
+    url: string;
+    errorText: string;
+    timestamp: number;
+};
+export type CaptureEvent = CaptureRequestEvent | CaptureResponseEvent | CaptureFailedEvent;
+export type CaptureArtifacts = {
+    /** Ordered list of captured network events. */
+    events: CaptureEvent[];
+    /** Content store containing captured bodies referenced by events. */
+    contentStore: ContentStore;
+    capabilities: CaptureCapabilities;
+};
+export type CaptureContext = {
+    capture?: CaptureArtifacts;
+};

package/dist/core/capture/internal/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/core/capture/memory-content-store.d.ts

@@ -0,0 +1,4 @@
+import type { ContentStore } from "../../types.js";
+export declare const createMemoryContentStore: (name?: string) => (ContentStore & {
+    _bytes: Map<string, Uint8Array>;
+}) | ContentStore;

package/dist/core/capture/memory-content-store.js

@@ -0,0 +1,38 @@
+import { streamFromUint8Array } from "../../utils/streams.js";
+import { toUint8Array } from "../../utils.js";
+export const createMemoryContentStore = (name = "memory") => {
+    const bytesById = new Map();
+    let nextId = 1;
+    const storeBytes = (bytes) => {
+        const id = String(nextId);
+        nextId += 1;
+        bytesById.set(id, bytes);
+        return id;
+    };
+    const getStoredBytes = (id) => {
+        const bytes = bytesById.get(id);
+        if (!bytes) {
+            throw new Error(`Missing memory content for ref id=${id}`);
+        }
+        return bytes;
+    };
+    return {
+        name,
+        _bytes: bytesById,
+        async put(body, _meta) {
+            const bytes = await toUint8Array(body);
+            const id = storeBytes(bytes);
+            return { kind: "store-ref", id };
+        },
+        async open(ref) {
+            if (ref.kind === "memory") {
+                return streamFromUint8Array(ref.data);
+            }
+            const bytes = getStoredBytes(ref.id);
+            return streamFromUint8Array(bytes);
+        },
+        async dispose() {
+            bytesById.clear();
+        }
+    };
+};

package/dist/core/capture/types.d.ts

@@ -0,0 +1,61 @@
+import type { ContentRef, ContentStore, ResourceType } from "../../types.js";
+export type Header = {
+    name: string;
+    value: string;
+};
+export type CaptureBodyRef = ContentRef;
+export type CaptureCapabilities = {
+    requestHeaders: "sent" | "approx" | "none";
+    responseHeaders: "raw" | "approx" | "none";
+    requestBodies: boolean;
+    responseBodies: "decoded" | "wire" | "none";
+    httpVersion: boolean;
+    remoteIp: boolean;
+    headerOrderPreserved: boolean;
+};
+export type CaptureRequestEvent = {
+    type: "http.request";
+    requestId: string;
+    url: string;
+    method: string;
+    headers: Header[];
+    timestamp: number;
+    frameId?: string;
+    resourceType?: ResourceType;
+    initiator?: {
+        type?: string;
+        url?: string;
+    };
+};
+export type CaptureResponseEvent = {
+    type: "http.response";
+    requestId: string;
+    url: string;
+    status: number;
+    statusText?: string;
+    headers: Header[];
+    timestamp: number;
+    mimeType?: string;
+    fromDiskCache?: boolean;
+    fromServiceWorker?: boolean;
+    bodyRef?: CaptureBodyRef;
+    bodySize?: number;
+};
+export type CaptureFailedEvent = {
+    type: "http.failed";
+    requestId: string;
+    url: string;
+    errorText: string;
+    timestamp: number;
+};
+export type CaptureEvent = CaptureRequestEvent | CaptureResponseEvent | CaptureFailedEvent;
+export type CaptureArtifacts = {
+    /** Ordered list of captured network events. */
+    events: CaptureEvent[];
+    /** Content store containing captured bodies referenced by events. */
+    contentStore: ContentStore;
+    capabilities: CaptureCapabilities;
+};
+export type CaptureContext = {
+    capture?: CaptureArtifacts;
+};

package/dist/core/capture/types.js

@@ -0,0 +1 @@
+export {};

package/dist/core/completion.d.ts

@@ -0,0 +1,4 @@
+import type { CompletionStrategy } from "../types.js";
+export declare const timeout: (ms: number) => CompletionStrategy;
+export declare const networkIdle: (ms: number, checkInterval?: number) => CompletionStrategy;
+export declare const normalizeCompletion: (completion?: CompletionStrategy | CompletionStrategy[]) => CompletionStrategy[];

package/dist/core/completion.js

@@ -0,0 +1,23 @@
+import { sleep } from "../utils.js";
+export const timeout = (ms) => ({
+    async wait() {
+        await sleep(ms);
+    }
+});
+export const networkIdle = (ms, checkInterval = 100) => ({
+    async wait(ctx) {
+        while (true) {
+            const stats = ctx.getStats();
+            const idleFor = ctx.now() - stats.lastNetworkTs;
+            if (stats.inflightRequests === 0 && idleFor >= ms) {
+                return;
+            }
+            await sleep(Math.min(checkInterval, ms));
+        }
+    }
+});
+export const normalizeCompletion = (completion) => {
+    if (!completion)
+        return [];
+    return Array.isArray(completion) ? completion : [completion];
+};

package/dist/core/content-store.d.ts

@@ -0,0 +1,21 @@
+import type { BodySource, ContentRef, ContentStore } from "../types.js";
+type HybridContentStoreOptions = {
+    thresholdBytes?: number;
+    baseDir?: string;
+};
+export declare class HybridContentStore implements ContentStore {
+    name: string;
+    private thresholdBytes;
+    private baseDir;
+    private storedIds;
+    constructor(options?: HybridContentStoreOptions);
+    put(body: BodySource, meta: {
+        url: string;
+        mimeType?: string;
+        sizeHint?: number;
+    }): Promise<ContentRef>;
+    open(ref: ContentRef): Promise<ReadableStream<Uint8Array>>;
+    dispose(): Promise<void>;
+    private removeBaseDir;
+}
+export {};