@pagepocket/lib 0.7.1 → 0.8.1

package/dist/core/plugin/context.js

@@ -0,0 +1,205 @@
+const createBarrier = () => {
+    let resolve = null;
+    let reject = null;
+    const promise = new Promise((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+    return {
+        promise,
+        resolve: (value) => resolve?.(value),
+        reject: (error) => reject?.(error)
+    };
+};
+const createHtmlBarrier = (ctxRef) => {
+    const barrier = createBarrier();
+    return {
+        whenHtml: async () => {
+            if (ctxRef.html) {
+                return ctxRef.html;
+            }
+            return barrier.promise;
+        },
+        resolveHtml: (html) => {
+            barrier.resolve(html);
+        },
+        rejectHtml: (error) => {
+            barrier.reject(error);
+        }
+    };
+};
+const createHtmlHooksDoneBarrier = () => {
+    const barrier = createBarrier();
+    return {
+        whenDone: async () => barrier.promise,
+        resolve: () => barrier.resolve(undefined),
+        reject: (error) => barrier.reject(error)
+    };
+};
+export const createDeferredTracker = () => {
+    const promises = new Set();
+    return {
+        defer(promise) {
+            promises.add(promise);
+            promise.finally(() => promises.delete(promise));
+        },
+        async awaitAll() {
+            await Promise.allSettled(Array.from(promises));
+        }
+    };
+};
+export const createHookRegistries = () => ({
+    init: [],
+    html: [],
+    beforeNetwork: [],
+    networkEvent: [],
+    afterNetwork: [],
+    finalize: []
+});
+export const createPagePocketContext = (input) => {
+    const state = {};
+    const hooks = createHookRegistries();
+    const deferred = createDeferredTracker();
+    const replaceElements = [];
+    let finished = false;
+    const ctx = {
+        entry: input.entry,
+        options: input.options,
+        pocketOptions: input.pocketOptions,
+        state,
+        _hooks: hooks,
+        _deferred: deferred,
+        _replaceElements: replaceElements,
+        _isFinished: () => finished,
+        _setFinished: () => {
+            finished = true;
+        }
+    };
+    // Optional network broadcaster surface.
+    // http-* capture plugins can emit to this so external observers can subscribe
+    // without any adapter/interceptor plumbing.
+    const createNetworkBroadcaster = () => {
+        const queues = new Set();
+        const emit = (value) => {
+            for (const q of queues) {
+                if (q.closed)
+                    continue;
+                if (q.pending.length > 0) {
+                    const resolve = q.pending.shift();
+                    resolve?.({ kind: "value", value });
+                    continue;
+                }
+                q.values.push({ kind: "value", value });
+            }
+        };
+        const close = () => {
+            for (const q of queues) {
+                if (q.closed)
+                    continue;
+                q.closed = true;
+                if (q.pending.length > 0) {
+                    for (const resolve of q.pending) {
+                        resolve({ kind: "close" });
+                    }
+                    q.pending = [];
+                    continue;
+                }
+                q.values.push({ kind: "close" });
+            }
+            queues.clear();
+        };
+        const subscribe = () => {
+            const queue = {
+                values: [],
+                pending: [],
+                closed: false
+            };
+            queues.add(queue);
+            return (async function* () {
+                try {
+                    while (true) {
+                        if (queue.values.length > 0) {
+                            const item = queue.values.shift();
+                            if (!item || item.kind === "close") {
+                                return;
+                            }
+                            yield item.value;
+                            continue;
+                        }
+                        if (queue.closed) {
+                            return;
+                        }
+                        const item = await new Promise((resolve) => {
+                            queue.pending.push(resolve);
+                        });
+                        if (item.kind === "close") {
+                            return;
+                        }
+                        yield item.value;
+                    }
+                }
+                finally {
+                    queue.closed = true;
+                    queues.delete(queue);
+                }
+            })();
+        };
+        return { emit, close, subscribe };
+    };
+    const network = createNetworkBroadcaster();
+    ctx.network = {
+        subscribe: () => network.subscribe()
+    };
+    ctx.emitNetworkEvent = (event) => {
+        network.emit(event);
+    };
+    const htmlBarrier = createHtmlBarrier(ctx);
+    const htmlHooksDoneBarrier = createHtmlHooksDoneBarrier();
+    ctx._htmlBarrier = htmlBarrier;
+    ctx._htmlHooksDoneBarrier = htmlHooksDoneBarrier;
+    ctx.whenHtml = async () => htmlBarrier.whenHtml();
+    ctx.whenHtmlHooksDone = async () => {
+        await htmlBarrier.whenHtml();
+        return htmlHooksDoneBarrier.whenDone();
+    };
+    ctx.onInit = (handler) => {
+        hooks.init.push(handler);
+    };
+    ctx.onHtml = (handler) => {
+        hooks.html.push(handler);
+    };
+    ctx.onBeforeNetwork = (handler) => {
+        hooks.beforeNetwork.push(handler);
+    };
+    ctx.onNetworkEvent = (handler) => {
+        hooks.networkEvent.push(handler);
+    };
+    ctx.onAfterNetwork = (handler) => {
+        hooks.afterNetwork.push(handler);
+    };
+    ctx.onFinalize = (handler) => {
+        hooks.finalize.push(handler);
+    };
+    ctx.setHtml = (next) => {
+        ctx.html = next;
+        htmlBarrier.resolveHtml(next);
+    };
+    ctx.addReplaceElements = (rules) => {
+        replaceElements.push(rules);
+    };
+    ctx.finish = (result) => {
+        if (finished) {
+            return;
+        }
+        ctx.result = result;
+        finished = true;
+    };
+    ctx.defer = (promise) => {
+        deferred.defer(promise);
+    };
+    return ctx;
+};
+export const resolveEnabledPlugins = async (plugins, _ctx) => {
+    // No gating: plugin execution is controlled by composition.
+    return plugins;
+};

package/dist/core/plugin/internal/context.d.ts

@@ -0,0 +1,47 @@
+import type { NetworkEvent } from "@pagepocket/interceptor";
+import type { ReplaceElementsConfig } from "../../../types";
+import type { CaptureWithPluginsOptions, HtmlArtifact, PagePocketCaptureResult, PagePocketContext, PagePocketPlugin } from "./types";
+type Hook<TArgs extends unknown[] = []> = (...args: TArgs) => void | Promise<void>;
+type FinalizeHook = () => void | PagePocketCaptureResult | Promise<void | PagePocketCaptureResult>;
+export type HookRegistries = {
+    init: Array<Hook>;
+    html: Array<Hook<[payload: {
+        html: HtmlArtifact;
+    }]>>;
+    beforeNetwork: Array<Hook>;
+    networkEvent: Array<Hook<[event: NetworkEvent]>>;
+    afterNetwork: Array<Hook>;
+    finalize: Array<FinalizeHook>;
+};
+export type DeferredTracker = {
+    defer(promise: Promise<unknown>): void;
+    awaitAll(): Promise<void>;
+};
+type HtmlBarrier = {
+    whenHtml(): Promise<HtmlArtifact>;
+    resolveHtml(html: HtmlArtifact): void;
+    rejectHtml(error: Error): void;
+};
+type HtmlHooksDoneBarrier = {
+    whenDone(): Promise<void>;
+    resolve(): void;
+    reject(error: Error): void;
+};
+export type PagePocketContextInternal = PagePocketContext & {
+    _hooks: HookRegistries;
+    _deferred: DeferredTracker;
+    _replaceElements: ReplaceElementsConfig[];
+    _htmlBarrier: HtmlBarrier;
+    _htmlHooksDoneBarrier: HtmlHooksDoneBarrier;
+    _isFinished(): boolean;
+    _setFinished(): void;
+};
+export declare const createDeferredTracker: () => DeferredTracker;
+export declare const createHookRegistries: () => HookRegistries;
+export declare const createPagePocketContext: (input: {
+    entry: PagePocketContext["entry"];
+    options: CaptureWithPluginsOptions;
+    pocketOptions: PagePocketContext["pocketOptions"];
+}) => PagePocketContextInternal;
+export declare const resolveEnabledPlugins: (plugins: PagePocketPlugin[], ctx: PagePocketContext) => Promise<PagePocketPlugin[]>;
+export {};

package/dist/core/plugin/internal/context.js

@@ -0,0 +1,142 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveEnabledPlugins = exports.createPagePocketContext = exports.createHookRegistries = exports.createDeferredTracker = void 0;
+const createBarrier = () => {
+    let resolve = null;
+    let reject = null;
+    const promise = new Promise((res, rej) => {
+        resolve = res;
+        reject = rej;
+    });
+    return {
+        promise,
+        resolve: (value) => resolve?.(value),
+        reject: (error) => reject?.(error)
+    };
+};
+const createHtmlBarrier = (ctxRef) => {
+    const barrier = createBarrier();
+    return {
+        whenHtml: async () => {
+            if (ctxRef.html) {
+                return ctxRef.html;
+            }
+            return barrier.promise;
+        },
+        resolveHtml: (html) => {
+            barrier.resolve(html);
+        },
+        rejectHtml: (error) => {
+            barrier.reject(error);
+        }
+    };
+};
+const createHtmlHooksDoneBarrier = () => {
+    const barrier = createBarrier();
+    return {
+        whenDone: async () => barrier.promise,
+        resolve: () => barrier.resolve(undefined),
+        reject: (error) => barrier.reject(error)
+    };
+};
+const createDeferredTracker = () => {
+    const promises = new Set();
+    return {
+        defer(promise) {
+            promises.add(promise);
+            promise.finally(() => promises.delete(promise));
+        },
+        async awaitAll() {
+            await Promise.allSettled(Array.from(promises));
+        }
+    };
+};
+exports.createDeferredTracker = createDeferredTracker;
+const createHookRegistries = () => ({
+    init: [],
+    html: [],
+    beforeNetwork: [],
+    networkEvent: [],
+    afterNetwork: [],
+    finalize: []
+});
+exports.createHookRegistries = createHookRegistries;
+const createPagePocketContext = (input) => {
+    const state = {};
+    const hooks = (0, exports.createHookRegistries)();
+    const deferred = (0, exports.createDeferredTracker)();
+    const replaceElements = [];
+    let finished = false;
+    const ctx = {
+        entry: input.entry,
+        options: input.options,
+        pocketOptions: input.pocketOptions,
+        state,
+        _hooks: hooks,
+        _deferred: deferred,
+        _replaceElements: replaceElements,
+        _isFinished: () => finished,
+        _setFinished: () => {
+            finished = true;
+        }
+    };
+    const htmlBarrier = createHtmlBarrier(ctx);
+    const htmlHooksDoneBarrier = createHtmlHooksDoneBarrier();
+    ctx._htmlBarrier = htmlBarrier;
+    ctx._htmlHooksDoneBarrier = htmlHooksDoneBarrier;
+    ctx.whenHtml = async () => htmlBarrier.whenHtml();
+    ctx.whenHtmlHooksDone = async () => {
+        await htmlBarrier.whenHtml();
+        return htmlHooksDoneBarrier.whenDone();
+    };
+    ctx.onInit = (handler) => {
+        hooks.init.push(handler);
+    };
+    ctx.onHtml = (handler) => {
+        hooks.html.push(handler);
+    };
+    ctx.onBeforeNetwork = (handler) => {
+        hooks.beforeNetwork.push(handler);
+    };
+    ctx.onNetworkEvent = (handler) => {
+        hooks.networkEvent.push(handler);
+    };
+    ctx.onAfterNetwork = (handler) => {
+        hooks.afterNetwork.push(handler);
+    };
+    ctx.onFinalize = (handler) => {
+        hooks.finalize.push(handler);
+    };
+    ctx.setHtml = (next) => {
+        ctx.html = next;
+        htmlBarrier.resolveHtml(next);
+    };
+    ctx.addReplaceElements = (rules) => {
+        replaceElements.push(rules);
+    };
+    ctx.finish = (result) => {
+        if (finished) {
+            return;
+        }
+        ctx.result = result;
+        finished = true;
+    };
+    ctx.defer = (promise) => {
+        deferred.defer(promise);
+    };
+    return ctx;
+};
+exports.createPagePocketContext = createPagePocketContext;
+const resolveEnabledPlugins = async (plugins, ctx) => {
+    const enabled = [];
+    for (const plugin of plugins) {
+        const gate = plugin.enabled;
+        if (gate === false) {
+            continue;
+        }
+        // Default enabled when gate is undefined.
+        enabled.push(plugin);
+    }
+    return enabled;
+};
+exports.resolveEnabledPlugins = resolveEnabledPlugins;

package/dist/core/plugin/internal/runner.d.ts

@@ -0,0 +1,12 @@
+import type { InterceptTarget, NetworkEvent } from "@pagepocket/interceptor";
+import type { CaptureWithPluginsOptions, PagePocketCaptureResult } from "./types";
+type RunnerInput = {
+    pocket: {
+        interceptedRequestEvents(): AsyncIterable<NetworkEvent>;
+    };
+    pocketOptions: import("../../../types").PagePocketOptions;
+    target: InterceptTarget;
+    options: CaptureWithPluginsOptions;
+};
+export declare const captureWithPlugins: (input: RunnerInput) => Promise<PagePocketCaptureResult>;
+export {};

package/dist/core/plugin/internal/runner.js

@@ -0,0 +1,232 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.captureWithPlugins = void 0;
+const completion_1 = require("../../completion");
+const debug_1 = require("../../debug");
+const inflight_tracker_1 = require("../../inflight-tracker");
+const context_1 = require("./context");
+const runHandlersSequentially = async (handlers, args, ctx) => {
+    for (const handler of handlers) {
+        if (ctx._isFinished()) {
+            return;
+        }
+        await handler(...args);
+    }
+};
+const runFinalizeHandlers = async (ctx) => {
+    for (const handler of ctx._hooks.finalize) {
+        if (ctx._isFinished()) {
+            return;
+        }
+        const result = await handler();
+        if (result) {
+            ctx.finish(result);
+            return;
+        }
+    }
+};
+const pluginLabel = (plugin) => {
+    const ctor = plugin.constructor;
+    const ctorName = ctor && typeof ctor.name === "string" ? ctor.name : undefined;
+    return {
+        name: plugin.name,
+        ctorName,
+        enabled: plugin.enabled
+    };
+};
+const captureWithPlugins = async (input) => {
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] capture start", {
+        targetKind: input.target.kind,
+        adapter: input.options.interceptor?.name,
+        pluginCount: input.options.plugins.length
+    });
+    const plugins = input.options.plugins;
+    const ctx = (0, context_1.createPagePocketContext)({
+        entry: input.target.kind === "url"
+            ? { kind: "url", url: input.target.url }
+            : input.target.kind === "puppeteer-page"
+                ? { kind: "puppeteer-page", page: input.target.page }
+                : input.target.kind === "cdp-tab"
+                    ? { kind: "cdp-tab", tabId: input.target.tabId }
+                    : input.target.kind === "html"
+                        ? { kind: "html-string", baseUrl: input.target.baseUrl, url: input.target.url }
+                        : (() => {
+                            throw new Error(`Unsupported target kind: ${input.target.kind}`);
+                        })(),
+        options: input.options,
+        pocketOptions: input.pocketOptions
+    });
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] plugins configured", plugins.map(pluginLabel));
+    const allPlugins = await (0, context_1.resolveEnabledPlugins)(plugins, ctx);
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] plugins enabled", allPlugins.map(pluginLabel));
+    for (const plugin of allPlugins) {
+        if (ctx._isFinished()) {
+            break;
+        }
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] plugin.apply start", pluginLabel(plugin));
+        await plugin.apply(ctx);
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] plugin.apply done", pluginLabel(plugin));
+    }
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=init start", { count: ctx._hooks.init.length });
+    await runHandlersSequentially(ctx._hooks.init, [], ctx);
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=init done");
+    const interceptor = input.options.interceptor;
+    if (!interceptor) {
+        // HTML-only flow.
+        if (input.target.kind !== "html") {
+            throw new Error("options.interceptor is required for non-HTML entrypoints.");
+        }
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=html (no interceptor) start");
+        ctx.setHtml({
+            htmlString: input.target.htmlString,
+            baseUrl: input.target.baseUrl,
+            url: input.target.url,
+            contentType: "text/html"
+        });
+        try {
+            await runHandlersSequentially(ctx._hooks.html, [{ html: ctx.html }], ctx);
+            ctx._htmlHooksDoneBarrier.resolve();
+        }
+        catch (error) {
+            const err = error instanceof Error ? error : new Error(String(error));
+            ctx._htmlHooksDoneBarrier.reject(err);
+            throw err;
+        }
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=html (no interceptor) done");
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=finalize start", {
+            count: ctx._hooks.finalize.length
+        });
+        await runFinalizeHandlers(ctx);
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=finalize done", { resultKind: ctx.result?.kind });
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] awaiting deferred work");
+        await ctx._deferred.awaitAll();
+        if (!ctx.result) {
+            throw new Error("No plugin produced a terminal result.");
+        }
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] capture done", { resultKind: ctx.result.kind });
+        return ctx.result;
+    }
+    const completionStrategies = (0, completion_1.normalizeCompletion)(input.options.completion);
+    const idleMs = input.options.timeoutMs ?? 5000;
+    const maxDurationMs = input.options.maxDurationMs;
+    const completion = completionStrategies.length > 0
+        ? completionStrategies
+        : [(0, completion_1.networkIdle)(idleMs), ...(maxDurationMs !== undefined ? [(0, completion_1.timeout)(maxDurationMs)] : [])];
+    let stopped = false;
+    const inflightTracker = new inflight_tracker_1.InflightTracker();
+    const onEvent = async (event) => {
+        inflightTracker.handleEvent(event);
+        await runHandlersSequentially(ctx._hooks.networkEvent, [event], ctx);
+    };
+    const handlers = {
+        onEvent(event) {
+            if (ctx._isFinished()) {
+                return;
+            }
+            input.pocket._emitNetworkEvent(event);
+            void onEvent(event);
+        },
+        onError(error) {
+            (0, debug_1.debugLog)("[pagepocket][plugin-runner] interceptor error", error);
+        }
+    };
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=session start");
+    const session = await interceptor.start(input.target, handlers, {
+        timeoutMs: input.options.timeoutMs,
+        maxDurationMs: input.options.maxDurationMs
+    });
+    try {
+        if (input.target.kind === "url" && session.navigate) {
+            (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=navigate start", { url: input.target.url });
+            await session.navigate(input.target.url);
+            (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=navigate done");
+        }
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=waitForHtml start");
+        const html = await session.waitForHtml();
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=waitForHtml done", {
+            url: html.url,
+            baseUrl: html.baseUrl,
+            contentType: html.contentType,
+            htmlBytes: html.htmlString.length
+        });
+        ctx.setHtml(html);
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=html start", { count: ctx._hooks.html.length });
+        try {
+            await runHandlersSequentially(ctx._hooks.html, [{ html }], ctx);
+            ctx._htmlHooksDoneBarrier.resolve();
+        }
+        catch (error) {
+            const err = error instanceof Error ? error : new Error(String(error));
+            ctx._htmlHooksDoneBarrier.reject(err);
+            throw err;
+        }
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=html done");
+        if (ctx._isFinished()) {
+            (0, debug_1.debugLog)("[pagepocket][plugin-runner] early finish after html", {
+                resultKind: ctx.result?.kind
+            });
+            (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=finalize start", {
+                count: ctx._hooks.finalize.length
+            });
+            await runFinalizeHandlers(ctx);
+            (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=finalize done", { resultKind: ctx.result?.kind });
+            (0, debug_1.debugLog)("[pagepocket][plugin-runner] awaiting deferred work");
+            await ctx._deferred.awaitAll();
+            if (!ctx.result) {
+                throw new Error("No plugin produced a terminal result.");
+            }
+            return ctx.result;
+        }
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=beforeNetwork start", {
+            count: ctx._hooks.beforeNetwork.length
+        });
+        await runHandlersSequentially(ctx._hooks.beforeNetwork, [], ctx);
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=beforeNetwork done");
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=startCapture start");
+        await session.startCapture();
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=startCapture done");
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=completion wait start", {
+            strategies: completion.length
+        });
+        if (completion.length === 1) {
+            await completion[0].wait({
+                now: () => Date.now(),
+                getStats: () => inflightTracker.getStats()
+            });
+        }
+        else {
+            await Promise.race(completion.map((strategy) => strategy.wait({
+                now: () => Date.now(),
+                getStats: () => inflightTracker.getStats()
+            })));
+        }
+        (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=completion wait done");
+    }
+    finally {
+        if (!stopped) {
+            stopped = true;
+            (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=session stop start");
+            await session.stop();
+            (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=session stop done");
+        }
+        input.pocket._closeNetworkEventStream();
+    }
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=afterNetwork start", {
+        count: ctx._hooks.afterNetwork.length
+    });
+    await runHandlersSequentially(ctx._hooks.afterNetwork, [], ctx);
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=afterNetwork done");
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=finalize start", {
+        count: ctx._hooks.finalize.length
+    });
+    await runFinalizeHandlers(ctx);
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] phase=finalize done", { resultKind: ctx.result?.kind });
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] awaiting deferred work");
+    await ctx._deferred.awaitAll();
+    if (!ctx.result) {
+        throw new Error("No plugin produced a terminal result.");
+    }
+    (0, debug_1.debugLog)("[pagepocket][plugin-runner] capture done", { resultKind: ctx.result.kind });
+    return ctx.result;
+};
+exports.captureWithPlugins = captureWithPlugins;