@pagepocket/capture-http-lighterceptor-unit 0.8.0

package/dist/capture-http-lighterceptor-plugin.d.ts

@@ -0,0 +1,9 @@
+import { type PagePocketContext, type PagePocketPlugin } from "@pagepocket/lib";
+import { type LighterceptorAdapterOptions } from "./internal/lighterceptor-adapter.js";
+export type CaptureHttpLighterceptorPluginOptions = LighterceptorAdapterOptions;
+export declare class CaptureHttpLighterceptorPlugin implements PagePocketPlugin {
+    readonly name = "plugin:capture-http-lighterceptor";
+    private adapterOptions;
+    constructor(options?: CaptureHttpLighterceptorPluginOptions);
+    apply(ctx: PagePocketContext): void;
+}

package/dist/capture-http-lighterceptor-plugin.js

@@ -0,0 +1,140 @@
+import { createMemoryContentStore } from "@pagepocket/lib";
+import { InflightTracker, networkIdle, normalizeCompletion, timeout } from "@pagepocket/lib";
+import { LighterceptorAdapter } from "./internal/lighterceptor-adapter.js";
+const headersRecordToList = (headers) => {
+    if (!headers)
+        return [];
+    return Object.keys(headers).map((name) => ({ name, value: headers[name] }));
+};
+export class CaptureHttpLighterceptorPlugin {
+    constructor(options) {
+        this.name = "plugin:capture-http-lighterceptor";
+        this.adapterOptions = options ?? {};
+    }
+    apply(ctx) {
+        const contentStore = createMemoryContentStore("capture-http-lighterceptor");
+        const events = [];
+        const capabilities = {
+            requestHeaders: "approx",
+            responseHeaders: "approx",
+            requestBodies: false,
+            responseBodies: "decoded",
+            httpVersion: false,
+            remoteIp: false,
+            headerOrderPreserved: false
+        };
+        ctx.capture = {
+            events,
+            contentStore,
+            capabilities
+        };
+        const inflightTracker = new InflightTracker();
+        const handleNetworkEvent = async (event) => {
+            inflightTracker.handleEvent(event);
+            ctx.emitNetworkEvent?.(event);
+            if (event.type === "request") {
+                events.push({
+                    type: "http.request",
+                    requestId: event.requestId,
+                    url: event.url,
+                    method: event.method,
+                    headers: headersRecordToList(event.headers),
+                    timestamp: event.timestamp,
+                    frameId: event.frameId,
+                    resourceType: event.resourceType,
+                    initiator: event.initiator
+                });
+                return;
+            }
+            if (event.type === "failed") {
+                events.push({
+                    type: "http.failed",
+                    requestId: event.requestId,
+                    url: event.url,
+                    errorText: event.errorText,
+                    timestamp: event.timestamp
+                });
+                return;
+            }
+            const bodyRef = event.body
+                ? await contentStore.put(event.body, {
+                    url: event.url,
+                    mimeType: event.mimeType,
+                    sizeHint: undefined
+                })
+                : undefined;
+            events.push({
+                type: "http.response",
+                requestId: event.requestId,
+                url: event.url,
+                status: event.status,
+                statusText: event.statusText,
+                headers: headersRecordToList(event.headers),
+                timestamp: event.timestamp,
+                mimeType: event.mimeType,
+                fromDiskCache: event.fromDiskCache,
+                fromServiceWorker: event.fromServiceWorker,
+                bodyRef,
+                bodySize: undefined
+            });
+        };
+        const stateKey = "captureHttpLighterceptor.session";
+        ctx.onInit(async () => {
+            const target = ctx.entry.kind === "url"
+                ? { kind: "url", url: ctx.entry.url }
+                : ctx.entry.kind === "html-string"
+                    ? {
+                        kind: "html",
+                        htmlString: await ctx.whenHtml().then((h) => h.htmlString),
+                        baseUrl: ctx.entry.baseUrl,
+                        ...(ctx.entry.url ? { url: ctx.entry.url } : {})
+                    }
+                    : (() => {
+                        throw new Error(`CaptureHttpLighterceptorPlugin does not support entry kind: ${String(ctx.entry.kind)}`);
+                    })();
+            const adapter = new LighterceptorAdapter(this.adapterOptions);
+            const session = await adapter.start(target, {
+                onEvent(event) {
+                    void handleNetworkEvent(event);
+                },
+                onError(error) {
+                    // eslint-disable-next-line no-console
+                    console.warn("[pagepocket][capture-http-lighterceptor] adapter error", error);
+                }
+            }, this.adapterOptions);
+            ctx.state[stateKey] = session;
+            // LighterceptorAdapter does not implement navigate; it fetches HTML itself.
+            // Allow tests to rely on an already-provided HTML artifact (no extra fetch).
+            if (!ctx.html) {
+                const html = await session.waitForHtml();
+                ctx.setHtml(html);
+            }
+        });
+        ctx.onBeforeNetwork(async () => {
+            const session = ctx.state[stateKey];
+            if (!session) {
+                throw new Error("CaptureHttpLighterceptorPlugin internal error: missing session");
+            }
+            await session.startCapture();
+            const completionStrategies = normalizeCompletion(ctx.options.completion);
+            const idleMs = ctx.options.timeoutMs ?? 5000;
+            const maxDurationMs = ctx.options.maxDurationMs;
+            const completion = completionStrategies.length > 0
+                ? completionStrategies
+                : [networkIdle(idleMs), ...(maxDurationMs !== undefined ? [timeout(maxDurationMs)] : [])];
+            if (completion.length === 1) {
+                await completion[0].wait({
+                    now: () => Date.now(),
+                    getStats: () => inflightTracker.getStats()
+                });
+            }
+            else {
+                await Promise.race(completion.map((strategy) => strategy.wait({
+                    now: () => Date.now(),
+                    getStats: () => inflightTracker.getStats()
+                })));
+            }
+            await session.stop();
+        });
+    }
+}

package/dist/capture-http-lighterceptor-unit.d.ts

@@ -0,0 +1,13 @@
+import { Unit, type CaptureArtifacts } from "@pagepocket/lib";
+import { type LighterceptorAdapterOptions } from "./internal/lighterceptor-adapter.js";
+export type CaptureHttpLighterceptorUnitOptions = LighterceptorAdapterOptions;
+export declare class CaptureHttpLighterceptorUnit extends Unit {
+    readonly id = "captureHttpLighterceptor";
+    readonly kind = "capture.http.lighterceptor";
+    private adapterOptions;
+    constructor(options?: CaptureHttpLighterceptorUnitOptions);
+    run(ctx: import("@pagepocket/lib").UnitContext, rt: import("@pagepocket/lib").UnitRuntime): Promise<{
+        capture: CaptureArtifacts;
+        html: {};
+    }>;
+}

package/dist/capture-http-lighterceptor-unit.js

@@ -0,0 +1,126 @@
+import { NETWORK } from "@pagepocket/contracts";
+import { Unit, createMemoryContentStore, InflightTracker, mapKind, networkIdle, normalizeCompletion, throwUnsupportedEntryKind, timeout } from "@pagepocket/lib";
+import { LighterceptorAdapter } from "./internal/lighterceptor-adapter.js";
+const headersRecordToList = (headers) => {
+    if (!headers)
+        return [];
+    return Object.keys(headers).map((name) => ({ name, value: headers[name] }));
+};
+const targetBuilders = {
+    url: (entry) => ({
+        kind: "url",
+        url: entry.url
+    }),
+    "html-string": (entry) => ({
+        kind: "html",
+        htmlString: entry.htmlString,
+        baseUrl: entry.baseUrl,
+        ...(entry.url ? { url: entry.url } : {})
+    })
+};
+export class CaptureHttpLighterceptorUnit extends Unit {
+    constructor(options) {
+        super();
+        this.id = "captureHttpLighterceptor";
+        this.kind = "capture.http.lighterceptor";
+        this.adapterOptions = options ?? {};
+    }
+    async run(ctx, rt) {
+        const contentStore = createMemoryContentStore("capture-http-lighterceptor");
+        const events = [];
+        const capabilities = {
+            requestHeaders: "approx",
+            responseHeaders: "approx",
+            requestBodies: false,
+            responseBodies: "decoded",
+            httpVersion: false,
+            remoteIp: false,
+            headerOrderPreserved: false
+        };
+        const inflightTracker = new InflightTracker();
+        const handleNetworkEvent = async (event) => {
+            inflightTracker.handleEvent(event);
+            rt.publish(NETWORK, event);
+            if (event.type === "request") {
+                events.push({
+                    type: "http.request",
+                    requestId: event.requestId,
+                    url: event.url,
+                    method: event.method,
+                    headers: headersRecordToList(event.headers),
+                    timestamp: event.timestamp,
+                    frameId: event.frameId,
+                    resourceType: event.resourceType,
+                    initiator: event.initiator
+                });
+                return;
+            }
+            if (event.type === "failed") {
+                events.push({
+                    type: "http.failed",
+                    requestId: event.requestId,
+                    url: event.url,
+                    errorText: event.errorText,
+                    timestamp: event.timestamp
+                });
+                return;
+            }
+            const bodyRef = event.body
+                ? await contentStore.put(event.body, {
+                    url: event.url,
+                    mimeType: event.mimeType,
+                    sizeHint: undefined
+                })
+                : undefined;
+            events.push({
+                type: "http.response",
+                requestId: event.requestId,
+                url: event.url,
+                status: event.status,
+                statusText: event.statusText,
+                headers: headersRecordToList(event.headers),
+                timestamp: event.timestamp,
+                mimeType: event.mimeType,
+                fromDiskCache: event.fromDiskCache,
+                fromServiceWorker: event.fromServiceWorker,
+                bodyRef,
+                bodySize: undefined
+            });
+        };
+        const capture = { events, contentStore, capabilities };
+        const adapter = new LighterceptorAdapter(this.adapterOptions);
+        const target = mapKind(rt.entry, targetBuilders, {
+            onUnsupportedKind: throwUnsupportedEntryKind("CaptureHttpLighterceptorUnit")
+        });
+        const session = await adapter.start(target, {
+            onEvent: (event) => {
+                void handleNetworkEvent(event);
+            },
+            onError: (error) => {
+                console.warn("[pagepocket][capture-http-lighterceptor] adapter error", error);
+            }
+        }, this.adapterOptions);
+        const html = ctx.value.html ?? (await session.waitForHtml());
+        await session.startCapture();
+        const completionStrategies = normalizeCompletion(rt.options.completion);
+        const idleMs = rt.options.timeoutMs ?? 5000;
+        const maxDurationMs = rt.options.maxDurationMs;
+        const completion = completionStrategies.length > 0
+            ? completionStrategies
+            : [networkIdle(idleMs), ...(maxDurationMs !== undefined ? [timeout(maxDurationMs)] : [])];
+        if (completion.length === 1) {
+            await completion[0].wait({
+                now: () => Date.now(),
+                getStats: () => inflightTracker.getStats()
+            });
+        }
+        else {
+            await Promise.race(completion.map((strategy) => strategy.wait({
+                now: () => Date.now(),
+                getStats: () => inflightTracker.getStats()
+            })));
+        }
+        await session.stop();
+        return { capture, html };
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { CaptureHttpLighterceptorUnit } from "./capture-http-lighterceptor-unit.js";
+export type { CaptureHttpLighterceptorUnitOptions } from "./capture-http-lighterceptor-unit.js";

package/dist/index.js

@@ -0,0 +1 @@
+export { CaptureHttpLighterceptorUnit } from "./capture-http-lighterceptor-unit.js";

package/dist/internal/lighterceptor-adapter.d.ts

@@ -0,0 +1,24 @@
+import type { InterceptSession, InterceptTarget, NetworkEventHandlers, NetworkInterceptorAdapter, TriggerAction } from "@pagepocket/lib";
+import { type LighterceptorOptions } from "@pagepocket/lighterceptor";
+export type LighterceptorAdapterOptions = LighterceptorOptions & {
+    triggerActions?: TriggerAction[];
+};
+/**
+ * Internal adapter for the capture plugin.
+ *
+ * Note: this is intentionally NOT published as a separate package.
+ */
+export declare class LighterceptorAdapter implements NetworkInterceptorAdapter {
+    readonly name = "lighterceptor";
+    readonly capabilities: {
+        canGetResponseBody: boolean;
+        canStreamResponseBody: boolean;
+        canGetRequestBody: boolean;
+        providesResourceType: boolean;
+        canWaitForHtml: boolean;
+        supportsStagedCapture: boolean;
+    };
+    private options;
+    constructor(options?: LighterceptorAdapterOptions);
+    start(target: InterceptTarget, handlers: NetworkEventHandlers, options?: LighterceptorAdapterOptions): Promise<InterceptSession>;
+}

package/dist/internal/lighterceptor-adapter.js

@@ -0,0 +1,200 @@
+import { Lighterceptor } from "@pagepocket/lighterceptor";
+import { runTriggerActions } from "./trigger-actions.js";
+import { decodeBase64 } from "./utils/base64.js";
+import { getHeaderValue } from "./utils/headers.js";
+import { inferResourceType } from "./utils/resource-type.js";
+const encodeUtf8 = (input) => new TextEncoder().encode(input);
+const toRequestEvent = (record) => {
+    const headers = record.headers ?? {};
+    const headersForType = record.responseHeaders && Object.keys(record.responseHeaders).length > 0
+        ? record.responseHeaders
+        : headers;
+    const resourceType = inferResourceType(record.source, headersForType);
+    const event = {
+        type: "request",
+        requestId: record.requestId,
+        url: record.url,
+        method: record.method,
+        headers,
+        resourceType: resourceType,
+        timestamp: record.timestamp
+    };
+    return event;
+};
+/**
+ * Internal adapter for the capture plugin.
+ *
+ * Note: this is intentionally NOT published as a separate package.
+ */
+export class LighterceptorAdapter {
+    constructor(options = {}) {
+        this.name = "lighterceptor";
+        this.capabilities = {
+            canGetResponseBody: true,
+            canStreamResponseBody: false,
+            canGetRequestBody: false,
+            providesResourceType: true,
+            canWaitForHtml: true,
+            supportsStagedCapture: true
+        };
+        this.options = options;
+    }
+    async start(target, handlers, options) {
+        const mergedOptions = {
+            ...this.options,
+            ...(options ?? {})
+        };
+        let stopped = false;
+        let startedCapture = false;
+        let htmlArtifact;
+        const htmlPromise = (async () => {
+            if (target.kind === "html") {
+                return {
+                    htmlString: target.htmlString,
+                    baseUrl: target.baseUrl,
+                    url: target.url,
+                    contentType: "text/html"
+                };
+            }
+            if (target.kind !== "url") {
+                throw new Error("LighterceptorAdapter only supports url or html targets.");
+            }
+            if (typeof fetch !== "function") {
+                throw new Error("Global fetch is required for LighterceptorAdapter url targets.");
+            }
+            const response = await fetch(target.url, {
+                headers: mergedOptions.headers
+            });
+            const htmlString = await response.text();
+            const contentType = response.headers.get("content-type") ?? undefined;
+            const url = response.url || target.url;
+            return {
+                htmlString,
+                baseUrl: url,
+                url,
+                contentType
+            };
+        })();
+        const waitForHtml = async () => {
+            if (htmlArtifact) {
+                return htmlArtifact;
+            }
+            htmlArtifact = await htmlPromise;
+            return htmlArtifact;
+        };
+        const emitPrimaryDocumentEvents = (artifact) => {
+            const docUrl = artifact.url ?? artifact.baseUrl;
+            if (!docUrl) {
+                return;
+            }
+            const now = Date.now();
+            const requestId = `doc:${docUrl}:${now}`;
+            const requestEvent = {
+                type: "request",
+                requestId,
+                url: docUrl,
+                method: "GET",
+                headers: {},
+                resourceType: "document",
+                timestamp: now
+            };
+            handlers.onEvent(requestEvent);
+            const headers = {};
+            if (artifact.contentType) {
+                headers["content-type"] = artifact.contentType;
+            }
+            const responseEvent = {
+                type: "response",
+                requestId,
+                url: docUrl,
+                status: 200,
+                statusText: "OK",
+                headers,
+                mimeType: artifact.contentType,
+                timestamp: now,
+                body: {
+                    kind: "buffer",
+                    data: new TextEncoder().encode(artifact.htmlString)
+                }
+            };
+            handlers.onEvent(responseEvent);
+        };
+        const startCapture = async () => {
+            if (startedCapture) {
+                return;
+            }
+            startedCapture = true;
+            const artifact = await waitForHtml();
+            if (stopped) {
+                return;
+            }
+            emitPrimaryDocumentEvents(artifact);
+            // Note: `timeoutMs` is defined as the library-level *network idle duration*.
+            // LighterceptorAdapter does not currently map this to an internal wait.
+            const settleTimeMs = mergedOptions.settleTimeMs;
+            const lighterceptor = new Lighterceptor(artifact.htmlString, {
+                recursion: true,
+                baseUrl: artifact.baseUrl,
+                ...mergedOptions,
+                ...(settleTimeMs !== undefined ? { settleTimeMs } : {})
+            });
+            const result = await lighterceptor.run();
+            const networkRecords = result.networkRecords ?? [];
+            let sequence = 0;
+            for (const record of networkRecords) {
+                if (stopped) {
+                    return;
+                }
+                const requestId = `${record.url}:${record.timestamp}:${sequence++}`;
+                const requestEvent = toRequestEvent({
+                    url: record.url,
+                    method: record.method || "GET",
+                    source: record.source,
+                    timestamp: record.timestamp,
+                    responseHeaders: record.response?.headers,
+                    requestId
+                });
+                handlers.onEvent(requestEvent);
+                if (record.response) {
+                    const headers = record.response.headers || {};
+                    const responseEvent = {
+                        type: "response",
+                        requestId: requestEvent.requestId,
+                        url: record.url,
+                        status: record.response.status,
+                        statusText: record.response.statusText,
+                        headers,
+                        mimeType: getHeaderValue(headers, "content-type"),
+                        timestamp: record.timestamp,
+                        body: {
+                            kind: "buffer",
+                            data: record.response.bodyEncoding === "base64"
+                                ? decodeBase64(record.response.body)
+                                : encodeUtf8(record.response.body)
+                        }
+                    };
+                    handlers.onEvent(responseEvent);
+                    continue;
+                }
+                if (record.error) {
+                    const failedEvent = {
+                        type: "failed",
+                        requestId: requestEvent.requestId,
+                        url: record.url,
+                        errorText: record.error,
+                        timestamp: record.timestamp
+                    };
+                    handlers.onEvent(failedEvent);
+                }
+            }
+            await runTriggerActions(mergedOptions.triggerActions);
+        };
+        return {
+            waitForHtml,
+            startCapture,
+            stop: async () => {
+                stopped = true;
+            }
+        };
+    }
+}

package/dist/internal/trigger-actions.d.ts

@@ -0,0 +1,2 @@
+import { type TriggerAction } from "@pagepocket/lib";
+export declare const runTriggerActions: (actions?: TriggerAction[]) => Promise<void>;

package/dist/internal/trigger-actions.js

@@ -0,0 +1,35 @@
+import { TriggerActionValues } from "@pagepocket/lib";
+export const runTriggerActions = async (actions = []) => {
+    if (typeof document === "undefined" || typeof window === "undefined") {
+        return;
+    }
+    for (const action of actions) {
+        if (action === TriggerActionValues.HOVER) {
+            await simulateHoverAll();
+        }
+        if (action === TriggerActionValues.SCROLL_TO_END) {
+            await simulateScrollToEnd();
+        }
+    }
+};
+const simulateHoverAll = async () => {
+    const elements = Array.from(document.querySelectorAll("*"));
+    for (const element of elements) {
+        const rect = element.getBoundingClientRect();
+        const x = rect.left + rect.width / 2;
+        const y = rect.top + rect.height / 2;
+        const event = new MouseEvent("mouseover", {
+            bubbles: true,
+            cancelable: true,
+            clientX: x,
+            clientY: y
+        });
+        element.dispatchEvent(event);
+    }
+};
+const simulateScrollToEnd = async () => {
+    const scrollHeight = document.documentElement?.scrollHeight ?? document.body?.scrollHeight;
+    if (typeof scrollHeight === "number") {
+        window.scrollTo({ top: scrollHeight });
+    }
+};

package/dist/internal/utils/base64.d.ts

@@ -0,0 +1 @@
+export declare const decodeBase64: (input: string) => Uint8Array;

package/dist/internal/utils/base64.js

@@ -0,0 +1,12 @@
+export const decodeBase64 = (input) => {
+    if (typeof atob === "function") {
+        const binary = atob(input);
+        const bytes = new Uint8Array(binary.length);
+        for (let i = 0; i < binary.length; i++) {
+            bytes[i] = binary.charCodeAt(i);
+        }
+        return bytes;
+    }
+    // Node fallback
+    return Uint8Array.from(Buffer.from(input, "base64"));
+};

package/dist/internal/utils/headers.d.ts

@@ -0,0 +1 @@
+export declare const getHeaderValue: (headers: Record<string, string>, name: string) => string | undefined;

package/dist/internal/utils/headers.js

@@ -0,0 +1,4 @@
+export const getHeaderValue = (headers, name) => {
+    const key = Object.keys(headers).find((k) => k.toLowerCase() === name.toLowerCase());
+    return key ? headers[key] : undefined;
+};

package/dist/internal/utils/resource-type.d.ts

@@ -0,0 +1,2 @@
+import type { ResourceType } from "@pagepocket/lib";
+export declare const inferResourceType: (source: string | undefined, headers: Record<string, string>) => ResourceType | undefined;

package/dist/internal/utils/resource-type.js

@@ -0,0 +1,25 @@
+import { getHeaderValue } from "./headers.js";
+export const inferResourceType = (source, headers) => {
+    if (source === "fetch")
+        return "fetch";
+    if (source === "xhr")
+        return "xhr";
+    if (source === "css")
+        return "stylesheet";
+    if (source === "img")
+        return "image";
+    const contentType = (getHeaderValue(headers, "content-type") || "").toLowerCase();
+    if (contentType.includes("text/html"))
+        return "document";
+    if (contentType.includes("text/css"))
+        return "stylesheet";
+    if (contentType.includes("javascript"))
+        return "script";
+    if (contentType.startsWith("image/"))
+        return "image";
+    if (contentType.startsWith("font/") || contentType.includes("woff"))
+        return "font";
+    if (contentType.startsWith("audio/") || contentType.startsWith("video/"))
+        return "media";
+    return undefined;
+};

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@pagepocket/capture-http-lighterceptor-unit",
+  "version": "0.8.0",
+  "description": "PagePocket plugin: capture HTTP events (lighterceptor adapter)",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "license": "ISC",
+  "dependencies": {
+    "@pagepocket/lib": "0.8.0",
+    "@pagepocket/contracts": "0.8.0",
+    "@pagepocket/lighterceptor": "0.8.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.4.5"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "node -e \"process.exit(0)\""
+  }
+}