@pagepocket/lib 0.7.1 → 0.8.0

package/dist/plugins/build-warc.js

@@ -0,0 +1,225 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BuildWarcPlugin = void 0;
+const normalizeHeaders = (headers) => {
+    const normalized = headers.map((h) => ({ name: h.name.toLowerCase(), value: h.value }));
+    normalized.sort((a, b) => {
+        if (a.name < b.name)
+            return -1;
+        if (a.name > b.name)
+            return 1;
+        return 0;
+    });
+    return normalized;
+};
+const headersToText = (headers) => headers.map((h) => `${h.name}: ${h.value}`).join("\r\n") + "\r\n";
+const urlToRequestTarget = (url) => {
+    try {
+        const parsed = new URL(url);
+        const path = parsed.pathname || "/";
+        const query = parsed.search || "";
+        return `${path}${query}`;
+    }
+    catch {
+        return url;
+    }
+};
+const statusTextFromStatus = (status) => {
+    // Small stable mapping; unknown -> empty.
+    switch (status) {
+        case 200:
+            return "OK";
+        case 201:
+            return "Created";
+        case 204:
+            return "No Content";
+        case 301:
+            return "Moved Permanently";
+        case 302:
+            return "Found";
+        case 304:
+            return "Not Modified";
+        case 400:
+            return "Bad Request";
+        case 401:
+            return "Unauthorized";
+        case 403:
+            return "Forbidden";
+        case 404:
+            return "Not Found";
+        case 500:
+            return "Internal Server Error";
+        default:
+            return "";
+    }
+};
+const streamToUint8Array = async (stream) => {
+    const reader = stream.getReader();
+    const chunks = [];
+    let total = 0;
+    while (true) {
+        const result = await reader.read();
+        if (result.done)
+            break;
+        if (result.value) {
+            chunks.push(result.value);
+            total += result.value.byteLength;
+        }
+    }
+    const output = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        output.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return output;
+};
+const encodeText = (text) => new TextEncoder().encode(text);
+const concatBytes = (chunks) => {
+    const total = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
+    const output = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        output.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return output;
+};
+const toWarcDate = (timestamp) => new Date(timestamp).toISOString();
+const randomWarcRecordId = () => {
+    // Best-effort UUIDv4 using crypto if available.
+    const g = globalThis;
+    const bytes = new Uint8Array(16);
+    if (g.crypto?.getRandomValues) {
+        g.crypto.getRandomValues(bytes);
+    }
+    else {
+        for (let i = 0; i < bytes.length; i += 1) {
+            bytes[i] = Math.floor(Math.random() * 256);
+        }
+    }
+    bytes[6] = (bytes[6] & 0x0f) | 0x40;
+    bytes[8] = (bytes[8] & 0x3f) | 0x80;
+    const hex = Array.from(bytes)
+        .map((b) => b.toString(16).padStart(2, "0"))
+        .join("");
+    const uuid = `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
+    return `<urn:uuid:${uuid}>`;
+};
+const makeWarcHeaders = (headers) => {
+    const lines = Object.entries(headers)
+        .map(([k, v]) => `${k}: ${v}`)
+        .join("\r\n");
+    return `${lines}\r\n\r\n`;
+};
+const makeWarcRecord = (input) => {
+    const baseHeaders = {
+        "WARC-Version": "WARC/1.1",
+        "WARC-Type": input.warcType,
+        "WARC-Record-ID": input.recordId,
+        "WARC-Date": input.date,
+        ...(input.targetUri ? { "WARC-Target-URI": input.targetUri } : {}),
+        "Content-Type": input.contentType,
+        "Content-Length": String(input.payload.byteLength),
+        ...(input.extraWarcHeaders ?? {})
+    };
+    const headerBytes = encodeText(makeWarcHeaders(baseHeaders));
+    const endBytes = encodeText("\r\n");
+    return concatBytes([headerBytes, input.payload, endBytes]);
+};
+const groupByRequestId = (events) => {
+    const requests = new Map();
+    const responses = new Map();
+    for (const e of events) {
+        if (e.type === "http.request") {
+            requests.set(e.requestId, e);
+            continue;
+        }
+        if (e.type === "http.response") {
+            responses.set(e.requestId, e);
+        }
+    }
+    return { requests, responses };
+};
+class BuildWarcPlugin {
+    constructor() {
+        this.name = "plugin:build-warc";
+    }
+    apply(ctx) {
+        ctx.onFinalize(async () => {
+            const capture = ctx.capture;
+            if (!capture) {
+                throw new Error("BuildWarcPlugin requires ctx.capture");
+            }
+            const { requests, responses } = groupByRequestId(capture.events);
+            const warcinfoPayload = encodeText([
+                "software: pagepocket",
+                `format: WARC File Format 1.1 (best-effort)`,
+                `pagepocket:headersNormalized=true`,
+                `pagepocket:bodyMode=decoded`,
+                `pagepocket:capabilities=${JSON.stringify(capture.capabilities)}`
+            ].join("\n") + "\n");
+            const records = [];
+            records.push(makeWarcRecord({
+                warcType: "warcinfo",
+                date: toWarcDate(Date.now()),
+                recordId: randomWarcRecordId(),
+                contentType: "application/warc-fields",
+                payload: warcinfoPayload
+            }));
+            for (const [requestId, reqEvent] of requests.entries()) {
+                if (reqEvent.type !== "http.request") {
+                    continue;
+                }
+                const resEvent = responses.get(requestId);
+                if (!resEvent || resEvent.type !== "http.response") {
+                    continue;
+                }
+                const reqHeaders = normalizeHeaders(reqEvent.headers);
+                const requestTarget = urlToRequestTarget(reqEvent.url);
+                const requestStartLine = `${reqEvent.method} ${requestTarget} HTTP/1.1\r\n`;
+                const requestHttpBytes = encodeText(requestStartLine + headersToText(reqHeaders) + "\r\n");
+                const resHeaders = normalizeHeaders(resEvent.headers);
+                const statusText = resEvent.statusText ?? statusTextFromStatus(resEvent.status);
+                const responseStartLine = `HTTP/1.1 ${resEvent.status} ${statusText}\r\n`;
+                const responseHeadersBytes = encodeText(responseStartLine + headersToText(resHeaders) + "\r\n");
+                const bodyBytes = resEvent.bodyRef
+                    ? await streamToUint8Array(await capture.contentStore.open(resEvent.bodyRef))
+                    : new Uint8Array();
+                const responseHttpBytes = concatBytes([responseHeadersBytes, bodyBytes]);
+                records.push(makeWarcRecord({
+                    warcType: "request",
+                    targetUri: reqEvent.url,
+                    date: toWarcDate(reqEvent.timestamp),
+                    recordId: randomWarcRecordId(),
+                    contentType: "application/http; msgtype=request",
+                    payload: requestHttpBytes
+                }));
+                records.push(makeWarcRecord({
+                    warcType: "response",
+                    targetUri: resEvent.url,
+                    date: toWarcDate(resEvent.timestamp),
+                    recordId: randomWarcRecordId(),
+                    contentType: "application/http; msgtype=response",
+                    payload: responseHttpBytes
+                }));
+            }
+            const warcBytes = concatBytes(records);
+            const files = {
+                root: {
+                    kind: "directory",
+                    path: "",
+                    entries: [
+                        {
+                            kind: "file",
+                            path: "capture.warc",
+                            source: { kind: "bytes", data: warcBytes }
+                        }
+                    ]
+                }
+            };
+            ctx.files = files;
+        });
+    }
+}
+exports.BuildWarcPlugin = BuildWarcPlugin;

package/dist/plugins/builtins/manifest.d.ts

@@ -0,0 +1,2 @@
+import type { PagePocketPlugin } from "../types";
+export declare const manifestPlugin: () => PagePocketPlugin;

package/dist/plugins/builtins/manifest.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.manifestPlugin = void 0;
+const manifestFromContext = (ctx) => {
+    const config = ctx.config;
+    return {
+        version: "1.0",
+        createdAt: ctx.bundle.createdAt,
+        entryUrl: ctx.bundle.entryUrl,
+        finalUrl: ctx.bundle.finalUrl,
+        emit: config.emit ?? [],
+        skip: config.skip,
+        plugins: config.plugins ?? [],
+        bundle: {
+            totals: ctx.bundle.totals,
+            warnings: ctx.bundle.warnings
+        },
+        snapshot: config.snapshot
+            ? {
+                built: config.snapshot.built,
+                options: config.snapshot.options,
+                meta: ctx.snapshot?.meta
+            }
+            : undefined
+    };
+};
+const manifestPlugin = () => {
+    return {
+        name: "built-in:manifest",
+        outputs: [
+            {
+                id: "manifest",
+                requires: "bundle",
+                async write(ctx) {
+                    const manifest = manifestFromContext(ctx);
+                    await ctx.sink.writeJson("manifest.json", manifest);
+                }
+            }
+        ]
+    };
+};
+exports.manifestPlugin = manifestPlugin;

package/dist/plugins/builtins/snapshot-directory.d.ts

@@ -0,0 +1,2 @@
+import type { PagePocketPlugin } from "../types";
+export declare const snapshotDirectoryPlugin: () => PagePocketPlugin;

package/dist/plugins/builtins/snapshot-directory.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.snapshotDirectoryPlugin = void 0;
+const snapshotDirectoryPlugin = () => {
+    return {
+        name: "built-in:snapshot-directory",
+        outputs: [
+            {
+                id: "directory",
+                requires: "snapshot",
+                async write(ctx) {
+                    if (!ctx.snapshot) {
+                        throw new Error("directory output requires ctx.snapshot");
+                    }
+                    if (!ctx.outputDir) {
+                        throw new Error("directory output requires ctx.outputDir");
+                    }
+                    await ctx.snapshot.toDirectory(ctx.outputDir);
+                }
+            }
+        ]
+    };
+};
+exports.snapshotDirectoryPlugin = snapshotDirectoryPlugin;

package/dist/plugins/builtins/snapshot-zip.d.ts

@@ -0,0 +1,2 @@
+import type { PagePocketPlugin } from "../types";
+export declare const snapshotZipPlugin: () => PagePocketPlugin;

package/dist/plugins/builtins/snapshot-zip.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.snapshotZipPlugin = void 0;
+const snapshotZipPlugin = () => {
+    return {
+        name: "built-in:snapshot-zip",
+        outputs: [
+            {
+                id: "zip",
+                requires: "snapshot",
+                async write(ctx) {
+                    if (!ctx.snapshot) {
+                        throw new Error("zip output requires ctx.snapshot");
+                    }
+                    const zipped = await ctx.snapshot.toZip();
+                    if (!(zipped instanceof Uint8Array)) {
+                        throw new Error("zip output expects snapshot.toZip() to return Uint8Array in node");
+                    }
+                    await ctx.sink.writeFile("snapshot.zip", zipped, "application/zip");
+                }
+            }
+        ]
+    };
+};
+exports.snapshotZipPlugin = snapshotZipPlugin;

package/dist/plugins/capture-http-lighterceptor.d.ts

@@ -0,0 +1,5 @@
+import type { PagePocketContext, PagePocketPlugin } from "../plugin/types";
+export declare class CaptureHttpLighterceptorPlugin implements PagePocketPlugin {
+    readonly name = "plugin:capture-http-lighterceptor";
+    apply(ctx: PagePocketContext): void;
+}

package/dist/plugins/capture-http-lighterceptor.js

@@ -0,0 +1,85 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CaptureHttpLighterceptorPlugin = void 0;
+const memory_content_store_1 = require("../capture/memory-content-store");
+const headersRecordToList = (headers) => {
+    if (!headers) {
+        return [];
+    }
+    const out = [];
+    for (const name of Object.keys(headers)) {
+        out.push({ name, value: headers[name] });
+    }
+    return out;
+};
+class CaptureHttpLighterceptorPlugin {
+    constructor() {
+        this.name = "plugin:capture-http-lighterceptor";
+    }
+    apply(ctx) {
+        const contentStore = (0, memory_content_store_1.createMemoryContentStore)("capture-http-lighterceptor");
+        const events = [];
+        const capabilities = {
+            requestHeaders: "approx",
+            responseHeaders: "approx",
+            requestBodies: false,
+            responseBodies: "decoded",
+            httpVersion: false,
+            remoteIp: false,
+            headerOrderPreserved: false
+        };
+        ctx.capture = {
+            events,
+            contentStore,
+            capabilities
+        };
+        ctx.onNetworkEvent(async (event) => {
+            if (event.type === "request") {
+                events.push({
+                    type: "http.request",
+                    requestId: event.requestId,
+                    url: event.url,
+                    method: event.method,
+                    headers: headersRecordToList(event.headers),
+                    timestamp: event.timestamp,
+                    frameId: event.frameId,
+                    resourceType: event.resourceType,
+                    initiator: event.initiator
+                });
+                return;
+            }
+            if (event.type === "failed") {
+                events.push({
+                    type: "http.failed",
+                    requestId: event.requestId,
+                    url: event.url,
+                    errorText: event.errorText,
+                    timestamp: event.timestamp
+                });
+                return;
+            }
+            const bodyRef = event.body
+                ? await contentStore.put(event.body, {
+                    url: event.url,
+                    mimeType: event.mimeType,
+                    sizeHint: undefined
+                })
+                : undefined;
+            events.push({
+                type: "http.response",
+                requestId: event.requestId,
+                url: event.url,
+                status: event.status,
+                statusText: event.statusText,
+                headers: headersRecordToList(event.headers),
+                timestamp: event.timestamp,
+                mimeType: event.mimeType,
+                fromDiskCache: event.fromDiskCache,
+                fromServiceWorker: event.fromServiceWorker,
+                bodyRef,
+                bodySize: undefined
+            });
+        });
+    }
+}
+exports.CaptureHttpLighterceptorPlugin = CaptureHttpLighterceptorPlugin;

package/dist/plugins/capture-http-puppeteer.d.ts

@@ -0,0 +1,5 @@
+import type { PagePocketContext, PagePocketPlugin } from "../plugin/types";
+export declare class CaptureHttpPuppeteerPlugin implements PagePocketPlugin {
+    readonly name = "plugin:capture-http-puppeteer";
+    apply(ctx: PagePocketContext): void;
+}

package/dist/plugins/capture-http-puppeteer.js

@@ -0,0 +1,85 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CaptureHttpPuppeteerPlugin = void 0;
+const memory_content_store_1 = require("../capture/memory-content-store");
+const headersRecordToList = (headers) => {
+    if (!headers) {
+        return [];
+    }
+    const out = [];
+    for (const name of Object.keys(headers)) {
+        out.push({ name, value: headers[name] });
+    }
+    return out;
+};
+class CaptureHttpPuppeteerPlugin {
+    constructor() {
+        this.name = "plugin:capture-http-puppeteer";
+    }
+    apply(ctx) {
+        const contentStore = (0, memory_content_store_1.createMemoryContentStore)("capture-http-puppeteer");
+        const events = [];
+        const capabilities = {
+            requestHeaders: "approx",
+            responseHeaders: "approx",
+            requestBodies: false,
+            responseBodies: "decoded",
+            httpVersion: false,
+            remoteIp: false,
+            headerOrderPreserved: false
+        };
+        ctx.capture = {
+            events,
+            contentStore,
+            capabilities
+        };
+        ctx.onNetworkEvent(async (event) => {
+            if (event.type === "request") {
+                events.push({
+                    type: "http.request",
+                    requestId: event.requestId,
+                    url: event.url,
+                    method: event.method,
+                    headers: headersRecordToList(event.headers),
+                    timestamp: event.timestamp,
+                    frameId: event.frameId,
+                    resourceType: event.resourceType,
+                    initiator: event.initiator
+                });
+                return;
+            }
+            if (event.type === "failed") {
+                events.push({
+                    type: "http.failed",
+                    requestId: event.requestId,
+                    url: event.url,
+                    errorText: event.errorText,
+                    timestamp: event.timestamp
+                });
+                return;
+            }
+            const bodyRef = event.body
+                ? await contentStore.put(event.body, {
+                    url: event.url,
+                    mimeType: event.mimeType,
+                    sizeHint: undefined
+                })
+                : undefined;
+            events.push({
+                type: "http.response",
+                requestId: event.requestId,
+                url: event.url,
+                status: event.status,
+                statusText: event.statusText,
+                headers: headersRecordToList(event.headers),
+                timestamp: event.timestamp,
+                mimeType: event.mimeType,
+                fromDiskCache: event.fromDiskCache,
+                fromServiceWorker: event.fromServiceWorker,
+                bodyRef,
+                bodySize: undefined
+            });
+        });
+    }
+}
+exports.CaptureHttpPuppeteerPlugin = CaptureHttpPuppeteerPlugin;

package/dist/plugins/host.d.ts

@@ -0,0 +1,37 @@
+import type { ReplaceElementsConfig } from "../types";
+import type { OutputWriter, PagePocketPlugin, PluginContext } from "./types";
+export type OutputPlan = {
+    /** Output IDs requested by the user (multiple allowed). */
+    emit: string[];
+    /** Output IDs to skip even if present in emit. */
+    skip?: string[];
+    /** Abort on first plugin failure. Default false. */
+    strict?: boolean;
+};
+export type ResolvedOutput = {
+    id: string;
+    requires: "bundle" | "snapshot";
+    write: OutputWriter["write"];
+    /** For diagnostics / ordering */
+    pluginName: string;
+};
+export interface PluginHost {
+    plugins(): PagePocketPlugin[];
+    init(ctx: PluginContext): Promise<void>;
+    /**
+     * Resolve all outputs by id and filter by emit/skip.
+     * MUST throw on:
+     * - duplicate output IDs
+     * - unknown emit IDs
+     */
+    resolveOutputs(plan: OutputPlan): {
+        outputs: ResolvedOutput[];
+        needsSnapshot: boolean;
+    };
+    /** Merge replaceElements from plugins in deterministic order. */
+    collectReplaceElements(ctx: PluginContext): Promise<ReplaceElementsConfig | undefined>;
+    shutdown(ctx: PluginContext): Promise<void>;
+}
+export declare const createPluginHost: (input: {
+    plugins: PagePocketPlugin[];
+}) => PluginHost;

package/dist/plugins/host.js

@@ -0,0 +1,105 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createPluginHost = void 0;
+const ordering_1 = require("./ordering");
+const collectAllOutputs = (plugins) => {
+    const items = [];
+    for (const plugin of plugins) {
+        const outputs = plugin.outputs ?? [];
+        for (const writer of outputs) {
+            items.push({ plugin, writer });
+        }
+    }
+    return items;
+};
+const formatAvailableOutputs = (plugins) => {
+    const byId = new Map();
+    for (const { plugin, writer } of collectAllOutputs(plugins)) {
+        byId.set(writer.id, { requires: writer.requires, pluginName: plugin.name });
+    }
+    const ids = Array.from(byId.keys()).sort();
+    return ids
+        .map((id) => {
+        const meta = byId.get(id);
+        return `- ${id} (${meta?.requires ?? "?"}) [${meta?.pluginName ?? "?"}]`;
+    })
+        .join("\n");
+};
+const createPluginHost = (input) => {
+    const ordered = (0, ordering_1.sortPluginsDeterministically)(input.plugins);
+    const outputsById = () => {
+        const map = new Map();
+        const duplicates = [];
+        for (const { plugin, writer } of collectAllOutputs(ordered)) {
+            if (map.has(writer.id)) {
+                duplicates.push(writer.id);
+                continue;
+            }
+            map.set(writer.id, { plugin, writer });
+        }
+        if (duplicates.length > 0) {
+            const unique = Array.from(new Set(duplicates)).sort();
+            throw new Error(`Duplicate output IDs: ${unique.join(", ")}`);
+        }
+        return map;
+    };
+    return {
+        plugins() {
+            return ordered.slice();
+        },
+        async init(ctx) {
+            for (const plugin of ordered) {
+                await plugin.hooks?.init?.(ctx);
+            }
+        },
+        resolveOutputs(plan) {
+            const emit = plan.emit ?? [];
+            const skip = new Set(plan.skip ?? []);
+            const all = outputsById();
+            const unknown = emit.filter((id) => !all.has(id));
+            if (unknown.length > 0) {
+                const available = formatAvailableOutputs(ordered);
+                throw new Error(`Unknown output IDs: ${unknown.sort().join(", ")}\n\nAvailable outputs:\n${available}`);
+            }
+            const resolved = [];
+            let needsSnapshot = false;
+            // Deterministic execution order: plugin order; within plugin outputs array order.
+            for (const plugin of ordered) {
+                for (const writer of plugin.outputs ?? []) {
+                    if (!emit.includes(writer.id))
+                        continue;
+                    if (skip.has(writer.id))
+                        continue;
+                    if (writer.requires === "snapshot") {
+                        needsSnapshot = true;
+                    }
+                    resolved.push({
+                        id: writer.id,
+                        requires: writer.requires,
+                        write: writer.write,
+                        pluginName: plugin.name
+                    });
+                }
+            }
+            return { outputs: resolved, needsSnapshot };
+        },
+        async collectReplaceElements(ctx) {
+            const merged = [];
+            for (const plugin of ordered) {
+                if (!plugin.replaceElements)
+                    continue;
+                const contributed = await plugin.replaceElements(ctx);
+                if (contributed && contributed.length > 0) {
+                    merged.push(...contributed);
+                }
+            }
+            return merged.length ? merged : undefined;
+        },
+        async shutdown(ctx) {
+            for (const plugin of ordered) {
+                await plugin.hooks?.shutdown?.(ctx);
+            }
+        }
+    };
+};
+exports.createPluginHost = createPluginHost;

package/dist/plugins/index.d.ts

@@ -0,0 +1,6 @@
+export type { ArtifactSink, LogLevel, OutputRequirement, OutputWriter, PagePocketPlugin, PluginContext, PluginOrder, RunContext } from "./types";
+export type { OutputPlan, ResolvedOutput, PluginHost } from "./host";
+export { createPluginHost } from "./host";
+export { manifestPlugin } from "./builtins/manifest";
+export { snapshotDirectoryPlugin } from "./builtins/snapshot-directory";
+export { snapshotZipPlugin } from "./builtins/snapshot-zip";

package/dist/plugins/index.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.snapshotZipPlugin = exports.snapshotDirectoryPlugin = exports.manifestPlugin = exports.createPluginHost = void 0;
+var host_1 = require("./host");
+Object.defineProperty(exports, "createPluginHost", { enumerable: true, get: function () { return host_1.createPluginHost; } });
+var manifest_1 = require("./builtins/manifest");
+Object.defineProperty(exports, "manifestPlugin", { enumerable: true, get: function () { return manifest_1.manifestPlugin; } });
+var snapshot_directory_1 = require("./builtins/snapshot-directory");
+Object.defineProperty(exports, "snapshotDirectoryPlugin", { enumerable: true, get: function () { return snapshot_directory_1.snapshotDirectoryPlugin; } });
+var snapshot_zip_1 = require("./builtins/snapshot-zip");
+Object.defineProperty(exports, "snapshotZipPlugin", { enumerable: true, get: function () { return snapshot_zip_1.snapshotZipPlugin; } });

package/dist/plugins/ordering.d.ts

@@ -0,0 +1,2 @@
+import type { PagePocketPlugin } from "./types";
+export declare const sortPluginsDeterministically: (plugins: PagePocketPlugin[]) => PagePocketPlugin[];