@pagepocket/lib 0.7.1 → 0.8.0

package/dist/units/runner.d.ts

@@ -0,0 +1,11 @@
+import type { ChannelToken } from "@pagepocket/contracts";
+import { type CaptureResult, type Plugin, type Unit } from "./contracts-bridge.js";
+import type { CaptureOptions, EntryInfo, PagePocketOptions } from "./types.js";
+export declare const runCapture: (input: {
+    entry: EntryInfo;
+    pocketOptions: PagePocketOptions;
+    options: CaptureOptions;
+    units: Unit[];
+    plugins?: Plugin[];
+    declaredChannels?: ChannelToken<unknown>[];
+}) => Promise<CaptureResult>;

package/dist/units/runner.js

@@ -0,0 +1,270 @@
+import { TERMINAL_RESULT_KEY } from "./contracts-bridge.js";
+const emptyAsyncIterable = () => ({
+    [Symbol.asyncIterator]() {
+        return {
+            next: async () => ({ value: undefined, done: true })
+        };
+    }
+});
+class AsyncQueue {
+    constructor() {
+        this.values = [];
+        this.pending = [];
+        this.done = false;
+    }
+    push(value) {
+        if (this.done) {
+            return;
+        }
+        const resolve = this.pending.shift();
+        if (resolve) {
+            resolve({ value, done: false });
+            return;
+        }
+        this.values.push(value);
+    }
+    close() {
+        if (this.done) {
+            return;
+        }
+        this.done = true;
+        for (const resolve of this.pending) {
+            resolve({ value: undefined, done: true });
+        }
+        this.pending = [];
+    }
+    async *iterate() {
+        while (true) {
+            if (this.values.length > 0) {
+                const v = this.values.shift();
+                if (v === undefined) {
+                    continue;
+                }
+                yield v;
+                continue;
+            }
+            if (this.done) {
+                return;
+            }
+            const next = await new Promise((resolve) => {
+                this.pending.push(resolve);
+            });
+            if (next.done) {
+                return;
+            }
+            yield next.value;
+        }
+    }
+}
+class DeferredTracker {
+    constructor() {
+        this.promises = [];
+    }
+    add(p) {
+        this.promises.push(p);
+    }
+    async awaitAll() {
+        const ps = this.promises;
+        this.promises = [];
+        await Promise.allSettled(ps);
+    }
+}
+class ElementPatchRegistryImpl {
+    constructor() {
+        this.items = [];
+    }
+    contribute(_source, rules) {
+        this.items.push({ rules });
+    }
+    contributeLazy(_source, build) {
+        this.items.push({ build });
+    }
+    async compile() {
+        const out = [];
+        for (const item of this.items) {
+            if (item.rules) {
+                out.push(...item.rules);
+                continue;
+            }
+            if (item.build) {
+                const rules = await item.build();
+                out.push(...rules);
+            }
+        }
+        return out;
+    }
+}
+class RuntimeImpl {
+    constructor(input) {
+        this.channels = new Map();
+        this.deferred = new DeferredTracker();
+        this.elements = new ElementPatchRegistryImpl();
+        this.entry = input.entry;
+        this.options = input.options;
+        this.pocketOptions = input.pocketOptions;
+    }
+    publish(t, value) {
+        const state = this.channels.get(t.id);
+        if (!state || state.closed) {
+            // No subscribers or closed: drop.
+            return;
+        }
+        state.hasPublisher = true;
+        for (const sub of state.subs) {
+            sub.push(value);
+        }
+    }
+    subscribe(t) {
+        // subscribe MUST be safe.
+        // Important: subscribe must also be able to receive future publishes.
+        // If no publisher ever publishes, the iterator will just complete on close.
+        if (!this.channels.has(t.id)) {
+            this.channels.set(t.id, { hasPublisher: false, subs: new Set(), closed: false });
+        }
+        const state = this.channels.get(t.id);
+        if (!state || state.closed) {
+            return emptyAsyncIterable();
+        }
+        const q = new AsyncQueue();
+        const sub = {
+            push: (v) => q.push(v),
+            close: () => q.close()
+        };
+        state.subs.add(sub);
+        const owner = this;
+        return (async function* () {
+            try {
+                for await (const v of q.iterate()) {
+                    yield v;
+                }
+            }
+            finally {
+                // best-effort cleanup
+                const s = owner.channels.get(t.id);
+                s?.subs.delete(sub);
+                q.close();
+            }
+        })();
+    }
+    hasPublisher(t) {
+        return this.channels.get(t.id)?.hasPublisher === true;
+    }
+    defer(promise) {
+        this.deferred.add(promise);
+    }
+    _ensureChannel(t) {
+        if (this.channels.has(t.id)) {
+            return;
+        }
+        this.channels.set(t.id, { hasPublisher: false, subs: new Set(), closed: false });
+    }
+    async _closeAllChannels() {
+        for (const state of this.channels.values()) {
+            if (state.closed) {
+                continue;
+            }
+            state.closed = true;
+            for (const sub of state.subs) {
+                sub.close();
+            }
+            state.subs.clear();
+        }
+    }
+    async _awaitDeferred() {
+        await this.deferred.awaitAll();
+    }
+}
+export const runCapture = async (input) => {
+    const rt = new RuntimeImpl({
+        entry: input.entry,
+        options: input.options,
+        pocketOptions: input.pocketOptions
+    });
+    for (const ch of input.declaredChannels ?? []) {
+        rt._ensureChannel(ch);
+    }
+    const pluginHost = {
+        entry: rt.entry,
+        options: rt.options,
+        subscribe: (t) => rt.subscribe(t),
+        hasPublisher: (t) => rt.hasPublisher(t),
+        elements: rt.elements,
+        defer: (p) => rt.defer(p)
+    };
+    const pluginSetupValues = new Map();
+    for (const plugin of input.plugins ?? []) {
+        const v = await plugin.setup(pluginHost);
+        if (typeof v !== "undefined") {
+            pluginSetupValues.set(plugin, v);
+        }
+    }
+    const mergePatch = (ctx, patch) => {
+        const next = { value: {} };
+        for (const [k, v] of Object.entries(patch)) {
+            if (k === TERMINAL_RESULT_KEY) {
+                continue;
+            }
+            next.value[k] = v;
+        }
+        return next;
+    };
+    let ctx = { value: {} };
+    let result;
+    try {
+        for (const unit of input.units) {
+            if (result) {
+                break;
+            }
+            const baseCtx = ctx;
+            const boundPlugins = (input.plugins ?? []).filter((plugin) => {
+                const unitId = plugin.constructor?.unitId;
+                return unitId === unit.id;
+            });
+            let pluginContributedValue = {};
+            if (boundPlugins.length > 0) {
+                const pluginResults = await Promise.allSettled(boundPlugins
+                    .filter((p) => typeof p.contribute === "function")
+                    .map(async (p) => p.contribute({ value: ctx.value, setupValue: pluginSetupValues.get(p) }, rt)));
+                for (const r of pluginResults) {
+                    if (r.status !== "fulfilled") {
+                        continue;
+                    }
+                    const patch = r.value;
+                    if (!patch || typeof patch !== "object") {
+                        continue;
+                    }
+                    const patchValue = patch;
+                    pluginContributedValue = {
+                        ...pluginContributedValue,
+                        ...mergePatch({ value: {} }, patchValue).value
+                    };
+                    const terminal = patchValue[TERMINAL_RESULT_KEY];
+                    if (terminal) {
+                        result = terminal;
+                    }
+                }
+            }
+            if (result) {
+                break;
+            }
+            const out = (await unit.run(baseCtx, rt)) ?? {};
+            const unitReturnValue = out && typeof out === "object" ? out : {};
+            const merged = unit.merge(unitReturnValue, pluginContributedValue);
+            const mergedPatch = merged && typeof merged === "object" ? merged : {};
+            ctx = { value: mergePatch({ value: {} }, mergedPatch).value };
+            const terminal = mergedPatch[TERMINAL_RESULT_KEY];
+            if (terminal) {
+                result = terminal;
+                break;
+            }
+        }
+    }
+    finally {
+        await rt._closeAllChannels();
+        await rt._awaitDeferred();
+    }
+    if (!result) {
+        throw new Error("No terminal result. Ensure a terminal unit returns a CaptureResult or calls finish().");
+    }
+    return result;
+};

package/dist/units/types.d.ts

@@ -0,0 +1,39 @@
+import type { CompletionStrategy, ContentStore, PathResolver, ResourceFilter } from "../types.js";
+export type EntryInfo = {
+    kind: "url";
+    url: string;
+} | {
+    kind: "puppeteer-page";
+    page: unknown;
+    url?: string;
+} | {
+    kind: "cdp-tab";
+    tabId: number;
+    url?: string;
+} | {
+    kind: "html-string";
+    htmlString: string;
+    baseUrl: string;
+    url?: string;
+} | {
+    kind: "document";
+    baseUrl: string;
+    url?: string;
+};
+export interface CaptureOptions {
+    completion?: CompletionStrategy | CompletionStrategy[];
+    timeoutMs?: number;
+    maxDurationMs?: number;
+    filter?: ResourceFilter;
+    blacklist?: RegExp[];
+    pathResolver?: PathResolver;
+    contentStore?: ContentStore;
+    rewriteEntry?: boolean;
+    rewriteCSS?: boolean;
+    limits?: {
+        maxTotalBytes?: number;
+        maxSingleResourceBytes?: number;
+        maxResources?: number;
+    };
+}
+export type PagePocketOptions = import("../types.js").PagePocketOptions;

package/dist/units/types.js

@@ -0,0 +1 @@
+export {};

package/dist/utils/streams.d.ts

@@ -0,0 +1,2 @@
+export declare const streamToUint8Array: (stream: ReadableStream<Uint8Array>) => Promise<Uint8Array<ArrayBuffer>>;
+export declare const streamFromUint8Array: (bytes: Uint8Array) => ReadableStream<Uint8Array>;

package/dist/utils/streams.js

@@ -0,0 +1,29 @@
+export const streamToUint8Array = async (stream) => {
+    const reader = stream.getReader();
+    const chunks = [];
+    let total = 0;
+    while (true) {
+        const result = await reader.read();
+        if (result.done) {
+            break;
+        }
+        const value = result.value;
+        if (value) {
+            chunks.push(value);
+            total += value.byteLength;
+        }
+    }
+    const output = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        output.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return output;
+};
+export const streamFromUint8Array = (bytes) => new ReadableStream({
+    start(controller) {
+        controller.enqueue(bytes);
+        controller.close();
+    }
+});

package/dist/utils.d.ts

@@ -1,4 +1,4 @@
-import type { BodySource } from "./types";
+import type { BodySource } from "./types.js";
 export declare const sleep: (ms: number) => Promise<unknown>;
 export declare const hashString: (value: string) => string;
 export declare const stripLeadingSlash: (value: string) => string;
@@ -17,3 +17,37 @@ export declare const bodyToTextOrBase64: (bytes: Uint8Array, mimeType?: string)
     base64: string;
     text?: undefined;
 };
+type UrlEquivalentOptions = {
+    /**
+     * Base URL used to resolve relative URLs.
+     *
+     * If omitted, both inputs must already be absolute URLs.
+     */
+    baseUrl?: string;
+    /**
+     * Ignore URL hash (fragment) when comparing.
+     *
+     * Default: true
+     */
+    ignoreHash?: boolean;
+    /**
+     * Treat default ports as equivalent (http:80, https:443).
+     *
+     * Default: true
+     */
+    ignoreDefaultPort?: boolean;
+    /**
+     * Normalize trailing slashes in pathname when comparing.
+     *
+     * Default: true
+     */
+    normalizeTrailingSlash?: boolean;
+};
+/**
+ * Compare two URLs for semantic equivalence under conservative normalization.
+ *
+ * This is intended to be shared by both static resource mapping (resources_path.json)
+ * and API matching (api.json), so it only returns boolean and avoids lossy heuristics.
+ */
+export declare const urlEquivalent: (a: string, b: string, options?: UrlEquivalentOptions) => boolean;
+export {};

package/dist/utils.js

@@ -1,12 +1,8 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.bodyToTextOrBase64 = exports.toUint8Array = exports.isUtf8Text = exports.decodeUtf8 = exports.bytesToBase64 = exports.sanitizePosixPath = exports.ensureLeadingSlash = exports.stripLeadingSlash = exports.hashString = exports.sleep = void 0;
-const content_type_1 = require("./content-type");
+import { isTextResponse } from "./content-type.js";
 const FNV_OFFSET = 0x811c9dc5;
 const FNV_PRIME = 0x01000193;
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-exports.sleep = sleep;
-const hashString = (value) => {
+export const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+export const hashString = (value) => {
     let hash = FNV_OFFSET;
     for (let i = 0; i < value.length; i += 1) {
         hash ^= value.charCodeAt(i);
@@ -14,12 +10,9 @@ const hashString = (value) => {
     }
     return hash.toString(16).padStart(8, "0");
 };
-exports.hashString = hashString;
-const stripLeadingSlash = (value) => value.replace(/^\/+/, "");
-exports.stripLeadingSlash = stripLeadingSlash;
-const ensureLeadingSlash = (value) => (value.startsWith("/") ? value : `/${value}`);
-exports.ensureLeadingSlash = ensureLeadingSlash;
-const sanitizePosixPath = (value) => {
+export const stripLeadingSlash = (value) => value.replace(/^\/+/, "");
+export const ensureLeadingSlash = (value) => (value.startsWith("/") ? value : `/${value}`);
+export const sanitizePosixPath = (value) => {
     const parts = value.split("/").filter(Boolean);
     const clean = [];
     for (const part of parts) {
@@ -31,11 +24,10 @@ const sanitizePosixPath = (value) => {
     }
     return clean.join("/");
 };
-exports.sanitizePosixPath = sanitizePosixPath;
 const getGlobalBuffer = () => {
     return globalThis.Buffer;
 };
-const bytesToBase64 = (bytes) => {
+export const bytesToBase64 = (bytes) => {
     const BufferCtor = getGlobalBuffer();
     if (BufferCtor) {
         return BufferCtor.from(bytes).toString("base64");
@@ -48,8 +40,7 @@ const bytesToBase64 = (bytes) => {
     }
     return btoa(binary);
 };
-exports.bytesToBase64 = bytesToBase64;
-const decodeUtf8 = (bytes) => {
+export const decodeUtf8 = (bytes) => {
     try {
         const decoder = new TextDecoder("utf-8", { fatal: true });
         return decoder.decode(bytes);
@@ -58,19 +49,17 @@ const decodeUtf8 = (bytes) => {
         return null;
     }
 };
-exports.decodeUtf8 = decodeUtf8;
-const isUtf8Text = (bytes, mimeType) => {
-    const decoded = (0, exports.decodeUtf8)(bytes);
+export const isUtf8Text = (bytes, mimeType) => {
+    const decoded = decodeUtf8(bytes);
     if (decoded === null) {
         return false;
     }
     if (mimeType) {
-        return (0, content_type_1.isTextResponse)(mimeType);
+        return isTextResponse(mimeType);
     }
     return true;
 };
-exports.isUtf8Text = isUtf8Text;
-const toUint8Array = async (body) => {
+export const toUint8Array = async (body) => {
     if (body.kind === "buffer") {
         return body.data;
     }
@@ -98,12 +87,101 @@ const toUint8Array = async (body) => {
     }
     return output;
 };
-exports.toUint8Array = toUint8Array;
-const bodyToTextOrBase64 = (bytes, mimeType) => {
-    if ((0, exports.isUtf8Text)(bytes, mimeType)) {
-        const text = (0, exports.decodeUtf8)(bytes) ?? "";
+// Prefer streamFromUint8Array from "./utils/streams".
+export const bodyToTextOrBase64 = (bytes, mimeType) => {
+    if (isUtf8Text(bytes, mimeType)) {
+        const text = decodeUtf8(bytes) ?? "";
         return { encoding: "text", text };
     }
-    return { encoding: "base64", base64: (0, exports.bytesToBase64)(bytes) };
+    return { encoding: "base64", base64: bytesToBase64(bytes) };
+};
+const toUrlOrNull = (input, baseUrl) => {
+    try {
+        if (baseUrl) {
+            return new URL(input, baseUrl);
+        }
+        return new URL(input);
+    }
+    catch {
+        return null;
+    }
+};
+const stripTrailingSlash = (value) => value.replace(/\/+$/, "");
+const normalizePortForCompare = (url, ignoreDefaultPort) => {
+    if (!ignoreDefaultPort) {
+        return url.port;
+    }
+    const protocol = url.protocol;
+    const port = url.port;
+    if (!port) {
+        return "";
+    }
+    if (protocol === "http:" && port === "80") {
+        return "";
+    }
+    if (protocol === "https:" && port === "443") {
+        return "";
+    }
+    return port;
+};
+const areProtocolsEquivalent = (a, b) => {
+    if (a === b) {
+        return true;
+    }
+    // Treat http/https as equivalent for matching.
+    if ((a === "http:" || a === "https:") && (b === "http:" || b === "https:")) {
+        return true;
+    }
+    return false;
+};
+/**
+ * Compare two URLs for semantic equivalence under conservative normalization.
+ *
+ * This is intended to be shared by both static resource mapping (resources_path.json)
+ * and API matching (api.json), so it only returns boolean and avoids lossy heuristics.
+ */
+export const urlEquivalent = (a, b, options = {}) => {
+    const { baseUrl, ignoreHash = true, ignoreDefaultPort = true, normalizeTrailingSlash: shouldNormalizeTrailingSlash = true } = options;
+    const left = toUrlOrNull(String(a), baseUrl);
+    const right = toUrlOrNull(String(b), baseUrl);
+    if (!left || !right) {
+        return false;
+    }
+    // Exclude non-network schemes from equivalence checks.
+    if (left.protocol === "data:" || left.protocol === "blob:") {
+        return false;
+    }
+    if (right.protocol === "data:" || right.protocol === "blob:") {
+        return false;
+    }
+    if (!areProtocolsEquivalent(left.protocol, right.protocol)) {
+        return false;
+    }
+    // Compare host and port (with optional default-port normalization).
+    const leftHost = left.hostname;
+    const rightHost = right.hostname;
+    if (leftHost !== rightHost) {
+        return false;
+    }
+    const leftPort = normalizePortForCompare(left, ignoreDefaultPort);
+    const rightPort = normalizePortForCompare(right, ignoreDefaultPort);
+    if (leftPort !== rightPort) {
+        return false;
+    }
+    const leftPath = shouldNormalizeTrailingSlash ? stripTrailingSlash(left.pathname) : left.pathname;
+    const rightPath = shouldNormalizeTrailingSlash
+        ? stripTrailingSlash(right.pathname)
+        : right.pathname;
+    if (leftPath !== rightPath) {
+        return false;
+    }
+    if (left.search !== right.search) {
+        return false;
+    }
+    if (!ignoreHash) {
+        if (left.hash !== right.hash) {
+            return false;
+        }
+    }
+    return true;
 };
-exports.bodyToTextOrBase64 = bodyToTextOrBase64;

package/dist/v3/contracts-bridge.d.ts

@@ -0,0 +1,69 @@
+import type { ArtifactToken, ChannelToken, ReplaceElementsConfig } from "@pagepocket/contracts";
+export type CaptureResult = {
+    kind: "raw";
+    outputDir: string;
+    meta?: unknown;
+} | {
+    kind: "zip";
+    zip: {
+        data: Uint8Array | Blob;
+        outputPath: string;
+    };
+    meta?: unknown;
+} | {
+    kind: "html";
+    html: string;
+    meta?: unknown;
+} | {
+    kind: "text";
+    text: string;
+    meta?: unknown;
+} | {
+    kind: "custom";
+    value: unknown;
+};
+export type DeferredHandle = Promise<unknown>;
+export interface ElementPatchRegistry {
+    contribute(source: {
+        type: "unit" | "plugin";
+        name: string;
+    }, rules: ReplaceElementsConfig): void;
+    contributeLazy(source: {
+        type: "unit" | "plugin";
+        name: string;
+    }, build: () => ReplaceElementsConfig | Promise<ReplaceElementsConfig>): void;
+    compile(): Promise<ReplaceElementsConfig>;
+}
+export interface UnitRuntime {
+    readonly entry: import("./types.js").EntryInfo;
+    readonly options: import("./types.js").CaptureOptions;
+    readonly pocketOptions: import("./types.js").PagePocketOptions;
+    provide<T>(t: ArtifactToken<T>, value: T): void;
+    get<T>(t: ArtifactToken<T>): T | undefined;
+    require<T>(t: ArtifactToken<T>): T;
+    publish<T>(t: ChannelToken<T>, value: T): void;
+    subscribe<T>(t: ChannelToken<T>): AsyncIterable<T>;
+    hasPublisher(t: ChannelToken<unknown>): boolean;
+    readonly elements: ElementPatchRegistry;
+    finish(result: CaptureResult): void;
+    defer(promise: DeferredHandle): void;
+}
+export interface PluginHost {
+    readonly entry: import("./types.js").EntryInfo;
+    readonly options: import("./types.js").CaptureOptions;
+    subscribe<T>(t: ChannelToken<T>): AsyncIterable<T>;
+    hasPublisher(t: ChannelToken<unknown>): boolean;
+    readonly elements: ElementPatchRegistry;
+    defer(promise: DeferredHandle): void;
+}
+export declare abstract class Unit {
+    abstract readonly id: string;
+    abstract readonly kind: string;
+    protected rt: UnitRuntime;
+    attach(rt: UnitRuntime): void;
+    abstract run(): Promise<void | CaptureResult>;
+}
+export interface Plugin {
+    readonly name: string;
+    setup(host: PluginHost): void | Promise<void>;
+}

package/dist/v3/contracts-bridge.js

@@ -0,0 +1,5 @@
+export class Unit {
+    attach(rt) {
+        this.rt = rt;
+    }
+}

package/dist/v3/index.d.ts

@@ -0,0 +1,4 @@
+export type { CaptureOptions, EntryInfo } from "./types.js";
+export type { CaptureResult, Plugin, PluginHost, UnitRuntime } from "./contracts-bridge.js";
+export { Unit } from "./contracts-bridge.js";
+export { runCaptureV3 as runCapture } from "./runner.js";

package/dist/v3/index.js

@@ -0,0 +1,2 @@
+export { Unit } from "./contracts-bridge.js";
+export { runCaptureV3 as runCapture } from "./runner.js";

package/dist/v3/runner.d.ts

@@ -0,0 +1,20 @@
+import type { ChannelToken } from "@pagepocket/contracts";
+import type { Plugin, Unit } from "./contracts-bridge.js";
+import { type CaptureResult } from "./contracts-bridge.js";
+import type { CaptureOptions, EntryInfo, PagePocketOptions } from "./types.js";
+export declare const runCaptureV3: (input: {
+    entry: EntryInfo;
+    pocketOptions: PagePocketOptions;
+    options: CaptureOptions;
+    units: Unit[];
+    plugins?: Plugin[];
+    declaredChannels?: ChannelToken<unknown>[];
+}) => Promise<CaptureResult>;
+export declare const runCapture: (input: {
+    entry: EntryInfo;
+    pocketOptions: PagePocketOptions;
+    options: CaptureOptions;
+    units: Unit[];
+    plugins?: Plugin[];
+    declaredChannels?: ChannelToken<unknown>[];
+}) => Promise<CaptureResult>;