@pagepocket/lib 0.7.0 → 0.8.0

package/dist/core/content-store.js

@@ -0,0 +1,54 @@
+import { readBinary, remove, write } from "@pagepocket/uni-fs";
+import { streamFromUint8Array } from "../utils/streams.js";
+import { hashString, toUint8Array } from "../utils.js";
+const DEFAULT_THRESHOLD = 256 * 1024;
+const DEFAULT_BASE_DIR = ".pagepocket_store";
+const nowId = () => `${Date.now()}_${Math.random().toString(16).slice(2)}`;
+const isNodeEnvironment = () => {
+    const globalProcess = globalThis.process;
+    return typeof globalProcess?.versions?.node === "string";
+};
+export class HybridContentStore {
+    constructor(options) {
+        this.name = "hybrid";
+        this.storedIds = new Set();
+        this.thresholdBytes = options?.thresholdBytes ?? DEFAULT_THRESHOLD;
+        this.baseDir = options?.baseDir ?? DEFAULT_BASE_DIR;
+    }
+    async put(body, meta) {
+        const data = await toUint8Array(body);
+        const size = data.byteLength;
+        if (size <= this.thresholdBytes) {
+            return { kind: "memory", data };
+        }
+        const id = `${hashString(meta.url)}_${nowId()}`;
+        await write(`${this.baseDir}/${id}`, "bin", data);
+        this.storedIds.add(id);
+        return { kind: "store-ref", id };
+    }
+    async open(ref) {
+        if (ref.kind === "memory") {
+            return streamFromUint8Array(ref.data);
+        }
+        const data = await readBinary(`${this.baseDir}/${ref.id}`, "bin");
+        return streamFromUint8Array(data);
+    }
+    async dispose() {
+        const entries = Array.from(this.storedIds);
+        this.storedIds.clear();
+        await Promise.all(entries.map((id) => remove(`${this.baseDir}/${id}`, "bin").catch(() => { })));
+        await this.removeBaseDir().catch(() => { });
+    }
+    async removeBaseDir() {
+        if (!isNodeEnvironment()) {
+            return;
+        }
+        if (!this.baseDir || this.baseDir === "/" || this.baseDir === ".") {
+            return;
+        }
+        const { resolve, isAbsolute } = await import("node:path");
+        const { rm } = await import("node:fs/promises");
+        const target = isAbsolute(this.baseDir) ? this.baseDir : resolve(this.baseDir);
+        await rm(target, { recursive: true, force: true });
+    }
+}

package/dist/core/debug.d.ts

@@ -0,0 +1 @@
+export declare const debugLog: (...args: unknown[]) => void;

package/dist/core/debug.js

@@ -0,0 +1,12 @@
+const isDebugEnabled = () => {
+    const globalProcess = globalThis
+        .process;
+    const value = globalProcess?.env?.PAGEPOCKET_DEBUG;
+    return Boolean(value);
+};
+export const debugLog = (...args) => {
+    if (!isDebugEnabled()) {
+        return;
+    }
+    console.log(...args);
+};

package/dist/core/file-tree-merge.d.ts

@@ -0,0 +1,2 @@
+import type { FileTree } from "./file-tree.js";
+export declare const mergeFileTrees: (base: FileTree, extra: FileTree) => FileTree;

package/dist/core/file-tree-merge.js

@@ -0,0 +1,27 @@
+const entryKey = (entry) => `${entry.kind}:${entry.path}`;
+const mergeDirectories = (a, b) => {
+    const outByKey = new Map();
+    for (const entry of a.entries) {
+        outByKey.set(entryKey(entry), entry);
+    }
+    for (const entry of b.entries) {
+        const key = entryKey(entry);
+        const prev = outByKey.get(key);
+        if (prev && prev.kind === "directory" && entry.kind === "directory") {
+            outByKey.set(key, mergeDirectories(prev, entry));
+            continue;
+        }
+        outByKey.set(key, entry);
+    }
+    return {
+        ...a,
+        entries: Array.from(outByKey.values())
+    };
+};
+export const mergeFileTrees = (base, extra) => {
+    const root = mergeDirectories(base.root, extra.root);
+    return {
+        root,
+        ...(base.content ? { content: base.content } : {})
+    };
+};

package/dist/core/file-tree.d.ts

@@ -0,0 +1,36 @@
+import type { ContentRef, ContentStoreHandle } from "../types.js";
+export type FileTreeSource = {
+    kind: "bytes";
+    data: Uint8Array;
+} | {
+    kind: "text";
+    text: string;
+} | {
+    kind: "content-ref";
+    ref: ContentRef;
+};
+export type FileTreeFile = {
+    kind: "file";
+    /** Path relative to the root (posix-ish, may include '/'). */
+    path: string;
+    source: FileTreeSource;
+};
+export type FileTreeDirectory = {
+    kind: "directory";
+    /** Path relative to the root (posix-ish, may include '/'). */
+    path: string;
+    entries: FileTreeEntry[];
+};
+export type FileTreeEntry = FileTreeFile | FileTreeDirectory;
+/**
+ * Core, plugin-agnostic filesystem tree.
+ *
+ * Builders should attach this to `ctx.value.files`.
+ * Writers should consume it.
+ */
+export type FileTree = {
+    /** Virtual root directory; its path should be "". */
+    root: FileTreeDirectory;
+    /** Optional content handle for `{ kind: "content-ref" }`. */
+    content?: ContentStoreHandle;
+};

package/dist/core/file-tree.js

@@ -0,0 +1 @@
+export {};

package/dist/core/inflight-tracker.d.ts

@@ -0,0 +1,19 @@
+import type { NetworkEvent } from "../types.js";
+type InflightStats = {
+    inflightRequests: number;
+    lastNetworkTs: number;
+    totalRequests: number;
+};
+export declare class InflightTracker {
+    private inflight;
+    private inflightRequests;
+    private lastNetworkTs;
+    private totalRequests;
+    private shouldIgnore;
+    constructor(options?: {
+        shouldIgnore?: (event: NetworkEvent) => boolean;
+    });
+    handleEvent(event: NetworkEvent): InflightStats;
+    getStats(): InflightStats;
+}
+export {};

package/dist/core/inflight-tracker.js

@@ -0,0 +1,44 @@
+export class InflightTracker {
+    constructor(options) {
+        this.inflight = new Set();
+        this.inflightRequests = 0;
+        this.lastNetworkTs = Date.now();
+        this.totalRequests = 0;
+        this.shouldIgnore = options?.shouldIgnore ?? (() => false);
+    }
+    handleEvent(event) {
+        if (this.shouldIgnore(event)) {
+            return this.getStats();
+        }
+        if (event?.timestamp) {
+            this.lastNetworkTs = event.timestamp;
+        }
+        else {
+            this.lastNetworkTs = Date.now();
+        }
+        if (event?.type === "request") {
+            this.totalRequests += 1;
+            if (!this.inflight.has(event.requestId)) {
+                this.inflight.add(event.requestId);
+                this.inflightRequests += 1;
+            }
+        }
+        if (event?.type === "response" || event?.type === "failed") {
+            if (this.inflight.delete(event.requestId)) {
+                this.inflightRequests = Math.max(0, this.inflightRequests - 1);
+            }
+        }
+        return {
+            inflightRequests: this.inflightRequests,
+            lastNetworkTs: this.lastNetworkTs,
+            totalRequests: this.totalRequests
+        };
+    }
+    getStats() {
+        return {
+            inflightRequests: this.inflightRequests,
+            lastNetworkTs: this.lastNetworkTs,
+            totalRequests: this.totalRequests
+        };
+    }
+}

package/dist/core/internal/completion.d.ts

@@ -0,0 +1,4 @@
+import type { CompletionStrategy } from "../../types";
+export declare const timeout: (ms: number) => CompletionStrategy;
+export declare const networkIdle: (ms: number, checkInterval?: number) => CompletionStrategy;
+export declare const normalizeCompletion: (completion?: CompletionStrategy | CompletionStrategy[]) => CompletionStrategy[];

package/dist/core/internal/completion.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeCompletion = exports.networkIdle = exports.timeout = void 0;
+const utils_1 = require("../../utils");
+const timeout = (ms) => ({
+    async wait() {
+        await (0, utils_1.sleep)(ms);
+    }
+});
+exports.timeout = timeout;
+const networkIdle = (ms, checkInterval = 100) => ({
+    async wait(ctx) {
+        while (true) {
+            const stats = ctx.getStats();
+            const idleFor = ctx.now() - stats.lastNetworkTs;
+            if (stats.inflightRequests === 0 && idleFor >= ms) {
+                return;
+            }
+            await (0, utils_1.sleep)(Math.min(checkInterval, ms));
+        }
+    }
+});
+exports.networkIdle = networkIdle;
+const normalizeCompletion = (completion) => {
+    if (!completion)
+        return [];
+    return Array.isArray(completion) ? completion : [completion];
+};
+exports.normalizeCompletion = normalizeCompletion;

package/dist/core/internal/content-store.d.ts

@@ -0,0 +1,21 @@
+import type { BodySource, ContentRef, ContentStore } from "../../types";
+type HybridContentStoreOptions = {
+    thresholdBytes?: number;
+    baseDir?: string;
+};
+export declare class HybridContentStore implements ContentStore {
+    name: string;
+    private thresholdBytes;
+    private baseDir;
+    private storedIds;
+    constructor(options?: HybridContentStoreOptions);
+    put(body: BodySource, meta: {
+        url: string;
+        mimeType?: string;
+        sizeHint?: number;
+    }): Promise<ContentRef>;
+    open(ref: ContentRef): Promise<ReadableStream<Uint8Array>>;
+    dispose(): Promise<void>;
+    private removeBaseDir;
+}
+export {};

package/dist/core/internal/content-store.js

@@ -0,0 +1,91 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.HybridContentStore = void 0;
+const uni_fs_1 = require("@pagepocket/uni-fs");
+const utils_1 = require("../../utils");
+const streams_1 = require("../../utils/streams");
+const DEFAULT_THRESHOLD = 256 * 1024;
+const DEFAULT_BASE_DIR = ".pagepocket_store";
+const nowId = () => `${Date.now()}_${Math.random().toString(16).slice(2)}`;
+const isNodeEnvironment = () => {
+    const globalProcess = globalThis.process;
+    return typeof globalProcess?.versions?.node === "string";
+};
+class HybridContentStore {
+    constructor(options) {
+        this.name = "hybrid";
+        this.storedIds = new Set();
+        this.thresholdBytes = options?.thresholdBytes ?? DEFAULT_THRESHOLD;
+        this.baseDir = options?.baseDir ?? DEFAULT_BASE_DIR;
+    }
+    async put(body, meta) {
+        const data = await (0, utils_1.toUint8Array)(body);
+        const size = data.byteLength;
+        if (size <= this.thresholdBytes) {
+            return { kind: "memory", data };
+        }
+        const id = `${(0, utils_1.hashString)(meta.url)}_${nowId()}`;
+        await (0, uni_fs_1.write)(`${this.baseDir}/${id}`, "bin", data);
+        this.storedIds.add(id);
+        return { kind: "store-ref", id };
+    }
+    async open(ref) {
+        if (ref.kind === "memory") {
+            return (0, streams_1.streamFromUint8Array)(ref.data);
+        }
+        const data = await (0, uni_fs_1.readBinary)(`${this.baseDir}/${ref.id}`, "bin");
+        return (0, streams_1.streamFromUint8Array)(data);
+    }
+    async dispose() {
+        const entries = Array.from(this.storedIds);
+        this.storedIds.clear();
+        await Promise.all(entries.map((id) => (0, uni_fs_1.remove)(`${this.baseDir}/${id}`, "bin").catch(() => { })));
+        await this.removeBaseDir().catch(() => { });
+    }
+    async removeBaseDir() {
+        if (!isNodeEnvironment()) {
+            return;
+        }
+        if (!this.baseDir || this.baseDir === "/" || this.baseDir === ".") {
+            return;
+        }
+        const { resolve, isAbsolute } = await Promise.resolve().then(() => __importStar(require("node:path")));
+        const { rm } = await Promise.resolve().then(() => __importStar(require("node:fs/promises")));
+        const target = isAbsolute(this.baseDir) ? this.baseDir : resolve(this.baseDir);
+        await rm(target, { recursive: true, force: true });
+    }
+}
+exports.HybridContentStore = HybridContentStore;

package/dist/core/internal/debug.d.ts

@@ -0,0 +1 @@
+export declare const debugLog: (...args: unknown[]) => void;

package/dist/core/internal/debug.js

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.debugLog = void 0;
+const isDebugEnabled = () => {
+    const globalProcess = globalThis
+        .process;
+    const value = globalProcess?.env?.PAGEPOCKET_DEBUG;
+    return Boolean(value);
+};
+const debugLog = (...args) => {
+    if (!isDebugEnabled()) {
+        return;
+    }
+    console.log(...args);
+};
+exports.debugLog = debugLog;

package/dist/core/internal/inflight-tracker.d.ts

@@ -0,0 +1,19 @@
+import type { NetworkEvent } from "@pagepocket/interceptor";
+type InflightStats = {
+    inflightRequests: number;
+    lastNetworkTs: number;
+    totalRequests: number;
+};
+export declare class InflightTracker {
+    private inflight;
+    private inflightRequests;
+    private lastNetworkTs;
+    private totalRequests;
+    private shouldIgnore;
+    constructor(options?: {
+        shouldIgnore?: (event: NetworkEvent) => boolean;
+    });
+    handleEvent(event: NetworkEvent): InflightStats;
+    getStats(): InflightStats;
+}
+export {};

package/dist/core/internal/inflight-tracker.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InflightTracker = void 0;
+class InflightTracker {
+    constructor(options) {
+        this.inflight = new Set();
+        this.inflightRequests = 0;
+        this.lastNetworkTs = Date.now();
+        this.totalRequests = 0;
+        this.shouldIgnore = options?.shouldIgnore ?? (() => false);
+    }
+    handleEvent(event) {
+        if (this.shouldIgnore(event)) {
+            return this.getStats();
+        }
+        if (event?.timestamp) {
+            this.lastNetworkTs = event.timestamp;
+        }
+        else {
+            this.lastNetworkTs = Date.now();
+        }
+        if (event?.type === "request") {
+            this.totalRequests += 1;
+            if (!this.inflight.has(event.requestId)) {
+                this.inflight.add(event.requestId);
+                this.inflightRequests += 1;
+            }
+        }
+        if (event?.type === "response" || event?.type === "failed") {
+            if (this.inflight.delete(event.requestId)) {
+                this.inflightRequests = Math.max(0, this.inflightRequests - 1);
+            }
+        }
+        return {
+            inflightRequests: this.inflightRequests,
+            lastNetworkTs: this.lastNetworkTs,
+            totalRequests: this.totalRequests
+        };
+    }
+    getStats() {
+        return {
+            inflightRequests: this.inflightRequests,
+            lastNetworkTs: this.lastNetworkTs,
+            totalRequests: this.totalRequests
+        };
+    }
+}
+exports.InflightTracker = InflightTracker;

package/dist/core/internal/pagepocket.d.ts

@@ -0,0 +1,27 @@
+import type { NetworkEvent } from "@pagepocket/interceptor";
+import type { NetworkEventStream, PagePocketOptions } from "../../types";
+import type { CaptureWithPluginsOptions, PagePocketCaptureResult } from "../plugin/types";
+export declare class PagePocket {
+    private target;
+    private options;
+    private eventStream;
+    private constructor();
+    static fromURL(url: string, options?: PagePocketOptions): PagePocket;
+    static fromPuppeteerPage(page: unknown, options?: PagePocketOptions): PagePocket;
+    static fromCDPTab(tabId: number, options?: PagePocketOptions): PagePocket;
+    static fromHTMLString(htmlString: string, options: {
+        baseUrl: string;
+        url?: string;
+    } & PagePocketOptions): PagePocket;
+    interceptedRequestEvents(): NetworkEventStream;
+    /** @internal */
+    _emitNetworkEvent(event: NetworkEvent): void;
+    /** @internal */
+    _closeNetworkEventStream(): void;
+    static fromDocument(doc: unknown, options: {
+        baseUrl: string;
+        url?: string;
+        serialize?: (doc: unknown) => string;
+    } & PagePocketOptions): PagePocket;
+    capture(options: CaptureWithPluginsOptions): Promise<PagePocketCaptureResult>;
+}

package/dist/core/internal/pagepocket.js

@@ -0,0 +1,155 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PagePocket = void 0;
+const runner_1 = require("../plugin/runner");
+class AsyncEventQueue {
+    constructor() {
+        this.values = [];
+        this.pending = [];
+        this.closed = false;
+    }
+    push(value) {
+        if (this.closed) {
+            return;
+        }
+        if (this.pending.length > 0) {
+            const resolve = this.pending.shift();
+            resolve?.({ kind: "value", value });
+            return;
+        }
+        this.values.push({ kind: "value", value });
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        if (this.pending.length > 0) {
+            for (const resolve of this.pending) {
+                resolve({ kind: "close" });
+            }
+            this.pending = [];
+            return;
+        }
+        this.values.push({ kind: "close" });
+    }
+    async *iterate() {
+        while (true) {
+            if (this.values.length > 0) {
+                const item = this.values.shift();
+                if (!item || item.kind === "close") {
+                    return;
+                }
+                yield item.value;
+                continue;
+            }
+            if (this.closed) {
+                return;
+            }
+            const item = await new Promise((resolve) => {
+                this.pending.push(resolve);
+            });
+            if (item.kind === "close") {
+                return;
+            }
+            yield item.value;
+        }
+    }
+}
+class BroadcastAsyncIterable {
+    constructor() {
+        this.queues = new Set();
+        this.closed = false;
+    }
+    subscribe() {
+        const queue = new AsyncEventQueue();
+        if (this.closed) {
+            queue.close();
+        }
+        this.queues.add(queue);
+        return (async function* (owner) {
+            try {
+                for await (const value of queue.iterate()) {
+                    yield value;
+                }
+            }
+            finally {
+                owner.queues.delete(queue);
+                queue.close();
+            }
+        })(this);
+    }
+    emit(value) {
+        if (this.queues.size === 0) {
+            return;
+        }
+        for (const queue of this.queues) {
+            queue.push(value);
+        }
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        for (const queue of this.queues) {
+            queue.close();
+        }
+        this.queues.clear();
+    }
+}
+class PagePocket {
+    constructor(target, options) {
+        this.eventStream = new BroadcastAsyncIterable();
+        this.target = target;
+        this.options = options ?? {};
+    }
+    static fromURL(url, options) {
+        return new PagePocket({ kind: "url", url }, options);
+    }
+    static fromPuppeteerPage(page, options) {
+        return new PagePocket({ kind: "puppeteer-page", page }, options);
+    }
+    static fromCDPTab(tabId, options) {
+        return new PagePocket({ kind: "cdp-tab", tabId }, options);
+    }
+    static fromHTMLString(htmlString, options) {
+        const { baseUrl, url, ...rest } = options;
+        return new PagePocket({ kind: "html", htmlString, baseUrl, ...(url ? { url } : {}) }, rest);
+    }
+    interceptedRequestEvents() {
+        return this.eventStream.subscribe();
+    }
+    /** @internal */
+    _emitNetworkEvent(event) {
+        this.eventStream.emit(event);
+    }
+    /** @internal */
+    _closeNetworkEventStream() {
+        this.eventStream.close();
+    }
+    static fromDocument(doc, options) {
+        const serialize = options.serialize ??
+            ((value) => {
+                if (value && typeof value === "object" && "documentElement" in value) {
+                    const el = value.documentElement;
+                    if (el && typeof el.outerHTML === "string") {
+                        return el.outerHTML;
+                    }
+                }
+                throw new Error("fromDocument requires options.serialize(doc) or doc.documentElement.outerHTML");
+            });
+        const htmlString = serialize(doc);
+        const { baseUrl, url, serialize: _serialize, ...rest } = options;
+        return new PagePocket({ kind: "html", htmlString, baseUrl, ...(url ? { url } : {}) }, rest);
+    }
+    async capture(options) {
+        return (0, runner_1.captureWithPlugins)({
+            pocket: this,
+            pocketOptions: this.options,
+            target: this.target,
+            options
+        });
+    }
+}
+exports.PagePocket = PagePocket;

package/dist/core/pagepocket.d.ts

@@ -0,0 +1,38 @@
+export type CaptureTarget = {
+    kind: "url";
+    url: string;
+} | {
+    kind: "puppeteer-page";
+    page: unknown;
+} | {
+    kind: "cdp-tab";
+    tabId: number;
+} | {
+    kind: "html";
+    htmlString: string;
+    baseUrl: string;
+    url?: string;
+};
+import type { PagePocketOptions } from "../types.js";
+import type { CaptureResult as PagePocketCaptureResult, Plugin as V3Plugin, Unit as V3Unit } from "../units/contracts-bridge.js";
+export declare class PagePocket {
+    private target;
+    private options;
+    private constructor();
+    static fromURL(url: string, options?: PagePocketOptions): PagePocket;
+    static fromPuppeteerPage(page: unknown, options?: PagePocketOptions): PagePocket;
+    static fromCDPTab(tabId: number, options?: PagePocketOptions): PagePocket;
+    static fromHTMLString(htmlString: string, options: {
+        baseUrl: string;
+        url?: string;
+    } & PagePocketOptions): PagePocket;
+    static fromDocument(doc: unknown, options: {
+        baseUrl: string;
+        url?: string;
+        serialize?: (doc: unknown) => string;
+    } & PagePocketOptions): PagePocket;
+    capture(options: {
+        units: V3Unit[];
+        plugins?: V3Plugin[];
+    } & import("../units/types.js").CaptureOptions): Promise<PagePocketCaptureResult>;
+}