@pagepocket/lib 0.12.0 → 0.13.0

package/dist/core/pagepocket.d.ts

@@ -13,12 +13,27 @@ export type CaptureTarget = {
     baseUrl: string;
     url?: string;
 };
+import type { ProgressEvent } from "@pagepocket/contracts";
 import type { PagePocketOptions } from "../types.js";
 import type { CaptureResult as PagePocketCaptureResult, Plugin as V3Plugin, Unit as V3Unit } from "../units/contracts-bridge.js";
 import type { CaptureOptions } from "../units/types.js";
+export type CaptureEventMap = {
+    "unit:start": Extract<ProgressEvent, {
+        type: "unit:start";
+    }>;
+    "unit:end": Extract<ProgressEvent, {
+        type: "unit:end";
+    }>;
+    "unit:log": Extract<ProgressEvent, {
+        type: "unit:log";
+    }>;
+};
+type CaptureEventName = keyof CaptureEventMap;
+type CaptureEventListener<K extends CaptureEventName> = (event: CaptureEventMap[K]) => void;
 export declare class PagePocket {
     private target;
     private options;
+    private listeners;
     private constructor();
     static fromURL(url: string, options?: PagePocketOptions): PagePocket;
     static fromPuppeteerPage(page: unknown, options?: PagePocketOptions): PagePocket;
@@ -32,8 +47,10 @@ export declare class PagePocket {
         url?: string;
         serialize?: (doc: unknown) => string;
     } & PagePocketOptions): PagePocket;
+    on<K extends CaptureEventName>(event: K, listener: CaptureEventListener<K>): this;
     capture(options: {
         units: V3Unit[];
         plugins?: V3Plugin[];
     } & CaptureOptions): Promise<PagePocketCaptureResult>;
 }
+export {};

package/dist/core/pagepocket.js

@@ -1,6 +1,7 @@
 import { runCapture } from "../units/index.js";
 export class PagePocket {
     constructor(target, options) {
+        this.listeners = new Map();
         this.target = target;
         this.options = options ?? {};
     }
@@ -32,6 +33,16 @@ export class PagePocket {
         const { baseUrl, url, serialize: _serialize, ...rest } = options;
         return new PagePocket({ kind: "html", htmlString, baseUrl, ...(url ? { url } : {}) }, rest);
     }
+    on(event, listener) {
+        const existing = this.listeners.get(event);
+        if (existing) {
+            existing.push(listener);
+        }
+        else {
+            this.listeners.set(event, [listener]);
+        }
+        return this;
+    }
     async capture(options) {
         const entry = this.target.kind === "url"
             ? { kind: "url", url: this.target.url }
@@ -45,12 +56,25 @@ export class PagePocket {
                         htmlString: this.target.htmlString,
                         ...(this.target.url ? { url: this.target.url } : {})
                     };
+        const hasListeners = this.listeners.size > 0;
+        const onProgress = hasListeners
+            ? (event) => {
+                const eventListeners = this.listeners.get(event.type);
+                if (!eventListeners) {
+                    return;
+                }
+                for (const listener of eventListeners) {
+                    listener(event);
+                }
+            }
+            : undefined;
         const result = await runCapture({
             entry,
             pocketOptions: this.options,
             options,
             units: options.units,
-            plugins: options.plugins
+            plugins: options.plugins,
+            onProgress
         });
         return result;
     }

package/dist/index.d.ts

@@ -1,4 +1,5 @@
 export { PagePocket } from "./core/pagepocket.js";
+export type { CaptureEventMap } from "./core/pagepocket.js";
 export * from "./units/index.js";
 export type { UnitContext, UnitContributeContext, UnitPatch, UnitRuntime } from "./units/contracts-bridge.js";
 export { TERMINAL_RESULT_KEY } from "./units/contracts-bridge.js";

package/dist/units/apply-replace-elements-to-file-tree.d.ts

@@ -0,0 +1,7 @@
+import type { FileTree } from "../core/file-tree.js";
+import type { ReplaceElementsConfig } from "../types.js";
+export declare const applyReplaceElementsToFileTree: (input: {
+    files: FileTree;
+    replaceElements: ReplaceElementsConfig;
+    entryUrl: string;
+}) => Promise<FileTree>;

package/dist/units/apply-replace-elements-to-file-tree.js

@@ -0,0 +1,63 @@
+import * as cheerio from "cheerio";
+import { applyReplaceElements } from "../replace-elements.js";
+import { streamToUint8Array } from "../utils/streams.js";
+import { decodeUtf8 } from "../utils.js";
+const isHtmlFile = (path) => path.endsWith(".html") || path.endsWith(".htm");
+const readFileSource = async (file, fileTree) => {
+    const source = file.source;
+    if (source.kind === "bytes") {
+        return source.data;
+    }
+    if (source.kind === "text") {
+        return new TextEncoder().encode(source.text);
+    }
+    if (source.kind === "content-ref" && fileTree.content) {
+        const stream = await fileTree.content.open(source.ref);
+        return streamToUint8Array(stream);
+    }
+    return undefined;
+};
+const processHtmlFile = async (file, fileTree, replaceElements, entryUrl) => {
+    const bytes = await readFileSource(file, fileTree);
+    if (!bytes) {
+        return file;
+    }
+    const decoded = decodeUtf8(bytes);
+    if (typeof decoded === "undefined") {
+        return file;
+    }
+    const $ = cheerio.load(decoded);
+    await applyReplaceElements({
+        $,
+        entryUrl,
+        url: entryUrl,
+        replaceElements,
+        isEntryDocument: true
+    });
+    const updatedHtml = $.html();
+    const updatedBytes = new TextEncoder().encode(updatedHtml);
+    return {
+        ...file,
+        source: { kind: "bytes", data: updatedBytes }
+    };
+};
+const processDirectory = async (dir, fileTree, replaceElements, entryUrl) => {
+    const updatedEntries = [];
+    for (const entry of dir.entries) {
+        if (entry.kind === "file" && isHtmlFile(entry.path)) {
+            updatedEntries.push(await processHtmlFile(entry, fileTree, replaceElements, entryUrl));
+            continue;
+        }
+        if (entry.kind === "directory") {
+            updatedEntries.push(await processDirectory(entry, fileTree, replaceElements, entryUrl));
+            continue;
+        }
+        updatedEntries.push(entry);
+    }
+    return { ...dir, entries: updatedEntries };
+};
+export const applyReplaceElementsToFileTree = async (input) => {
+    const { files, replaceElements, entryUrl } = input;
+    const updatedRoot = await processDirectory(files.root, files, replaceElements, entryUrl);
+    return { ...files, root: updatedRoot };
+};

package/dist/units/contracts-bridge.d.ts

@@ -55,6 +55,14 @@ export interface UnitRuntime {
     hasPublisher(t: ChannelToken<unknown>): boolean;
     readonly elements: ElementPatchRegistry;
     defer(promise: DeferredHandle): void;
+    /**
+     * Emit a log message from the currently executing unit.
+     *
+     * The message is published on the well-known PROGRESS channel as a
+     * `unit:log` event so external consumers (e.g. `PagePocket.on("unit:log", …)`)
+     * can observe it.
+     */
+    log(message: string, data?: unknown): void;
 }
 export interface PluginHost {
     readonly entry: EntryInfo;
@@ -66,7 +74,7 @@ export interface PluginHost {
 }
 export declare abstract class Unit {
     abstract readonly id: string;
-    abstract readonly kind: string;
+    abstract readonly description: string;
     abstract run(ctx: UnitContext, rt: UnitRuntime): Promise<void | UnitPatch>;
     merge(returnValue: UnitPatch, pluginContributedValue?: UnitPatch): UnitPatch;
 }

package/dist/units/file-tree-unit.d.ts

@@ -0,0 +1,28 @@
+import { Unit, type UnitPatch } from "./contracts-bridge.js";
+/**
+ * Abstract base class for units whose `run()` produces a `FileTree`.
+ *
+ * Provides a default `merge` implementation that deep-merges the `files`
+ * property when both the unit return value and the plugin-contributed value
+ * contain a valid `FileTree`.  All other properties are shallow-spread
+ * (plugin wins on conflict), matching the base `Unit.merge` behaviour.
+ *
+ * Subclasses only need to implement `id` and `run()`.
+ *
+ * Usage:
+ * ```ts
+ * import { FileTreeUnit } from "@pagepocket/lib";
+ *
+ * export class MyUnit extends FileTreeUnit {
+ *   readonly id = "my";
+ *
+ *   async run(ctx, rt) {
+ *     const files = buildFiles();
+ *     return { files };
+ *   }
+ * }
+ * ```
+ */
+export declare abstract class FileTreeUnit extends Unit {
+    merge(returnValue: UnitPatch, pluginContributedValue?: UnitPatch): UnitPatch;
+}

package/dist/units/file-tree-unit.js

@@ -0,0 +1,53 @@
+import { mergeFileTrees } from "../core/file-tree-merge.js";
+import { Unit } from "./contracts-bridge.js";
+/**
+ * Abstract base class for units whose `run()` produces a `FileTree`.
+ *
+ * Provides a default `merge` implementation that deep-merges the `files`
+ * property when both the unit return value and the plugin-contributed value
+ * contain a valid `FileTree`.  All other properties are shallow-spread
+ * (plugin wins on conflict), matching the base `Unit.merge` behaviour.
+ *
+ * Subclasses only need to implement `id` and `run()`.
+ *
+ * Usage:
+ * ```ts
+ * import { FileTreeUnit } from "@pagepocket/lib";
+ *
+ * export class MyUnit extends FileTreeUnit {
+ *   readonly id = "my";
+ *
+ *   async run(ctx, rt) {
+ *     const files = buildFiles();
+ *     return { files };
+ *   }
+ * }
+ * ```
+ */
+export class FileTreeUnit extends Unit {
+    merge(returnValue, pluginContributedValue = {}) {
+        const mergedValue = { ...returnValue, ...pluginContributedValue };
+        const returnFiles = returnValue.files;
+        const pluginFiles = pluginContributedValue.files;
+        if (!isFileTree(returnFiles) || !isFileTree(pluginFiles)) {
+            return mergedValue;
+        }
+        return { ...mergedValue, files: mergeFileTrees(returnFiles, pluginFiles) };
+    }
+}
+const isFileTree = (value) => {
+    if (!value || typeof value !== "object") {
+        return false;
+    }
+    if (!("root" in value)) {
+        return false;
+    }
+    const root = value.root;
+    if (!root || typeof root !== "object") {
+        return false;
+    }
+    const rootRecord = root;
+    return (rootRecord.kind === "directory" &&
+        typeof rootRecord.path === "string" &&
+        Array.isArray(rootRecord.entries));
+};

package/dist/units/index.d.ts

@@ -1,4 +1,7 @@
 export type { CaptureOptions, EntryInfo } from "./types.js";
 export type { CaptureResult, Plugin, PluginHost, UnitRuntime } from "./contracts-bridge.js";
 export { Unit } from "./contracts-bridge.js";
+export { FileTreeUnit } from "./file-tree-unit.js";
+export { SnapshotUnit } from "./snapshot-unit.js";
 export { runCapture } from "./runner.js";
+export type { ProgressListener } from "./runner.js";

package/dist/units/index.js

@@ -1,2 +1,4 @@
 export { Unit } from "./contracts-bridge.js";
+export { FileTreeUnit } from "./file-tree-unit.js";
+export { SnapshotUnit } from "./snapshot-unit.js";
 export { runCapture } from "./runner.js";

package/dist/units/internal/runtime.d.ts

@@ -1,4 +1,4 @@
-import type { ChannelToken, ReplaceElementsConfig } from "@pagepocket/contracts";
+import { type ChannelToken, type ReplaceElementsConfig } from "@pagepocket/contracts";
 import { type ElementPatchRegistry, type UnitContext, type UnitPatch, type UnitRuntime } from "../contracts-bridge.js";
 import type { CaptureOptions, EntryInfo, PagePocketOptions } from "../types.js";
 declare class ElementPatchRegistryImpl implements ElementPatchRegistry {
@@ -19,17 +19,20 @@ export declare class RuntimeImpl implements UnitRuntime {
     readonly pocketOptions: PagePocketOptions;
     private channels;
     private deferred;
+    private currentUnitId;
     readonly elements: ElementPatchRegistryImpl;
     constructor(input: {
         entry: EntryInfo;
         options: CaptureOptions;
         pocketOptions: PagePocketOptions;
     });
+    _setCurrentUnitId(unitId: string): void;
+    log(message: string, data?: unknown): void;
     publish<T>(channelToken: ChannelToken<T>, value: T): void;
     subscribe<T>(channelToken: ChannelToken<T>): AsyncIterable<T>;
     hasPublisher(channelToken: ChannelToken<unknown>): boolean;
     defer(promise: Promise<unknown>): void;
-    _ensureChannel(channelToken: ChannelToken<unknown>): void;
+    _ensureChannel<T>(channelToken: ChannelToken<T>): void;
     _closeAllChannels(): Promise<void>;
     _awaitDeferred(): Promise<void>;
 }

package/dist/units/internal/runtime.js

@@ -1,3 +1,4 @@
+import { PROGRESS } from "@pagepocket/contracts";
 import { TERMINAL_RESULT_KEY } from "../contracts-bridge.js";
 import { AsyncQueue, emptyAsyncIterable } from "./async-queue.js";
 import { DeferredTracker } from "./deferred-tracker.js";
@@ -30,11 +31,24 @@ export class RuntimeImpl {
     constructor(input) {
         this.channels = new Map();
         this.deferred = new DeferredTracker();
+        this.currentUnitId = "";
         this.elements = new ElementPatchRegistryImpl();
         this.entry = input.entry;
         this.options = input.options;
         this.pocketOptions = input.pocketOptions;
     }
+    _setCurrentUnitId(unitId) {
+        this.currentUnitId = unitId;
+    }
+    log(message, data) {
+        const event = {
+            type: "unit:log",
+            unitId: this.currentUnitId,
+            message,
+            ...(data !== undefined ? { data } : {})
+        };
+        this.publish(PROGRESS, event);
+    }
     publish(channelToken, value) {
         const state = this.channels.get(channelToken.id);
         if (!state || state.closed) {

package/dist/units/runner.d.ts

@@ -1,6 +1,7 @@
-import type { ChannelToken } from "@pagepocket/contracts";
+import { type ChannelToken, type ProgressEvent } from "@pagepocket/contracts";
 import { type CaptureResult, type Plugin, type Unit } from "./contracts-bridge.js";
 import type { CaptureOptions, EntryInfo, PagePocketOptions } from "./types.js";
+export type ProgressListener = (event: ProgressEvent) => void;
 export declare const runCapture: (input: {
     entry: EntryInfo;
     pocketOptions: PagePocketOptions;
@@ -8,4 +9,5 @@ export declare const runCapture: (input: {
     units: Unit[];
     plugins?: Plugin[];
     declaredChannels?: ChannelToken<unknown>[];
+    onProgress?: ProgressListener;
 }) => Promise<CaptureResult>;

package/dist/units/runner.js

@@ -1,3 +1,5 @@
+import { PROGRESS } from "@pagepocket/contracts";
+import { debugLog } from "../core/debug.js";
 import { TERMINAL_RESULT_KEY } from "./contracts-bridge.js";
 import { mergePatchIntoFreshContext, RuntimeImpl } from "./internal/runtime.js";
 export const runCapture = async (input) => {
@@ -9,6 +11,16 @@ export const runCapture = async (input) => {
     for (const channel of input.declaredChannels ?? []) {
         rt._ensureChannel(channel);
     }
+    rt._ensureChannel(PROGRESS);
+    if (input.onProgress) {
+        const listener = input.onProgress;
+        const progressTask = (async () => {
+            for await (const event of rt.subscribe(PROGRESS)) {
+                listener(event);
+            }
+        })();
+        rt.defer(progressTask);
+    }
     const pluginHost = {
         entry: rt.entry,
         options: rt.options,
@@ -27,11 +39,23 @@ export const runCapture = async (input) => {
     const mergePatch = (_ctx, patch) => mergePatchIntoFreshContext(patch);
     let ctx = { value: {} };
     let result;
+    const totalUnits = input.units.length;
     try {
-        for (const unit of input.units) {
+        for (let i = 0; i < input.units.length; i++) {
             if (result) {
                 break;
             }
+            const unit = input.units[i];
+            const unitIndex = i;
+            rt._setCurrentUnitId(unit.id);
+            rt.publish(PROGRESS, {
+                type: "unit:start",
+                unitId: unit.id,
+                unitDescription: unit.description,
+                index: unitIndex,
+                total: totalUnits
+            });
+            const unitStartTime = Date.now();
             const baseCtx = ctx;
             const boundPlugins = (input.plugins ?? []).filter((plugin) => {
                 const unitId = plugin.constructor?.unitId;
@@ -62,13 +86,44 @@ export const runCapture = async (input) => {
                 }
             }
             if (result) {
+                rt.publish(PROGRESS, {
+                    type: "unit:end",
+                    unitId: unit.id,
+                    unitDescription: unit.description,
+                    index: unitIndex,
+                    total: totalUnits,
+                    durationMs: Date.now() - unitStartTime
+                });
                 break;
             }
-            const out = (await unit.run(baseCtx, rt)) ?? {};
+            let out;
+            try {
+                out = (await unit.run(baseCtx, rt)) ?? {};
+            }
+            catch (err) {
+                debugLog(`[runner] unit "${unit.id}" threw:`, err);
+                rt.publish(PROGRESS, {
+                    type: "unit:end",
+                    unitId: unit.id,
+                    unitDescription: unit.description,
+                    index: unitIndex,
+                    total: totalUnits,
+                    durationMs: Date.now() - unitStartTime
+                });
+                continue;
+            }
             const unitReturnValue = out && typeof out === "object" ? out : {};
             const merged = unit.merge(unitReturnValue, pluginContributedValue);
             const mergedPatch = merged && typeof merged === "object" ? merged : {};
             ctx = { value: mergePatch({ value: {} }, mergedPatch).value };
+            rt.publish(PROGRESS, {
+                type: "unit:end",
+                unitId: unit.id,
+                unitDescription: unit.description,
+                index: unitIndex,
+                total: totalUnits,
+                durationMs: Date.now() - unitStartTime
+            });
             const terminal = mergedPatch[TERMINAL_RESULT_KEY];
             if (terminal) {
                 result = terminal;

package/dist/units/snapshot-unit.d.ts

@@ -0,0 +1,31 @@
+import type { UnitContext, UnitPatch, UnitRuntime } from "./contracts-bridge.js";
+import { FileTreeUnit } from "./file-tree-unit.js";
+/**
+ * Base class for units that produce the primary snapshot FileTree.
+ *
+ * `run()` delegates to `build()` and automatically:
+ * - Injects `snapshotType` into the returned patch.
+ * - Compiles all plugin-contributed element-replacement rules
+ *   (`rt.elements.compile()`) and applies them to every HTML file
+ *   in the returned FileTree.  Subclasses never need to call
+ *   `rt.elements.compile()` themselves.
+ *
+ * Subclasses implement `id`, `snapshotType`, and `build()`.
+ *
+ * ```ts
+ * export class MySnapshotUnit extends SnapshotUnit {
+ *   readonly id = "mySnapshot";
+ *   readonly snapshotType = "my-type";
+ *
+ *   async build(ctx, rt) {
+ *     return { files: buildFiles(), html: ctx.value.html };
+ *   }
+ * }
+ * ```
+ */
+export declare abstract class SnapshotUnit extends FileTreeUnit {
+    /** Identifier for the kind of snapshot this unit produces (e.g. "full", "main-content"). */
+    abstract readonly snapshotType: string;
+    abstract build(ctx: UnitContext, rt: UnitRuntime): Promise<void | UnitPatch>;
+    run(ctx: UnitContext, rt: UnitRuntime): Promise<void | UnitPatch>;
+}

package/dist/units/snapshot-unit.js

@@ -0,0 +1,58 @@
+import { applyReplaceElementsToFileTree } from "./apply-replace-elements-to-file-tree.js";
+import { FileTreeUnit } from "./file-tree-unit.js";
+/**
+ * Base class for units that produce the primary snapshot FileTree.
+ *
+ * `run()` delegates to `build()` and automatically:
+ * - Injects `snapshotType` into the returned patch.
+ * - Compiles all plugin-contributed element-replacement rules
+ *   (`rt.elements.compile()`) and applies them to every HTML file
+ *   in the returned FileTree.  Subclasses never need to call
+ *   `rt.elements.compile()` themselves.
+ *
+ * Subclasses implement `id`, `snapshotType`, and `build()`.
+ *
+ * ```ts
+ * export class MySnapshotUnit extends SnapshotUnit {
+ *   readonly id = "mySnapshot";
+ *   readonly snapshotType = "my-type";
+ *
+ *   async build(ctx, rt) {
+ *     return { files: buildFiles(), html: ctx.value.html };
+ *   }
+ * }
+ * ```
+ */
+export class SnapshotUnit extends FileTreeUnit {
+    async run(ctx, rt) {
+        const patch = await this.build(ctx, rt);
+        if (!patch) {
+            return;
+        }
+        const files = patch.files;
+        const replaceElements = await rt.elements.compile();
+        if (files && replaceElements.length > 0) {
+            const entryUrl = resolveEntryUrl(rt);
+            const updatedFiles = await applyReplaceElementsToFileTree({
+                files,
+                replaceElements,
+                entryUrl
+            });
+            return { ...patch, files: updatedFiles, snapshotType: this.snapshotType };
+        }
+        return { ...patch, snapshotType: this.snapshotType };
+    }
+}
+const resolveEntryUrl = (rt) => {
+    const entry = rt.entry;
+    if (entry.kind === "url") {
+        return entry.url;
+    }
+    if (entry.kind === "html-string" || entry.kind === "document") {
+        return entry.url ?? entry.baseUrl;
+    }
+    if (entry.kind === "puppeteer-page" || entry.kind === "cdp-tab") {
+        return entry.url ?? "";
+    }
+    return "";
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pagepocket/lib",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "Library for rewriting HTML snapshots and inlining local resources.",
   "type": "module",
   "main": "dist/index.js",
@@ -20,9 +20,9 @@
   "dependencies": {
     "cheerio": "^1.0.0-rc.12",
     "domhandler": "^5.0.3",
-    "@pagepocket/contracts": "0.12.0",
-    "@pagepocket/uni-fs": "0.12.0",
-    "@pagepocket/shared": "0.12.0"
+    "@pagepocket/contracts": "0.13.0",
+    "@pagepocket/shared": "0.13.0",
+    "@pagepocket/uni-fs": "0.13.0"
   },
   "devDependencies": {
     "@playwright/test": "^1.50.1",