@pagepocket/main-content-unit 0.12.0

package/dist/asset-filename.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Derive a human-readable, filesystem-safe filename from a URL and MIME type.
+ *
+ * Attempts to preserve the original filename from the URL path. Falls back
+ * to a hash-based name. Appends the correct extension from the MIME type
+ * when the URL-derived name lacks one.
+ *
+ * Usage:
+ * ```ts
+ * assetFilename("https://cdn.example.com/images/photo.jpg", "image/jpeg")
+ * // => "photo.jpg"
+ * assetFilename("https://cdn.example.com/api/image?id=42", "image/png")
+ * // => "image_id_42.png"
+ * ```
+ */
+export declare const assetFilename: (url: string, mimeType?: string) => string;

package/dist/asset-filename.js

@@ -0,0 +1,62 @@
+import { extensionFromContentType } from "@pagepocket/shared";
+const extractPathBasename = (url) => {
+    try {
+        const pathname = new URL(url).pathname;
+        const segments = pathname.split("/");
+        const last = segments[segments.length - 1];
+        if (!last || last === "/") {
+            return undefined;
+        }
+        return decodeURIComponent(last);
+    }
+    catch {
+        return undefined;
+    }
+};
+const sanitize = (name) => name
+    .replace(/[^a-zA-Z0-9._-]/g, "_")
+    .replace(/_+/g, "_")
+    .replace(/^_+|_+$/g, "")
+    .slice(0, 120);
+/**
+ * Derive a human-readable, filesystem-safe filename from a URL and MIME type.
+ *
+ * Attempts to preserve the original filename from the URL path. Falls back
+ * to a hash-based name. Appends the correct extension from the MIME type
+ * when the URL-derived name lacks one.
+ *
+ * Usage:
+ * ```ts
+ * assetFilename("https://cdn.example.com/images/photo.jpg", "image/jpeg")
+ * // => "photo.jpg"
+ * assetFilename("https://cdn.example.com/api/image?id=42", "image/png")
+ * // => "image_id_42.png"
+ * ```
+ */
+export const assetFilename = (url, mimeType) => {
+    const basename = extractPathBasename(url);
+    const mimeExt = extensionFromContentType(mimeType);
+    if (basename) {
+        const sanitized = sanitize(basename);
+        const hasExt = sanitized.includes(".");
+        if (sanitized && hasExt) {
+            return sanitized;
+        }
+        if (sanitized && mimeExt) {
+            return `${sanitized}${mimeExt}`;
+        }
+        if (sanitized) {
+            return sanitized;
+        }
+    }
+    const hash = simpleHash(url);
+    return mimeExt ? `${hash}${mimeExt}` : hash;
+};
+const simpleHash = (input) => {
+    let hash = 0;
+    for (let index = 0; index < input.length; index++) {
+        const char = input.charCodeAt(index);
+        hash = ((hash << 5) - hash + char) | 0;
+    }
+    return `asset_${Math.abs(hash).toString(36)}`;
+};

package/dist/build-content-tree.d.ts

@@ -0,0 +1,29 @@
+import { type CaptureArtifacts, type FileTree } from "@pagepocket/lib";
+export type ContentTreeInput = {
+    contentHtml: string;
+    baseUrl: string;
+    capture: CaptureArtifacts;
+};
+export type ContentTreeResult = {
+    files: FileTree;
+    rewrittenHtml: string;
+};
+/**
+ * Build a self-contained FileTree from extracted main content HTML.
+ *
+ * 1. Scans `contentHtml` for media asset URLs (img, video, audio, source, poster).
+ * 2. Resolves each URL against the capture's ContentStore to download the body.
+ * 3. Assigns each asset a filesystem-safe name under `/assets/`.
+ * 4. Rewrites all asset URLs in the HTML to `/assets/{filename}`.
+ * 5. Returns a FileTree with `/index.html` + `/assets/*`.
+ *
+ * Usage:
+ * ```ts
+ * const { files, rewrittenHtml } = await buildContentTree({
+ *   contentHtml: '<img src="https://example.com/photo.jpg">',
+ *   baseUrl: 'https://example.com/article',
+ *   capture,
+ * });
+ * ```
+ */
+export declare const buildContentTree: (input: ContentTreeInput) => Promise<ContentTreeResult>;

package/dist/build-content-tree.js

@@ -0,0 +1,86 @@
+import { streamToUint8Array } from "@pagepocket/lib";
+import { assetFilename } from "./asset-filename.js";
+import { buildCaptureLookup } from "./capture-lookup.js";
+import { collectAssetUrls } from "./collect-asset-urls.js";
+import { rewriteContentUrls } from "./rewrite-content-urls.js";
+/**
+ * Build a self-contained FileTree from extracted main content HTML.
+ *
+ * 1. Scans `contentHtml` for media asset URLs (img, video, audio, source, poster).
+ * 2. Resolves each URL against the capture's ContentStore to download the body.
+ * 3. Assigns each asset a filesystem-safe name under `/assets/`.
+ * 4. Rewrites all asset URLs in the HTML to `/assets/{filename}`.
+ * 5. Returns a FileTree with `/index.html` + `/assets/*`.
+ *
+ * Usage:
+ * ```ts
+ * const { files, rewrittenHtml } = await buildContentTree({
+ *   contentHtml: '<img src="https://example.com/photo.jpg">',
+ *   baseUrl: 'https://example.com/article',
+ *   capture,
+ * });
+ * ```
+ */
+export const buildContentTree = async (input) => {
+    const { contentHtml, baseUrl, capture } = input;
+    const assetUrls = collectAssetUrls(contentHtml, baseUrl);
+    const captureLookup = buildCaptureLookup(capture);
+    const urlToAssetPath = new Map();
+    const assetFiles = [];
+    const usedFilenames = new Set();
+    for (const assetUrl of assetUrls) {
+        const entry = captureLookup.get(assetUrl);
+        if (!entry || !entry.bodyRef) {
+            continue;
+        }
+        const rawName = assetFilename(assetUrl, entry.mimeType);
+        const uniqueName = deduplicateFilename(rawName, usedFilenames);
+        usedFilenames.add(uniqueName);
+        const assetPath = `/assets/${uniqueName}`;
+        urlToAssetPath.set(assetUrl, assetPath);
+        const stream = await capture.contentStore.open(entry.bodyRef);
+        const bytes = await streamToUint8Array(stream);
+        assetFiles.push({
+            kind: "file",
+            path: assetPath,
+            source: { kind: "bytes", data: bytes }
+        });
+    }
+    const resolve = (absoluteUrl) => urlToAssetPath.get(absoluteUrl);
+    const rewrittenHtml = rewriteContentUrls({
+        html: contentHtml,
+        baseUrl,
+        resolve
+    });
+    const indexHtmlBytes = new TextEncoder().encode(rewrittenHtml);
+    const indexFile = {
+        kind: "file",
+        path: "/index.html",
+        source: { kind: "bytes", data: indexHtmlBytes }
+    };
+    return {
+        rewrittenHtml,
+        files: {
+            root: {
+                kind: "directory",
+                path: "",
+                entries: [indexFile, ...assetFiles]
+            }
+        }
+    };
+};
+const deduplicateFilename = (name, used) => {
+    if (!used.has(name)) {
+        return name;
+    }
+    const dotIndex = name.lastIndexOf(".");
+    const stem = dotIndex > 0 ? name.slice(0, dotIndex) : name;
+    const ext = dotIndex > 0 ? name.slice(dotIndex) : "";
+    for (let suffix = 2; suffix <= 9999; suffix++) {
+        const candidate = `${stem}-${suffix}${ext}`;
+        if (!used.has(candidate)) {
+            return candidate;
+        }
+    }
+    return `${stem}-${Date.now()}${ext}`;
+};

package/dist/capture-lookup.d.ts

@@ -0,0 +1,28 @@
+import type { CaptureArtifacts } from "@pagepocket/lib";
+type ResponseEntry = {
+    url: string;
+    mimeType?: string;
+    bodyRef?: {
+        kind: "memory";
+        data: Uint8Array;
+    } | {
+        kind: "store-ref";
+        id: string;
+    };
+    bodySize?: number;
+};
+/**
+ * Build a URL-keyed lookup of captured HTTP responses.
+ *
+ * Matches response events to their request URLs, deduplicating by URL
+ * (last response wins). Only successful responses (status 2xx) with
+ * a body reference are included.
+ *
+ * Usage:
+ * ```ts
+ * const lookup = buildCaptureLookup(capture);
+ * const entry = lookup.get("https://example.com/photo.jpg");
+ * ```
+ */
+export declare const buildCaptureLookup: (capture: CaptureArtifacts) => Map<string, ResponseEntry>;
+export {};

package/dist/capture-lookup.js

@@ -0,0 +1,40 @@
+/**
+ * Build a URL-keyed lookup of captured HTTP responses.
+ *
+ * Matches response events to their request URLs, deduplicating by URL
+ * (last response wins). Only successful responses (status 2xx) with
+ * a body reference are included.
+ *
+ * Usage:
+ * ```ts
+ * const lookup = buildCaptureLookup(capture);
+ * const entry = lookup.get("https://example.com/photo.jpg");
+ * ```
+ */
+export const buildCaptureLookup = (capture) => {
+    const requestUrls = new Map();
+    const lookup = new Map();
+    for (const event of capture.events) {
+        if (event.type === "http.request") {
+            requestUrls.set(event.requestId, event.url);
+            continue;
+        }
+        if (event.type !== "http.response") {
+            continue;
+        }
+        if (event.status < 200 || event.status >= 300) {
+            continue;
+        }
+        if (!event.bodyRef) {
+            continue;
+        }
+        const url = requestUrls.get(event.requestId) ?? event.url;
+        lookup.set(url, {
+            url,
+            mimeType: event.mimeType,
+            bodyRef: event.bodyRef,
+            bodySize: event.bodySize
+        });
+    }
+    return lookup;
+};

package/dist/collect-asset-urls.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Collect all unique absolute asset URLs referenced in an HTML fragment.
+ *
+ * Scans `img`, `video`, `audio`, `source`, and `poster` attributes
+ * including `srcset`. Returns a deduplicated list of absolute URLs.
+ *
+ * Usage:
+ * ```ts
+ * const urls = collectAssetUrls('<img src="/photo.jpg">', 'https://example.com');
+ * // => ["https://example.com/photo.jpg"]
+ * ```
+ */
+export declare const collectAssetUrls: (html: string, baseUrl: string) => string[];

package/dist/collect-asset-urls.js

@@ -0,0 +1,79 @@
+import * as cheerio from "cheerio";
+const ASSET_SELECTORS = [
+    { selector: "img[src]", attr: "src" },
+    { selector: "video[src]", attr: "src" },
+    { selector: "audio[src]", attr: "src" },
+    { selector: "source[src]", attr: "src" },
+    { selector: "video[poster]", attr: "poster" },
+    { selector: "img[srcset]", attr: "srcset" },
+    { selector: "source[srcset]", attr: "srcset" }
+];
+const shouldSkip = (value) => {
+    const trimmed = value.trim();
+    return (!trimmed ||
+        trimmed.startsWith("data:") ||
+        trimmed.startsWith("blob:") ||
+        trimmed.startsWith("javascript:") ||
+        trimmed.startsWith("#"));
+};
+const toAbsolute = (value, baseUrl) => {
+    if (shouldSkip(value)) {
+        return undefined;
+    }
+    try {
+        return new URL(value, baseUrl).toString();
+    }
+    catch {
+        return undefined;
+    }
+};
+const parseSrcsetUrls = (srcset, baseUrl) => {
+    const urls = [];
+    for (const part of srcset.split(",")) {
+        const segments = part.trim().split(/\s+/);
+        const rawUrl = segments[0];
+        if (!rawUrl) {
+            continue;
+        }
+        const absolute = toAbsolute(rawUrl, baseUrl);
+        if (absolute) {
+            urls.push(absolute);
+        }
+    }
+    return urls;
+};
+/**
+ * Collect all unique absolute asset URLs referenced in an HTML fragment.
+ *
+ * Scans `img`, `video`, `audio`, `source`, and `poster` attributes
+ * including `srcset`. Returns a deduplicated list of absolute URLs.
+ *
+ * Usage:
+ * ```ts
+ * const urls = collectAssetUrls('<img src="/photo.jpg">', 'https://example.com');
+ * // => ["https://example.com/photo.jpg"]
+ * ```
+ */
+export const collectAssetUrls = (html, baseUrl) => {
+    const $ = cheerio.load(html, undefined, false);
+    const seen = new Set();
+    for (const { selector, attr } of ASSET_SELECTORS) {
+        $(selector).each((_, element) => {
+            const value = $(element).attr(attr);
+            if (!value) {
+                return;
+            }
+            if (attr === "srcset") {
+                for (const url of parseSrcsetUrls(value, baseUrl)) {
+                    seen.add(url);
+                }
+                return;
+            }
+            const absolute = toAbsolute(value, baseUrl);
+            if (absolute) {
+                seen.add(absolute);
+            }
+        });
+    }
+    return [...seen];
+};

package/dist/extract-url-map.d.ts

@@ -0,0 +1,13 @@
+import type { FileTree } from "@pagepocket/lib";
+/**
+ * Extract the URL-to-local-path mapping from a snapshot FileTree by reading
+ * the embedded `resources_path.json` file.
+ *
+ * Usage:
+ * ```ts
+ * const urlToPath = await extractUrlMap(fileTree);
+ * const localPath = urlToPath.get("https://example.com/img.jpg");
+ * // => "/resources/img.jpg"
+ * ```
+ */
+export declare const extractUrlMap: (files: FileTree) => Promise<Map<string, string>>;

package/dist/extract-url-map.js

@@ -0,0 +1,75 @@
+/**
+ * Extract the URL-to-local-path mapping from a snapshot FileTree by reading
+ * the embedded `resources_path.json` file.
+ *
+ * Usage:
+ * ```ts
+ * const urlToPath = await extractUrlMap(fileTree);
+ * const localPath = urlToPath.get("https://example.com/img.jpg");
+ * // => "/resources/img.jpg"
+ * ```
+ */
+export const extractUrlMap = async (files) => {
+    const resourcesPathEntry = findFile(files.root, "/resources_path.json");
+    if (!resourcesPathEntry) {
+        return new Map();
+    }
+    const bytes = await readFileEntry(files, resourcesPathEntry);
+    const text = new TextDecoder().decode(bytes);
+    const snapshot = JSON.parse(text);
+    const urlToPath = new Map();
+    for (const item of snapshot.items) {
+        urlToPath.set(item.url, item.path);
+    }
+    return urlToPath;
+};
+const findFile = (dir, targetPath) => {
+    for (const entry of dir.entries) {
+        if (entry.kind === "file" && entry.path === targetPath) {
+            return entry;
+        }
+        if (entry.kind === "directory") {
+            const found = findFile(entry, targetPath);
+            if (found) {
+                return found;
+            }
+        }
+    }
+    return undefined;
+};
+const readFileEntry = async (files, entry) => {
+    if (entry.kind !== "file") {
+        throw new Error("Expected file entry");
+    }
+    const source = entry.source;
+    if (source.kind === "bytes") {
+        return source.data;
+    }
+    if (source.kind === "text") {
+        return new TextEncoder().encode(source.text);
+    }
+    if (source.kind === "content-ref") {
+        if (!files.content) {
+            throw new Error("Cannot resolve content-ref without files.content handle");
+        }
+        const stream = await files.content.open(source.ref);
+        const reader = stream.getReader();
+        const chunks = [];
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done) {
+                break;
+            }
+            chunks.push(value);
+        }
+        const totalLength = chunks.reduce((sum, chunk) => sum + chunk.byteLength, 0);
+        const result = new Uint8Array(totalLength);
+        let offset = 0;
+        for (const chunk of chunks) {
+            result.set(chunk, offset);
+            offset += chunk.byteLength;
+        }
+        return result;
+    }
+    throw new Error(`Unhandled file source kind: ${JSON.stringify(source)}`);
+};

package/dist/index.d.ts

@@ -0,0 +1,2 @@
+export { MainContentUnit, MainContentUnit as default } from "./main-content-unit.js";
+export type { MainContentUnitOptions } from "./main-content-unit.js";

package/dist/index.js

@@ -0,0 +1 @@
+export { MainContentUnit, MainContentUnit as default } from "./main-content-unit.js";

package/dist/main-content-unit.d.ts

@@ -0,0 +1,20 @@
+import { Unit, type FileTree, type UnitContext, type UnitPatch, type UnitRuntime } from "@pagepocket/lib";
+export type MainContentUnitOptions = {
+    markdown?: boolean;
+};
+export declare class MainContentUnit extends Unit {
+    readonly id = "mainContent";
+    readonly kind = "extract.mainContent";
+    private options;
+    constructor(options?: MainContentUnitOptions);
+    merge(returnValue: UnitPatch, pluginContributedValue?: UnitPatch): UnitPatch;
+    run(ctx: UnitContext, _rt: UnitRuntime): Promise<{
+        files: FileTree;
+        html: {
+            htmlString: string;
+            baseUrl: string;
+            url?: string;
+        };
+    }>;
+    private extractMainContent;
+}

package/dist/main-content-unit.js

@@ -0,0 +1,62 @@
+import { mergeFileTrees, Unit } from "@pagepocket/lib";
+import { Defuddle } from "defuddle/node";
+import { buildContentTree } from "./build-content-tree.js";
+export class MainContentUnit extends Unit {
+    constructor(options) {
+        super();
+        this.id = "mainContent";
+        this.kind = "extract.mainContent";
+        this.options = options ?? {};
+    }
+    merge(returnValue, pluginContributedValue = {}) {
+        const mergedValue = { ...returnValue, ...pluginContributedValue };
+        const returnFiles = returnValue.files;
+        const pluginFiles = pluginContributedValue.files;
+        if (!isFileTree(returnFiles) || !isFileTree(pluginFiles)) {
+            return mergedValue;
+        }
+        return { ...mergedValue, files: mergeFileTrees(returnFiles, pluginFiles) };
+    }
+    async run(ctx, _rt) {
+        const capture = ctx.value.capture;
+        if (!capture) {
+            throw new Error("MainContentUnit requires ctx.value.capture");
+        }
+        const html = ctx.value.html;
+        if (!html) {
+            throw new Error("MainContentUnit requires ctx.value.html");
+        }
+        const domHtml = ctx.value.domHtml;
+        const sourceHtml = domHtml?.htmlString ?? html.htmlString;
+        const baseUrl = domHtml?.baseUrl ?? html.baseUrl;
+        const contentHtml = await this.extractMainContent(sourceHtml, baseUrl);
+        const { files } = await buildContentTree({
+            contentHtml,
+            baseUrl,
+            capture
+        });
+        return { files, html };
+    }
+    async extractMainContent(htmlString, baseUrl) {
+        const result = await Defuddle(htmlString, baseUrl, {
+            markdown: this.options.markdown ?? false
+        });
+        return result.content;
+    }
+}
+const isFileTree = (value) => {
+    if (!value || typeof value !== "object") {
+        return false;
+    }
+    if (!("root" in value)) {
+        return false;
+    }
+    const root = value.root;
+    if (!root || typeof root !== "object") {
+        return false;
+    }
+    const rootRecord = root;
+    return (rootRecord.kind === "directory" &&
+        typeof rootRecord.path === "string" &&
+        Array.isArray(rootRecord.entries));
+};

package/dist/rewrite-content-urls.d.ts

@@ -0,0 +1,23 @@
+type UrlResolver = (absoluteUrl: string) => string | undefined;
+/**
+ * Rewrite resource URLs in an HTML fragment so they point to local snapshot paths.
+ *
+ * This handles `src`, `href`, `srcset`, `poster`, `data-src`, `data-href`,
+ * `data-poster`, `data-url`, and inline `style` background URLs — the same
+ * set of attributes that the snapshot builder's full-page rewriter handles.
+ *
+ * Usage:
+ * ```ts
+ * const rewritten = rewriteContentUrls({
+ *   html: '<img src="https://example.com/photo.jpg">',
+ *   baseUrl: 'https://example.com/article',
+ *   resolve: (url) => urlToPath.get(url),
+ * });
+ * ```
+ */
+export declare const rewriteContentUrls: (input: {
+    html: string;
+    baseUrl: string;
+    resolve: UrlResolver;
+}) => string;
+export {};

package/dist/rewrite-content-urls.js

@@ -0,0 +1,87 @@
+import * as cheerio from "cheerio";
+/**
+ * Rewrite resource URLs in an HTML fragment so they point to local snapshot paths.
+ *
+ * This handles `src`, `href`, `srcset`, `poster`, `data-src`, `data-href`,
+ * `data-poster`, `data-url`, and inline `style` background URLs — the same
+ * set of attributes that the snapshot builder's full-page rewriter handles.
+ *
+ * Usage:
+ * ```ts
+ * const rewritten = rewriteContentUrls({
+ *   html: '<img src="https://example.com/photo.jpg">',
+ *   baseUrl: 'https://example.com/article',
+ *   resolve: (url) => urlToPath.get(url),
+ * });
+ * ```
+ */
+export const rewriteContentUrls = (input) => {
+    const { html, baseUrl, resolve } = input;
+    const $ = cheerio.load(html, undefined, false);
+    const rewriteAttr = (selector, attr) => {
+        $(selector).each((_, element) => {
+            const value = $(element).attr(attr);
+            if (!value) {
+                return;
+            }
+            const resolved = resolveValue(value, baseUrl, resolve);
+            if (resolved) {
+                $(element).attr(attr, resolved);
+            }
+        });
+    };
+    rewriteAttr("[src]", "src");
+    rewriteAttr("link[href]", "href");
+    rewriteAttr("a[href]", "href");
+    rewriteAttr("[poster]", "poster");
+    rewriteAttr("[data-src]", "data-src");
+    rewriteAttr("[data-href]", "data-href");
+    rewriteAttr("[data-poster]", "data-poster");
+    rewriteAttr("[data-url]", "data-url");
+    $("[srcset]").each((_, element) => {
+        const value = $(element).attr("srcset");
+        if (!value) {
+            return;
+        }
+        const rewritten = rewriteSrcset(value, baseUrl, resolve);
+        $(element).attr("srcset", rewritten);
+    });
+    return $.html();
+};
+const resolveValue = (value, baseUrl, resolve) => {
+    if (shouldSkip(value)) {
+        return undefined;
+    }
+    try {
+        const absolute = new URL(value, baseUrl).toString();
+        return resolve(absolute);
+    }
+    catch {
+        return undefined;
+    }
+};
+const rewriteSrcset = (srcset, baseUrl, resolve) => {
+    const parts = srcset.split(",").map((part) => part.trim());
+    const rewritten = parts.map((part) => {
+        const [url, ...descriptors] = part.split(/\s+/);
+        if (!url) {
+            return part;
+        }
+        const resolved = resolveValue(url, baseUrl, resolve);
+        if (!resolved) {
+            return part;
+        }
+        return [resolved, ...descriptors].join(" ");
+    });
+    return rewritten.join(", ");
+};
+const shouldSkip = (value) => {
+    const trimmed = value.trim();
+    return (!trimmed ||
+        trimmed.startsWith("data:") ||
+        trimmed.startsWith("blob:") ||
+        trimmed.startsWith("javascript:") ||
+        trimmed.startsWith("#") ||
+        trimmed.startsWith("mailto:") ||
+        trimmed.startsWith("tel:"));
+};

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@pagepocket/main-content-unit",
+  "version": "0.12.0",
+  "description": "PagePocket unit: extract main content from captured HTML using defuddle and rewrite resource URLs to local paths",
+  "type": "module",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "license": "ISC",
+  "dependencies": {
+    "cheerio": "^1.0.0",
+    "defuddle": "^0.6.0",
+    "jsdom": "^26.0.0",
+    "@pagepocket/lib": "0.12.0",
+    "@pagepocket/shared": "0.12.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.4.5",
+    "tsx": "^4.19.3",
+    "@types/node": "^20.17.12"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "tsx --test specs/*.spec.ts"
+  }
+}