@pagepocket/lib 0.7.1 → 0.8.1

package/dist/snapshot-builder/capture-index.d.ts

@@ -0,0 +1,12 @@
+import type { ResourceFilter } from "../types";
+import type { ApiEntry, BuildLimits, StoredResource } from "./types";
+import type { CaptureArtifacts } from "../capture/types";
+export declare const indexCapture: (input: {
+    capture: CaptureArtifacts;
+    filter: ResourceFilter;
+    limits?: BuildLimits;
+    warnings: string[];
+}) => Promise<{
+    resources: StoredResource[];
+    apiEntries: ApiEntry[];
+}>;

package/dist/snapshot-builder/capture-index.js

@@ -0,0 +1,173 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.indexCapture = void 0;
+const utils_1 = require("../utils");
+const streams_1 = require("../utils/streams");
+const http_1 = require("./http");
+const isApiResource = (request) => {
+    const type = request?.resourceType;
+    return type === "fetch" || type === "xhr";
+};
+const buildByRequestId = (events) => {
+    const byId = new Map();
+    const ensure = (requestId) => {
+        const existing = byId.get(requestId);
+        if (existing) {
+            return existing;
+        }
+        const created = {};
+        byId.set(requestId, created);
+        return created;
+    };
+    for (const event of events) {
+        if (event.type === "http.request") {
+            ensure(event.requestId).request = event;
+            continue;
+        }
+        if (event.type === "http.response") {
+            ensure(event.requestId).response = event;
+            continue;
+        }
+        if (event.type === "http.failed") {
+            ensure(event.requestId).failed = event;
+        }
+    }
+    return byId;
+};
+const indexCapture = async (input) => {
+    const byId = buildByRequestId(input.capture.events);
+    const resources = [];
+    const apiEntries = [];
+    const apiSeen = new Set();
+    let totalBytes = 0;
+    const recordApiFailure = (request, failed) => {
+        if (apiSeen.has(request.requestId))
+            return;
+        apiSeen.add(request.requestId);
+        apiEntries.push({
+            request,
+            record: {
+                url: request.url,
+                method: request.method,
+                requestHeaders: request.headers,
+                error: failed.errorText,
+                timestamp: failed.timestamp
+            }
+        });
+    };
+    const recordApiResponse = async (request, response, bodyRef) => {
+        if (apiSeen.has(request.requestId))
+            return;
+        apiSeen.add(request.requestId);
+        const record = {
+            url: request.url,
+            method: request.method,
+            requestHeaders: request.headers,
+            status: response.status,
+            statusText: response.statusText,
+            responseHeaders: response.headers,
+            timestamp: response.timestamp
+        };
+        if (bodyRef) {
+            const stream = await input.capture.contentStore.open(bodyRef);
+            const bytes = await (0, streams_1.streamToUint8Array)(stream);
+            if (bytes.byteLength > 0) {
+                const mimeType = (0, http_1.responseMimeType)(response);
+                const decoded = (0, utils_1.bodyToTextOrBase64)(bytes, mimeType);
+                if (decoded.encoding === "text") {
+                    record.responseBody = decoded.text;
+                    record.responseEncoding = "text";
+                }
+                else {
+                    record.responseBodyBase64 = decoded.base64;
+                    record.responseEncoding = "base64";
+                }
+            }
+        }
+        apiEntries.push({ request, record });
+    };
+    for (const record of byId.values()) {
+        if (!record.request || record.request.type !== "http.request") {
+            continue;
+        }
+        const requestEvent = record.request;
+        const request = {
+            type: "request",
+            requestId: requestEvent.requestId,
+            url: requestEvent.url,
+            method: requestEvent.method,
+            headers: (0, http_1.headersListToRecord)(requestEvent.headers),
+            timestamp: requestEvent.timestamp,
+            frameId: requestEvent.frameId,
+            resourceType: requestEvent.resourceType,
+            initiator: requestEvent.initiator
+        };
+        if (record.failed && record.failed.type === "http.failed") {
+            const failedEvent = record.failed;
+            const failed = {
+                type: "failed",
+                requestId: failedEvent.requestId,
+                url: failedEvent.url,
+                errorText: failedEvent.errorText,
+                timestamp: failedEvent.timestamp
+            };
+            if (isApiResource(request)) {
+                recordApiFailure(request, failed);
+            }
+            continue;
+        }
+        if (!record.response || record.response.type !== "http.response") {
+            continue;
+        }
+        const responseEvent = record.response;
+        const response = {
+            type: "response",
+            requestId: responseEvent.requestId,
+            url: responseEvent.url,
+            status: responseEvent.status,
+            statusText: responseEvent.statusText,
+            headers: (0, http_1.headersListToRecord)(responseEvent.headers),
+            timestamp: responseEvent.timestamp,
+            mimeType: responseEvent.mimeType,
+            fromDiskCache: responseEvent.fromDiskCache,
+            fromServiceWorker: responseEvent.fromServiceWorker,
+            body: undefined
+        };
+        const bodyRef = responseEvent.bodyRef;
+        if (isApiResource(request)) {
+            await recordApiResponse(request, response, bodyRef);
+        }
+        if (!input.filter.shouldSave(request, response)) {
+            continue;
+        }
+        if (!bodyRef) {
+            input.warnings.push(`Missing body for ${request.url}`);
+            continue;
+        }
+        const stream = await input.capture.contentStore.open(bodyRef);
+        const bytes = await (0, streams_1.streamToUint8Array)(stream);
+        const byteLength = bytes.byteLength;
+        if (input.limits?.maxSingleResourceBytes && byteLength > input.limits.maxSingleResourceBytes) {
+            input.warnings.push(`Resource too large: ${request.url}`);
+            continue;
+        }
+        if (input.limits?.maxResources && resources.length >= input.limits.maxResources) {
+            input.warnings.push(`Resource limit reached at ${request.url}`);
+            continue;
+        }
+        if (input.limits?.maxTotalBytes && totalBytes + byteLength > input.limits.maxTotalBytes) {
+            input.warnings.push(`Total byte limit reached at ${request.url}`);
+            continue;
+        }
+        totalBytes += byteLength;
+        resources.push({
+            request,
+            response,
+            contentRef: bodyRef,
+            size: (0, http_1.parseContentLength)(response.headers || {}) ?? byteLength,
+            mimeType: (0, http_1.responseMimeType)(response)
+        });
+    }
+    return { resources, apiEntries };
+};
+exports.indexCapture = indexCapture;

package/dist/snapshot-builder/emit-document.d.ts

@@ -0,0 +1,24 @@
+import type { BuildOptions, StoredResource } from "./types.js";
+export declare const emitDocumentFile: (input: {
+    resource: StoredResource;
+    path: string;
+    entryUrl: string;
+    groupUrl: string;
+    apiPath: string;
+    resolve: (absoluteUrl: string) => string | null;
+    rewriteEntry: boolean;
+    replaceElements: BuildOptions["replaceElements"];
+    contentStore: BuildOptions["capture"]["contentStore"];
+    snapshotEntryUrl: string;
+}) => Promise<{
+    file: {
+        path: string;
+        mimeType: string;
+        size: number;
+        source: import("../types.js").ContentRef;
+        originalUrl: string;
+        resourceType: import("../types.js").ResourceType | undefined;
+        headers: Record<string, string>;
+    };
+    title: string | undefined;
+}>;

package/dist/snapshot-builder/emit-document.js

@@ -0,0 +1,50 @@
+import { rewriteEntryHtml } from "../rewrite-links.js";
+import { streamToUint8Array } from "../utils/streams.js";
+import { decodeUtf8 } from "../utils.js";
+export const emitDocumentFile = async (input) => {
+    const stream = await input.contentStore.open(input.resource.contentRef);
+    const bytes = await streamToUint8Array(stream);
+    const decoded = decodeUtf8(bytes);
+    if (decoded === null) {
+        return {
+            file: {
+                path: input.path,
+                mimeType: input.resource.mimeType ?? "text/html",
+                size: bytes.byteLength,
+                source: input.resource.contentRef,
+                originalUrl: input.resource.request.url,
+                resourceType: input.resource.request.resourceType,
+                headers: input.resource.response.headers
+            },
+            title: undefined
+        };
+    }
+    const rewritten = await rewriteEntryHtml({
+        html: decoded,
+        entryUrl: input.groupUrl,
+        apiPath: input.apiPath,
+        resolve: input.resolve,
+        rewriteLinks: input.rewriteEntry,
+        replaceElements: input.replaceElements,
+        isEntryDocument: input.groupUrl === input.entryUrl,
+        snapshotEntryUrl: input.snapshotEntryUrl
+    });
+    const encoded = new TextEncoder().encode(rewritten.html);
+    const contentRef = await input.contentStore.put({ kind: "buffer", data: encoded }, {
+        url: input.resource.request.url,
+        mimeType: input.resource.mimeType,
+        sizeHint: encoded.byteLength
+    });
+    return {
+        file: {
+            path: input.path,
+            mimeType: input.resource.mimeType ?? "text/html",
+            size: encoded.byteLength,
+            source: contentRef,
+            originalUrl: input.resource.request.url,
+            resourceType: input.resource.request.resourceType,
+            headers: input.resource.response.headers
+        },
+        title: rewritten.title
+    };
+};

package/dist/snapshot-builder/grouping.d.ts

@@ -0,0 +1,8 @@
+import type { ApiEntry, DocumentGroup, StoredResource } from "./types.js";
+export declare const docDirFromUrl: (url: string) => string;
+export declare const groupResources: (input: {
+    entryUrl: string;
+    resources: StoredResource[];
+    apiEntries: ApiEntry[];
+    warnings: string[];
+}) => DocumentGroup[];

package/dist/snapshot-builder/grouping.js

@@ -0,0 +1,87 @@
+import { sanitizePosixPath } from "../utils.js";
+export const docDirFromUrl = (url) => {
+    try {
+        const parsed = new URL(url);
+        const clean = sanitizePosixPath(parsed.pathname || "");
+        if (!clean) {
+            return "root";
+        }
+        return clean;
+    }
+    catch {
+        return "root";
+    }
+};
+export const groupResources = (input) => {
+    const documents = input.resources.filter((resource) => resource.request.resourceType === "document");
+    const hasFrameId = input.resources.some((resource) => !!resource.request.frameId);
+    const primaryDoc = documents.find((doc) => doc.request.url === input.entryUrl) ?? documents[0];
+    if (!hasFrameId) {
+        if (documents.length > 1) {
+            input.warnings.push("Multiple documents captured without frameId; using the first document.");
+        }
+        const primaryGroup = {
+            id: primaryDoc?.request.requestId ?? "root",
+            url: primaryDoc?.request.url ?? input.entryUrl,
+            resources: [],
+            apiEntries: []
+        };
+        for (const resource of input.resources) {
+            if (resource.request.resourceType === "document" && resource !== primaryDoc) {
+                continue;
+            }
+            primaryGroup.resources.push(resource);
+            if (resource.request.resourceType === "document") {
+                primaryGroup.docResource = resource;
+            }
+        }
+        for (const apiEntry of input.apiEntries) {
+            primaryGroup.apiEntries.push(apiEntry);
+        }
+        return [primaryGroup];
+    }
+    const groups = new Map();
+    for (const doc of documents) {
+        const id = doc.request.frameId ?? doc.request.requestId;
+        groups.set(id, {
+            id,
+            url: doc.request.url,
+            resources: [doc],
+            apiEntries: [],
+            docResource: doc
+        });
+    }
+    const primaryGroup = primaryDoc
+        ? (groups.get(primaryDoc.request.frameId ?? primaryDoc.request.requestId) ?? null)
+        : null;
+    const groupByUrl = new Map();
+    for (const group of groups.values()) {
+        groupByUrl.set(group.url, group);
+    }
+    for (const resource of input.resources) {
+        if (resource.request.resourceType === "document") {
+            continue;
+        }
+        const frameId = resource.request.frameId;
+        const byFrame = frameId ? groups.get(frameId) : undefined;
+        const byInitiator = resource.request.initiator?.url
+            ? groupByUrl.get(resource.request.initiator.url)
+            : undefined;
+        const target = byFrame ?? byInitiator ?? primaryGroup ?? Array.from(groups.values())[0];
+        if (target) {
+            target.resources.push(resource);
+        }
+    }
+    for (const entry of input.apiEntries) {
+        const frameId = entry.request.frameId;
+        const byFrame = frameId ? groups.get(frameId) : undefined;
+        const byInitiator = entry.request.initiator?.url
+            ? groupByUrl.get(entry.request.initiator.url)
+            : undefined;
+        const target = byFrame ?? byInitiator ?? primaryGroup ?? Array.from(groups.values())[0];
+        if (target) {
+            target.apiEntries.push(entry);
+        }
+    }
+    return Array.from(groups.values());
+};

package/dist/snapshot-builder/http.d.ts

@@ -0,0 +1,6 @@
+import type { Header } from "../core/capture/types.js";
+import type { NetworkResponseEvent } from "../types.js";
+export declare const headersListToRecord: (headers: Header[]) => Record<string, string>;
+export declare const getHeaderValue: (headers: Record<string, string>, name: string) => string | undefined;
+export declare const parseContentLength: (headers: Record<string, string>) => number | undefined;
+export declare const responseMimeType: (response: NetworkResponseEvent) => string | undefined;

package/dist/snapshot-builder/http.js

@@ -0,0 +1,28 @@
+export const headersListToRecord = (headers) => {
+    const out = {};
+    for (const h of headers) {
+        out[h.name] = h.value;
+    }
+    return out;
+};
+export const getHeaderValue = (headers, name) => {
+    const target = name.toLowerCase();
+    for (const key in headers) {
+        if (key.toLowerCase() === target) {
+            return headers[key];
+        }
+    }
+    return undefined;
+};
+export const parseContentLength = (headers) => {
+    const raw = getHeaderValue(headers, "content-length");
+    if (!raw) {
+        return undefined;
+    }
+    const parsed = Number(raw);
+    if (!Number.isFinite(parsed) || parsed <= 0) {
+        return undefined;
+    }
+    return parsed;
+};
+export const responseMimeType = (response) => response.mimeType || getHeaderValue(response.headers || {}, "content-type");

package/dist/snapshot-builder/index.d.ts

@@ -0,0 +1,4 @@
+export type { BuildOptions } from "./types.js";
+export type { ApiEntry, StoredResource } from "./capture-index/index.js";
+export { indexCapture } from "./capture-index/index.js";
+export { buildSnapshot } from "./build-snapshot.js";

package/dist/snapshot-builder/index.js

@@ -0,0 +1,2 @@
+export { indexCapture } from "./capture-index/index.js";
+export { buildSnapshot } from "./build-snapshot.js";

package/dist/snapshot-builder/path-map.d.ts

@@ -0,0 +1,3 @@
+export declare const escapePercentForStaticServers: (value: string) => string;
+export declare const docDirFromUrl: (url: string) => string;
+export declare const resolveSnapshotPath: (urlToPath: Map<string, string>, absoluteUrl: string) => string | null;

package/dist/snapshot-builder/path-map.js

@@ -0,0 +1,35 @@
+import { sanitizePosixPath } from "../utils.js";
+export const escapePercentForStaticServers = (value) => {
+    // Many static servers decode percent-encoding in the request path before
+    // resolving it to a filesystem path.
+    //
+    // Our snapshots can contain literal "%2F" sequences in filenames (e.g.
+    // Substack image URLs embedded into a path segment). When a server decodes
+    // "%2F" to "/", it changes the path structure and causes 404s.
+    //
+    // Escaping "%" to "%25" makes the request decode back to the original
+    // filename on disk.
+    return value.split("%").join("%25");
+};
+export const docDirFromUrl = (url) => {
+    try {
+        const parsed = new URL(url);
+        const clean = sanitizePosixPath(parsed.pathname || "");
+        return clean || "root";
+    }
+    catch {
+        return "root";
+    }
+};
+export const resolveSnapshotPath = (urlToPath, absoluteUrl) => {
+    const resolved = urlToPath.get(absoluteUrl);
+    if (!resolved) {
+        return null;
+    }
+    // Defensive: resolved should be a path, but avoid breaking any unexpected
+    // absolute URLs.
+    if (resolved.includes("://")) {
+        return resolved;
+    }
+    return escapePercentForStaticServers(resolved);
+};

package/dist/snapshot-builder/resources-path.d.ts

@@ -0,0 +1,23 @@
+import type { ResourceType } from "../types.js";
+export type ResourcesPathSnapshotItem = {
+    url: string;
+    path: string;
+    resourceType?: ResourceType;
+    mimeType?: string;
+    size?: number;
+};
+export type ResourcesPathSnapshot = {
+    version: "1.0";
+    createdAt: number;
+    items: ResourcesPathSnapshotItem[];
+};
+export declare const buildResourcesPathSnapshot: (createdAt: number, files: Array<{
+    path: string;
+    originalUrl?: string;
+    resourceType?: ResourceType;
+    mimeType?: string;
+    size?: number;
+    source?: {
+        kind: string;
+    };
+}>) => ResourcesPathSnapshot;

package/dist/snapshot-builder/resources-path.js

@@ -0,0 +1,47 @@
+export const buildResourcesPathSnapshot = (createdAt, files) => {
+    const items = [];
+    for (const file of files) {
+        const originalUrl = file.originalUrl;
+        if (!originalUrl) {
+            continue;
+        }
+        // Exclude HTML documents. We only want static resources.
+        if (file.resourceType === "document") {
+            continue;
+        }
+        // Skip snapshot-local pseudo URLs.
+        if (originalUrl.startsWith("/")) {
+            continue;
+        }
+        // Be defensive: only include valid absolute URLs.
+        try {
+            new URL(originalUrl);
+        }
+        catch {
+            continue;
+        }
+        items.push({
+            url: originalUrl,
+            path: file.path,
+            resourceType: file.resourceType,
+            mimeType: file.mimeType,
+            size: file.size
+        });
+    }
+    items.sort((left, right) => {
+        if (left.path < right.path)
+            return -1;
+        if (left.path > right.path)
+            return 1;
+        if (left.url < right.url)
+            return -1;
+        if (left.url > right.url)
+            return 1;
+        return 0;
+    });
+    return {
+        version: "1.0",
+        createdAt,
+        items
+    };
+};

package/dist/snapshot-builder/rewrite-resource.d.ts

@@ -0,0 +1,18 @@
+import type { BuildOptions, StoredResource } from "./types.js";
+export declare const maybeRewriteStylesheet: (input: {
+    resource: StoredResource;
+    resolve: (absoluteUrl: string) => string | null;
+    contentStore: BuildOptions["capture"]["contentStore"];
+    rewriteCSS: boolean;
+}) => Promise<{
+    contentRef: import("../types.js").ContentRef;
+    size: number;
+}>;
+export declare const maybeRewriteScript: (input: {
+    resource: StoredResource;
+    resolve: (absoluteUrl: string) => string | null;
+    contentStore: BuildOptions["capture"]["contentStore"];
+}) => Promise<{
+    contentRef: import("../types.js").ContentRef;
+    size: number;
+}>;

package/dist/snapshot-builder/rewrite-resource.js

@@ -0,0 +1,52 @@
+import { rewriteCssText } from "../css-rewrite.js";
+import { rewriteJsText } from "../rewrite-links.js";
+import { streamToUint8Array } from "../utils/streams.js";
+import { decodeUtf8 } from "../utils.js";
+export const maybeRewriteStylesheet = async (input) => {
+    if (input.resource.request.resourceType !== "stylesheet" || !input.rewriteCSS) {
+        return { contentRef: input.resource.contentRef, size: input.resource.size };
+    }
+    const stream = await input.contentStore.open(input.resource.contentRef);
+    const bytes = await streamToUint8Array(stream);
+    const decoded = decodeUtf8(bytes);
+    if (decoded === null) {
+        return { contentRef: input.resource.contentRef, size: input.resource.size };
+    }
+    const rewritten = await rewriteCssText({
+        cssText: decoded,
+        cssUrl: input.resource.request.url,
+        resolveUrl: input.resolve
+    });
+    if (rewritten === decoded) {
+        return { contentRef: input.resource.contentRef, size: input.resource.size };
+    }
+    const encoded = new TextEncoder().encode(rewritten);
+    const contentRef = await input.contentStore.put({ kind: "buffer", data: encoded }, {
+        url: input.resource.request.url,
+        mimeType: input.resource.mimeType,
+        sizeHint: encoded.byteLength
+    });
+    return { contentRef, size: encoded.byteLength };
+};
+export const maybeRewriteScript = async (input) => {
+    if (input.resource.request.resourceType !== "script") {
+        return { contentRef: input.resource.contentRef, size: input.resource.size };
+    }
+    const stream = await input.contentStore.open(input.resource.contentRef);
+    const bytes = await streamToUint8Array(stream);
+    const decoded = decodeUtf8(bytes);
+    if (decoded === null) {
+        return { contentRef: input.resource.contentRef, size: input.resource.size };
+    }
+    const rewritten = await rewriteJsText(decoded, input.resolve, input.resource.request.url);
+    if (rewritten === decoded) {
+        return { contentRef: input.resource.contentRef, size: input.resource.size };
+    }
+    const encoded = new TextEncoder().encode(rewritten);
+    const contentRef = await input.contentStore.put({ kind: "buffer", data: encoded }, {
+        url: input.resource.request.url,
+        mimeType: input.resource.mimeType,
+        sizeHint: encoded.byteLength
+    });
+    return { contentRef, size: encoded.byteLength };
+};

package/dist/snapshot-builder/types.d.ts

@@ -0,0 +1,37 @@
+import type { CaptureArtifacts } from "../core/capture/types.js";
+import type { ApiRecord, ContentRef, NetworkRequestEvent, NetworkResponseEvent, PathResolver, ResourceFilter, ReplaceElementsConfig } from "../types.js";
+export type BuildLimits = {
+    maxTotalBytes?: number;
+    maxSingleResourceBytes?: number;
+    maxResources?: number;
+};
+export type BuildOptions = {
+    entryUrl: string;
+    createdAt: number;
+    capture: CaptureArtifacts;
+    filter: ResourceFilter;
+    pathResolver?: PathResolver;
+    rewriteEntry: boolean;
+    rewriteCSS: boolean;
+    replaceElements?: ReplaceElementsConfig;
+    limits?: BuildLimits;
+    warnings: string[];
+};
+export type StoredResource = {
+    request: NetworkRequestEvent;
+    response: NetworkResponseEvent;
+    contentRef: ContentRef;
+    size: number;
+    mimeType?: string;
+};
+export type ApiEntry = {
+    record: ApiRecord;
+    request: NetworkRequestEvent;
+};
+export type DocumentGroup = {
+    id: string;
+    url: string;
+    resources: StoredResource[];
+    apiEntries: ApiEntry[];
+    docResource?: StoredResource;
+};

package/dist/snapshot-builder/types.js

@@ -0,0 +1,2 @@
+export {};
+// NOTE: ResourcesPathSnapshot types are defined in resources-path.ts.

package/dist/snapshot-builder.d.ts

@@ -1,16 +1,20 @@
-import type { ApiEntry, StoredResource } from "./network-store";
-import type { ContentStore, PageSnapshot, PathResolver, ReplaceElementsConfig } from "./types";
-type BuildOptions = {
+import type { FileTree } from "./core/file-tree";
+import type { CaptureArtifacts } from "./capture/types";
+import type { PathResolver, ResourceFilter, ReplaceElementsConfig } from "./types";
+export type BuildOptions = {
     entryUrl: string;
     createdAt: number;
-    resources: StoredResource[];
-    apiEntries: ApiEntry[];
-    contentStore: ContentStore;
+    capture: CaptureArtifacts;
+    filter: ResourceFilter;
     pathResolver?: PathResolver;
     rewriteEntry: boolean;
     rewriteCSS: boolean;
     replaceElements?: ReplaceElementsConfig;
+    limits?: {
+        maxTotalBytes?: number;
+        maxSingleResourceBytes?: number;
+        maxResources?: number;
+    };
     warnings: string[];
 };
-export declare const buildSnapshot: (input: BuildOptions) => Promise<PageSnapshot>;
-export {};
+export declare const buildSnapshot: (input: BuildOptions) => Promise<FileTree>;