@pagepocket/lib 0.4.2 → 0.5.1

package/dist/snapshot-builder.js

@@ -0,0 +1,275 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildSnapshot = void 0;
+const path_resolver_1 = require("./path-resolver");
+const snapshot_1 = require("./snapshot");
+const css_rewrite_1 = require("./css-rewrite");
+const rewrite_links_1 = require("./rewrite-links");
+const utils_1 = require("./utils");
+const streamToUint8Array = async (stream) => {
+    const reader = stream.getReader();
+    const chunks = [];
+    let total = 0;
+    while (true) {
+        const result = await reader.read();
+        if (result.done)
+            break;
+        if (result.value) {
+            chunks.push(result.value);
+            total += result.value.byteLength;
+        }
+    }
+    const output = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        output.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return output;
+};
+const docDirFromUrl = (url) => {
+    try {
+        const parsed = new URL(url);
+        const clean = (0, utils_1.sanitizePosixPath)(parsed.pathname || "");
+        if (!clean) {
+            return "root";
+        }
+        return clean;
+    }
+    catch {
+        return "root";
+    }
+};
+const groupResources = (input) => {
+    const documents = input.resources.filter((resource) => resource.request.resourceType === "document");
+    const hasFrameId = input.resources.some((resource) => !!resource.request.frameId);
+    const primaryDoc = documents.find((doc) => doc.request.url === input.entryUrl) ?? documents[0];
+    if (!hasFrameId) {
+        if (documents.length > 1) {
+            input.warnings.push("Multiple documents captured without frameId; using the first document.");
+        }
+        const primaryGroup = {
+            id: primaryDoc?.request.requestId ?? "root",
+            url: primaryDoc?.request.url ?? input.entryUrl,
+            resources: [],
+            apiEntries: []
+        };
+        for (const resource of input.resources) {
+            if (resource.request.resourceType === "document" && resource !== primaryDoc) {
+                continue;
+            }
+            primaryGroup.resources.push(resource);
+            if (resource.request.resourceType === "document") {
+                primaryGroup.docResource = resource;
+            }
+        }
+        for (const apiEntry of input.apiEntries) {
+            primaryGroup.apiEntries.push(apiEntry);
+        }
+        return [primaryGroup];
+    }
+    const groups = new Map();
+    for (const doc of documents) {
+        const id = doc.request.frameId ?? doc.request.requestId;
+        groups.set(id, {
+            id,
+            url: doc.request.url,
+            resources: [doc],
+            apiEntries: [],
+            docResource: doc
+        });
+    }
+    const primaryGroup = primaryDoc
+        ? groups.get(primaryDoc.request.frameId ?? primaryDoc.request.requestId) ?? null
+        : null;
+    const groupByUrl = new Map();
+    for (const group of groups.values()) {
+        groupByUrl.set(group.url, group);
+    }
+    for (const resource of input.resources) {
+        if (resource.request.resourceType === "document") {
+            continue;
+        }
+        const frameId = resource.request.frameId;
+        const byFrame = frameId ? groups.get(frameId) : undefined;
+        const byInitiator = resource.request.initiator?.url ? groupByUrl.get(resource.request.initiator.url) : undefined;
+        const target = byFrame ?? byInitiator ?? primaryGroup ?? Array.from(groups.values())[0];
+        if (target) {
+            target.resources.push(resource);
+        }
+    }
+    for (const entry of input.apiEntries) {
+        const frameId = entry.request.frameId;
+        const byFrame = frameId ? groups.get(frameId) : undefined;
+        const byInitiator = entry.request.initiator?.url ? groupByUrl.get(entry.request.initiator.url) : undefined;
+        const target = byFrame ?? byInitiator ?? primaryGroup ?? Array.from(groups.values())[0];
+        if (target) {
+            target.apiEntries.push(entry);
+        }
+    }
+    return Array.from(groups.values());
+};
+const buildApiSnapshot = (url, createdAt, entries) => ({
+    version: "1.0",
+    url,
+    createdAt,
+    records: entries.map((entry) => entry.record)
+});
+const buildSnapshot = async (input) => {
+    const warnings = input.warnings;
+    const groups = groupResources({
+        entryUrl: input.entryUrl,
+        resources: input.resources,
+        apiEntries: input.apiEntries,
+        warnings
+    });
+    const multiDoc = groups.length > 1;
+    const files = [];
+    let entryPath = "";
+    let title;
+    for (const group of groups) {
+        const docDir = multiDoc ? docDirFromUrl(group.url) : "";
+        const baseResolver = input.pathResolver ?? (0, path_resolver_1.createDefaultPathResolver)();
+        const resolver = multiDoc ? (0, path_resolver_1.withPrefixPathResolver)(baseResolver, docDir) : baseResolver;
+        const urlToPath = new Map();
+        for (const resource of group.resources) {
+            const path = resolver.resolve({
+                url: resource.request.url,
+                resourceType: resource.request.resourceType,
+                mimeType: resource.mimeType,
+                suggestedFilename: undefined,
+                isCrossOrigin: (0, path_resolver_1.resolveCrossOrigin)(resource.request.url, group.url),
+                entryUrl: group.url
+            });
+            urlToPath.set(resource.request.url, path);
+        }
+        const resolve = (absoluteUrl) => urlToPath.get(absoluteUrl) ?? null;
+        const apiPath = (0, utils_1.ensureLeadingSlash)(multiDoc ? `${(0, utils_1.sanitizePosixPath)(docDir)}/api.json` : "/api.json");
+        for (const resource of group.resources) {
+            if (resource.request.resourceType === "document") {
+                const path = urlToPath.get(resource.request.url) ?? "/index.html";
+                const stream = await input.contentStore.open(resource.contentRef);
+                const bytes = await streamToUint8Array(stream);
+                const decoded = (0, utils_1.decodeUtf8)(bytes) ?? "";
+                let html = decoded;
+                const rewritten = await (0, rewrite_links_1.rewriteEntryHtml)({
+                    html,
+                    entryUrl: group.url,
+                    apiPath,
+                    resolve,
+                    rewriteLinks: input.rewriteEntry
+                });
+                html = rewritten.html;
+                if (!title) {
+                    title = rewritten.title;
+                }
+                const encoded = new TextEncoder().encode(html);
+                const contentRef = await input.contentStore.put({ kind: "buffer", data: encoded }, { url: resource.request.url, mimeType: resource.mimeType, sizeHint: encoded.byteLength });
+                files.push({
+                    path,
+                    mimeType: resource.mimeType ?? "text/html",
+                    size: encoded.byteLength,
+                    source: contentRef,
+                    originalUrl: resource.request.url,
+                    resourceType: resource.request.resourceType,
+                    headers: resource.response.headers
+                });
+                if (resource.request.url === input.entryUrl || !entryPath) {
+                    entryPath = path;
+                }
+                continue;
+            }
+            let contentRef = resource.contentRef;
+            let size = resource.size;
+            if (resource.request.resourceType === "stylesheet" && input.rewriteCSS) {
+                const stream = await input.contentStore.open(resource.contentRef);
+                const bytes = await streamToUint8Array(stream);
+                const decoded = (0, utils_1.decodeUtf8)(bytes);
+                if (decoded !== null) {
+                    const rewritten = await (0, css_rewrite_1.rewriteCssText)({
+                        cssText: decoded,
+                        cssUrl: resource.request.url,
+                        resolveUrl: resolve
+                    });
+                    if (rewritten !== decoded) {
+                        const encoded = new TextEncoder().encode(rewritten);
+                        contentRef = await input.contentStore.put({ kind: "buffer", data: encoded }, {
+                            url: resource.request.url,
+                            mimeType: resource.mimeType,
+                            sizeHint: encoded.byteLength
+                        });
+                        size = encoded.byteLength;
+                    }
+                }
+            }
+            if (resource.request.resourceType === "script") {
+                const stream = await input.contentStore.open(contentRef);
+                const bytes = await streamToUint8Array(stream);
+                const decoded = (0, utils_1.decodeUtf8)(bytes);
+                if (decoded !== null) {
+                    const rewritten = await (0, rewrite_links_1.rewriteJsText)(decoded, resolve, resource.request.url);
+                    if (rewritten !== decoded) {
+                        const encoded = new TextEncoder().encode(rewritten);
+                        contentRef = await input.contentStore.put({ kind: "buffer", data: encoded }, {
+                            url: resource.request.url,
+                            mimeType: resource.mimeType,
+                            sizeHint: encoded.byteLength
+                        });
+                        size = encoded.byteLength;
+                    }
+                }
+            }
+            const path = urlToPath.get(resource.request.url) ??
+                resolver.resolve({
+                    url: resource.request.url,
+                    resourceType: resource.request.resourceType,
+                    mimeType: resource.mimeType,
+                    suggestedFilename: undefined,
+                    isCrossOrigin: (0, path_resolver_1.resolveCrossOrigin)(resource.request.url, group.url),
+                    entryUrl: group.url
+                });
+            files.push({
+                path,
+                mimeType: resource.mimeType,
+                size,
+                source: contentRef,
+                originalUrl: resource.request.url,
+                resourceType: resource.request.resourceType,
+                headers: resource.response.headers
+            });
+        }
+        const apiSnapshot = buildApiSnapshot(group.url, input.createdAt, group.apiEntries);
+        const apiBytes = new TextEncoder().encode(JSON.stringify(apiSnapshot, null, 2));
+        const apiRef = await input.contentStore.put({ kind: "buffer", data: apiBytes }, { url: apiPath, mimeType: "application/json", sizeHint: apiBytes.byteLength });
+        files.push({
+            path: apiPath,
+            mimeType: "application/json",
+            size: apiBytes.byteLength,
+            source: apiRef,
+            originalUrl: apiPath
+        });
+    }
+    const totalBytes = files.reduce((sum, file) => sum + (file.size ?? 0), 0);
+    const totalFiles = files.length;
+    const snapshotUrl = input.entryUrl || groups[0]?.url || "";
+    return (0, snapshot_1.createPageSnapshot)({
+        version: "1.0",
+        createdAt: input.createdAt,
+        url: snapshotUrl,
+        title,
+        entry: entryPath || "/index.html",
+        files,
+        meta: {
+            totalBytes,
+            totalFiles,
+            warnings: warnings.length ? warnings : undefined
+        },
+        content: {
+            open: (ref) => input.contentStore.open(ref),
+            dispose: async () => {
+                await input.contentStore.dispose?.();
+            }
+        }
+    });
+};
+exports.buildSnapshot = buildSnapshot;

package/dist/snapshot.d.ts

@@ -0,0 +1,2 @@
+import type { PageSnapshot } from "./types";
+export declare const createPageSnapshot: (data: Omit<PageSnapshot, "toDirectory" | "toZip">) => PageSnapshot;

package/dist/snapshot.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createPageSnapshot = void 0;
+const writers_1 = require("./writers");
+const createPageSnapshot = (data) => {
+    return {
+        ...data,
+        toDirectory: (outDir, options) => (0, writers_1.writeToFS)(data, outDir, options),
+        toZip: (options) => (0, writers_1.toZip)(data, options)
+    };
+};
+exports.createPageSnapshot = createPageSnapshot;

package/dist/types.d.ts

@@ -1,20 +1,188 @@
-export type FetchRecord = {
-    kind: "fetch" | "xhr";
+export type ResourceType = "document" | "stylesheet" | "script" | "image" | "font" | "media" | "xhr" | "fetch" | "other" | (string & {});
+export type BodySource = {
+    kind: "buffer";
+    data: Uint8Array;
+} | {
+    kind: "stream";
+    stream: ReadableStream<Uint8Array>;
+} | {
+    kind: "late";
+    read: () => Promise<Uint8Array>;
+};
+export interface NetworkRequestEvent {
+    type: "request";
+    requestId: string;
     url: string;
     method: string;
-    requestBody?: string;
-    status?: number;
+    headers: Record<string, string>;
+    frameId?: string;
+    resourceType?: ResourceType;
+    initiator?: {
+        type?: string;
+        url?: string;
+    };
+    timestamp: number;
+}
+export interface NetworkResponseEvent {
+    type: "response";
+    requestId: string;
+    url: string;
+    status: number;
     statusText?: string;
-    responseHeaders?: Record<string, string>;
-    responseBody?: string;
-    error?: string;
+    headers: Record<string, string>;
+    mimeType?: string;
+    fromDiskCache?: boolean;
+    fromServiceWorker?: boolean;
+    timestamp: number;
+    body?: BodySource;
+}
+export interface NetworkRequestFailedEvent {
+    type: "failed";
+    requestId: string;
+    url: string;
+    errorText: string;
     timestamp: number;
+}
+export type NetworkEvent = NetworkRequestEvent | NetworkResponseEvent | NetworkRequestFailedEvent;
+export interface NetworkEventHandlers {
+    onEvent(event: NetworkEvent): void;
+    onError?(error: Error): void;
+    onLog?(msg: string, meta?: unknown): void;
+}
+export interface InterceptorCapabilities {
+    canGetResponseBody: boolean;
+    canStreamResponseBody: boolean;
+    canGetRequestBody: boolean;
+    providesResourceType: boolean;
+}
+export type InterceptTarget = {
+    kind: "url";
+    url: string;
+} | {
+    kind: "puppeteer-page";
+    page: unknown;
+} | {
+    kind: "cdp-tab";
+    tabId: number;
+} | {
+    kind: "cdp-session";
+    session: unknown;
+};
+export type InterceptOptions = Record<string, unknown>;
+export type NavigateOptions = Record<string, unknown>;
+export interface InterceptSession {
+    navigate?(url: string, options?: NavigateOptions): Promise<void>;
+    stop(): Promise<void>;
+}
+export interface NetworkInterceptorAdapter {
+    readonly name: string;
+    readonly capabilities: InterceptorCapabilities;
+    start(target: InterceptTarget, handlers: NetworkEventHandlers, options?: InterceptOptions): Promise<InterceptSession>;
+}
+export interface PathResolver {
+    resolve(input: {
+        url: string;
+        resourceType?: ResourceType;
+        mimeType?: string;
+        suggestedFilename?: string;
+        isCrossOrigin: boolean;
+        entryUrl: string;
+    }): string;
+}
+export interface ResourceFilter {
+    shouldSave(req: NetworkRequestEvent, res?: NetworkResponseEvent): boolean;
+}
+export type ContentRef = {
+    kind: "memory";
+    data: Uint8Array;
+} | {
+    kind: "store-ref";
+    id: string;
 };
-export type NetworkRecord = {
+export interface ContentStore {
+    name: string;
+    put(body: BodySource, meta: {
+        url: string;
+        mimeType?: string;
+        sizeHint?: number;
+    }): Promise<ContentRef>;
+    open(ref: ContentRef): Promise<ReadableStream<Uint8Array>>;
+    dispose?(): Promise<void>;
+}
+export interface ContentStoreHandle {
+    open(ref: ContentRef): Promise<ReadableStream<Uint8Array>>;
+    dispose?(): Promise<void>;
+}
+export interface CompletionContext {
+    now(): number;
+    getStats(): {
+        inflightRequests: number;
+        lastNetworkTs: number;
+        totalRequests: number;
+    };
+}
+export interface CompletionStrategy {
+    wait(ctx: CompletionContext): Promise<void>;
+}
+export interface PagePocketOptions {
+}
+export interface CaptureOptions {
+    interceptor: NetworkInterceptorAdapter;
+    completion?: CompletionStrategy | CompletionStrategy[];
+    filter?: ResourceFilter;
+    pathResolver?: PathResolver;
+    contentStore?: ContentStore;
+    rewriteEntry?: boolean;
+    rewriteCSS?: boolean;
+    limits?: {
+        maxTotalBytes?: number;
+        maxSingleResourceBytes?: number;
+        maxResources?: number;
+    };
+}
+export interface SnapshotFile {
+    path: string;
+    mimeType?: string;
+    size?: number;
+    source: ContentRef;
+    originalUrl?: string;
+    resourceType?: ResourceType;
+    headers?: Record<string, string>;
+}
+export interface PageSnapshot {
+    version: "1.0";
+    createdAt: number;
+    url: string;
+    title?: string;
+    entry: string;
+    files: SnapshotFile[];
+    meta?: {
+        totalBytes?: number;
+        totalFiles?: number;
+        warnings?: string[];
+    };
+    content: ContentStoreHandle;
+    toDirectory(outDir: string, options?: WriteFSOptions): Promise<WriteResult>;
+    toZip(options?: ZipOptions): Promise<Uint8Array | Blob>;
+}
+export interface WriteFSOptions {
+    clearCache?: boolean;
+}
+export interface WriteResult {
+    filesWritten: number;
+    totalBytes: number;
+}
+export interface ZipOptions {
+    asBlob?: boolean;
+    clearCache?: boolean;
+}
+export interface ApiRecord {
     url: string;
     method: string;
     requestHeaders?: Record<string, string>;
     requestBody?: string;
+    requestBodyBase64?: string;
+    requestEncoding?: "text" | "base64";
     status?: number;
     statusText?: string;
     responseHeaders?: Record<string, string>;
@@ -23,35 +191,10 @@ export type NetworkRecord = {
     responseEncoding?: "text" | "base64";
     error?: string;
     timestamp: number;
-};
-export type CapturedResponseRecord = {
-    status: number;
-    statusText: string;
-    headers: Record<string, string>;
-    body: string;
-    bodyEncoding: "text" | "base64";
-};
-export type CapturedNetworkRecord = {
-    url: string;
-    source?: string;
-    method: string;
-    timestamp: number;
-    response?: CapturedResponseRecord;
-    error?: string;
-};
-export type SnapshotData = {
+}
+export interface ApiSnapshot {
+    version: "1.0";
     url: string;
-    title: string;
-    capturedAt: string;
-    fetchXhrRecords: FetchRecord[];
-    networkRecords: CapturedNetworkRecord[];
-    resources: Array<{
-        url: string;
-        localPath: string;
-        contentType?: string | null;
-        size?: number;
-    }>;
-};
-export interface NetworkInterceptorAdapter {
-    run(url: string): Promise<SnapshotData>;
+    createdAt: number;
+    records: ApiRecord[];
 }

package/dist/utils.d.ts

@@ -0,0 +1,19 @@
+import type { BodySource } from "./types";
+export declare const sleep: (ms: number) => Promise<unknown>;
+export declare const hashString: (value: string) => string;
+export declare const stripLeadingSlash: (value: string) => string;
+export declare const ensureLeadingSlash: (value: string) => string;
+export declare const sanitizePosixPath: (value: string) => string;
+export declare const bytesToBase64: (bytes: Uint8Array) => string;
+export declare const decodeUtf8: (bytes: Uint8Array) => string | null;
+export declare const isUtf8Text: (bytes: Uint8Array, mimeType?: string) => boolean;
+export declare const toUint8Array: (body: BodySource) => Promise<Uint8Array>;
+export declare const bodyToTextOrBase64: (bytes: Uint8Array, mimeType?: string) => {
+    encoding: "text";
+    text: string;
+    base64?: undefined;
+} | {
+    encoding: "base64";
+    base64: string;
+    text?: undefined;
+};

package/dist/utils.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.bodyToTextOrBase64 = exports.toUint8Array = exports.isUtf8Text = exports.decodeUtf8 = exports.bytesToBase64 = exports.sanitizePosixPath = exports.ensureLeadingSlash = exports.stripLeadingSlash = exports.hashString = exports.sleep = void 0;
+const content_type_1 = require("./content-type");
+const FNV_OFFSET = 0x811c9dc5;
+const FNV_PRIME = 0x01000193;
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+exports.sleep = sleep;
+const hashString = (value) => {
+    let hash = FNV_OFFSET;
+    for (let i = 0; i < value.length; i += 1) {
+        hash ^= value.charCodeAt(i);
+        hash = (hash * FNV_PRIME) >>> 0;
+    }
+    return hash.toString(16).padStart(8, "0");
+};
+exports.hashString = hashString;
+const stripLeadingSlash = (value) => value.replace(/^\/+/, "");
+exports.stripLeadingSlash = stripLeadingSlash;
+const ensureLeadingSlash = (value) => value.startsWith("/") ? value : `/${value}`;
+exports.ensureLeadingSlash = ensureLeadingSlash;
+const sanitizePosixPath = (value) => {
+    const parts = value.split("/").filter(Boolean);
+    const clean = [];
+    for (const part of parts) {
+        if (part === ".")
+            continue;
+        if (part === "..")
+            continue;
+        clean.push(part);
+    }
+    return clean.join("/");
+};
+exports.sanitizePosixPath = sanitizePosixPath;
+const getGlobalBuffer = () => {
+    return globalThis.Buffer;
+};
+const bytesToBase64 = (bytes) => {
+    const BufferCtor = getGlobalBuffer();
+    if (BufferCtor) {
+        return BufferCtor.from(bytes).toString("base64");
+    }
+    let binary = "";
+    const chunkSize = 0x8000;
+    for (let i = 0; i < bytes.length; i += chunkSize) {
+        const chunk = bytes.subarray(i, i + chunkSize);
+        binary += String.fromCharCode(...chunk);
+    }
+    return btoa(binary);
+};
+exports.bytesToBase64 = bytesToBase64;
+const decodeUtf8 = (bytes) => {
+    try {
+        const decoder = new TextDecoder("utf-8", { fatal: true });
+        return decoder.decode(bytes);
+    }
+    catch {
+        return null;
+    }
+};
+exports.decodeUtf8 = decodeUtf8;
+const isUtf8Text = (bytes, mimeType) => {
+    const decoded = (0, exports.decodeUtf8)(bytes);
+    if (decoded === null) {
+        return false;
+    }
+    if (mimeType) {
+        return (0, content_type_1.isTextResponse)(mimeType);
+    }
+    return true;
+};
+exports.isUtf8Text = isUtf8Text;
+const toUint8Array = async (body) => {
+    if (body.kind === "buffer") {
+        return body.data;
+    }
+    if (body.kind === "late") {
+        return body.read();
+    }
+    const reader = body.stream.getReader();
+    const chunks = [];
+    let total = 0;
+    while (true) {
+        const result = await reader.read();
+        if (result.done)
+            break;
+        const value = result.value;
+        if (value) {
+            chunks.push(value);
+            total += value.byteLength;
+        }
+    }
+    const output = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        output.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return output;
+};
+exports.toUint8Array = toUint8Array;
+const bodyToTextOrBase64 = (bytes, mimeType) => {
+    if ((0, exports.isUtf8Text)(bytes, mimeType)) {
+        const text = (0, exports.decodeUtf8)(bytes) ?? "";
+        return { encoding: "text", text };
+    }
+    return { encoding: "base64", base64: (0, exports.bytesToBase64)(bytes) };
+};
+exports.bodyToTextOrBase64 = bodyToTextOrBase64;

package/dist/writers.d.ts

@@ -0,0 +1,3 @@
+import type { PageSnapshot, WriteFSOptions, WriteResult, ZipOptions } from "./types";
+export declare const writeToFS: (snapshot: PageSnapshot, outDir: string, options?: WriteFSOptions) => Promise<WriteResult>;
+export declare const toZip: (snapshot: PageSnapshot, options?: ZipOptions) => Promise<Uint8Array | Blob>;