@pagepocket/lib 0.5.1 → 0.6.1

package/README.md

@@ -17,7 +17,7 @@ import { PagePocket } from "@pagepocket/lib";
 import { CdpAdapter } from "@pagepocket/cdp-adapter";
 
 const interceptor = new CdpAdapter();
-const snapshot = await PagePocket.fromURL("https://example.com").capture({
+const snapshot = await PagePocket.fromTarget({ kind: "cdp-tab", tabId: 123 }).capture({
   interceptor
 });
 

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 export { PagePocket } from "./pagepocket";
-export type { ApiRecord, ApiSnapshot, CaptureOptions, CompletionContext, CompletionStrategy, ContentRef, ContentStore, ContentStoreHandle, InterceptOptions, InterceptSession, InterceptTarget, NetworkEvent, NetworkEventHandlers, NetworkInterceptorAdapter, NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent, PagePocketOptions, PageSnapshot, PathResolver, ResourceFilter, ResourceType, SnapshotFile, WriteFSOptions, WriteResult, ZipOptions } from "./types";
+export type { ApiRecord, ApiSnapshot, CaptureOptions, CompletionContext, CompletionStrategy, ContentRef, ContentStore, ContentStoreHandle, NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent, PagePocketOptions, PageSnapshot, PathResolver, ResourceFilter, SnapshotFile, WriteFSOptions, WriteResult, ZipOptions } from "./types";
 export { HybridContentStore } from "./content-store";
 export { createDefaultPathResolver, withPrefixPathResolver } from "./path-resolver";
 export { createDefaultResourceFilter } from "./resource-filter";

package/dist/network-store.d.ts

@@ -1,4 +1,5 @@
-import type { ApiRecord, ContentRef, ContentStore, NetworkEvent, NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent, ResourceFilter } from "./types";
+import type { NetworkEvent, NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent } from "@pagepocket/interceptor";
+import type { ApiRecord, ContentRef, ContentStore, ResourceFilter } from "./types";
 type Limits = {
     maxTotalBytes?: number;
     maxSingleResourceBytes?: number;

package/dist/pagepocket.d.ts

@@ -1,4 +1,5 @@
-import type { CaptureOptions, InterceptTarget, PagePocketOptions, PageSnapshot } from "./types";
+import type { InterceptTarget } from "@pagepocket/interceptor";
+import type { CaptureOptions, PagePocketOptions, PageSnapshot } from "./types";
 export declare class PagePocket {
     private target;
     private options;

package/dist/pagepocket.js

@@ -1,12 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PagePocket = void 0;
-const content_store_1 = require("./content-store");
 const completion_1 = require("./completion");
+const content_store_1 = require("./content-store");
+const network_store_1 = require("./network-store");
 const path_resolver_1 = require("./path-resolver");
 const resource_filter_1 = require("./resource-filter");
 const snapshot_builder_1 = require("./snapshot-builder");
-const network_store_1 = require("./network-store");
 class PagePocket {
     constructor(target, options) {
         this.target = target;
@@ -29,9 +29,7 @@ class PagePocket {
         const rewriteCSS = options?.rewriteCSS ?? true;
         const limits = options?.limits;
         const completionStrategies = (0, completion_1.normalizeCompletion)(options?.completion);
-        const completion = completionStrategies.length > 0
-            ? completionStrategies
-            : [(0, completion_1.networkIdle)(1000), (0, completion_1.timeout)(5000)];
+        const completion = completionStrategies.length > 0 ? completionStrategies : [(0, completion_1.networkIdle)(1000), (0, completion_1.timeout)(5000)];
         const store = new network_store_1.NetworkStore({
             contentStore,
             filter,

package/dist/path-resolver.d.ts

@@ -1,4 +1,5 @@
-import type { PathResolver, ResourceType } from "./types";
+import type { ResourceType } from "@pagepocket/interceptor";
+import type { PathResolver } from "./types";
 export declare const createDefaultPathResolver: () => PathResolver;
 export declare const resolveCrossOrigin: (url: string, entryUrl: string) => boolean;
 export declare const withPrefixPathResolver: (resolver: PathResolver, prefix: string) => PathResolver;

package/dist/path-resolver.js

@@ -43,9 +43,7 @@ const createDefaultPathResolver = () => {
             const pathname = normalizePathname(parsed?.pathname || "/");
             const queryHash = `${parsed?.search || ""}${parsed?.hash || ""}`;
             const suffix = queryHash ? `__ppq_${(0, utils_1.hashString)(queryHash)}` : "";
-            const basePath = input.isCrossOrigin
-                ? `/external_resources${pathname}`
-                : `${pathname}`;
+            const basePath = input.isCrossOrigin ? `/external_resources${pathname}` : `${pathname}`;
             let resolvedPath = suffix ? withSuffix(basePath, suffix) : basePath;
             const collisionKey = resolvedPath;
             const existingUrl = usedPaths.get(collisionKey);

package/dist/resource-filter.js

@@ -1,14 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createDefaultResourceFilter = void 0;
-const DEFAULT_ALLOWED = new Set([
-    "document",
-    "stylesheet",
-    "script",
-    "image",
-    "font",
-    "media"
-]);
+const DEFAULT_ALLOWED = new Set(["document", "stylesheet", "script", "image", "font", "media"]);
 const isSkippableUrl = (url) => url.startsWith("data:") ||
     url.startsWith("blob:") ||
     url.startsWith("mailto:") ||

package/dist/snapshot-builder.d.ts

@@ -1,5 +1,5 @@
-import type { ContentStore, PageSnapshot, PathResolver } from "./types";
 import type { ApiEntry, StoredResource } from "./network-store";
+import type { ContentStore, PageSnapshot, PathResolver } from "./types";
 type BuildOptions = {
     entryUrl: string;
     createdAt: number;

package/dist/snapshot-builder.js

@@ -1,10 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.buildSnapshot = void 0;
-const path_resolver_1 = require("./path-resolver");
-const snapshot_1 = require("./snapshot");
 const css_rewrite_1 = require("./css-rewrite");
+const path_resolver_1 = require("./path-resolver");
 const rewrite_links_1 = require("./rewrite-links");
+const snapshot_1 = require("./snapshot");
 const utils_1 = require("./utils");
 const streamToUint8Array = async (stream) => {
     const reader = stream.getReader();
@@ -80,7 +80,7 @@ const groupResources = (input) => {
         });
     }
     const primaryGroup = primaryDoc
-        ? groups.get(primaryDoc.request.frameId ?? primaryDoc.request.requestId) ?? null
+        ? (groups.get(primaryDoc.request.frameId ?? primaryDoc.request.requestId) ?? null)
         : null;
     const groupByUrl = new Map();
     for (const group of groups.values()) {
@@ -92,7 +92,9 @@ const groupResources = (input) => {
         }
         const frameId = resource.request.frameId;
         const byFrame = frameId ? groups.get(frameId) : undefined;
-        const byInitiator = resource.request.initiator?.url ? groupByUrl.get(resource.request.initiator.url) : undefined;
+        const byInitiator = resource.request.initiator?.url
+            ? groupByUrl.get(resource.request.initiator.url)
+            : undefined;
         const target = byFrame ?? byInitiator ?? primaryGroup ?? Array.from(groups.values())[0];
         if (target) {
             target.resources.push(resource);
@@ -101,7 +103,9 @@ const groupResources = (input) => {
     for (const entry of input.apiEntries) {
         const frameId = entry.request.frameId;
         const byFrame = frameId ? groups.get(frameId) : undefined;
-        const byInitiator = entry.request.initiator?.url ? groupByUrl.get(entry.request.initiator.url) : undefined;
+        const byInitiator = entry.request.initiator?.url
+            ? groupByUrl.get(entry.request.initiator.url)
+            : undefined;
         const target = byFrame ?? byInitiator ?? primaryGroup ?? Array.from(groups.values())[0];
         if (target) {
             target.apiEntries.push(entry);

package/dist/types.d.ts

@@ -1,84 +1,5 @@
-export type ResourceType = "document" | "stylesheet" | "script" | "image" | "font" | "media" | "xhr" | "fetch" | "other" | (string & {});
-export type BodySource = {
-    kind: "buffer";
-    data: Uint8Array;
-} | {
-    kind: "stream";
-    stream: ReadableStream<Uint8Array>;
-} | {
-    kind: "late";
-    read: () => Promise<Uint8Array>;
-};
-export interface NetworkRequestEvent {
-    type: "request";
-    requestId: string;
-    url: string;
-    method: string;
-    headers: Record<string, string>;
-    frameId?: string;
-    resourceType?: ResourceType;
-    initiator?: {
-        type?: string;
-        url?: string;
-    };
-    timestamp: number;
-}
-export interface NetworkResponseEvent {
-    type: "response";
-    requestId: string;
-    url: string;
-    status: number;
-    statusText?: string;
-    headers: Record<string, string>;
-    mimeType?: string;
-    fromDiskCache?: boolean;
-    fromServiceWorker?: boolean;
-    timestamp: number;
-    body?: BodySource;
-}
-export interface NetworkRequestFailedEvent {
-    type: "failed";
-    requestId: string;
-    url: string;
-    errorText: string;
-    timestamp: number;
-}
-export type NetworkEvent = NetworkRequestEvent | NetworkResponseEvent | NetworkRequestFailedEvent;
-export interface NetworkEventHandlers {
-    onEvent(event: NetworkEvent): void;
-    onError?(error: Error): void;
-    onLog?(msg: string, meta?: unknown): void;
-}
-export interface InterceptorCapabilities {
-    canGetResponseBody: boolean;
-    canStreamResponseBody: boolean;
-    canGetRequestBody: boolean;
-    providesResourceType: boolean;
-}
-export type InterceptTarget = {
-    kind: "url";
-    url: string;
-} | {
-    kind: "puppeteer-page";
-    page: unknown;
-} | {
-    kind: "cdp-tab";
-    tabId: number;
-} | {
-    kind: "cdp-session";
-    session: unknown;
-};
-export type InterceptOptions = Record<string, unknown>;
-export type NavigateOptions = Record<string, unknown>;
-export interface InterceptSession {
-    navigate?(url: string, options?: NavigateOptions): Promise<void>;
-    stop(): Promise<void>;
-}
-export interface NetworkInterceptorAdapter {
-    readonly name: string;
-    readonly capabilities: InterceptorCapabilities;
-    start(target: InterceptTarget, handlers: NetworkEventHandlers, options?: InterceptOptions): Promise<InterceptSession>;
-}
+import type { BodySource, NetworkInterceptorAdapter, NetworkRequestEvent, NetworkResponseEvent, ResourceType } from "@pagepocket/interceptor";
+export type { BodySource, InterceptOptions, InterceptSession, InterceptTarget, InterceptorActions, InterceptorCapabilities, NavigateOptions, NetworkEvent, NetworkEventHandlers, NetworkInterceptorAdapter, NetworkRequestEvent, NetworkRequestFailedEvent, NetworkResponseEvent, ResourceType, TriggerAction } from "@pagepocket/interceptor";
 export interface PathResolver {
     resolve(input: {
         url: string;

package/dist/utils.js

@@ -17,7 +17,7 @@ const hashString = (value) => {
 exports.hashString = hashString;
 const stripLeadingSlash = (value) => value.replace(/^\/+/, "");
 exports.stripLeadingSlash = stripLeadingSlash;
-const ensureLeadingSlash = (value) => value.startsWith("/") ? value : `/${value}`;
+const ensureLeadingSlash = (value) => (value.startsWith("/") ? value : `/${value}`);
 exports.ensureLeadingSlash = ensureLeadingSlash;
 const sanitizePosixPath = (value) => {
     const parts = value.split("/").filter(Boolean);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pagepocket/lib",
-  "version": "0.5.1",
+  "version": "0.6.1",
   "description": "Library for rewriting HTML snapshots and inlining local resources.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -12,10 +12,13 @@
   "license": "ISC",
   "dependencies": {
     "cheerio": "^1.0.0-rc.12",
-    "@pagepocket/uni-fs": "0.5.1"
+    "@pagepocket/interceptor": "0.6.1",
+    "@pagepocket/uni-fs": "0.6.1"
   },
   "devDependencies": {
+    "@playwright/test": "^1.50.1",
     "@types/node": "^20.11.30",
+    "esbuild": "^0.25.1",
     "prettier": "^3.7.4",
     "tsx": "^4.19.3",
     "typescript": "^5.4.5"
@@ -24,6 +27,7 @@
     "build": "tsc",
     "format": "prettier --write .",
     "format:check": "prettier --check .",
-    "test": "tsx --test specs/**/*.test.ts"
+    "test": "tsx --test specs/*.spec.ts",
+    "test:e2e": "playwright test -c e2e/playwright.config.ts"
   }
 }

package/dist/download-resources.d.ts

@@ -1,25 +0,0 @@
-import { type ResourceReference, type SrcsetReference } from "./resources";
-import type { SnapshotData } from "./types";
-export type DownloadedResource = {
-    url: string;
-    filename: string;
-    extension: string;
-    localPath: string;
-    contentType?: string | null;
-    size?: number;
-};
-type DownloadResourcesInput = {
-    baseUrl: string;
-    assetsDirName: string;
-    resourceUrls: ResourceReference[];
-    srcsetItems: SrcsetReference[];
-    referer?: string;
-};
-type DownloadResourcesResult = {
-    resourceMap: Map<string, DownloadedResource>;
-    resourceMeta: SnapshotData["resources"];
-    downloadedCount: number;
-    failedCount: number;
-};
-export declare const downloadResources: (input: DownloadResourcesInput) => Promise<DownloadResourcesResult>;
-export {};

package/dist/download-resources.js

@@ -1,160 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.downloadResources = void 0;
-const content_type_1 = require("./content-type");
-const resources_1 = require("./resources");
-const FNV_OFFSET = 0x811c9dc5;
-const FNV_PRIME = 0x01000193;
-const shouldSkipUrl = (value) => {
-    const trimmed = value.trim();
-    return (!trimmed ||
-        trimmed.startsWith("data:") ||
-        trimmed.startsWith("blob:") ||
-        trimmed.startsWith("mailto:") ||
-        trimmed.startsWith("tel:") ||
-        trimmed.startsWith("javascript:") ||
-        trimmed.startsWith("#"));
-};
-const hashUrl = (value) => {
-    let hash = FNV_OFFSET;
-    for (let i = 0; i < value.length; i += 1) {
-        hash ^= value.charCodeAt(i);
-        hash = (hash * FNV_PRIME) >>> 0;
-    }
-    return hash.toString(16).padStart(8, "0");
-};
-const extensionFromUrl = (url) => {
-    try {
-        const pathname = new URL(url).pathname;
-        const lastDot = pathname.lastIndexOf(".");
-        if (lastDot === -1) {
-            return "";
-        }
-        return pathname.slice(lastDot);
-    }
-    catch {
-        return "";
-    }
-};
-const normalizeExtension = (extension) => {
-    if (!extension) {
-        return "";
-    }
-    return extension.startsWith(".") ? extension.slice(1) : extension;
-};
-const buildLocalPath = (assetsDirName, filename, extension) => {
-    const normalizedExtension = normalizeExtension(extension);
-    if (!normalizedExtension) {
-        return `${assetsDirName}/${filename}`;
-    }
-    return `${assetsDirName}/${filename}.${normalizedExtension}`;
-};
-const collectSrcsetUrls = (items, baseUrl) => {
-    const urls = [];
-    for (const item of items) {
-        const parts = item.value.split(",").map((part) => part.trim());
-        for (const part of parts) {
-            const [rawUrl] = part.split(/\s+/, 2);
-            if (!rawUrl) {
-                continue;
-            }
-            const absolute = (0, resources_1.toAbsoluteUrl)(baseUrl, rawUrl);
-            urls.push(absolute);
-        }
-    }
-    return urls;
-};
-const downloadResources = async (input) => {
-    const { write, exists } = await Promise.resolve().then(() => __importStar(require("@pagepocket/uni-fs")));
-    const resourceMap = new Map();
-    const resourceMeta = [];
-    let downloadedCount = 0;
-    let failedCount = 0;
-    const srcsetUrls = collectSrcsetUrls(input.srcsetItems, input.baseUrl);
-    const candidateUrls = [...input.resourceUrls.map((resource) => resource.url), ...srcsetUrls];
-    for (const candidate of candidateUrls) {
-        if (shouldSkipUrl(candidate)) {
-            continue;
-        }
-        const url = (0, resources_1.toAbsoluteUrl)(input.baseUrl, candidate);
-        if (!url || resourceMap.has(url)) {
-            continue;
-        }
-        try {
-            const headers = {};
-            if (input.referer) {
-                headers.referer = input.referer;
-            }
-            const response = await fetch(url, { redirect: "follow", headers });
-            const contentType = response.headers.get("content-type");
-            const buffer = new Uint8Array(await response.arrayBuffer());
-            const extFromUrl = extensionFromUrl(url);
-            const extFromType = (0, content_type_1.extensionFromContentType)(contentType);
-            const extension = normalizeExtension(extFromUrl || extFromType);
-            const filename = hashUrl(url);
-            const localPath = buildLocalPath(input.assetsDirName, filename, extension);
-            if (!(await exists(input.assetsDirName + "/" + filename, extension))) {
-                await write(input.assetsDirName + "/" + filename, extension, buffer);
-            }
-            const entry = {
-                url,
-                filename,
-                extension,
-                localPath,
-                contentType,
-                size: buffer.byteLength
-            };
-            resourceMap.set(url, entry);
-            resourceMeta.push({
-                url,
-                localPath,
-                contentType,
-                size: buffer.byteLength
-            });
-            downloadedCount += 1;
-        }
-        catch {
-            failedCount += 1;
-        }
-    }
-    return {
-        resourceMap,
-        resourceMeta,
-        downloadedCount,
-        failedCount
-    };
-};
-exports.downloadResources = downloadResources;

package/dist/network-records.d.ts

@@ -1,4 +0,0 @@
-import type { CapturedNetworkRecord, NetworkRecord } from "./types";
-export declare const toDataUrlFromRecord: (record: NetworkRecord) => string | null;
-export declare const findFaviconDataUrl: (records: NetworkRecord[]) => string | null;
-export declare const mapCapturedNetworkRecords: (records: CapturedNetworkRecord[] | undefined) => NetworkRecord[];

package/dist/network-records.js

@@ -1,83 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.mapCapturedNetworkRecords = exports.findFaviconDataUrl = exports.toDataUrlFromRecord = void 0;
-const getHeaderValue = (headers, name) => {
-    for (const key in headers) {
-        if (key.toLowerCase() === name.toLowerCase()) {
-            return headers[key];
-        }
-    }
-    return undefined;
-};
-const toBase64 = (value) => {
-    const bufferConstructor = globalThis.Buffer;
-    if (bufferConstructor) {
-        return bufferConstructor.from(value, "utf-8").toString("base64");
-    }
-    if (typeof btoa === "function") {
-        return btoa(value);
-    }
-    return "";
-};
-const toDataUrlFromRecord = (record) => {
-    if (!record)
-        return null;
-    const headers = record.responseHeaders || {};
-    const contentType = getHeaderValue(headers, "content-type") || "application/octet-stream";
-    if (record.responseEncoding === "base64" && record.responseBodyBase64) {
-        return `data:${contentType};base64,${record.responseBodyBase64}`;
-    }
-    if (record.responseBody) {
-        const encoded = toBase64(record.responseBody);
-        if (!encoded) {
-            return null;
-        }
-        return `data:${contentType};base64,${encoded}`;
-    }
-    return null;
-};
-exports.toDataUrlFromRecord = toDataUrlFromRecord;
-const findFaviconDataUrl = (records) => {
-    for (const record of records) {
-        if (!record || !record.url)
-            continue;
-        const headers = record.responseHeaders || {};
-        const contentType = (getHeaderValue(headers, "content-type") || "").toLowerCase();
-        const pathname = (() => {
-            try {
-                return new URL(record.url).pathname;
-            }
-            catch {
-                return record.url;
-            }
-        })();
-        const looksLikeFavicon = contentType.includes("icon") || /favicon(\.[a-z0-9]+)?$/i.test(pathname || "");
-        if (!looksLikeFavicon)
-            continue;
-        const dataUrl = (0, exports.toDataUrlFromRecord)(record);
-        if (dataUrl)
-            return dataUrl;
-    }
-    return null;
-};
-exports.findFaviconDataUrl = findFaviconDataUrl;
-const mapCapturedNetworkRecords = (records) => {
-    if (!records)
-        return [];
-    return records.map((record) => {
-        const response = record.response;
-        return {
-            url: record.url,
-            method: record.method,
-            status: response?.status,
-            statusText: response?.statusText,
-            responseHeaders: response?.headers,
-            responseBody: response?.bodyEncoding === "text" ? response.body : undefined,
-            responseBodyBase64: response?.bodyEncoding === "base64" ? response.body : undefined,
-            responseEncoding: response?.bodyEncoding,
-            error: record.error,
-            timestamp: record.timestamp
-        };
-    });
-};
-exports.mapCapturedNetworkRecords = mapCapturedNetworkRecords;