@pagepocket/lib 0.4.2 → 0.5.0

package/dist/network-store.js

@@ -0,0 +1,159 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NetworkStore = void 0;
+const utils_1 = require("./utils");
+const isApiResource = (request) => {
+    const type = request?.resourceType;
+    return type === "fetch" || type === "xhr";
+};
+const getHeaderValue = (headers, name) => {
+    const target = name.toLowerCase();
+    for (const key in headers) {
+        if (key.toLowerCase() === target) {
+            return headers[key];
+        }
+    }
+    return undefined;
+};
+const responseMimeType = (response) => response.mimeType || getHeaderValue(response.headers || {}, "content-type");
+class NetworkStore {
+    constructor(options) {
+        this.requests = new Map();
+        this.storedResources = [];
+        this.apiEntries = [];
+        this.apiRecordIds = new Set();
+        this.warnings = [];
+        this.totalBytes = 0;
+        this.contentStore = options.contentStore;
+        this.filter = options.filter;
+        this.limits = options.limits ?? {};
+    }
+    getWarnings() {
+        return this.warnings.slice();
+    }
+    getTotals() {
+        return {
+            totalBytes: this.totalBytes,
+            totalFiles: this.storedResources.length
+        };
+    }
+    getResources() {
+        return this.storedResources.slice();
+    }
+    getApiRecords() {
+        return this.apiEntries.map((entry) => entry.record);
+    }
+    getApiEntries() {
+        return this.apiEntries.slice();
+    }
+    getRequestRecords() {
+        return new Map(this.requests);
+    }
+    async handleEvent(event) {
+        if (event.type === "request") {
+            this.requests.set(event.requestId, { request: event });
+            return;
+        }
+        const record = this.requests.get(event.requestId);
+        if (!record) {
+            return;
+        }
+        if (event.type === "failed") {
+            record.failed = event;
+            if (isApiResource(record.request)) {
+                this.recordApiFailure(record.request, event);
+            }
+            return;
+        }
+        record.response = event;
+        const request = record.request;
+        const response = event;
+        const isApi = isApiResource(request);
+        const shouldSave = this.filter.shouldSave(request, response);
+        let bodyBytes = null;
+        if (response.body) {
+            bodyBytes = await (0, utils_1.toUint8Array)(response.body);
+        }
+        if (isApi) {
+            await this.recordApiResponse(request, response, bodyBytes);
+        }
+        if (!shouldSave) {
+            return;
+        }
+        if (!bodyBytes) {
+            this.warnings.push(`Missing body for ${request.url}`);
+            return;
+        }
+        if (this.limits.maxSingleResourceBytes &&
+            bodyBytes.byteLength > this.limits.maxSingleResourceBytes) {
+            this.warnings.push(`Resource too large: ${request.url}`);
+            return;
+        }
+        if (this.limits.maxResources && this.storedResources.length >= this.limits.maxResources) {
+            this.warnings.push(`Resource limit reached at ${request.url}`);
+            return;
+        }
+        if (this.limits.maxTotalBytes &&
+            this.totalBytes + bodyBytes.byteLength > this.limits.maxTotalBytes) {
+            this.warnings.push(`Total byte limit reached at ${request.url}`);
+            return;
+        }
+        const contentRef = await this.contentStore.put({ kind: "buffer", data: bodyBytes }, {
+            url: request.url,
+            mimeType: responseMimeType(response),
+            sizeHint: bodyBytes.byteLength
+        });
+        const stored = {
+            request,
+            response,
+            contentRef,
+            size: bodyBytes.byteLength,
+            mimeType: responseMimeType(response)
+        };
+        this.storedResources.push(stored);
+        this.totalBytes += bodyBytes.byteLength;
+    }
+    recordApiFailure(request, failed) {
+        if (this.apiRecordIds.has(request.requestId)) {
+            return;
+        }
+        this.apiRecordIds.add(request.requestId);
+        const record = {
+            url: request.url,
+            method: request.method,
+            requestHeaders: request.headers,
+            error: failed.errorText,
+            timestamp: failed.timestamp
+        };
+        this.apiEntries.push({ record, request });
+    }
+    async recordApiResponse(request, response, bodyBytes) {
+        if (this.apiRecordIds.has(request.requestId)) {
+            return;
+        }
+        this.apiRecordIds.add(request.requestId);
+        const record = {
+            url: request.url,
+            method: request.method,
+            requestHeaders: request.headers,
+            status: response.status,
+            statusText: response.statusText,
+            responseHeaders: response.headers,
+            timestamp: response.timestamp
+        };
+        if (bodyBytes && bodyBytes.byteLength > 0) {
+            const mimeType = responseMimeType(response);
+            const decoded = (0, utils_1.bodyToTextOrBase64)(bodyBytes, mimeType);
+            if (decoded.encoding === "text") {
+                record.responseBody = decoded.text;
+                record.responseEncoding = "text";
+            }
+            else {
+                record.responseBodyBase64 = decoded.base64;
+                record.responseEncoding = "base64";
+            }
+        }
+        this.apiEntries.push({ record, request });
+    }
+}
+exports.NetworkStore = NetworkStore;

package/dist/pagepocket.d.ts

@@ -1,23 +1,9 @@
-import type { NetworkInterceptorAdapter, SnapshotData } from "./types";
-export type PagePocketOptions = {
-    assetsDirName?: string;
-    baseUrl?: string;
-    requestsPath?: string;
-};
-interface PageContent extends PagePocketOptions {
-    content: string;
-    title: string;
-}
-type RequestsInput = SnapshotData | string;
+import type { CaptureOptions, InterceptTarget, PagePocketOptions, PageSnapshot } from "./types";
 export declare class PagePocket {
-    private htmlString;
-    private requestsJSON;
+    private target;
     private options;
-    resources: SnapshotData["resources"];
-    downloadedCount: number;
-    failedCount: number;
-    constructor(htmlString: string, requestsJSON: RequestsInput, options?: PagePocketOptions);
-    static fromNetworkIntercetor(htmlString: string, url: string, interceptorAdapter: NetworkInterceptorAdapter, options?: PagePocketOptions): Promise<PagePocket>;
-    put(): Promise<PageContent>;
+    private constructor();
+    static fromURL(url: string, options?: PagePocketOptions): PagePocket;
+    static fromTarget(target: InterceptTarget, options?: PagePocketOptions): PagePocket;
+    capture(options?: CaptureOptions): Promise<PageSnapshot>;
 }
-export {};

package/dist/pagepocket.js

@@ -1,82 +1,108 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PagePocket = void 0;
-const download_resources_1 = require("./download-resources");
-const hack_html_1 = require("./hack-html");
-const network_records_1 = require("./network-records");
-const resources_1 = require("./resources");
-const rewrite_links_1 = require("./rewrite-links");
-const safeFilename = (input) => {
-    const trimmed = input.trim();
-    if (!trimmed) {
-        return "snapshot";
-    }
-    return (trimmed
-        .replace(/[^a-zA-Z0-9._-]+/g, "_")
-        .replace(/^_+|_+$/g, "")
-        .slice(0, 120) || "snapshot");
-};
-const parseRequestsJson = (requestsJSON) => {
-    const snapshot = typeof requestsJSON === "string" ? JSON.parse(requestsJSON) : requestsJSON;
-    const rawNetworkRecords = (snapshot.networkRecords || []);
-    const mappedNetworkRecords = (0, network_records_1.mapCapturedNetworkRecords)(rawNetworkRecords);
-    return {
-        snapshot,
-        networkRecords: mappedNetworkRecords
-    };
-};
+const content_store_1 = require("./content-store");
+const completion_1 = require("./completion");
+const path_resolver_1 = require("./path-resolver");
+const resource_filter_1 = require("./resource-filter");
+const snapshot_builder_1 = require("./snapshot-builder");
+const network_store_1 = require("./network-store");
 class PagePocket {
-    constructor(htmlString, requestsJSON, options) {
-        this.resources = [];
-        this.downloadedCount = 0;
-        this.failedCount = 0;
-        this.htmlString = htmlString;
-        this.requestsJSON = requestsJSON;
+    constructor(target, options) {
+        this.target = target;
         this.options = options ?? {};
     }
-    static async fromNetworkIntercetor(htmlString, url, interceptorAdapter, options) {
-        return new PagePocket(htmlString, await interceptorAdapter.run(url), options);
+    static fromURL(url, options) {
+        return new PagePocket({ kind: "url", url }, options);
     }
-    async put() {
-        const { snapshot, networkRecords } = parseRequestsJson(this.requestsJSON);
-        const safeTitle = safeFilename(snapshot.title || "snapshot");
-        const assetsDirName = this.options.assetsDirName ?? `${safeTitle}_files`;
-        const baseUrl = this.options.baseUrl ?? snapshot.url ?? "";
-        const requestsPath = this.options.requestsPath ?? `${safeTitle}.requests.json`;
-        const { $, resourceUrls, srcsetItems } = (0, resources_1.extractResourceUrls)(this.htmlString, baseUrl);
-        const downloadResult = await (0, download_resources_1.downloadResources)({
-            baseUrl,
-            assetsDirName,
-            resourceUrls,
-            srcsetItems,
-            referer: baseUrl
-        });
-        this.resources = downloadResult.resourceMeta;
-        this.downloadedCount = downloadResult.downloadedCount;
-        this.failedCount = downloadResult.failedCount;
-        await (0, rewrite_links_1.rewriteLinks)({
-            $,
-            resourceUrls,
-            srcsetItems,
-            baseUrl,
-            assetsDirName,
-            resourceMap: downloadResult.resourceMap,
-            networkRecords
-        });
-        const faviconDataUrl = (0, network_records_1.findFaviconDataUrl)(networkRecords);
-        (0, hack_html_1.hackHtml)({
-            $,
-            baseUrl,
-            requestsPath,
-            faviconDataUrl
+    static fromTarget(target, options) {
+        return new PagePocket(target, options);
+    }
+    async capture(options) {
+        if (!options?.interceptor) {
+            throw new Error("CaptureOptions.interceptor is required.");
+        }
+        const contentStore = options?.contentStore ?? new content_store_1.HybridContentStore();
+        const filter = options?.filter ?? (0, resource_filter_1.createDefaultResourceFilter)();
+        const pathResolver = options?.pathResolver ?? (0, path_resolver_1.createDefaultPathResolver)();
+        const rewriteEntry = options?.rewriteEntry ?? true;
+        const rewriteCSS = options?.rewriteCSS ?? true;
+        const limits = options?.limits;
+        const completionStrategies = (0, completion_1.normalizeCompletion)(options?.completion);
+        const completion = completionStrategies.length > 0
+            ? completionStrategies
+            : [(0, completion_1.networkIdle)(1000), (0, completion_1.timeout)(5000)];
+        const store = new network_store_1.NetworkStore({
+            contentStore,
+            filter,
+            limits
         });
-        return {
-            content: $.html(),
-            title: safeTitle,
-            assetsDirName,
-            requestsPath,
-            baseUrl
+        const inflight = new Set();
+        let inflightRequests = 0;
+        let lastNetworkTs = Date.now();
+        let totalRequests = 0;
+        const pendingEvents = new Set();
+        const onEvent = (event) => {
+            if (event?.timestamp) {
+                lastNetworkTs = event.timestamp;
+            }
+            else {
+                lastNetworkTs = Date.now();
+            }
+            if (event?.type === "request") {
+                totalRequests += 1;
+                if (!inflight.has(event.requestId)) {
+                    inflight.add(event.requestId);
+                    inflightRequests += 1;
+                }
+            }
+            if (event?.type === "response" || event?.type === "failed") {
+                if (inflight.delete(event.requestId)) {
+                    inflightRequests = Math.max(0, inflightRequests - 1);
+                }
+            }
+            const task = store.handleEvent(event);
+            pendingEvents.add(task);
+            task.finally(() => pendingEvents.delete(task));
         };
+        const session = await options.interceptor.start(this.target, { onEvent });
+        if (this.target.kind === "url" && session?.navigate) {
+            await session.navigate(this.target.url);
+        }
+        if (completion.length === 1) {
+            await completion[0].wait({
+                now: () => Date.now(),
+                getStats: () => ({
+                    inflightRequests,
+                    lastNetworkTs,
+                    totalRequests
+                })
+            });
+        }
+        else if (completion.length > 1) {
+            await Promise.race(completion.map((strategy) => strategy.wait({
+                now: () => Date.now(),
+                getStats: () => ({
+                    inflightRequests,
+                    lastNetworkTs,
+                    totalRequests
+                })
+            })));
+        }
+        await session.stop();
+        await Promise.all(pendingEvents);
+        const entryUrl = this.target.kind === "url" ? this.target.url : "";
+        return (0, snapshot_builder_1.buildSnapshot)({
+            entryUrl,
+            createdAt: Date.now(),
+            resources: store.getResources(),
+            apiEntries: store.getApiEntries(),
+            contentStore,
+            pathResolver,
+            rewriteEntry,
+            rewriteCSS,
+            warnings: store.getWarnings()
+        });
     }
 }
 exports.PagePocket = PagePocket;

package/dist/path-resolver.d.ts

@@ -0,0 +1,5 @@
+import type { PathResolver, ResourceType } from "./types";
+export declare const createDefaultPathResolver: () => PathResolver;
+export declare const resolveCrossOrigin: (url: string, entryUrl: string) => boolean;
+export declare const withPrefixPathResolver: (resolver: PathResolver, prefix: string) => PathResolver;
+export declare const isDocumentType: (resourceType?: ResourceType) => boolean;

package/dist/path-resolver.js

@@ -0,0 +1,92 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isDocumentType = exports.withPrefixPathResolver = exports.resolveCrossOrigin = exports.createDefaultPathResolver = void 0;
+const utils_1 = require("./utils");
+const normalizePathname = (pathname) => {
+    const normalized = pathname || "/";
+    const clean = (0, utils_1.sanitizePosixPath)(normalized);
+    const leading = clean ? `/${clean}` : "/";
+    if (leading.endsWith("/")) {
+        return `${leading}index`;
+    }
+    return leading;
+};
+const withSuffix = (path, suffix) => {
+    const lastSlash = path.lastIndexOf("/");
+    const lastDot = path.lastIndexOf(".");
+    if (lastDot > lastSlash) {
+        return `${path.slice(0, lastDot)}${suffix}${path.slice(lastDot)}`;
+    }
+    return `${path}${suffix}`;
+};
+const sameDomain = (left, right) => left.hostname === right.hostname;
+const createDefaultPathResolver = () => {
+    const resolvedByUrl = new Map();
+    const usedPaths = new Map();
+    return {
+        resolve(input) {
+            if (resolvedByUrl.has(input.url)) {
+                return resolvedByUrl.get(input.url) ?? "/index.html";
+            }
+            if (input.resourceType === "document") {
+                const entryPath = "/index.html";
+                resolvedByUrl.set(input.url, entryPath);
+                return entryPath;
+            }
+            let parsed = null;
+            try {
+                parsed = new URL(input.url);
+            }
+            catch {
+                parsed = null;
+            }
+            const pathname = normalizePathname(parsed?.pathname || "/");
+            const queryHash = `${parsed?.search || ""}${parsed?.hash || ""}`;
+            const suffix = queryHash ? `__ppq_${(0, utils_1.hashString)(queryHash)}` : "";
+            const basePath = input.isCrossOrigin
+                ? `/external_resources${pathname}`
+                : `${pathname}`;
+            let resolvedPath = suffix ? withSuffix(basePath, suffix) : basePath;
+            const collisionKey = resolvedPath;
+            const existingUrl = usedPaths.get(collisionKey);
+            if (existingUrl && existingUrl !== input.url) {
+                const collisionSuffix = `__ppc_${(0, utils_1.hashString)(input.url)}`;
+                resolvedPath = withSuffix(resolvedPath, collisionSuffix);
+            }
+            usedPaths.set(resolvedPath, input.url);
+            resolvedByUrl.set(input.url, resolvedPath);
+            return resolvedPath;
+        }
+    };
+};
+exports.createDefaultPathResolver = createDefaultPathResolver;
+const resolveCrossOrigin = (url, entryUrl) => {
+    try {
+        const parsed = new URL(url);
+        const entry = new URL(entryUrl);
+        return !sameDomain(parsed, entry);
+    }
+    catch {
+        return false;
+    }
+};
+exports.resolveCrossOrigin = resolveCrossOrigin;
+const withPrefixPathResolver = (resolver, prefix) => {
+    const normalizedPrefix = (0, utils_1.sanitizePosixPath)(prefix);
+    if (!normalizedPrefix) {
+        return resolver;
+    }
+    const prefixWithSlash = `/${normalizedPrefix}`;
+    return {
+        resolve(input) {
+            const resolved = resolver.resolve(input);
+            if (!resolved.startsWith("/")) {
+                return `${prefixWithSlash}/${resolved}`;
+            }
+            return `${prefixWithSlash}${resolved}`;
+        }
+    };
+};
+exports.withPrefixPathResolver = withPrefixPathResolver;
+const isDocumentType = (resourceType) => resourceType === "document";
+exports.isDocumentType = isDocumentType;

package/dist/replay-script.d.ts

@@ -1 +1,11 @@
-export declare const buildReplayScript: (requestsPath: string, baseUrl: string) => string;
+import type { ApiRecord } from "./types";
+export type MatchApiOptions = {
+    records: ApiRecord[];
+    byKey?: Map<string, ApiRecord>;
+    baseUrl: string;
+    method: string;
+    url: string;
+    body?: unknown;
+};
+export declare function matchAPI(options: MatchApiOptions): ApiRecord | undefined;
+export declare const buildReplayScript: (apiPath: string, baseUrl: string) => string;