@pagepocket/lib 0.4.0

package/dist/content-type.d.ts

@@ -0,0 +1,2 @@
+export declare const extensionFromContentType: (contentType?: string | null) => "" | ".css" | ".js" | ".png" | ".jpg" | ".gif" | ".svg" | ".woff2" | ".woff";
+export declare const isTextResponse: (contentType: string) => boolean;

package/dist/content-type.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isTextResponse = exports.extensionFromContentType = void 0;
+const extensionFromContentType = (contentType) => {
+    if (!contentType) {
+        return "";
+    }
+    if (contentType.includes("text/css"))
+        return ".css";
+    if (contentType.includes("javascript"))
+        return ".js";
+    if (contentType.includes("image/png"))
+        return ".png";
+    if (contentType.includes("image/jpeg"))
+        return ".jpg";
+    if (contentType.includes("image/gif"))
+        return ".gif";
+    if (contentType.includes("image/svg"))
+        return ".svg";
+    if (contentType.includes("font/woff2"))
+        return ".woff2";
+    if (contentType.includes("font/woff"))
+        return ".woff";
+    return "";
+};
+exports.extensionFromContentType = extensionFromContentType;
+const isTextResponse = (contentType) => {
+    const lowered = contentType.toLowerCase();
+    return (lowered.startsWith("text/") ||
+        lowered.includes("json") ||
+        lowered.includes("javascript") ||
+        lowered.includes("xml") ||
+        lowered.includes("svg") ||
+        lowered.includes("html"));
+};
+exports.isTextResponse = isTextResponse;

package/dist/css-rewrite.d.ts

@@ -0,0 +1,9 @@
+export type CssUrlResolver = (absoluteUrl: string) => Promise<string | null>;
+type RewriteCssInput = {
+    filename: string;
+    extension: string;
+    cssUrl: string;
+    resolveUrl: CssUrlResolver;
+};
+export declare const rewriteCssUrls: (input: RewriteCssInput) => Promise<boolean>;
+export {};

package/dist/css-rewrite.js

@@ -0,0 +1,76 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rewriteCssUrls = void 0;
+const URL_PATTERN = /url\(\s*(['"]?)([^'")]+)\1\s*\)/g;
+const rewriteCssUrls = async (input) => {
+    const { readText, write } = await Promise.resolve().then(() => __importStar(require("uni-fs")));
+    const original = await readText(input.filename, input.extension);
+    let updated = "";
+    let lastIndex = 0;
+    let changed = false;
+    for (const match of original.matchAll(URL_PATTERN)) {
+        const index = match.index ?? 0;
+        updated += original.slice(lastIndex, index);
+        const quote = match[1] || "";
+        const rawUrl = String(match[2] || "").trim();
+        let replacement = match[0];
+        if (rawUrl && !rawUrl.startsWith("data:") && !rawUrl.startsWith("blob:")) {
+            const absolute = (() => {
+                try {
+                    return new URL(rawUrl, input.cssUrl).toString();
+                }
+                catch {
+                    return null;
+                }
+            })();
+            if (absolute) {
+                const resolved = await input.resolveUrl(absolute);
+                if (resolved) {
+                    replacement = `url(${quote}${resolved}${quote})`;
+                    changed = true;
+                }
+            }
+        }
+        updated += replacement;
+        lastIndex = index + match[0].length;
+    }
+    updated += original.slice(lastIndex);
+    if (changed) {
+        await write(input.filename, input.extension, updated);
+    }
+    return changed;
+};
+exports.rewriteCssUrls = rewriteCssUrls;

package/dist/download-resources.d.ts

@@ -0,0 +1,25 @@
+import { type ResourceReference, type SrcsetReference } from "./resources";
+import type { SnapshotData } from "./types";
+export type DownloadedResource = {
+    url: string;
+    filename: string;
+    extension: string;
+    localPath: string;
+    contentType?: string | null;
+    size?: number;
+};
+type DownloadResourcesInput = {
+    baseUrl: string;
+    assetsDirName: string;
+    resourceUrls: ResourceReference[];
+    srcsetItems: SrcsetReference[];
+    referer?: string;
+};
+type DownloadResourcesResult = {
+    resourceMap: Map<string, DownloadedResource>;
+    resourceMeta: SnapshotData["resources"];
+    downloadedCount: number;
+    failedCount: number;
+};
+export declare const downloadResources: (input: DownloadResourcesInput) => Promise<DownloadResourcesResult>;
+export {};

package/dist/download-resources.js

@@ -0,0 +1,163 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.downloadResources = void 0;
+const content_type_1 = require("./content-type");
+const resources_1 = require("./resources");
+const FNV_OFFSET = 0x811c9dc5;
+const FNV_PRIME = 0x01000193;
+const shouldSkipUrl = (value) => {
+    const trimmed = value.trim();
+    return (!trimmed ||
+        trimmed.startsWith("data:") ||
+        trimmed.startsWith("blob:") ||
+        trimmed.startsWith("mailto:") ||
+        trimmed.startsWith("tel:") ||
+        trimmed.startsWith("javascript:") ||
+        trimmed.startsWith("#"));
+};
+const hashUrl = (value) => {
+    let hash = FNV_OFFSET;
+    for (let i = 0; i < value.length; i += 1) {
+        hash ^= value.charCodeAt(i);
+        hash = (hash * FNV_PRIME) >>> 0;
+    }
+    return hash.toString(16).padStart(8, "0");
+};
+const extensionFromUrl = (url) => {
+    try {
+        const pathname = new URL(url).pathname;
+        const lastDot = pathname.lastIndexOf(".");
+        if (lastDot === -1) {
+            return "";
+        }
+        return pathname.slice(lastDot);
+    }
+    catch {
+        return "";
+    }
+};
+const normalizeExtension = (extension) => {
+    if (!extension) {
+        return "";
+    }
+    return extension.startsWith(".") ? extension.slice(1) : extension;
+};
+const buildLocalPath = (assetsDirName, filename, extension) => {
+    const normalizedExtension = normalizeExtension(extension);
+    if (!normalizedExtension) {
+        return `${assetsDirName}/${filename}`;
+    }
+    return `${assetsDirName}/${filename}.${normalizedExtension}`;
+};
+const collectSrcsetUrls = (items, baseUrl) => {
+    const urls = [];
+    for (const item of items) {
+        const parts = item.value.split(",").map((part) => part.trim());
+        for (const part of parts) {
+            const [rawUrl] = part.split(/\s+/, 2);
+            if (!rawUrl) {
+                continue;
+            }
+            const absolute = (0, resources_1.toAbsoluteUrl)(baseUrl, rawUrl);
+            urls.push(absolute);
+        }
+    }
+    return urls;
+};
+const downloadResources = async (input) => {
+    const { write, exists } = await Promise.resolve().then(() => __importStar(require("uni-fs")));
+    const resourceMap = new Map();
+    const resourceMeta = [];
+    let downloadedCount = 0;
+    let failedCount = 0;
+    const srcsetUrls = collectSrcsetUrls(input.srcsetItems, input.baseUrl);
+    const candidateUrls = [
+        ...input.resourceUrls.map((resource) => resource.url),
+        ...srcsetUrls
+    ];
+    for (const candidate of candidateUrls) {
+        if (shouldSkipUrl(candidate)) {
+            continue;
+        }
+        const url = (0, resources_1.toAbsoluteUrl)(input.baseUrl, candidate);
+        if (!url || resourceMap.has(url)) {
+            continue;
+        }
+        try {
+            const headers = {};
+            if (input.referer) {
+                headers.referer = input.referer;
+            }
+            const response = await fetch(url, { redirect: "follow", headers });
+            const contentType = response.headers.get("content-type");
+            const buffer = new Uint8Array(await response.arrayBuffer());
+            const extFromUrl = extensionFromUrl(url);
+            const extFromType = (0, content_type_1.extensionFromContentType)(contentType);
+            const extension = normalizeExtension(extFromUrl || extFromType);
+            const filename = hashUrl(url);
+            const localPath = buildLocalPath(input.assetsDirName, filename, extension);
+            if (!(await exists(input.assetsDirName + "/" + filename, extension))) {
+                await write(input.assetsDirName + "/" + filename, extension, buffer);
+            }
+            const entry = {
+                url,
+                filename,
+                extension,
+                localPath,
+                contentType,
+                size: buffer.byteLength
+            };
+            resourceMap.set(url, entry);
+            resourceMeta.push({
+                url,
+                localPath,
+                contentType,
+                size: buffer.byteLength
+            });
+            downloadedCount += 1;
+        }
+        catch {
+            failedCount += 1;
+        }
+    }
+    return {
+        resourceMap,
+        resourceMeta,
+        downloadedCount,
+        failedCount
+    };
+};
+exports.downloadResources = downloadResources;

package/dist/hack-html.d.ts

@@ -0,0 +1,9 @@
+import type { CheerioAPI } from "cheerio";
+type HackHtmlInput = {
+    $: CheerioAPI;
+    baseUrl: string;
+    requestsPath: string;
+    faviconDataUrl?: string | null;
+};
+export declare const hackHtml: (input: HackHtmlInput) => void;
+export {};

package/dist/hack-html.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hackHtml = void 0;
+const preload_1 = require("./preload");
+const replay_script_1 = require("./replay-script");
+const hackHtml = (input) => {
+    const replayScript = (0, replay_script_1.buildReplayScript)(input.requestsPath, input.baseUrl);
+    const preloadScript = `<script>${(0, preload_1.buildPreloadScript)()}</script>`;
+    const head = input.$("head");
+    const root = input.$.root();
+    if (head.length) {
+        head.prepend(replayScript);
+        head.prepend(preloadScript);
+    }
+    else {
+        root.prepend(replayScript);
+        root.prepend(preloadScript);
+    }
+    if (input.faviconDataUrl) {
+        const existingIcon = input.$('link[rel="icon"]');
+        if (existingIcon.length) {
+            existingIcon.attr("href", input.faviconDataUrl);
+        }
+        else if (head.length) {
+            head.append(`<link rel="icon" href="${input.faviconDataUrl}" />`);
+        }
+        else {
+            root.append(`<link rel="icon" href="${input.faviconDataUrl}" />`);
+        }
+    }
+};
+exports.hackHtml = hackHtml;

package/dist/hackers/index.d.ts

@@ -0,0 +1,3 @@
+import type { ScriptHacker } from "./types";
+export declare const preloadHackers: ScriptHacker[];
+export declare const replayHackers: ScriptHacker[];

package/dist/hackers/index.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.replayHackers = exports.preloadHackers = void 0;
+const preload_fetch_1 = require("./preload-fetch");
+const preload_xhr_1 = require("./preload-xhr");
+const replay_beacon_1 = require("./replay-beacon");
+const replay_dom_rewrite_1 = require("./replay-dom-rewrite");
+const replay_eventsource_1 = require("./replay-eventsource");
+const replay_fetch_1 = require("./replay-fetch");
+const replay_svg_image_1 = require("./replay-svg-image");
+const replay_websocket_1 = require("./replay-websocket");
+const replay_xhr_1 = require("./replay-xhr");
+exports.preloadHackers = [preload_fetch_1.preloadFetchRecorder, preload_xhr_1.preloadXhrRecorder];
+exports.replayHackers = [
+    replay_fetch_1.replayFetchResponder,
+    replay_xhr_1.replayXhrResponder,
+    replay_dom_rewrite_1.replayDomRewriter,
+    replay_svg_image_1.replaySvgImageRewriter,
+    replay_beacon_1.replayBeaconStub,
+    replay_websocket_1.replayWebSocketStub,
+    replay_eventsource_1.replayEventSourceStub
+];

package/dist/hackers/preload-fetch.d.ts

@@ -0,0 +1,2 @@
+import type { ScriptHacker } from "./types";
+export declare const preloadFetchRecorder: ScriptHacker;

package/dist/hackers/preload-fetch.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.preloadFetchRecorder = void 0;
+exports.preloadFetchRecorder = {
+    id: "preload-fetch-recorder",
+    stage: "preload",
+    build: () => `
+  // Record fetch calls and responses for replay.
+  const originalFetch = window.fetch.bind(window);
+  const recordFetch = async (input, init) => {
+    trackPendingStart();
+    const url = typeof input === "string" ? toAbsoluteUrl(input) : toAbsoluteUrl(input.url);
+    const method = init && init.method ? init.method : (typeof input === "string" ? "GET" : input.method || "GET");
+    const requestBody = normalizeBody(init && init.body ? init.body : (typeof input === "string" ? undefined : input.body));
+
+    try {
+      const response = await originalFetch(input, init);
+      const clone = response.clone();
+      const responseBody = await clone.text();
+      const headers = {};
+      response.headers.forEach((value, key) => {
+        headers[key] = value;
+      });
+      records.push({
+        kind: "fetch",
+        url,
+        method,
+        requestBody,
+        status: response.status,
+        statusText: response.statusText,
+        responseHeaders: headers,
+        responseBody,
+        timestamp: Date.now()
+      });
+      return response;
+    } catch (error) {
+      records.push({
+        kind: "fetch",
+        url,
+        method,
+        requestBody,
+        error: String(error),
+        timestamp: Date.now()
+      });
+      throw error;
+    } finally {
+      trackPendingEnd();
+    }
+  };
+
+  window.fetch = (input, init) => {
+    return recordFetch(input, init);
+  };
+  window.fetch.__pagepocketOriginal = originalFetch;
+`
+};

package/dist/hackers/preload-xhr.d.ts

@@ -0,0 +1,2 @@
+import type { ScriptHacker } from "./types";
+export declare const preloadXhrRecorder: ScriptHacker;

package/dist/hackers/preload-xhr.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.preloadXhrRecorder = void 0;
+exports.preloadXhrRecorder = {
+    id: "preload-xhr-recorder",
+    stage: "preload",
+    build: () => `
+  // Record XHR calls and responses for replay.
+  const originalOpen = XMLHttpRequest.prototype.open;
+  const originalSend = XMLHttpRequest.prototype.send;
+
+  XMLHttpRequest.prototype.open = function(method, url, ...rest) {
+    this.__pagepocketMethod = method;
+    this.__pagepocketUrl = toAbsoluteUrl(url);
+    return originalOpen.call(this, method, url, ...rest);
+  };
+
+  XMLHttpRequest.prototype.send = function(body) {
+    const xhr = this;
+    const requestBody = normalizeBody(body);
+
+    const onLoadEnd = () => {
+      const responseHeadersRaw = xhr.getAllResponseHeaders();
+      const headers = {};
+      responseHeadersRaw
+        .trim()
+        .split(/\\r?\\n/)
+        .filter(Boolean)
+        .forEach((line) => {
+          const index = line.indexOf(":");
+          if (index > -1) {
+            const key = line.slice(0, index).trim().toLowerCase();
+            const value = line.slice(index + 1).trim();
+            headers[key] = value;
+          }
+        });
+
+      records.push({
+        kind: "xhr",
+        url: xhr.__pagepocketUrl || "",
+        method: xhr.__pagepocketMethod || "GET",
+        requestBody,
+        status: xhr.status,
+        statusText: xhr.statusText,
+        responseHeaders: headers,
+        responseBody: xhr.responseText || "",
+        timestamp: Date.now()
+      });
+
+      xhr.removeEventListener("loadend", onLoadEnd);
+      trackPendingEnd();
+    };
+
+    trackPendingStart();
+    xhr.addEventListener("loadend", onLoadEnd);
+    return originalSend.call(xhr, body);
+  };
+`
+};

package/dist/hackers/replay-beacon.d.ts

@@ -0,0 +1,2 @@
+import type { ScriptHacker } from "./types";
+export declare const replayBeaconStub: ScriptHacker;

package/dist/hackers/replay-beacon.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.replayBeaconStub = void 0;
+exports.replayBeaconStub = {
+    id: "replay-beacon-stub",
+    stage: "replay",
+    build: () => `
+  // Stub beacon calls so analytics doesn't leak outside the snapshot.
+  if (navigator.sendBeacon) {
+    const originalBeacon = navigator.sendBeacon.bind(navigator);
+    navigator.sendBeacon = (url, data) => {
+      const record = findRecord("POST", url, data);
+      if (record) {
+        return true;
+      }
+      return true;
+    };
+    navigator.sendBeacon.__pagepocketOriginal = originalBeacon;
+  }
+`
+};

package/dist/hackers/replay-dom-rewrite.d.ts

@@ -0,0 +1,2 @@
+import type { ScriptHacker } from "./types";
+export declare const replayDomRewriter: ScriptHacker;