@pagepocket/lib 0.4.0

package/dist/network-records.js

@@ -0,0 +1,83 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapLighterceptorRecords = exports.findFaviconDataUrl = exports.toDataUrlFromRecord = void 0;
+const getHeaderValue = (headers, name) => {
+    for (const key in headers) {
+        if (key.toLowerCase() === name.toLowerCase()) {
+            return headers[key];
+        }
+    }
+    return undefined;
+};
+const toBase64 = (value) => {
+    const bufferConstructor = globalThis.Buffer;
+    if (bufferConstructor) {
+        return bufferConstructor.from(value, "utf-8").toString("base64");
+    }
+    if (typeof btoa === "function") {
+        return btoa(value);
+    }
+    return "";
+};
+const toDataUrlFromRecord = (record) => {
+    if (!record)
+        return null;
+    const headers = record.responseHeaders || {};
+    const contentType = getHeaderValue(headers, "content-type") || "application/octet-stream";
+    if (record.responseEncoding === "base64" && record.responseBodyBase64) {
+        return `data:${contentType};base64,${record.responseBodyBase64}`;
+    }
+    if (record.responseBody) {
+        const encoded = toBase64(record.responseBody);
+        if (!encoded) {
+            return null;
+        }
+        return `data:${contentType};base64,${encoded}`;
+    }
+    return null;
+};
+exports.toDataUrlFromRecord = toDataUrlFromRecord;
+const findFaviconDataUrl = (records) => {
+    for (const record of records) {
+        if (!record || !record.url)
+            continue;
+        const headers = record.responseHeaders || {};
+        const contentType = (getHeaderValue(headers, "content-type") || "").toLowerCase();
+        const pathname = (() => {
+            try {
+                return new URL(record.url).pathname;
+            }
+            catch {
+                return record.url;
+            }
+        })();
+        const looksLikeFavicon = contentType.includes("icon") || /favicon(\.[a-z0-9]+)?$/i.test(pathname || "");
+        if (!looksLikeFavicon)
+            continue;
+        const dataUrl = (0, exports.toDataUrlFromRecord)(record);
+        if (dataUrl)
+            return dataUrl;
+    }
+    return null;
+};
+exports.findFaviconDataUrl = findFaviconDataUrl;
+const mapLighterceptorRecords = (records) => {
+    if (!records)
+        return [];
+    return records.map((record) => {
+        const response = record.response;
+        return {
+            url: record.url,
+            method: record.method,
+            status: response?.status,
+            statusText: response?.statusText,
+            responseHeaders: response?.headers,
+            responseBody: response?.bodyEncoding === "text" ? response.body : undefined,
+            responseBodyBase64: response?.bodyEncoding === "base64" ? response.body : undefined,
+            responseEncoding: response?.bodyEncoding,
+            error: record.error,
+            timestamp: record.timestamp
+        };
+    });
+};
+exports.mapLighterceptorRecords = mapLighterceptorRecords;

package/dist/pagepocket.d.ts

@@ -0,0 +1,18 @@
+import type { SnapshotData } from "./types";
+export type PagePocketOptions = {
+    assetsDirName?: string;
+    baseUrl?: string;
+    requestsPath?: string;
+};
+type RequestsInput = SnapshotData | string;
+export declare class PagePocket {
+    private htmlString;
+    private requestsJSON;
+    private options;
+    resources: SnapshotData["resources"];
+    downloadedCount: number;
+    failedCount: number;
+    constructor(htmlString: string, requestsJSON: RequestsInput, options?: PagePocketOptions);
+    put(): Promise<string>;
+}
+export {};

package/dist/pagepocket.js

@@ -0,0 +1,73 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PagePocket = void 0;
+const download_resources_1 = require("./download-resources");
+const hack_html_1 = require("./hack-html");
+const network_records_1 = require("./network-records");
+const resources_1 = require("./resources");
+const rewrite_links_1 = require("./rewrite-links");
+const safeFilename = (input) => {
+    const trimmed = input.trim();
+    if (!trimmed) {
+        return "snapshot";
+    }
+    return (trimmed
+        .replace(/[^a-zA-Z0-9._-]+/g, "_")
+        .replace(/^_+|_+$/g, "")
+        .slice(0, 120) || "snapshot");
+};
+const parseRequestsJson = (requestsJSON) => {
+    const snapshot = typeof requestsJSON === "string" ? JSON.parse(requestsJSON) : requestsJSON;
+    const rawNetworkRecords = (snapshot.networkRecords || []);
+    const mappedNetworkRecords = (0, network_records_1.mapLighterceptorRecords)(rawNetworkRecords);
+    return {
+        snapshot,
+        networkRecords: mappedNetworkRecords
+    };
+};
+class PagePocket {
+    constructor(htmlString, requestsJSON, options) {
+        this.resources = [];
+        this.downloadedCount = 0;
+        this.failedCount = 0;
+        this.htmlString = htmlString;
+        this.requestsJSON = requestsJSON;
+        this.options = options ?? {};
+    }
+    async put() {
+        const { snapshot, networkRecords } = parseRequestsJson(this.requestsJSON);
+        const safeTitle = safeFilename(snapshot.title || "snapshot");
+        const assetsDirName = this.options.assetsDirName ?? `${safeTitle}_files`;
+        const baseUrl = this.options.baseUrl ?? snapshot.url ?? "";
+        const requestsPath = this.options.requestsPath ?? `${safeTitle}.requests.json`;
+        const { $, resourceUrls, srcsetItems } = (0, resources_1.extractResourceUrls)(this.htmlString, baseUrl);
+        const downloadResult = await (0, download_resources_1.downloadResources)({
+            baseUrl,
+            assetsDirName,
+            resourceUrls,
+            srcsetItems,
+            referer: baseUrl
+        });
+        this.resources = downloadResult.resourceMeta;
+        this.downloadedCount = downloadResult.downloadedCount;
+        this.failedCount = downloadResult.failedCount;
+        await (0, rewrite_links_1.rewriteLinks)({
+            $,
+            resourceUrls,
+            srcsetItems,
+            baseUrl,
+            assetsDirName,
+            resourceMap: downloadResult.resourceMap,
+            networkRecords
+        });
+        const faviconDataUrl = (0, network_records_1.findFaviconDataUrl)(networkRecords);
+        (0, hack_html_1.hackHtml)({
+            $,
+            baseUrl,
+            requestsPath,
+            faviconDataUrl
+        });
+        return $.html();
+    }
+}
+exports.PagePocket = PagePocket;

package/dist/preload.d.ts

@@ -0,0 +1 @@
+export declare const buildPreloadScript: () => string;

package/dist/preload.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildPreloadScript = void 0;
+const hackers_1 = require("./hackers");
+const buildPreloadScript = () => {
+    const context = { stage: "preload" };
+    const hackerScripts = hackers_1.preloadHackers
+        .map((hacker) => `  // hacker:${hacker.id}\n${hacker.build(context)}`)
+        .join("\n");
+    return `
+(function () {
+  if (window.__pagepocketPatched) {
+    return;
+  }
+  Object.defineProperty(window, "__pagepocketPatched", { value: true });
+
+  const records = [];
+  window.__pagepocketRecords = records;
+  window.__pagepocketPendingRequests = 0;
+
+  const toAbsoluteUrl = (input) => {
+    try {
+      return new URL(input, window.location.href).toString();
+    } catch {
+      return input;
+    }
+  };
+
+  const normalizeBody = (body) => {
+    if (body === undefined || body === null) {
+      return "";
+    }
+    if (typeof body === "string") {
+      return body;
+    }
+    if (body instanceof ArrayBuffer) {
+      try {
+        return new TextDecoder().decode(body);
+      } catch {
+        return "";
+      }
+    }
+    if (body instanceof Blob) {
+      return "";
+    }
+    return String(body);
+  };
+
+  const trackPendingStart = () => {
+    window.__pagepocketPendingRequests += 1;
+  };
+  const trackPendingEnd = () => {
+    window.__pagepocketPendingRequests = Math.max(0, window.__pagepocketPendingRequests - 1);
+  };
+
+${hackerScripts}
+})();
+`;
+};
+exports.buildPreloadScript = buildPreloadScript;

package/dist/replay-script.d.ts

@@ -0,0 +1 @@
+export declare const buildReplayScript: (requestsPath: string, baseUrl: string) => string;

package/dist/replay-script.js

@@ -0,0 +1,347 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildReplayScript = void 0;
+const hackers_1 = require("./hackers");
+const buildReplayScript = (requestsPath, baseUrl) => {
+    const basePayload = JSON.stringify(baseUrl);
+    const requestsPayload = JSON.stringify(requestsPath);
+    const context = { stage: "replay" };
+    const hackerScripts = hackers_1.replayHackers
+        .map((hacker) => `  // hacker:${hacker.id}\n${hacker.build(context)}`)
+        .join("\n");
+    return `
+<script>
+(function(){
+  // Load the snapshot metadata before patching runtime APIs.
+  const baseUrl = ${basePayload};
+  const requestsUrl = ${requestsPayload};
+  const __pagepocketOriginalFetch = window.fetch ? window.fetch.bind(window) : null;
+
+  const loadSnapshot = async () => {
+    try {
+      if (!__pagepocketOriginalFetch) {
+        throw new Error("Fetch is unavailable");
+      }
+      const response = await __pagepocketOriginalFetch(requestsUrl);
+      if (!response.ok) {
+        throw new Error("Failed to load snapshot metadata");
+      }
+      return await response.json();
+    } catch {
+      return {
+        url: baseUrl,
+        title: "",
+        capturedAt: "",
+        fetchXhrRecords: [],
+        networkRecords: [],
+        resources: []
+      };
+    }
+  };
+
+  // Soften JSON parse failures to avoid halting replay flows.
+  const originalResponseJson = Response && Response.prototype && Response.prototype.json;
+  if (originalResponseJson) {
+    Response.prototype.json = function(...args) {
+      try {
+        return originalResponseJson.apply(this, args).catch(() => null);
+      } catch {
+        return Promise.resolve(null);
+      }
+    };
+  }
+
+  // Guard to reapply patches if overwritten later.
+  const ensureReplayPatches = () => {
+    try {
+      if (!window.fetch.__pagepocketOriginal && typeof __pagepocketOriginalFetch === "function") {
+        window.fetch.__pagepocketOriginal = __pagepocketOriginalFetch;
+      }
+    } catch {}
+    try {
+      if (!XMLHttpRequest.prototype.send.__pagepocketOriginal) {
+        XMLHttpRequest.prototype.send.__pagepocketOriginal = XMLHttpRequest.prototype.send;
+      }
+    } catch {}
+  };
+
+  let records = [];
+  let networkRecords = [];
+  const byKey = new Map();
+
+  const localResourceSet = new Set();
+  const resourceUrlMap = new Map();
+
+  const normalizeUrl = (input) => {
+    try { return new URL(input, baseUrl).toString(); } catch { return input; }
+  };
+
+  let baseOrigin = "";
+  let baseDir = "";
+  try {
+    const parsedBase = new URL(baseUrl);
+    baseOrigin = parsedBase.origin;
+    baseDir = new URL(".", parsedBase).toString().replace(/\\/$/, "");
+  } catch {}
+
+
+  const expandUrlVariants = (value) => {
+    const variants = [];
+    if (typeof value === "string") {
+      variants.push(value);
+      variants.push(normalizeUrl(value));
+      if (baseOrigin && value.startsWith("/")) {
+        variants.push(baseOrigin + value);
+        if (baseDir) variants.push(baseDir + value);
+      } else if (baseDir) {
+        variants.push(baseDir + (value.startsWith("/") ? value : "/" + value));
+      }
+      try {
+        const parsed = new URL(value, baseUrl);
+        const pathWithSearch = (parsed.pathname || "") + (parsed.search || "");
+        if (baseOrigin && parsed.origin !== baseOrigin) {
+          variants.push(baseOrigin + pathWithSearch);
+          if (baseDir) {
+            const path = pathWithSearch.startsWith("/") ? pathWithSearch : "/" + pathWithSearch;
+            variants.push(baseDir + path);
+          }
+        }
+      } catch {}
+    }
+    return Array.from(new Set(variants.filter(Boolean)));
+  };
+
+  const normalizeBody = (body) => {
+    if (body === undefined || body === null) return "";
+    if (typeof body === "string") return body;
+    try { return String(body); } catch { return ""; }
+  };
+
+  // Build a stable key so requests with identical method/url/body match the same response.
+  const makeKey = (method, url, body) => method.toUpperCase() + " " + normalizeUrl(url) + " " + normalizeBody(body);
+  const makeVariantKeys = (method, url, body) => {
+    return expandUrlVariants(url).map((variant) => makeKey(method, variant, body));
+  };
+  const normalizeNetworkRecord = (record) => {
+    if (!record || typeof record !== "object") {
+      return record;
+    }
+    if (record.response && record.response.body !== undefined) {
+      const response = record.response || {};
+      const encoding = response.bodyEncoding || "text";
+      return {
+        url: record.url,
+        method: record.method || "GET",
+        requestBody: record.requestBody || "",
+        status: response.status,
+        statusText: response.statusText,
+        responseHeaders: response.headers,
+        responseBody: encoding === "text" ? response.body : undefined,
+        responseBodyBase64: encoding === "base64" ? response.body : undefined,
+        responseEncoding: encoding,
+        error: record.error,
+        timestamp: record.timestamp
+      };
+    }
+    return record;
+  };
+
+  const primeLookups = (snapshot) => {
+    records = snapshot.fetchXhrRecords || [];
+    networkRecords = (snapshot.networkRecords || []).map(normalizeNetworkRecord);
+    byKey.clear();
+    localResourceSet.clear();
+    resourceUrlMap.clear();
+
+    for (const record of records) {
+      if (!record || !record.url || !record.method) continue;
+      const keys = makeVariantKeys(record.method, record.url, record.requestBody || "");
+      for (const key of keys) {
+        if (!byKey.has(key)) {
+          byKey.set(key, record);
+        }
+      }
+    }
+
+    for (const record of networkRecords) {
+      if (!record || !record.url || !record.method) continue;
+      const keys = makeVariantKeys(record.method, record.url, record.requestBody || "");
+      for (const key of keys) {
+        if (!byKey.has(key)) {
+          byKey.set(key, record);
+        }
+      }
+    }
+
+    // Track local resource files and map original URLs to local paths.
+    const resourceList = snapshot.resources || [];
+    for (const item of resourceList) {
+      if (!item || !item.localPath) continue;
+      localResourceSet.add(item.localPath);
+      localResourceSet.add("./" + item.localPath);
+      localResourceSet.add("/" + item.localPath);
+
+      if (item.url) {
+        const variants = expandUrlVariants(item.url);
+        for (const variant of variants) {
+          resourceUrlMap.set(variant, item.localPath);
+        }
+      }
+    }
+  };
+
+
+  const ready = (async () => {
+    // Deserialize the snapshot and prepare lookup tables for offline responses.
+    const snapshot = (await loadSnapshot()) || {};
+    primeLookups(snapshot);
+    return snapshot;
+  })();
+
+  const isLocalResource = (value) => {
+    if (!value) return false;
+    if (value.startsWith("data:") || value.startsWith("blob:")) return true;
+    return localResourceSet.has(value);
+  };
+
+  // Lookup helpers for request records and local assets.
+  const findRecord = (method, url, body) => {
+    const variants = expandUrlVariants(url);
+    for (const variant of variants) {
+      const key = makeKey(method, variant, body);
+      if (byKey.has(key)) return byKey.get(key);
+    }
+    for (const variant of variants) {
+      const fallbackKey = makeKey(method, variant, "");
+      if (byKey.has(fallbackKey)) return byKey.get(fallbackKey);
+    }
+    for (const variant of variants) {
+      const getKey = makeKey("GET", variant, "");
+      if (byKey.has(getKey)) return byKey.get(getKey);
+    }
+    return null;
+  };
+
+  const findByUrl = (url) => {
+    if (isLocalResource(url)) return null;
+    const variants = expandUrlVariants(url);
+    for (const variant of variants) {
+      const direct = byKey.get(makeKey("GET", variant, ""));
+      if (direct) return direct;
+    }
+    // Attempt a looser match: ignore querystring if needed.
+    for (const variant of variants) {
+      try {
+        const withoutQuery = new URL(variant).origin + new URL(variant).pathname;
+        const direct = byKey.get(makeKey("GET", withoutQuery, ""));
+        if (direct) return direct;
+      } catch {}
+    }
+    return null;
+  };
+
+  const findLocalPath = (url) => {
+    if (!url) return null;
+    const variants = expandUrlVariants(url);
+    for (const variant of variants) {
+      const hit = resourceUrlMap.get(variant);
+      if (hit) return hit;
+    }
+    for (const variant of variants) {
+      try {
+        const withoutQuery = new URL(variant).origin + new URL(variant).pathname;
+        const hit = resourceUrlMap.get(withoutQuery);
+        if (hit) return hit;
+      } catch {}
+    }
+    // If still not found, fallback to data URLs if present.
+    return null;
+  };
+
+  // Safe property injection for emulating XHR state transitions.
+  const defineProp = (obj, key, value) => {
+    try {
+      Object.defineProperty(obj, key, { value, configurable: true });
+    } catch {}
+  };
+
+  // Base64 helpers for binary payloads.
+  const decodeBase64 = (input) => {
+    try {
+      const binary = atob(input || "");
+      const bytes = new Uint8Array(binary.length);
+
+      Array.from(binary).forEach((char, index) => {
+        bytes[index] = char.charCodeAt(0);
+      });
+
+      return bytes;
+    } catch {
+      return new Uint8Array();
+    }
+  };
+
+  const bytesToBase64 = (bytes) => {
+    const binary = Array.from(bytes, (value) => String.fromCharCode(value)).join("");
+    return btoa(binary);
+  };
+
+  const textToBase64 = (text) => {
+    try {
+      const bytes = new TextEncoder().encode(text || "");
+      return bytesToBase64(bytes);
+    } catch {
+      return btoa(text || "");
+    }
+  };
+
+  // Resolve a content type from recorded response headers.
+  const getContentType = (record) => {
+    const headers = record.responseHeaders || {};
+    for (const key in headers) {
+      if (key.toLowerCase() === "content-type") {
+        return headers[key] || "application/octet-stream";
+      }
+    }
+    return "application/octet-stream";
+  };
+
+  // Turn a recorded response into a data URL for inline usage.
+  const toDataUrl = (record, fallbackType) => {
+    if (!record) return "";
+    const contentType = getContentType(record) || fallbackType || "application/octet-stream";
+    if (record.responseEncoding === "base64" && record.responseBodyBase64) {
+      return "data:" + contentType + ";base64," + record.responseBodyBase64;
+    }
+
+    if (record.responseBody) {
+      return "data:" + contentType + ";base64," + textToBase64(record.responseBody);
+    }
+    return "data:" + (fallbackType || "application/octet-stream") + ",";
+  };
+
+  // Build a real Response object from the recorded payload.
+  const responseFromRecord = (record) => {
+    const headers = new Headers(record.responseHeaders || {});
+    if (record.responseEncoding === "base64" && record.responseBodyBase64) {
+      const bytes = decodeBase64(record.responseBodyBase64);
+      return new Response(bytes, {
+        status: record.status || 200,
+        statusText: record.statusText || "OK",
+        headers
+      });
+    }
+    const bodyText = record.responseBody || "";
+    return new Response(bodyText, {
+      status: record.status || 200,
+      statusText: record.statusText || "OK",
+      headers
+    });
+  };
+
+${hackerScripts}
+})();
+</script>
+`;
+};
+exports.buildReplayScript = buildReplayScript;

package/dist/resources.d.ts

@@ -0,0 +1,16 @@
+import * as cheerio from "cheerio";
+export type ResourceReference = {
+    attr: string;
+    element: any;
+    url: string;
+};
+export type SrcsetReference = {
+    element: any;
+    value: string;
+};
+export declare const toAbsoluteUrl: (baseUrl: string, resourceUrl: string) => string;
+export declare const extractResourceUrls: (html: string, baseUrl: string) => {
+    $: cheerio.CheerioAPI;
+    resourceUrls: ResourceReference[];
+    srcsetItems: SrcsetReference[];
+};

package/dist/resources.js

@@ -0,0 +1,82 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractResourceUrls = exports.toAbsoluteUrl = void 0;
+const cheerio = __importStar(require("cheerio"));
+const toAbsoluteUrl = (baseUrl, resourceUrl) => {
+    try {
+        return new URL(resourceUrl, baseUrl).toString();
+    }
+    catch {
+        return resourceUrl;
+    }
+};
+exports.toAbsoluteUrl = toAbsoluteUrl;
+const extractResourceUrls = (html, baseUrl) => {
+    const $ = cheerio.load(html);
+    const urls = [];
+    const collect = (selector, attr) => {
+        $(selector).each((_, element) => {
+            const value = $(element).attr(attr);
+            if (value) {
+                urls.push({ attr, element });
+            }
+        });
+    };
+    collect("script[src]", "src");
+    collect("link[rel=stylesheet][href]", "href");
+    collect("link[rel=icon][href]", "href");
+    collect("img[src]", "src");
+    collect("source[src]", "src");
+    collect("video[src]", "src");
+    collect("audio[src]", "src");
+    const srcsetItems = [];
+    $("img[srcset], source[srcset]").each((_, element) => {
+        const value = $(element).attr("srcset");
+        if (value) {
+            srcsetItems.push({ element, value });
+        }
+    });
+    const resourceUrls = urls.map(({ attr, element }) => {
+        const value = $(element).attr(attr) || "";
+        return {
+            attr,
+            element,
+            url: (0, exports.toAbsoluteUrl)(baseUrl, value)
+        };
+    });
+    return { $, resourceUrls, srcsetItems };
+};
+exports.extractResourceUrls = extractResourceUrls;