@pagepocket/cli 0.6.0 → 0.6.1

package/dist/lib/replay-script.js

@@ -1,346 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildReplayScript = void 0;
-const hackers_1 = require("./hackers");
-const buildReplayScript = (requestsPath, baseUrl) => {
-    const basePayload = JSON.stringify(baseUrl);
-    const requestsPayload = JSON.stringify(requestsPath);
-    const context = { stage: "replay" };
-    const hackerScripts = hackers_1.replayHackers
-        .map((hacker) => `  // hacker:${hacker.id}\n${hacker.build(context)}`)
-        .join("\n");
-    return `
-<script>
-(function(){
-  // Load the snapshot metadata before patching runtime APIs.
-  const baseUrl = ${basePayload};
-  const requestsUrl = ${requestsPayload};
-  const __pagepocketOriginalFetch = window.fetch ? window.fetch.bind(window) : null;
-
-  const loadSnapshot = async () => {
-    try {
-      if (!__pagepocketOriginalFetch) {
-        throw new Error("Fetch is unavailable");
-      }
-      const response = await __pagepocketOriginalFetch(requestsUrl);
-      if (!response.ok) {
-        throw new Error("Failed to load snapshot metadata");
-      }
-      return await response.json();
-    } catch {
-      return {
-        url: baseUrl,
-        title: "",
-        capturedAt: "",
-        fetchXhrRecords: [],
-        networkRecords: [],
-        resources: []
-      };
-    }
-  };
-
-  // Soften JSON parse failures to avoid halting replay flows.
-  const originalResponseJson = Response && Response.prototype && Response.prototype.json;
-  if (originalResponseJson) {
-    Response.prototype.json = function(...args) {
-      try {
-        return originalResponseJson.apply(this, args).catch(() => null);
-      } catch {
-        return Promise.resolve(null);
-      }
-    };
-  }
-
-  // Guard to reapply patches if overwritten later.
-  const ensureReplayPatches = () => {
-    try {
-      if (!window.fetch.__pagepocketOriginal && typeof __pagepocketOriginalFetch === "function") {
-        window.fetch.__pagepocketOriginal = __pagepocketOriginalFetch;
-      }
-    } catch {}
-    try {
-      if (!XMLHttpRequest.prototype.send.__pagepocketOriginal) {
-        XMLHttpRequest.prototype.send.__pagepocketOriginal = XMLHttpRequest.prototype.send;
-      }
-    } catch {}
-  };
-
-  let records = [];
-  let networkRecords = [];
-  const byKey = new Map();
-
-  const localResourceSet = new Set();
-  const resourceUrlMap = new Map();
-
-  const normalizeUrl = (input) => {
-    try { return new URL(input, baseUrl).toString(); } catch { return input; }
-  };
-
-  let baseOrigin = "";
-  let baseDir = "";
-  try {
-    const parsedBase = new URL(baseUrl);
-    baseOrigin = parsedBase.origin;
-    baseDir = new URL(".", parsedBase).toString().replace(/\\/$/, "");
-  } catch {}
-
-
-  const expandUrlVariants = (value) => {
-    const variants = [];
-    if (typeof value === "string") {
-      variants.push(value);
-      variants.push(normalizeUrl(value));
-      if (baseOrigin && value.startsWith("/")) {
-        variants.push(baseOrigin + value);
-        if (baseDir) variants.push(baseDir + value);
-      } else if (baseDir) {
-        variants.push(baseDir + (value.startsWith("/") ? value : "/" + value));
-      }
-      try {
-        const parsed = new URL(value, baseUrl);
-        const pathWithSearch = (parsed.pathname || "") + (parsed.search || "");
-        if (baseOrigin && parsed.origin !== baseOrigin) {
-          variants.push(baseOrigin + pathWithSearch);
-          if (baseDir) {
-            const path = pathWithSearch.startsWith("/") ? pathWithSearch : "/" + pathWithSearch;
-            variants.push(baseDir + path);
-          }
-        }
-      } catch {}
-    }
-    return Array.from(new Set(variants.filter(Boolean)));
-  };
-
-  const normalizeBody = (body) => {
-    if (body === undefined || body === null) return "";
-    if (typeof body === "string") return body;
-    try { return String(body); } catch { return ""; }
-  };
-
-  // Build a stable key so requests with identical method/url/body match the same response.
-  const makeKey = (method, url, body) => method.toUpperCase() + " " + normalizeUrl(url) + " " + normalizeBody(body);
-  const makeVariantKeys = (method, url, body) => {
-    return expandUrlVariants(url).map((variant) => makeKey(method, variant, body));
-  };
-  const normalizeNetworkRecord = (record) => {
-    if (!record || typeof record !== "object") {
-      return record;
-    }
-    if (record.response && record.response.body !== undefined) {
-      const response = record.response || {};
-      const encoding = response.bodyEncoding || "text";
-      return {
-        url: record.url,
-        method: record.method || "GET",
-        requestBody: record.requestBody || "",
-        status: response.status,
-        statusText: response.statusText,
-        responseHeaders: response.headers,
-        responseBody: encoding === "text" ? response.body : undefined,
-        responseBodyBase64: encoding === "base64" ? response.body : undefined,
-        responseEncoding: encoding,
-        error: record.error,
-        timestamp: record.timestamp
-      };
-    }
-    return record;
-  };
-
-  const primeLookups = (snapshot) => {
-    records = snapshot.fetchXhrRecords || [];
-    networkRecords = (snapshot.networkRecords || []).map(normalizeNetworkRecord);
-    byKey.clear();
-    localResourceSet.clear();
-    resourceUrlMap.clear();
-
-    for (const record of records) {
-      if (!record || !record.url || !record.method) continue;
-      const keys = makeVariantKeys(record.method, record.url, record.requestBody || "");
-      for (const key of keys) {
-        if (!byKey.has(key)) {
-          byKey.set(key, record);
-        }
-      }
-    }
-
-    for (const record of networkRecords) {
-      if (!record || !record.url || !record.method) continue;
-      const keys = makeVariantKeys(record.method, record.url, record.requestBody || "");
-      for (const key of keys) {
-        if (!byKey.has(key)) {
-          byKey.set(key, record);
-        }
-      }
-    }
-
-    // Track local resource files and map original URLs to local paths.
-    const resourceList = snapshot.resources || [];
-    for (const item of resourceList) {
-      if (!item || !item.localPath) continue;
-      localResourceSet.add(item.localPath);
-      localResourceSet.add("./" + item.localPath);
-
-      if (item.url) {
-        const variants = expandUrlVariants(item.url);
-        for (const variant of variants) {
-          resourceUrlMap.set(variant, item.localPath);
-        }
-      }
-    }
-  };
-
-
-  const ready = (async () => {
-    // Deserialize the snapshot and prepare lookup tables for offline responses.
-    const snapshot = (await loadSnapshot()) || {};
-    primeLookups(snapshot);
-    return snapshot;
-  })();
-
-  const isLocalResource = (value) => {
-    if (!value) return false;
-    if (value.startsWith("data:") || value.startsWith("blob:")) return true;
-    return localResourceSet.has(value);
-  };
-
-  // Lookup helpers for request records and local assets.
-  const findRecord = (method, url, body) => {
-    const variants = expandUrlVariants(url);
-    for (const variant of variants) {
-      const key = makeKey(method, variant, body);
-      if (byKey.has(key)) return byKey.get(key);
-    }
-    for (const variant of variants) {
-      const fallbackKey = makeKey(method, variant, "");
-      if (byKey.has(fallbackKey)) return byKey.get(fallbackKey);
-    }
-    for (const variant of variants) {
-      const getKey = makeKey("GET", variant, "");
-      if (byKey.has(getKey)) return byKey.get(getKey);
-    }
-    return null;
-  };
-
-  const findByUrl = (url) => {
-    if (isLocalResource(url)) return null;
-    const variants = expandUrlVariants(url);
-    for (const variant of variants) {
-      const direct = byKey.get(makeKey("GET", variant, ""));
-      if (direct) return direct;
-    }
-    // Attempt a looser match: ignore querystring if needed.
-    for (const variant of variants) {
-      try {
-        const withoutQuery = new URL(variant).origin + new URL(variant).pathname;
-        const direct = byKey.get(makeKey("GET", withoutQuery, ""));
-        if (direct) return direct;
-      } catch {}
-    }
-    return null;
-  };
-
-  const findLocalPath = (url) => {
-    if (!url) return null;
-    const variants = expandUrlVariants(url);
-    for (const variant of variants) {
-      const hit = resourceUrlMap.get(variant);
-      if (hit) return hit;
-    }
-    for (const variant of variants) {
-      try {
-        const withoutQuery = new URL(variant).origin + new URL(variant).pathname;
-        const hit = resourceUrlMap.get(withoutQuery);
-        if (hit) return hit;
-      } catch {}
-    }
-    // If still not found, fallback to data URLs if present.
-    return null;
-  };
-
-  // Safe property injection for emulating XHR state transitions.
-  const defineProp = (obj, key, value) => {
-    try {
-      Object.defineProperty(obj, key, { value, configurable: true });
-    } catch {}
-  };
-
-  // Base64 helpers for binary payloads.
-  const decodeBase64 = (input) => {
-    try {
-      const binary = atob(input || "");
-      const bytes = new Uint8Array(binary.length);
-
-      Array.from(binary).forEach((char, index) => {
-        bytes[index] = char.charCodeAt(0);
-      });
-
-      return bytes;
-    } catch {
-      return new Uint8Array();
-    }
-  };
-
-  const bytesToBase64 = (bytes) => {
-    const binary = Array.from(bytes, (value) => String.fromCharCode(value)).join("");
-    return btoa(binary);
-  };
-
-  const textToBase64 = (text) => {
-    try {
-      const bytes = new TextEncoder().encode(text || "");
-      return bytesToBase64(bytes);
-    } catch {
-      return btoa(text || "");
-    }
-  };
-
-  // Resolve a content type from recorded response headers.
-  const getContentType = (record) => {
-    const headers = record.responseHeaders || {};
-    for (const key in headers) {
-      if (key.toLowerCase() === "content-type") {
-        return headers[key] || "application/octet-stream";
-      }
-    }
-    return "application/octet-stream";
-  };
-
-  // Turn a recorded response into a data URL for inline usage.
-  const toDataUrl = (record, fallbackType) => {
-    if (!record) return "";
-    const contentType = getContentType(record) || fallbackType || "application/octet-stream";
-    if (record.responseEncoding === "base64" && record.responseBodyBase64) {
-      return "data:" + contentType + ";base64," + record.responseBodyBase64;
-    }
-
-    if (record.responseBody) {
-      return "data:" + contentType + ";base64," + textToBase64(record.responseBody);
-    }
-    return "data:" + (fallbackType || "application/octet-stream") + ",";
-  };
-
-  // Build a real Response object from the recorded payload.
-  const responseFromRecord = (record) => {
-    const headers = new Headers(record.responseHeaders || {});
-    if (record.responseEncoding === "base64" && record.responseBodyBase64) {
-      const bytes = decodeBase64(record.responseBodyBase64);
-      return new Response(bytes, {
-        status: record.status || 200,
-        statusText: record.statusText || "OK",
-        headers
-      });
-    }
-    const bodyText = record.responseBody || "";
-    return new Response(bodyText, {
-      status: record.status || 200,
-      statusText: record.statusText || "OK",
-      headers
-    });
-  };
-
-${hackerScripts}
-})();
-</script>
-`;
-};
-exports.buildReplayScript = buildReplayScript;

package/dist/lib/resources.js

@@ -1,131 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.applyResourceMapToDom = exports.downloadResource = exports.extractResourceUrls = exports.toAbsoluteUrl = void 0;
-const node_crypto_1 = __importDefault(require("node:crypto"));
-const promises_1 = __importDefault(require("node:fs/promises"));
-const node_path_1 = __importDefault(require("node:path"));
-const cheerio = __importStar(require("cheerio"));
-const content_type_1 = require("./content-type");
-const toAbsoluteUrl = (baseUrl, resourceUrl) => {
-    try {
-        return new URL(resourceUrl, baseUrl).toString();
-    }
-    catch {
-        return resourceUrl;
-    }
-};
-exports.toAbsoluteUrl = toAbsoluteUrl;
-const extractResourceUrls = (html, baseUrl) => {
-    const $ = cheerio.load(html);
-    const urls = [];
-    const collect = (selector, attr) => {
-        $(selector).each((_, element) => {
-            const value = $(element).attr(attr);
-            if (value) {
-                urls.push({ attr, element });
-            }
-        });
-    };
-    collect("script[src]", "src");
-    collect("link[rel=stylesheet][href]", "href");
-    collect("link[rel=icon][href]", "href");
-    collect("img[src]", "src");
-    collect("source[src]", "src");
-    collect("video[src]", "src");
-    collect("audio[src]", "src");
-    const srcsetItems = [];
-    $("img[srcset], source[srcset]").each((_, element) => {
-        const value = $(element).attr("srcset");
-        if (value) {
-            srcsetItems.push({ element, value });
-        }
-    });
-    const resourceUrls = urls.map(({ attr, element }) => {
-        const value = $(element).attr(attr) || "";
-        return {
-            attr,
-            element,
-            url: (0, exports.toAbsoluteUrl)(baseUrl, value)
-        };
-    });
-    return { $, resourceUrls, srcsetItems };
-};
-exports.extractResourceUrls = extractResourceUrls;
-const downloadResource = async (url, outputDir, referer) => {
-    const headers = {};
-    if (referer) {
-        headers.referer = referer;
-    }
-    const response = await fetch(url, { redirect: "follow", headers });
-    const contentType = response.headers.get("content-type");
-    const buffer = Buffer.from(await response.arrayBuffer());
-    const urlPath = new URL(url).pathname;
-    const ext = node_path_1.default.extname(urlPath) || (0, content_type_1.extensionFromContentType)(contentType);
-    const filename = `${node_crypto_1.default.createHash("sha1").update(url).digest("hex")}${ext}`;
-    const outputPath = node_path_1.default.join(outputDir, filename);
-    await promises_1.default.writeFile(outputPath, buffer);
-    return { outputPath, filename, contentType, size: buffer.length };
-};
-exports.downloadResource = downloadResource;
-const applyResourceMapToDom = ($, resourceUrls, srcsetItems, baseUrl, resourceMap, assetsDirName) => {
-    for (const resource of resourceUrls) {
-        const local = resourceMap.get(resource.url);
-        if (!local) {
-            continue;
-        }
-        $(resource.element).attr(resource.attr, node_path_1.default.join(assetsDirName, local));
-    }
-    for (const item of srcsetItems) {
-        const parts = item.value.split(",").map((part) => part.trim());
-        const rewritten = parts
-            .map((part) => {
-            const [url, descriptor] = part.split(/\s+/, 2);
-            const absolute = (0, exports.toAbsoluteUrl)(baseUrl, url);
-            const local = resourceMap.get(absolute);
-            if (!local) {
-                return part;
-            }
-            const nextUrl = node_path_1.default.join(assetsDirName, local);
-            return descriptor ? `${nextUrl} ${descriptor}` : nextUrl;
-        })
-            .join(", ");
-        $(item.element).attr("srcset", rewritten);
-    }
-};
-exports.applyResourceMapToDom = applyResourceMapToDom;

package/dist/lib/stages/download.js

@@ -1,61 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runDownloadStage = void 0;
-const ora_1 = __importDefault(require("ora"));
-const node_path_1 = __importDefault(require("node:path"));
-const resources_1 = require("../resources");
-const css_rewrite_1 = require("../css-rewrite");
-const runDownloadStage = async (html, targetUrl, networkRecords, resourcesDir, assetsDirName) => {
-    const downloadSpinner = (0, ora_1.default)("Downloading resources").start();
-    const dataUrlMap = (0, css_rewrite_1.buildDataUrlMap)(networkRecords);
-    const { $, resourceUrls, srcsetItems } = (0, resources_1.extractResourceUrls)(html, targetUrl);
-    const resourceMap = new Map();
-    const resourceMeta = [];
-    let downloadedCount = 0;
-    let failedCount = 0;
-    for (const resource of resourceUrls) {
-        const url = resource.url;
-        if (!url || resourceMap.has(url)) {
-            continue;
-        }
-        try {
-            const resourceLabel = (() => {
-                try {
-                    const pathname = new URL(url).pathname;
-                    const basename = node_path_1.default.basename(pathname);
-                    return basename || url;
-                }
-                catch {
-                    return url;
-                }
-            })();
-            downloadSpinner.text = `Downloading ${resourceLabel}`;
-            const { filename, contentType, size, outputPath } = await (0, resources_1.downloadResource)(url, resourcesDir, targetUrl);
-            if ((contentType && contentType.includes("text/css")) || outputPath.endsWith(".css")) {
-                await (0, css_rewrite_1.rewriteCssUrls)(outputPath, url, dataUrlMap);
-            }
-            resourceMap.set(url, filename);
-            resourceMeta.push({
-                url,
-                localPath: node_path_1.default.join(assetsDirName, filename),
-                contentType,
-                size
-            });
-            downloadedCount += 1;
-        }
-        catch {
-            failedCount += 1;
-            continue;
-        }
-    }
-    const downloadSummary = failedCount > 0
-        ? `Resources downloaded (${downloadedCount} saved, ${failedCount} failed)`
-        : `Resources downloaded (${downloadedCount} saved)`;
-    downloadSpinner.succeed(downloadSummary);
-    (0, resources_1.applyResourceMapToDom)($, resourceUrls, srcsetItems, targetUrl, resourceMap, assetsDirName);
-    return { resourceMap, resourceMeta, html: $.html(), $ };
-};
-exports.runDownloadStage = runDownloadStage;