@pagepocket/cli 0.4.0

package/dist/cli.js

@@ -0,0 +1,135 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_path_1 = __importDefault(require("node:path"));
+const core_1 = require("@oclif/core");
+const chalk_1 = __importDefault(require("chalk"));
+const lib_1 = require("@pagepocket/lib");
+const with_spinner_1 = require("./utils/with-spinner");
+const build_snapshot_data_1 = require("./stages/build-snapshot-data");
+const capture_network_1 = require("./stages/capture-network");
+const fetch_html_1 = require("./stages/fetch-html");
+const prepare_output_1 = require("./stages/prepare-output");
+const write_snapshot_1 = require("./stages/write-snapshot");
+class PagepocketCommand extends core_1.Command {
+    async run() {
+        const { args, flags } = await this.parse(PagepocketCommand);
+        const targetUrl = args.url;
+        const outputFlag = flags.output ? flags.output.trim() : undefined;
+        const fetchTimeoutMs = Number(process.env.PAGEPOCKET_FETCH_TIMEOUT_MS || "60000");
+        const headersOverride = (() => {
+            const raw = process.env.PAGEPOCKET_FETCH_HEADERS;
+            if (!raw) {
+                return undefined;
+            }
+            try {
+                const parsed = JSON.parse(raw);
+                const headers = {};
+                for (const [key, value] of Object.entries(parsed)) {
+                    if (value === undefined || value === null) {
+                        continue;
+                    }
+                    headers[key] = String(value);
+                }
+                return headers;
+            }
+            catch {
+                throw new Error("Invalid PAGEPOCKET_FETCH_HEADERS JSON.");
+            }
+        })();
+        const fetchXhrRecords = [];
+        const fetched = await (0, with_spinner_1.withSpinner)(async () => (0, fetch_html_1.fetchHtml)(targetUrl, fetchTimeoutMs, headersOverride), "Fetching the target HTML");
+        const networkStage = await (async () => {
+            try {
+                return await (0, with_spinner_1.withSpinner)(async () => (0, capture_network_1.captureNetwork)(targetUrl, fetched.title), "Capturing network requests with lighterceptor");
+            }
+            catch {
+                return {
+                    networkRecords: [],
+                    lighterceptorNetworkRecords: [],
+                    capturedTitle: undefined,
+                    title: fetched.title
+                };
+            }
+        })();
+        const outputPaths = await (0, with_spinner_1.withSpinner)(async () => (0, prepare_output_1.prepareOutputPaths)(networkStage.title, outputFlag), "Preparing output paths");
+        const downloadStage = await (0, with_spinner_1.withSpinner)(async () => {
+            const originalCwd = process.cwd();
+            const shouldRestoreCwd = outputPaths.baseDir !== originalCwd;
+            try {
+                if (shouldRestoreCwd) {
+                    process.chdir(outputPaths.baseDir);
+                }
+                const seedSnapshot = {
+                    url: targetUrl,
+                    title: networkStage.title,
+                    capturedAt: new Date().toISOString(),
+                    fetchXhrRecords,
+                    networkRecords: networkStage.lighterceptorNetworkRecords,
+                    resources: []
+                };
+                const pagepocket = new lib_1.PagePocket(fetched.html, seedSnapshot, {
+                    assetsDirName: outputPaths.assetsDirName,
+                    baseUrl: targetUrl,
+                    requestsPath: node_path_1.default.basename(outputPaths.outputRequestsPath)
+                });
+                const snapshotHtml = await pagepocket.put();
+                return {
+                    snapshotHtml,
+                    resourceMeta: pagepocket.resources,
+                    downloadedCount: pagepocket.downloadedCount,
+                    failedCount: pagepocket.failedCount
+                };
+            }
+            finally {
+                if (shouldRestoreCwd) {
+                    try {
+                        process.chdir(originalCwd);
+                    }
+                    catch {
+                        // Ignore restore errors to preserve original failure.
+                    }
+                }
+            }
+        }, "Downloading resources");
+        const snapshotData = await (0, with_spinner_1.withSpinner)(async () => (0, build_snapshot_data_1.buildSnapshotData)({
+            targetUrl,
+            title: networkStage.title,
+            fetchXhrRecords,
+            lighterceptorNetworkRecords: networkStage.lighterceptorNetworkRecords,
+            resources: downloadStage.resourceMeta
+        }), "Preparing snapshot HTML");
+        await (0, with_spinner_1.withSpinner)(async () => {
+            await (0, write_snapshot_1.writeSnapshotFiles)({
+                outputRequestsPath: outputPaths.outputRequestsPath,
+                outputHtmlPath: outputPaths.outputHtmlPath,
+                snapshotData,
+                snapshotHtml: downloadStage.snapshotHtml
+            });
+        }, "Writing snapshot files");
+        this.log(chalk_1.default.green("All done! Snapshot created."));
+        this.log(`HTML saved to ${chalk_1.default.cyan(outputPaths.outputHtmlPath)}`);
+        this.log(`Requests saved to ${chalk_1.default.cyan(outputPaths.outputRequestsPath)}`);
+        this.log(`Resources saved to ${chalk_1.default.cyan(outputPaths.resourcesDir)}`);
+        process.exit();
+    }
+}
+PagepocketCommand.description = "Save a snapshot of a web page.";
+PagepocketCommand.args = {
+    url: core_1.Args.string({
+        description: "URL to snapshot",
+        required: true
+    })
+};
+PagepocketCommand.flags = {
+    help: core_1.Flags.help({
+        char: "h"
+    }),
+    output: core_1.Flags.string({
+        char: "o",
+        description: "Output path for the snapshot HTML file"
+    })
+};
+exports.default = PagepocketCommand;

package/dist/index.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const core_1 = require("@oclif/core");
+(0, core_1.run)().catch((error) => {
+    const message = error && typeof error.message === "string" ? error.message : String(error);
+    console.error(message);
+    const exitCode = error && typeof error.exitCode === "number" ? error.exitCode : 1;
+    process.exit(exitCode);
+});

package/dist/lib/content-type.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isTextResponse = exports.extensionFromContentType = void 0;
+const extensionFromContentType = (contentType) => {
+    if (!contentType) {
+        return "";
+    }
+    if (contentType.includes("text/css"))
+        return ".css";
+    if (contentType.includes("javascript"))
+        return ".js";
+    if (contentType.includes("image/png"))
+        return ".png";
+    if (contentType.includes("image/jpeg"))
+        return ".jpg";
+    if (contentType.includes("image/gif"))
+        return ".gif";
+    if (contentType.includes("image/svg"))
+        return ".svg";
+    if (contentType.includes("font/woff2"))
+        return ".woff2";
+    if (contentType.includes("font/woff"))
+        return ".woff";
+    return "";
+};
+exports.extensionFromContentType = extensionFromContentType;
+const isTextResponse = (contentType) => {
+    const lowered = contentType.toLowerCase();
+    return (lowered.startsWith("text/") ||
+        lowered.includes("json") ||
+        lowered.includes("javascript") ||
+        lowered.includes("xml") ||
+        lowered.includes("svg") ||
+        lowered.includes("html"));
+};
+exports.isTextResponse = isTextResponse;

package/dist/lib/css-rewrite.js

@@ -0,0 +1,62 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rewriteCssUrls = exports.buildDataUrlMap = void 0;
+const promises_1 = __importDefault(require("node:fs/promises"));
+const getHeaderValue = (headers, name) => {
+    // Normalize header names to find the requested value.
+    for (const key in headers) {
+        if (key.toLowerCase() === name.toLowerCase()) {
+            return headers[key];
+        }
+    }
+    return undefined;
+};
+const buildDataUrlMap = (records) => {
+    // Build a lookup of recorded binary responses so CSS url() can be inlined.
+    const map = new Map();
+    for (const record of records) {
+        if (!record || !record.url || !record.responseBodyBase64) {
+            continue;
+        }
+        const headers = record.responseHeaders || {};
+        const contentType = getHeaderValue(headers, "content-type") || "application/octet-stream";
+        map.set(record.url, `data:${contentType};base64,${record.responseBodyBase64}`);
+    }
+    return map;
+};
+exports.buildDataUrlMap = buildDataUrlMap;
+const rewriteCssUrls = async (filePath, cssUrl, dataUrlMap) => {
+    // Replace external URLs in CSS with data URLs to preserve rendering offline.
+    const css = await promises_1.default.readFile(filePath, "utf-8");
+    const urlPattern = /url\(\s*(['"]?)([^'")]+)\1\s*\)/g;
+    const rewritten = css.replace(urlPattern, (match, quote, rawUrl) => {
+        const trimmed = String(rawUrl || "").trim();
+        if (!trimmed || trimmed.startsWith("data:") || trimmed.startsWith("blob:")) {
+            return match;
+        }
+        const absolute = (() => {
+            try {
+                return new URL(trimmed, cssUrl).toString();
+            }
+            catch {
+                return null;
+            }
+        })();
+        if (!absolute) {
+            return match;
+        }
+        const dataUrl = dataUrlMap.get(absolute);
+        if (!dataUrl) {
+            return match;
+        }
+        const safeQuote = quote || "";
+        return `url(${safeQuote}${dataUrl}${safeQuote})`;
+    });
+    if (rewritten !== css) {
+        await promises_1.default.writeFile(filePath, rewritten, "utf-8");
+    }
+};
+exports.rewriteCssUrls = rewriteCssUrls;

package/dist/lib/filename.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.safeFilename = void 0;
+const safeFilename = (input) => {
+    const trimmed = input.trim();
+    if (!trimmed) {
+        return "snapshot";
+    }
+    return (trimmed
+        .replace(/[^a-zA-Z0-9._-]+/g, "_")
+        .replace(/^_+|_+$/g, "")
+        .slice(0, 120) || "snapshot");
+};
+exports.safeFilename = safeFilename;

package/dist/lib/hackers/capture-network.js

@@ -0,0 +1,64 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.captureNetworkRecorder = void 0;
+const content_type_1 = require("../content-type");
+exports.captureNetworkRecorder = {
+    id: "capture-network-recorder",
+    stage: "capture",
+    apply: async ({ page, networkRecords }) => {
+        await page.setRequestInterception(true);
+        page.on("request", (request) => {
+            request.continue().catch(() => undefined);
+        });
+        page.on("response", async (response) => {
+            const request = response.request();
+            const url = response.url();
+            const headers = response.headers();
+            const requestHeaders = request.headers();
+            const requestBody = request.postData() || "";
+            // Capture the response body while preserving encoding for replay.
+            const { responseBody, responseBodyBase64, responseEncoding, error } = await (async () => {
+                try {
+                    const buffer = await response.buffer();
+                    const contentType = headers["content-type"] || "";
+                    if ((0, content_type_1.isTextResponse)(contentType)) {
+                        return {
+                            responseBody: buffer.toString("utf-8"),
+                            responseBodyBase64: undefined,
+                            responseEncoding: "text",
+                            error: undefined
+                        };
+                    }
+                    return {
+                        responseBody: undefined,
+                        responseBodyBase64: buffer.toString("base64"),
+                        responseEncoding: "base64",
+                        error: undefined
+                    };
+                }
+                catch (err) {
+                    return {
+                        responseBody: undefined,
+                        responseBodyBase64: undefined,
+                        responseEncoding: undefined,
+                        error: String(err)
+                    };
+                }
+            })();
+            networkRecords.push({
+                url,
+                method: request.method(),
+                requestHeaders,
+                requestBody,
+                status: response.status(),
+                statusText: response.statusText(),
+                responseHeaders: headers,
+                responseBody,
+                responseBodyBase64,
+                responseEncoding,
+                error,
+                timestamp: Date.now()
+            });
+        });
+    }
+};

package/dist/lib/hackers/index.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.replayHackers = exports.preloadHackers = void 0;
+const preload_fetch_1 = require("./preload-fetch");
+const preload_xhr_1 = require("./preload-xhr");
+const replay_beacon_1 = require("./replay-beacon");
+const replay_dom_rewrite_1 = require("./replay-dom-rewrite");
+const replay_eventsource_1 = require("./replay-eventsource");
+const replay_fetch_1 = require("./replay-fetch");
+const replay_svg_image_1 = require("./replay-svg-image");
+const replay_websocket_1 = require("./replay-websocket");
+const replay_xhr_1 = require("./replay-xhr");
+exports.preloadHackers = [preload_fetch_1.preloadFetchRecorder, preload_xhr_1.preloadXhrRecorder];
+exports.replayHackers = [
+    replay_fetch_1.replayFetchResponder,
+    replay_xhr_1.replayXhrResponder,
+    replay_dom_rewrite_1.replayDomRewriter,
+    replay_svg_image_1.replaySvgImageRewriter,
+    replay_beacon_1.replayBeaconStub,
+    replay_websocket_1.replayWebSocketStub,
+    replay_eventsource_1.replayEventSourceStub
+];

package/dist/lib/hackers/preload-fetch.js

@@ -0,0 +1,56 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.preloadFetchRecorder = void 0;
+exports.preloadFetchRecorder = {
+    id: "preload-fetch-recorder",
+    stage: "preload",
+    build: () => `
+  // Record fetch calls and responses for replay.
+  const originalFetch = window.fetch.bind(window);
+  const recordFetch = async (input, init) => {
+    trackPendingStart();
+    const url = typeof input === "string" ? toAbsoluteUrl(input) : toAbsoluteUrl(input.url);
+    const method = init && init.method ? init.method : (typeof input === "string" ? "GET" : input.method || "GET");
+    const requestBody = normalizeBody(init && init.body ? init.body : (typeof input === "string" ? undefined : input.body));
+
+    try {
+      const response = await originalFetch(input, init);
+      const clone = response.clone();
+      const responseBody = await clone.text();
+      const headers = {};
+      response.headers.forEach((value, key) => {
+        headers[key] = value;
+      });
+      records.push({
+        kind: "fetch",
+        url,
+        method,
+        requestBody,
+        status: response.status,
+        statusText: response.statusText,
+        responseHeaders: headers,
+        responseBody,
+        timestamp: Date.now()
+      });
+      return response;
+    } catch (error) {
+      records.push({
+        kind: "fetch",
+        url,
+        method,
+        requestBody,
+        error: String(error),
+        timestamp: Date.now()
+      });
+      throw error;
+    } finally {
+      trackPendingEnd();
+    }
+  };
+
+  window.fetch = (input, init) => {
+    return recordFetch(input, init);
+  };
+  window.fetch.__pagepocketOriginal = originalFetch;
+`
+};

package/dist/lib/hackers/preload-image.js

@@ -0,0 +1,61 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.preloadImageRecorder = void 0;
+exports.preloadImageRecorder = {
+    id: "preload-image-recorder",
+    stage: "preload",
+    build: () => `
+  // Intercept image src assignments and record them without triggering network requests.
+  const NativeImage = window.Image;
+  const placeholderSrc =
+    "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///ywAAAAAAQABAAACAUwAOw==";
+  const nativeDescriptor = Object.getOwnPropertyDescriptor(HTMLImageElement.prototype, "src");
+  const nativeSet = nativeDescriptor && nativeDescriptor.set ? nativeDescriptor.set : null;
+  const nativeGet = nativeDescriptor && nativeDescriptor.get ? nativeDescriptor.get : null;
+
+  const recordImage = (input) => {
+    if (!input) return;
+    const url = toAbsoluteUrl(input);
+    records.push({
+      kind: "image",
+      url,
+      method: "GET",
+      requestBody: "",
+      timestamp: Date.now()
+    });
+  };
+
+  const shouldBypass = (value) => {
+    const raw = String(value || "");
+    if (!raw) return true;
+    return raw.startsWith("data:") || raw.startsWith("blob:");
+  };
+
+  Object.defineProperty(HTMLImageElement.prototype, "src", {
+    set(value) {
+      console.info(value)
+
+      if (shouldBypass(value)) {
+        if (nativeSet) nativeSet.call(this, value);
+        return;
+      }
+      const url = toAbsoluteUrl(value);
+      this.__webechoImageSrc = url;
+      recordImage(url);
+      if (nativeSet) nativeSet.call(this, placeholderSrc);
+    },
+    get() {
+      if (this.__webechoImageSrc) return this.__webechoImageSrc;
+      return nativeGet ? nativeGet.call(this) : "";
+    },
+    configurable: true,
+    enumerable: true
+  });
+
+  window.Image = function(...args) {
+    return new NativeImage(...args);
+  };
+  window.Image.prototype = NativeImage.prototype;
+  Object.defineProperty(window.Image, "name", { value: "Image" });
+`
+};

package/dist/lib/hackers/preload-xhr.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.preloadXhrRecorder = void 0;
+exports.preloadXhrRecorder = {
+    id: "preload-xhr-recorder",
+    stage: "preload",
+    build: () => `
+  // Record XHR calls and responses for replay.
+  const originalOpen = XMLHttpRequest.prototype.open;
+  const originalSend = XMLHttpRequest.prototype.send;
+
+  XMLHttpRequest.prototype.open = function(method, url, ...rest) {
+    this.__pagepocketMethod = method;
+    this.__pagepocketUrl = toAbsoluteUrl(url);
+    return originalOpen.call(this, method, url, ...rest);
+  };
+
+  XMLHttpRequest.prototype.send = function(body) {
+    const xhr = this;
+    const requestBody = normalizeBody(body);
+
+    const onLoadEnd = () => {
+      const responseHeadersRaw = xhr.getAllResponseHeaders();
+      const headers = {};
+      responseHeadersRaw
+        .trim()
+        .split(/\\r?\\n/)
+        .filter(Boolean)
+        .forEach((line) => {
+          const index = line.indexOf(":");
+          if (index > -1) {
+            const key = line.slice(0, index).trim().toLowerCase();
+            const value = line.slice(index + 1).trim();
+            headers[key] = value;
+          }
+        });
+
+      records.push({
+        kind: "xhr",
+        url: xhr.__pagepocketUrl || "",
+        method: xhr.__pagepocketMethod || "GET",
+        requestBody,
+        status: xhr.status,
+        statusText: xhr.statusText,
+        responseHeaders: headers,
+        responseBody: xhr.responseText || "",
+        timestamp: Date.now()
+      });
+
+      xhr.removeEventListener("loadend", onLoadEnd);
+      trackPendingEnd();
+    };
+
+    trackPendingStart();
+    xhr.addEventListener("loadend", onLoadEnd);
+    return originalSend.call(xhr, body);
+  };
+`
+};

package/dist/lib/hackers/replay-beacon.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.replayBeaconStub = void 0;
+exports.replayBeaconStub = {
+    id: "replay-beacon-stub",
+    stage: "replay",
+    build: () => `
+  // Stub beacon calls so analytics doesn't leak outside the snapshot.
+  if (navigator.sendBeacon) {
+    const originalBeacon = navigator.sendBeacon.bind(navigator);
+    navigator.sendBeacon = (url, data) => {
+      const record = findRecord("POST", url, data);
+      if (record) {
+        return true;
+      }
+      return true;
+    };
+    navigator.sendBeacon.__pagepocketOriginal = originalBeacon;
+  }
+`
+};