@pagepocket/cli 0.6.0 → 0.6.1

package/dist/lib/stages/index.js

@@ -1,235 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runDownloadStage = exports.runTriggerStage = exports.runVisitStage = exports.runInterceptStage = exports.setupInterception = void 0;
-const ora_1 = __importDefault(require("ora"));
-const node_path_1 = __importDefault(require("node:path"));
-const resources_1 = require("../resources");
-const css_rewrite_1 = require("../css-rewrite");
-const hackers_1 = require("../hackers");
-const setupInterception = async (page, networkRecords) => {
-    await (0, hackers_1.applyCaptureHackers)({ stage: "capture", page, networkRecords });
-};
-exports.setupInterception = setupInterception;
-const runInterceptStage = () => {
-    const spinner = (0, ora_1.default)("Intercepting network requests").start();
-    spinner.succeed("Intercepted network requests");
-};
-exports.runInterceptStage = runInterceptStage;
-const runVisitStage = async (page, targetUrl, navigationTimeoutMs) => {
-    const spinner = (0, ora_1.default)("Visiting the target site").start();
-    const response = await page.goto(targetUrl, {
-        waitUntil: "domcontentloaded",
-        timeout: navigationTimeoutMs
-    });
-    await page.waitForSelector("body", { timeout: 15000 });
-    spinner.succeed("Visited the target site");
-    return response;
-};
-exports.runVisitStage = runVisitStage;
-const runTriggerStage = async (page, pendingTimeoutMs) => {
-    const spinner = (0, ora_1.default)("Triggering additional requests").start();
-    await page.waitForSelector("body", { timeout: 15000 });
-    await page.waitForNetworkIdle({ idleTime: 5000, timeout: 30000 }).catch(() => undefined);
-    await page
-        .waitForFunction(() => window.__pagepocketPendingRequests === 0, {
-        timeout: pendingTimeoutMs
-    })
-        .catch(() => undefined);
-    await new Promise((resolve) => setTimeout(resolve, 2000));
-    await page.evaluate(() => {
-        const events = ["mouseover", "mouseenter", "mousemove"];
-        const elements = Array.from(document.querySelectorAll("*"));
-        for (const el of elements) {
-            try {
-                const rect = el.getBoundingClientRect();
-                const hasSize = rect && rect.width >= 1 && rect.height >= 1;
-                const clientX = hasSize ? rect.left + rect.width / 2 : 0;
-                const clientY = hasSize ? rect.top + rect.height / 2 : 0;
-                for (const type of events) {
-                    const evt = new MouseEvent(type, {
-                        bubbles: true,
-                        cancelable: true,
-                        view: window,
-                        clientX,
-                        clientY
-                    });
-                    el.dispatchEvent(evt);
-                }
-            }
-            catch { }
-        }
-    });
-    await page.waitForNetworkIdle({ idleTime: 5000, timeout: 30000 }).catch(() => undefined);
-    await page
-        .waitForFunction(() => window.__pagepocketPendingRequests === 0, {
-        timeout: pendingTimeoutMs
-    })
-        .catch(() => undefined);
-    await new Promise((resolve) => setTimeout(resolve, 2000));
-    spinner.succeed("Triggered additional requests");
-};
-exports.runTriggerStage = runTriggerStage;
-const runDownloadStage = async (html, targetUrl, networkRecords, resourcesDir, assetsDirName) => {
-    const downloadSpinner = (0, ora_1.default)("Downloading resources").start();
-    const dataUrlMap = (0, css_rewrite_1.buildDataUrlMap)(networkRecords);
-    const { $, resourceUrls, srcsetItems } = (0, resources_1.extractResourceUrls)(html, targetUrl);
-    const resourceMap = new Map();
-    const resourceMeta = [];
-    let downloadedCount = 0;
-    let failedCount = 0;
-    for (const resource of resourceUrls) {
-        const url = resource.url;
-        if (!url || resourceMap.has(url)) {
-            continue;
-        }
-        try {
-            const resourceLabel = (() => {
-                try {
-                    const pathname = new URL(url).pathname;
-                    const basename = node_path_1.default.basename(pathname);
-                    return basename || url;
-                }
-                catch {
-                    return url;
-                }
-            })();
-            downloadSpinner.text = `Downloading ${resourceLabel}`;
-            const { filename, contentType, size, outputPath } = await (0, resources_1.downloadResource)(url, resourcesDir, targetUrl);
-            if ((contentType && contentType.includes("text/css")) || outputPath.endsWith(".css")) {
-                await (0, css_rewrite_1.rewriteCssUrls)(outputPath, url, dataUrlMap);
-            }
-            resourceMap.set(url, filename);
-            resourceMeta.push({
-                url,
-                localPath: node_path_1.default.join(assetsDirName, filename),
-                contentType,
-                size
-            });
-            downloadedCount += 1;
-        }
-        catch {
-            failedCount += 1;
-            continue;
-        }
-    }
-    const downloadSummary = failedCount > 0
-        ? `Resources downloaded (${downloadedCount} saved, ${failedCount} failed)`
-        : `Resources downloaded (${downloadedCount} saved)`;
-    downloadSpinner.succeed(downloadSummary);
-    (0, resources_1.applyResourceMapToDom)($, resourceUrls, srcsetItems, targetUrl, resourceMap, assetsDirName);
-    return { resourceMap, resourceMeta, html: $.html() };
-};
-exports.runDownloadStage = runDownloadStage;
-const runInterceptStage = async () => {
-    const spinner = (0, ora_1.default)("Intercepting network requests").start();
-    spinner.succeed("Intercepted network requests");
-};
-exports.runInterceptStage = runInterceptStage;
-const runVisitStage = async (page, targetUrl, navigationTimeoutMs) => {
-    const spinner = (0, ora_1.default)("Visiting the target site").start();
-    const response = await page.goto(targetUrl, {
-        waitUntil: "domcontentloaded",
-        timeout: navigationTimeoutMs
-    });
-    await page.waitForSelector("body", { timeout: 15000 });
-    spinner.succeed("Visited the target site");
-    return response;
-};
-exports.runVisitStage = runVisitStage;
-const runTriggerStage = async (page, pendingTimeoutMs) => {
-    const spinner = (0, ora_1.default)("Triggering additional requests").start();
-    await page.waitForSelector("body", { timeout: 15000 });
-    await page.waitForNetworkIdle({ idleTime: 5000, timeout: 30000 }).catch(() => undefined);
-    await page
-        .waitForFunction(() => window.__pagepocketPendingRequests === 0, {
-        timeout: pendingTimeoutMs
-    })
-        .catch(() => undefined);
-    await new Promise((resolve) => setTimeout(resolve, 2000));
-    await page.evaluate(() => {
-        const events = ["mouseover", "mouseenter", "mousemove"];
-        const elements = Array.from(document.querySelectorAll("*"));
-        for (const el of elements) {
-            try {
-                const rect = el.getBoundingClientRect();
-                const hasSize = rect && rect.width >= 1 && rect.height >= 1;
-                const clientX = hasSize ? rect.left + rect.width / 2 : 0;
-                const clientY = hasSize ? rect.top + rect.height / 2 : 0;
-                for (const type of events) {
-                    const evt = new MouseEvent(type, {
-                        bubbles: true,
-                        cancelable: true,
-                        view: window,
-                        clientX,
-                        clientY
-                    });
-                    el.dispatchEvent(evt);
-                }
-            }
-            catch { }
-        }
-    });
-    await page.waitForNetworkIdle({ idleTime: 5000, timeout: 30000 }).catch(() => undefined);
-    await page
-        .waitForFunction(() => window.__pagepocketPendingRequests === 0, {
-        timeout: pendingTimeoutMs
-    })
-        .catch(() => undefined);
-    await new Promise((resolve) => setTimeout(resolve, 2000));
-    spinner.succeed("Triggered additional requests");
-};
-exports.runTriggerStage = runTriggerStage;
-const runDownloadStage = async (html, targetUrl, networkRecords, resourcesDir, assetsDirName) => {
-    const downloadSpinner = (0, ora_1.default)("Downloading resources").start();
-    const dataUrlMap = (0, css_rewrite_1.buildDataUrlMap)(networkRecords);
-    const { $, resourceUrls, srcsetItems } = (0, resources_1.extractResourceUrls)(html, targetUrl);
-    const resourceMap = new Map();
-    const resourceMeta = [];
-    let downloadedCount = 0;
-    let failedCount = 0;
-    for (const resource of resourceUrls) {
-        const url = resource.url;
-        if (!url || resourceMap.has(url)) {
-            continue;
-        }
-        try {
-            const resourceLabel = (() => {
-                try {
-                    const pathname = new URL(url).pathname;
-                    const basename = node_path_1.default.basename(pathname);
-                    return basename || url;
-                }
-                catch {
-                    return url;
-                }
-            })();
-            downloadSpinner.text = `Downloading ${resourceLabel}`;
-            const { filename, contentType, size, outputPath } = await (0, resources_1.downloadResource)(url, resourcesDir, targetUrl);
-            if ((contentType && contentType.includes("text/css")) || outputPath.endsWith(".css")) {
-                await (0, css_rewrite_1.rewriteCssUrls)(outputPath, url, dataUrlMap);
-            }
-            resourceMap.set(url, filename);
-            resourceMeta.push({
-                url,
-                localPath: node_path_1.default.join(assetsDirName, filename),
-                contentType,
-                size
-            });
-            downloadedCount += 1;
-        }
-        catch {
-            failedCount += 1;
-            continue;
-        }
-    }
-    const downloadSummary = failedCount > 0
-        ? `Resources downloaded (${downloadedCount} saved, ${failedCount} failed)`
-        : `Resources downloaded (${downloadedCount} saved)`;
-    downloadSpinner.succeed(downloadSummary);
-    (0, resources_1.applyResourceMapToDom)($, resourceUrls, srcsetItems, targetUrl, resourceMap, assetsDirName);
-    return { resourceMap, resourceMeta, html: $.html() };
-};
-exports.runDownloadStage = runDownloadStage;

package/dist/lib/stages/intercept.js

@@ -1,23 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runInterceptStage = exports.setupInterception = void 0;
-const ora_1 = __importDefault(require("ora"));
-const hackers_1 = require("../hackers");
-const setupInterception = async (page, networkRecords) => {
-    await (0, hackers_1.applyCaptureHackers)({ stage: "capture", page, networkRecords });
-};
-exports.setupInterception = setupInterception;
-const runInterceptStage = async () => {
-    const spinner = (0, ora_1.default)("Intercepting network requests").start();
-    try {
-        spinner.succeed("Intercepted network requests");
-    }
-    catch (error) {
-        spinner.fail("Failed to intercept network requests");
-        throw error;
-    }
-};
-exports.runInterceptStage = runInterceptStage;

package/dist/lib/stages/trigger.js

@@ -1,56 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runTriggerStage = void 0;
-const ora_1 = __importDefault(require("ora"));
-const runTriggerStage = async (page, pendingTimeoutMs) => {
-    const spinner = (0, ora_1.default)("Triggering additional requests").start();
-    try {
-        await page.waitForSelector("body", { timeout: 15000 });
-        await page.waitForNetworkIdle({ idleTime: 5000, timeout: 30000 }).catch(() => undefined);
-        await page
-            .waitForFunction(() => window.__pagepocketPendingRequests === 0, {
-            timeout: pendingTimeoutMs
-        })
-            .catch(() => undefined);
-        await new Promise((resolve) => setTimeout(resolve, 2000));
-        await page.evaluate(() => {
-            const events = ["mouseover", "mouseenter", "mousemove"];
-            const elements = Array.from(document.querySelectorAll("*"));
-            for (const el of elements) {
-                try {
-                    const rect = el.getBoundingClientRect();
-                    const hasSize = rect && rect.width >= 1 && rect.height >= 1;
-                    const clientX = hasSize ? rect.left + rect.width / 2 : 0;
-                    const clientY = hasSize ? rect.top + rect.height / 2 : 0;
-                    for (const type of events) {
-                        const evt = new MouseEvent(type, {
-                            bubbles: true,
-                            cancelable: true,
-                            view: window,
-                            clientX,
-                            clientY
-                        });
-                        el.dispatchEvent(evt);
-                    }
-                }
-                catch { }
-            }
-        });
-        await page.waitForNetworkIdle({ idleTime: 5000, timeout: 30000 }).catch(() => undefined);
-        await page
-            .waitForFunction(() => window.__pagepocketPendingRequests === 0, {
-            timeout: pendingTimeoutMs
-        })
-            .catch(() => undefined);
-        await new Promise((resolve) => setTimeout(resolve, 2000));
-        spinner.succeed("Triggered additional requests");
-    }
-    catch (error) {
-        spinner.fail("Failed to trigger additional requests");
-        throw error;
-    }
-};
-exports.runTriggerStage = runTriggerStage;

package/dist/lib/stages/visit.js

@@ -1,24 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.runVisitStage = void 0;
-const ora_1 = __importDefault(require("ora"));
-const runVisitStage = async (page, targetUrl, navigationTimeoutMs) => {
-    const spinner = (0, ora_1.default)("Visiting the target site").start();
-    try {
-        const response = await page.goto(targetUrl, {
-            waitUntil: "domcontentloaded",
-            timeout: navigationTimeoutMs
-        });
-        await page.waitForSelector("body", { timeout: 15000 });
-        spinner.succeed("Visited the target site");
-        return response;
-    }
-    catch (error) {
-        spinner.fail("Failed to visit the target site");
-        throw error;
-    }
-};
-exports.runVisitStage = runVisitStage;

package/dist/lib/types.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/preload.js

@@ -1,60 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildPreloadScript = void 0;
-const hackers_1 = require("./lib/hackers");
-const buildPreloadScript = () => {
-    const context = { stage: "preload" };
-    const hackerScripts = hackers_1.preloadHackers
-        .map((hacker) => `  // hacker:${hacker.id}\n${hacker.build(context)}`)
-        .join("\n");
-    return `
-(function () {
-  if (window.__pagepocketPatched) {
-    return;
-  }
-  Object.defineProperty(window, "__pagepocketPatched", { value: true });
-
-  const records = [];
-  window.__pagepocketRecords = records;
-  window.__pagepocketPendingRequests = 0;
-
-  const toAbsoluteUrl = (input) => {
-    try {
-      return new URL(input, window.location.href).toString();
-    } catch {
-      return input;
-    }
-  };
-
-  const normalizeBody = (body) => {
-    if (body === undefined || body === null) {
-      return "";
-    }
-    if (typeof body === "string") {
-      return body;
-    }
-    if (body instanceof ArrayBuffer) {
-      try {
-        return new TextDecoder().decode(body);
-      } catch {
-        return "";
-      }
-    }
-    if (body instanceof Blob) {
-      return "";
-    }
-    return String(body);
-  };
-
-  const trackPendingStart = () => {
-    window.__pagepocketPendingRequests += 1;
-  };
-  const trackPendingEnd = () => {
-    window.__pagepocketPendingRequests = Math.max(0, window.__pagepocketPendingRequests - 1);
-  };
-
-${hackerScripts}
-})();
-`;
-};
-exports.buildPreloadScript = buildPreloadScript;

package/dist/stages/build-snapshot-data.js

@@ -1,14 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildSnapshotData = void 0;
-const buildSnapshotData = (input) => {
-    return {
-        url: input.targetUrl,
-        title: input.title,
-        capturedAt: new Date().toISOString(),
-        fetchXhrRecords: input.fetchXhrRecords,
-        networkRecords: input.capturedNetworkRecords,
-        resources: input.resources
-    };
-};
-exports.buildSnapshotData = buildSnapshotData;

package/dist/stages/build-snapshot.js

@@ -1,30 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.buildSnapshotHtml = void 0;
-const node_path_1 = __importDefault(require("node:path"));
-const replay_script_1 = require("../lib/replay-script");
-const buildSnapshotHtml = (input) => {
-    const replayScript = (0, replay_script_1.buildReplayScript)(node_path_1.default.basename(input.outputRequestsPath), input.targetUrl);
-    const head = input.$("head");
-    if (head.length) {
-        head.prepend(replayScript);
-    }
-    else {
-        input.$.root().prepend(replayScript);
-    }
-    if (input.faviconDataUrl) {
-        const existingIcon = input.$('link[rel="icon"]');
-        if (existingIcon.length) {
-            existingIcon.attr("href", input.faviconDataUrl);
-        }
-        else {
-            const link = '<link rel="icon" href="' + input.faviconDataUrl + '" />';
-            head.length ? head.append(link) : input.$.root().append(link);
-        }
-    }
-    return input.$.html();
-};
-exports.buildSnapshotHtml = buildSnapshotHtml;

package/dist/stages/capture-network.js

@@ -1,19 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.captureNetwork = void 0;
-const lib_1 = require("@pagepocket/lib");
-const lighterceptor_1 = require("@pagepocket/lighterceptor");
-const captureNetwork = async (targetUrl, currentTitle) => {
-    const result = await new lighterceptor_1.Lighterceptor(targetUrl, { recursion: true }).run();
-    const capturedNetworkRecords = (result.networkRecords ?? []);
-    const networkRecords = (0, lib_1.mapCapturedNetworkRecords)(capturedNetworkRecords);
-    const capturedTitle = result.title;
-    const title = currentTitle === "snapshot" && capturedTitle ? capturedTitle : currentTitle;
-    return {
-        networkRecords,
-        capturedNetworkRecords,
-        capturedTitle,
-        title
-    };
-};
-exports.captureNetwork = captureNetwork;

package/dist/stages/download-resources.js

@@ -1,48 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.downloadResources = void 0;
-const node_path_1 = __importDefault(require("node:path"));
-const css_rewrite_1 = require("../lib/css-rewrite");
-const resources_1 = require("../lib/resources");
-const downloadResources = async (input) => {
-    const dataUrlMap = (0, css_rewrite_1.buildDataUrlMap)(input.networkRecords);
-    const { $, resourceUrls, srcsetItems } = (0, resources_1.extractResourceUrls)(input.html, input.targetUrl);
-    const resourceMap = new Map();
-    const resourceMeta = [];
-    let downloadedCount = 0;
-    let failedCount = 0;
-    for (const resource of resourceUrls) {
-        const url = resource.url;
-        if (!url || resourceMap.has(url)) {
-            continue;
-        }
-        try {
-            const { filename, contentType, size, outputPath } = await (0, resources_1.downloadResource)(url, input.resourcesDir, input.targetUrl);
-            if ((contentType && contentType.includes("text/css")) || outputPath.endsWith(".css")) {
-                await (0, css_rewrite_1.rewriteCssUrls)(outputPath, url, dataUrlMap);
-            }
-            resourceMap.set(url, filename);
-            resourceMeta.push({
-                url,
-                localPath: node_path_1.default.join(input.assetsDirName, filename),
-                contentType,
-                size
-            });
-            downloadedCount += 1;
-        }
-        catch {
-            failedCount += 1;
-        }
-    }
-    (0, resources_1.applyResourceMapToDom)($, resourceUrls, srcsetItems, input.targetUrl, resourceMap, input.assetsDirName);
-    return {
-        $,
-        resourceMeta,
-        downloadedCount,
-        failedCount
-    };
-};
-exports.downloadResources = downloadResources;

package/dist/stages/fetch-html.js

@@ -1,69 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.fetchHtml = void 0;
-const cheerio = __importStar(require("cheerio"));
-const got_1 = __importDefault(require("got"));
-const fetchHtml = async (targetUrl, timeoutMs, headersOverride) => {
-    const headers = {
-        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
-        accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-        "accept-language": "en-US,en;q=0.9",
-        referer: targetUrl,
-        "Accept-Language": "zh-CN,zh;q=0.9",
-        "Sec-Fetch-Site": "same-origin",
-        "Sec-Fetch-Mode": "navigate",
-        "Sec-Fetch-Dest": "document",
-        ...headersOverride
-    };
-    const response = await (0, got_1.default)(targetUrl, {
-        headers,
-        followRedirect: true,
-        throwHttpErrors: false,
-        timeout: { request: timeoutMs }
-    });
-    if (response.statusCode < 200 || response.statusCode >= 300) {
-        const statusText = response.statusMessage ? ` ${response.statusMessage}` : "";
-        throw new Error(`HTTP ${response.statusCode}${statusText}`);
-    }
-    const html = response.body;
-    const $initial = cheerio.load(html);
-    const title = $initial("title").first().text() || "snapshot";
-    return { html, title };
-};
-exports.fetchHtml = fetchHtml;