@pagepocket/cli 0.4.1 → 0.5.0

package/README.md

@@ -1,8 +1,7 @@
 # @pagepocket/cli
 
-CLI for capturing offline snapshots of web pages. It fetches HTML, records network
-responses, downloads assets, rewrites links to local files, and injects a replay
-script so the snapshot works offline.
+CLI for capturing offline snapshots of web pages using the PagePocket library and
+NetworkInterceptorAdapter event streams.
 
 ## Install
 
@@ -19,18 +18,15 @@ pp https://example.com -o ./snapshots
 
 ## Output
 
-Snapshots are written to the current directory by default:
+Snapshots are written to a folder named after the page title (or `snapshot`) inside
+the output directory (default: current directory). Example layout:
 
-- `*.html`: offline snapshot page
-- `*.requests.json`: recorded requests/responses
-- `*_files/`: downloaded assets
-
-## Configuration
-
-Environment variables:
-
-- `PAGEPOCKET_FETCH_TIMEOUT_MS` (default: `60000`)
-- `PAGEPOCKET_FETCH_HEADERS` (JSON string of extra headers)
+```
+<output>/<title>/index.html
+<output>/<title>/api.json
+<output>/<title>/<same-origin paths>
+<output>/<title>/external_resources/<cross-origin paths>
+```
 
 ## Development
 

package/dist/cli.js

@@ -3,116 +3,41 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const node_path_1 = __importDefault(require("node:path"));
 const core_1 = require("@oclif/core");
-const chalk_1 = __importDefault(require("chalk"));
 const lib_1 = require("@pagepocket/lib");
-const with_spinner_1 = require("./utils/with-spinner");
-const build_snapshot_data_1 = require("./stages/build-snapshot-data");
-const capture_network_1 = require("./stages/capture-network");
-const fetch_html_1 = require("./stages/fetch-html");
+const lighterceptor_adapter_1 = require("@pagepocket/lighterceptor-adapter");
+const chalk_1 = __importDefault(require("chalk"));
 const prepare_output_1 = require("./stages/prepare-output");
 const write_snapshot_1 = require("./stages/write-snapshot");
+const with_spinner_1 = require("./utils/with-spinner");
 class PagepocketCommand extends core_1.Command {
     async run() {
         const { args, flags } = await this.parse(PagepocketCommand);
         const targetUrl = args.url;
         const outputFlag = flags.output ? flags.output.trim() : undefined;
-        const fetchTimeoutMs = Number(process.env.PAGEPOCKET_FETCH_TIMEOUT_MS || "60000");
-        const headersOverride = (() => {
-            const raw = process.env.PAGEPOCKET_FETCH_HEADERS;
-            if (!raw) {
-                return undefined;
-            }
-            try {
-                const parsed = JSON.parse(raw);
-                const headers = {};
-                for (const [key, value] of Object.entries(parsed)) {
-                    if (value === undefined || value === null) {
-                        continue;
-                    }
-                    headers[key] = String(value);
-                }
-                return headers;
-            }
-            catch {
-                throw new Error("Invalid PAGEPOCKET_FETCH_HEADERS JSON.");
-            }
-        })();
-        const fetchXhrRecords = [];
-        const fetched = await (0, with_spinner_1.withSpinner)(async () => (0, fetch_html_1.fetchHtml)(targetUrl, fetchTimeoutMs, headersOverride), "Fetching the target HTML");
-        const networkStage = await (async () => {
-            try {
-                return await (0, with_spinner_1.withSpinner)(async () => (0, capture_network_1.captureNetwork)(targetUrl, fetched.title), "Capturing network requests with lighterceptor");
-            }
-            catch {
-                return {
-                    networkRecords: [],
-                    lighterceptorNetworkRecords: [],
-                    capturedTitle: undefined,
-                    title: fetched.title
-                };
-            }
-        })();
-        const outputPaths = await (0, with_spinner_1.withSpinner)(async () => (0, prepare_output_1.prepareOutputPaths)(networkStage.title, outputFlag), "Preparing output paths");
-        const downloadStage = await (0, with_spinner_1.withSpinner)(async () => {
-            const originalCwd = process.cwd();
-            const shouldRestoreCwd = outputPaths.baseDir !== originalCwd;
-            try {
-                if (shouldRestoreCwd) {
-                    process.chdir(outputPaths.baseDir);
-                }
-                const seedSnapshot = {
-                    url: targetUrl,
-                    title: networkStage.title,
-                    capturedAt: new Date().toISOString(),
-                    fetchXhrRecords,
-                    networkRecords: networkStage.lighterceptorNetworkRecords,
-                    resources: []
-                };
-                const pagepocket = new lib_1.PagePocket(fetched.html, seedSnapshot, {
-                    assetsDirName: outputPaths.assetsDirName,
-                    baseUrl: targetUrl,
-                    requestsPath: node_path_1.default.basename(outputPaths.outputRequestsPath)
-                });
-                const snapshotHtml = await pagepocket.put();
-                return {
-                    snapshotHtml,
-                    resourceMeta: pagepocket.resources,
-                    downloadedCount: pagepocket.downloadedCount,
-                    failedCount: pagepocket.failedCount
-                };
-            }
-            finally {
-                if (shouldRestoreCwd) {
-                    try {
-                        process.chdir(originalCwd);
-                    }
-                    catch {
-                        // Ignore restore errors to preserve original failure.
-                    }
-                }
-            }
-        }, "Downloading resources");
-        const snapshotData = await (0, with_spinner_1.withSpinner)(async () => (0, build_snapshot_data_1.buildSnapshotData)({
-            targetUrl,
-            title: networkStage.title,
-            fetchXhrRecords,
-            lighterceptorNetworkRecords: networkStage.lighterceptorNetworkRecords,
-            resources: downloadStage.resourceMeta
-        }), "Preparing snapshot HTML");
+        const headers = {
+            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
+            accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+            "accept-language": "en-US,en;q=0.9",
+            referer: targetUrl
+        };
+        const snapshot = await (0, with_spinner_1.withSpinner)(async () => {
+            const interceptor = new lighterceptor_adapter_1.LighterceptorAdapter({ headers });
+            const pagepocket = lib_1.PagePocket.fromURL(targetUrl);
+            return pagepocket.capture({
+                interceptor,
+                completion: { wait: async () => { } }
+            });
+        }, "Capturing snapshot");
+        const { outputDir } = await (0, with_spinner_1.withSpinner)(() => (0, prepare_output_1.prepareOutputDir)(snapshot.title ?? "snapshot", outputFlag), "Preparing output directory");
         await (0, with_spinner_1.withSpinner)(async () => {
             await (0, write_snapshot_1.writeSnapshotFiles)({
-                outputRequestsPath: outputPaths.outputRequestsPath,
-                outputHtmlPath: outputPaths.outputHtmlPath,
-                snapshotData,
-                snapshotHtml: downloadStage.snapshotHtml
+                outputDir,
+                snapshot
             });
         }, "Writing snapshot files");
         this.log(chalk_1.default.green("All done! Snapshot created."));
-        this.log(`HTML saved to ${chalk_1.default.cyan(outputPaths.outputHtmlPath)}`);
-        this.log(`Requests saved to ${chalk_1.default.cyan(outputPaths.outputRequestsPath)}`);
-        this.log(`Resources saved to ${chalk_1.default.cyan(outputPaths.resourcesDir)}`);
+        this.log(`Snapshot saved to ${chalk_1.default.cyan(outputDir)}`);
         process.exit();
     }
 }

package/dist/stages/build-snapshot-data.js

@@ -7,7 +7,7 @@ const buildSnapshotData = (input) => {
         title: input.title,
         capturedAt: new Date().toISOString(),
         fetchXhrRecords: input.fetchXhrRecords,
-        networkRecords: input.lighterceptorNetworkRecords,
+        networkRecords: input.capturedNetworkRecords,
         resources: input.resources
     };
 };

package/dist/stages/capture-network.js

@@ -1,17 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.captureNetwork = void 0;
-const lighterceptor_1 = require("lighterceptor");
 const lib_1 = require("@pagepocket/lib");
+const lighterceptor_1 = require("@pagepocket/lighterceptor");
 const captureNetwork = async (targetUrl, currentTitle) => {
     const result = await new lighterceptor_1.Lighterceptor(targetUrl, { recursion: true }).run();
-    const lighterceptorNetworkRecords = (result.networkRecords ?? []);
-    const networkRecords = (0, lib_1.mapLighterceptorRecords)(lighterceptorNetworkRecords);
+    const capturedNetworkRecords = (result.networkRecords ?? []);
+    const networkRecords = (0, lib_1.mapCapturedNetworkRecords)(capturedNetworkRecords);
     const capturedTitle = result.title;
     const title = currentTitle === "snapshot" && capturedTitle ? capturedTitle : currentTitle;
     return {
         networkRecords,
-        lighterceptorNetworkRecords,
+        capturedNetworkRecords,
         capturedTitle,
         title
     };

package/dist/stages/prepare-output.js

@@ -3,25 +3,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.prepareOutputPaths = void 0;
+exports.prepareOutputDir = void 0;
 const promises_1 = __importDefault(require("node:fs/promises"));
 const node_path_1 = __importDefault(require("node:path"));
 const filename_1 = require("../lib/filename");
-const prepareOutputPaths = async (title, outputFlag) => {
+const prepareOutputDir = async (title, outputFlag) => {
     const safeTitle = (0, filename_1.safeFilename)(title || "snapshot");
     const baseDir = outputFlag ? node_path_1.default.resolve(outputFlag) : process.cwd();
-    const outputHtmlPath = node_path_1.default.join(baseDir, `${safeTitle}.html`);
-    const outputRequestsPath = node_path_1.default.join(baseDir, `${safeTitle}.requests.json`);
-    const assetsDirName = `${safeTitle}_files`;
-    const resourcesDir = node_path_1.default.join(baseDir, assetsDirName);
-    await promises_1.default.mkdir(resourcesDir, { recursive: true });
+    const outputDir = node_path_1.default.join(baseDir, safeTitle);
+    await promises_1.default.mkdir(outputDir, { recursive: true });
     return {
         safeTitle,
-        baseDir,
-        outputHtmlPath,
-        outputRequestsPath,
-        assetsDirName,
-        resourcesDir
+        outputDir
     };
 };
-exports.prepareOutputPaths = prepareOutputPaths;
+exports.prepareOutputDir = prepareOutputDir;

package/dist/stages/write-snapshot.js

@@ -1,12 +1,7 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.writeSnapshotFiles = void 0;
-const promises_1 = __importDefault(require("node:fs/promises"));
 const writeSnapshotFiles = async (input) => {
-    await promises_1.default.writeFile(input.outputRequestsPath, JSON.stringify(input.snapshotData, null, 2), "utf-8");
-    await promises_1.default.writeFile(input.outputHtmlPath, input.snapshotHtml, "utf-8");
+    await input.snapshot.toDirectory(input.outputDir);
 };
 exports.writeSnapshotFiles = writeSnapshotFiles;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pagepocket/cli",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "description": "CLI for capturing offline snapshots of web pages.",
   "main": "dist/index.js",
   "bin": {
@@ -14,16 +14,13 @@
   "license": "ISC",
   "dependencies": {
     "@oclif/core": "^4.0.9",
-    "cheerio": "^1.0.0-rc.12",
     "chalk": "^4.1.2",
-    "got": "^11.8.6",
     "ora": "^9.0.0",
-    "lighterceptor": "npm:@pagepocket/lighterceptor@0.4.1",
-    "@pagepocket/lib": "0.4.1"
+    "@pagepocket/lib": "0.5.0",
+    "@pagepocket/lighterceptor-adapter": "0.5.0"
   },
   "devDependencies": {
     "@types/node": "^20.11.30",
-    "prettier": "^3.7.4",
     "tsx": "^4.19.3",
     "typescript": "^5.4.5"
   },
@@ -36,8 +33,6 @@
   },
   "scripts": {
     "build": "tsc",
-    "format": "prettier --write .",
-    "format:check": "prettier --check .",
     "start": "node dist/index.js",
     "test": "tsx --test specs/**/*.test.ts"
   }