@link-assistant/web-capture 1.11.2 → 1.12.0

package/.dockerignore

@@ -0,0 +1,6 @@
+node_modules
+reports
+coverage
+*.log
+.captures
+capture-replay.json

package/.jscpd.json

@@ -1,8 +1,6 @@
 {
-  "threshold": 0,
   "minTokens": 30,
   "minLines": 5,
-  "skipComments": true,
   "ignore": [
     "**/node_modules/**",
     "**/build/**",
@@ -12,9 +10,10 @@
     "**/.changeset/**",
     "**/package-lock.json",
     "**/pnpm-lock.yaml",
-    "**/yarn.lock"
+    "**/yarn.lock",
+    "**/reports/**"
   ],
-  "format": "console",
   "reporters": ["console", "html"],
-  "output": "./reports/jscpd"
+  "output": "./reports/jscpd",
+  "mode": "weak"
 }

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @link-assistant/web-capture
 
+## 1.12.0
+
+### Minor Changes
+
+- a6c74ba: Add a SHA-256 disk capture cache with TTL, offline stale replay, portable receipts, listing and pruning. Upgrade Puppeteer to 25 and forward packaged browser paths through browser-commander 0.10.0. Update other direct dependencies and require audit/freshness checks in CI. Node.js 22.22.1 or later is now required. Bundled consumers must externalize Playwright and Playwright Core.
+
 ## 1.11.2
 
 ### Patch Changes

package/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:22-bullseye
+FROM node:24-bookworm
 
 WORKDIR /app
 
@@ -27,12 +27,14 @@ RUN apt-get update && \
         libxshmfence1 \
         xdg-utils \
         --no-install-recommends && break || \
-      { echo "apt-get install attempt $i failed, retrying..."; sleep 5; }; \
+      { if [ "$i" -eq 3 ]; then exit 1; fi; \
+        echo "apt-get install attempt $i failed, retrying..."; \
+        apt-get update && sleep 5; }; \
     done && \
     rm -rf /var/lib/apt/lists/*
 
 # Set Puppeteer environment variables
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
+ENV PUPPETEER_SKIP_DOWNLOAD=true
 ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
 ENV PUPPETEER_ARGS="--no-sandbox --disable-setuid-sandbox --disable-dev-shm-usage"
 
@@ -40,8 +42,8 @@ ENV PUPPETEER_ARGS="--no-sandbox --disable-setuid-sandbox --disable-dev-shm-usag
 ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
 ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium
 
-COPY package.json yarn.lock ./
-RUN yarn install --frozen-lockfile
+COPY package.json package-lock.json ./
+RUN npm ci
 
 COPY . .
 

package/README.md

@@ -19,6 +19,16 @@ A CLI and microservice to fetch URLs and render them as:
 
 ## Installation
 
+Node.js 22.22.1 or newer is required. CI and Docker use Node.js 24. Use `npm ci`
+for a locked repository install; Docker and CI share `package-lock.json` and its
+security overrides.
+
+Bundled applications must externalize `playwright` and `playwright-core` and
+ship them as production packages. Playwright does not support bundling its
+internal modules. Packaged Chromium is supported through
+`createBrowser(engine, { executablePath: packagedPath, channel: 'chrome' })`
+for both engines. See [the upstream bundling discussion](https://github.com/microsoft/playwright/issues/33031).
+
 ### From npm
 
 ```bash
@@ -518,14 +528,14 @@ with environment variables:
 
 ```bash
 # Download the Wikipedia page (markdown + image) in every supported engine
-WIKIPEDIA_INTEGRATION=true npm test -- --testPathPattern="wikipedia-download"
+WIKIPEDIA_INTEGRATION=true npm test -- --testPathPatterns="wikipedia-download"
 
 # Download a GitHub repository page as compact txt/markdown, original HTML, and screenshots
-GITHUB_REPOSITORY_INTEGRATION=true npm test -- --testPathPattern="github-readme"
+GITHUB_REPOSITORY_INTEGRATION=true npm test -- --testPathPatterns="github-readme"
 
 # Habr articles and public Google Docs live suites
-HABR_INTEGRATION=true npm test -- --testPathPattern="habr-article"
-GDOCS_INTEGRATION=true npm test -- --testPathPattern="gdocs-public-doc"
+HABR_INTEGRATION=true npm test -- --testPathPatterns="habr-article"
+GDOCS_INTEGRATION=true npm test -- --testPathPatterns="gdocs-public-doc"
 ```
 
 ## Built With
@@ -598,3 +608,11 @@ JSON response does not duplicate the full provider body.
 ## License
 
 [Unlicense](../LICENSE) — This is free and unencumbered software released into the public domain. You are free to copy, modify, publish, use, compile, sell, or distribute this software for any purpose, commercial or non-commercial, and by any means. See [https://unlicense.org](https://unlicense.org) for details.
+
+## Capture cache
+
+Both packages provide `CaptureStore` and `CachedTransport` over caller-owned transports. Captures keep the exact body bytes, final URL, status, response headers and diagnostics. Request keys include the exact URL, method and every request header in sorted order. Identical bodies share a SHA-256 file. The default TTL is 60 days; expiry is measured from fetch completion, in Unix milliseconds.
+
+Online mode reuses fresh captures and refetches expired entries. Offline mode never invokes the transport: it returns cached captures with `cached: true` and a `stale` flag, or an `offline_cache_miss` error. Use `capture` to read these flags; the transport adapter returns the original receipt unchanged. Transport errors are not stored. Corrupt or unsupported records produce cache errors instead of returning unverified bytes.
+
+`list` returns request metadata; `prune(now, ttl)` removes expired request records and unreferenced body files. `export(request)` and `import(json)` exchange version 1 JSON replay fixtures with byte arrays, shared between Rust and JavaScript. See [the cache format and examples](../docs/capture-cache.md).

package/examples/js/capture_cache.js

@@ -0,0 +1,12 @@
+import { CaptureStore, CachedTransport } from '../../src/cache.js';
+import { fetchTransport } from '../../src/transport.js';
+
+const store = new CaptureStore('.captures');
+const request = { url: 'https://example.com', method: 'GET', headers: {} };
+const online = new CachedTransport(fetchTransport, store);
+const first = await online.capture(request);
+const offline = new CachedTransport(fetchTransport, store, { offline: true });
+const replay = await offline.capture(request);
+console.log(replay.sha256, replay.cached, replay.stale);
+console.log('Exact replay:', first.receipt.body.equals(replay.receipt.body));
+console.log('Portable fixture:', await store.export(request));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.11.2",
+  "version": "1.12.0",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {
@@ -11,19 +11,19 @@
     "dev": "nodemon bin/web-capture.js -- --serve",
     "test": "node --experimental-vm-modules ./node_modules/.bin/jest",
     "test:watch": "node --experimental-vm-modules ./node_modules/.bin/jest --watch",
-    "test:e2e:process": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e.process.test.js",
-    "test:e2e:docker": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e.docker.test.js",
-    "test:e2e": "yarn test:e2e:process && yarn test:e2e:docker",
-    "test:all": "yarn build && yarn test && yarn test:e2e",
-    "build": "docker compose down && docker compose build && yarn start",
+    "test:e2e:process": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e/process.test.js",
+    "test:e2e:docker": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e/docker.test.js",
+    "test:e2e": "npm run test:e2e:process && npm run test:e2e:docker",
+    "test:all": "npm run build && npm test && npm run test:e2e",
+    "build": "docker compose down && docker compose build && npm start",
     "examples:python": "python3 examples/python/html_download.py && python3 examples/python/markdown_download.py && python3 examples/python/image_download.py",
     "examples:javascript": "node examples/js/html_download.js && node examples/js/markdown_download.js && node examples/js/image_download.js",
-    "examples": "yarn build && yarn examples:javascript && yarn examples:python",
+    "examples": "npm run build && npm run examples:javascript && npm run examples:python",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "format": "prettier --write .",
     "format:check": "prettier --check .",
-    "check:duplication": "jscpd .",
+    "check:duplication": "jscpd . --baseline-from-ref origin/main --fail-on-new-clones --fail-on-empty",
     "check": "npm run lint && npm run format:check && npm run check:duplication",
     "prepare": "husky || true",
     "changeset": "changeset",
@@ -32,42 +32,43 @@
     "changeset:status": "changeset status --since=origin/main"
   },
   "dependencies": {
-    "@kreuzberg/html-to-markdown-node": "^3.5.5",
+    "@kreuzberg/html-to-markdown-node": "^3.7.2",
     "archiver": "^8.0.0",
-    "browser-commander": "^0.8.0",
-    "cheerio": "^1.0.0",
-    "docx": "^9.6.1",
-    "express": "^4.18.2",
-    "he": "^1.2.0",
-    "iconv-lite": "^0.6.3",
+    "browser-commander": "0.10.0",
+    "cheerio": "^1.2.0",
+    "docx": "^9.9.0",
+    "express": "^5.2.1",
+    "he": "^2.0.0",
+    "iconv-lite": "^0.7.3",
     "lino-arguments": "^0.3.0",
     "log-lazy": "^1.0.4",
-    "node-fetch": "^2.7.0",
-    "playwright": "^1.49.0",
-    "puppeteer": "^24.8.2",
-    "puppeteer-core": "^24.40.0",
-    "turndown": "^7.1.1",
+    "node-fetch": "^3.3.2",
+    "playwright": "^1.63.0",
+    "puppeteer": "^25.12.0",
+    "puppeteer-core": "^25.12.0",
+    "turndown": "^7.2.4",
     "turndown-plugin-gfm": "^1.0.2"
   },
   "devDependencies": {
-    "@babel/core": "^7.24.0",
-    "@babel/preset-env": "^7.24.0",
-    "@changesets/cli": "^2.29.7",
-    "@types/archiver": "^7.0.0",
-    "babel-jest": "^29.7.0",
-    "eslint": "^9.38.0",
+    "@babel/core": "^8.0.7",
+    "@babel/preset-env": "^8.0.7",
+    "@changesets/cli": "^3.0.3",
+    "@types/archiver": "^8.0.0",
+    "babel-jest": "^30.5.2",
+    "eslint": "^10.12.0",
     "eslint-config-prettier": "^10.1.8",
-    "eslint-plugin-prettier": "^5.5.4",
-    "get-port": "^7.1.0",
+    "eslint-plugin-prettier": "^5.5.6",
+    "get-port": "^7.2.0",
     "husky": "^9.1.7",
-    "jest": "^29.7.0",
-    "jscpd": "^4.0.5",
-    "lint-staged": "^16.2.6",
-    "markdown-it": "^14.1.1",
-    "nock": "^13.5.4",
-    "prettier": "^3.6.2",
-    "supertest": "^6.3.4",
-    "unzipper": "^0.12.3"
+    "jest": "^30.5.2",
+    "jscpd": "^5.4.0",
+    "lint-staged": "^17.6.0",
+    "markdown-it": "^15.0.2",
+    "nock": "^15.0.1",
+    "prettier": "^3.9.9",
+    "supertest": "^7.3.1",
+    "unzipper": "^0.12.5",
+    "@eslint/js": "^10.0.1"
   },
   "type": "module",
   "keywords": [
@@ -82,7 +83,7 @@
     "url": "https://github.com/link-assistant/web-capture"
   },
   "engines": {
-    "node": ">=22.0.0 <23.0.0"
+    "node": ">=22.22.1"
   },
   "lint-staged": {
     "*.{js,mjs,cjs}": [
@@ -94,5 +95,16 @@
       "prettier --write",
       "prettier --check"
     ]
+  },
+  "dependencyBlockers": {
+    "browser-commander": {
+      "issue": "https://github.com/link-assistant/web-capture/issues/160",
+      "reason": "Keep the first compatible packaged-browser release; later versions add native better-sqlite3 dependencies."
+    }
+  },
+  "overrides": {
+    "@istanbuljs/load-nyc-config": {
+      "js-yaml": "^4.1.1"
+    }
   }
 }

package/src/cache.js

@@ -0,0 +1,313 @@
+import { createHash, randomUUID } from 'node:crypto';
+import {
+  mkdir,
+  readFile,
+  writeFile,
+  rename,
+  unlink,
+  readdir,
+} from 'node:fs/promises';
+import path from 'node:path';
+import { captureResponse } from './transport.js';
+
+const VERSION = 1;
+const queues = new Map();
+const digest = (bytes) => createHash('sha256').update(bytes).digest('hex');
+
+export class CaptureCacheError extends Error {
+  constructor(message, kind = 'cache', sourceUrl = '') {
+    super(message);
+    this.name = 'CaptureCacheError';
+    this.kind = kind;
+    this.sourceUrl = sourceUrl;
+  }
+}
+
+export function captureRequestKey({ url, method = 'GET', headers = {} }) {
+  if (
+    typeof url !== 'string' ||
+    typeof method !== 'string' ||
+    !headers ||
+    typeof headers !== 'object' ||
+    Array.isArray(headers) ||
+    Object.values(headers).some((value) => typeof value !== 'string')
+  ) {
+    throw new CaptureCacheError('Invalid capture request');
+  }
+  return digest(
+    JSON.stringify([
+      VERSION,
+      url,
+      method,
+      Object.entries(headers).sort(([a], [b]) =>
+        Buffer.compare(Buffer.from(a), Buffer.from(b))
+      ),
+    ])
+  );
+}
+
+function validate(record) {
+  if (
+    !record?.request ||
+    record.version !== VERSION ||
+    !/^[a-f0-9]{64}$/.test(record.sha256) ||
+    record.requestKey !== captureRequestKey(record.request) ||
+    !Number.isSafeInteger(record.fetchedAt) ||
+    record.fetchedAt < 0 ||
+    !record.receipt?.finalUrl ||
+    !record.receipt?.diagnostics
+  ) {
+    throw new CaptureCacheError('Unsupported or invalid capture record');
+  }
+}
+
+function capture(record, cached = true) {
+  return {
+    receipt: { ...record.receipt, body: Buffer.from(record.receipt.body) },
+    requestKey: record.requestKey,
+    fetchedAt: record.fetchedAt,
+    sha256: record.sha256,
+    cached,
+    stale: false,
+  };
+}
+
+/** Shared disk format; operations are serialized within this Node process. */
+export class CaptureStore {
+  constructor(directory) {
+    this.directory = path.resolve(directory);
+  }
+
+  async locked(operation) {
+    const previous = queues.get(this.directory) || Promise.resolve();
+    const next = previous.catch(() => {}).then(operation);
+    queues.set(this.directory, next);
+    try {
+      return await next;
+    } catch (error) {
+      if (error instanceof CaptureCacheError) {
+        throw error;
+      }
+      throw new CaptureCacheError(error.message);
+    } finally {
+      if (queues.get(this.directory) === next) {
+        queues.delete(this.directory);
+      }
+    }
+  }
+  recordPath(key) {
+    return path.join(this.directory, 'requests', `${key}.json`);
+  }
+  bodyPath(sha256) {
+    return path.join(this.directory, 'bodies', sha256);
+  }
+
+  async read(request) {
+    let record;
+    try {
+      record = JSON.parse(
+        await readFile(this.recordPath(captureRequestKey(request)), 'utf8')
+      );
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        return null;
+      }
+      throw error;
+    }
+    validate(record);
+    if (record.requestKey !== captureRequestKey(request)) {
+      throw new CaptureCacheError('Request does not match capture');
+    }
+    record.receipt.body = await readFile(this.bodyPath(record.sha256));
+    if (digest(record.receipt.body) !== record.sha256) {
+      throw new CaptureCacheError('Capture SHA-256 mismatch');
+    }
+    return record;
+  }
+  load(request) {
+    return this.locked(async () => {
+      const record = await this.read(request);
+      return record ? capture(record) : null;
+    });
+  }
+  async atomicWrite(target, bytes) {
+    await mkdir(path.dirname(target), { recursive: true, mode: 0o700 });
+    const temporary = path.join(path.dirname(target), `.tmp-${randomUUID()}`);
+    try {
+      await writeFile(temporary, bytes, { flag: 'wx', mode: 0o600 });
+      await rename(temporary, target);
+    } finally {
+      await unlink(temporary).catch((error) => {
+        if (error.code !== 'ENOENT') {
+          throw error;
+        }
+      });
+    }
+  }
+  store(request, receipt, fetchedAt) {
+    return this.locked(async () => {
+      const record = {
+        version: VERSION,
+        requestKey: captureRequestKey(request),
+        request: {
+          url: request.url,
+          method: request.method ?? 'GET',
+          headers: { ...request.headers },
+        },
+        fetchedAt,
+        sha256: digest(receipt.body),
+        receipt: { ...receipt, body: [] },
+      };
+      validate(record);
+      await this.atomicWrite(this.bodyPath(record.sha256), receipt.body);
+      await this.atomicWrite(
+        this.recordPath(record.requestKey),
+        JSON.stringify(record, null, 2)
+      );
+      record.receipt.body = receipt.body;
+      return capture(record, false);
+    });
+  }
+  export(request) {
+    return this.locked(async () => {
+      const record = await this.read(request);
+      if (!record) {
+        throw new CaptureCacheError('No capture to export');
+      }
+      record.receipt.body = [...record.receipt.body];
+      return JSON.stringify(record, null, 2);
+    });
+  }
+  async import(replay) {
+    let record;
+    try {
+      record = JSON.parse(replay);
+    } catch (error) {
+      throw new CaptureCacheError(`Invalid replay JSON: ${error.message}`);
+    }
+    validate(record);
+    if (
+      !Array.isArray(record.receipt.body) ||
+      record.receipt.body.some(
+        (byte) => !Number.isInteger(byte) || byte < 0 || byte > 255
+      )
+    ) {
+      throw new CaptureCacheError('Invalid replay bytes');
+    }
+    const body = Buffer.from(record.receipt.body);
+    if (digest(body) !== record.sha256) {
+      throw new CaptureCacheError('Replay SHA-256 mismatch');
+    }
+    return await this.store(
+      record.request,
+      { ...record.receipt, body },
+      record.fetchedAt
+    );
+  }
+  async records() {
+    let names;
+    try {
+      names = await readdir(path.join(this.directory, 'requests'));
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        return [];
+      }
+      throw error;
+    }
+    const result = [];
+    for (const name of names.filter((name) => name.endsWith('.json')).sort()) {
+      const record = JSON.parse(
+        await readFile(path.join(this.directory, 'requests', name), 'utf8')
+      );
+      validate(record);
+      if (name !== `${record.requestKey}.json`) {
+        throw new CaptureCacheError('Capture filename mismatch');
+      }
+      result.push(record);
+    }
+    return result;
+  }
+  list() {
+    return this.locked(() => this.records());
+  }
+  prune(now, ttl) {
+    return this.locked(async () => {
+      const records = await this.records();
+      const live = new Set();
+      let removed = 0;
+      for (const record of records) {
+        if (Math.max(0, now - record.fetchedAt) >= ttl) {
+          await unlink(this.recordPath(record.requestKey));
+          removed++;
+        } else {
+          live.add(record.sha256);
+        }
+      }
+      let bodies = [];
+      try {
+        bodies = await readdir(path.join(this.directory, 'bodies'));
+      } catch (error) {
+        if (error.code !== 'ENOENT') {
+          throw error;
+        }
+      }
+      for (const body of bodies) {
+        if (!live.has(body)) {
+          await unlink(this.bodyPath(body));
+        }
+      }
+      return removed;
+    });
+  }
+}
+
+/** Decorate any injectable transport; `execute` is safe to pass as a callback. */
+export class CachedTransport {
+  constructor(
+    inner,
+    store,
+    { ttl = 60 * 24 * 60 * 60 * 1000, offline = false, now = Date.now } = {}
+  ) {
+    if (!Number.isFinite(ttl) || ttl < 0) {
+      throw new TypeError('TTL must be non-negative milliseconds');
+    }
+    this.inner = inner;
+    this.store = store;
+    this.ttl = ttl;
+    this.offline = offline;
+    this.now = now;
+    this.execute = this.execute.bind(this);
+  }
+  async capture(request) {
+    request.signal?.throwIfAborted();
+    const identity = {
+      url: request.url,
+      method: request.method ?? 'GET',
+      headers: { ...request.headers },
+    };
+    const saved = await this.store.load(identity);
+    if (saved) {
+      saved.stale = Math.max(0, this.now() - saved.fetchedAt) >= this.ttl;
+      if (!saved.stale || this.offline) {
+        return saved;
+      }
+    }
+    if (this.offline) {
+      throw new CaptureCacheError(
+        'No cached capture in offline mode',
+        'offline_cache_miss',
+        identity.url
+      );
+    }
+    const receipt = await captureResponse(identity.url, {
+      ...identity,
+      headers: { ...identity.headers },
+      signal: request.signal,
+      transport: this.inner,
+    });
+    return this.store.store(identity, receipt, this.now());
+  }
+  async execute(request) {
+    return (await this.capture(request)).receipt;
+  }
+}

package/src/gdocs-model-wait.js

@@ -12,7 +12,6 @@ export async function waitForGoogleDocsModelQuiescence(
   const started = Date.now();
   let lastFingerprint = null;
   let stableSince = null;
-  let lastModelData = null;
   let lastStableForMs = 0;
   let pollCount = 0;
 
@@ -20,7 +19,6 @@ export async function waitForGoogleDocsModelQuiescence(
     const modelData = await readGoogleDocsModelData(page);
     pollCount += 1;
     const fingerprint = googleDocsModelFingerprint(modelData);
-    lastModelData = modelData;
 
     if (
       fingerprint.chunks > 0 &&
@@ -56,8 +54,8 @@ export async function waitForGoogleDocsModelQuiescence(
 
     const elapsedMs = Date.now() - started;
     if (elapsedMs >= maxWaitMs) {
-      const lastChunks = lastModelData?.chunks?.length || 0;
-      const lastCidUrls = Object.keys(lastModelData?.cidUrlMap || {}).length;
+      const lastChunks = modelData?.chunks?.length || 0;
+      const lastCidUrls = Object.keys(modelData?.cidUrlMap || {}).length;
       throw googleDocsBrowserModelUnavailableError(
         `Google Docs DOCS_modelChunk stream did not quiesce within ${maxWaitMs} ms for document ${documentId} (last chunks=${lastChunks}, payload_bytes=${fingerprint.payloadBytes}, cid_urls=${lastCidUrls}, poll_count=${pollCount}, stable_for_ms=${lastStableForMs})`
       );

package/src/gdocs.js

@@ -8,7 +8,7 @@
 // Browser-model capture loads /edit and extracts DOCS_modelChunk data.
 
 import fetch from 'node-fetch';
-import he from 'he';
+import * as he from 'he';
 import * as cheerio from 'cheerio';
 import { convertHtmlToMarkdown } from './lib.js';
 import { createBrowser as defaultCreateBrowser } from './browser.js';

package/src/index.js

@@ -97,3 +97,10 @@ export {
   parseSharedDialog,
   sharedDialogHandler,
 } from './shared-dialog.js';
+
+export {
+  CaptureStore,
+  CachedTransport,
+  CaptureCacheError,
+  captureRequestKey,
+} from './cache.js';

package/src/lib.js

@@ -6,7 +6,7 @@ import TurndownService from 'turndown';
 import iconv from 'iconv-lite';
 import { URL, URLSearchParams } from 'url';
 import turndownPluginGfm from 'turndown-plugin-gfm';
-import he from 'he';
+import * as he from 'he';
 import { isFormulaImage, isMathElement, extractFormula } from './latex.js';
 import { extractMetadata } from './metadata.js';
 import { postProcessMarkdown } from './postprocess.js';

package/src/search.js

@@ -20,7 +20,7 @@
  */
 
 import * as cheerio from 'cheerio';
-import he from 'he';
+import * as he from 'he';
 import { URL } from 'url';
 import { captureResponse, fetchTransport } from './transport.js';
 

package/src/shared-dialog.js

@@ -9,7 +9,7 @@
  */
 
 import fetch from 'node-fetch';
-import he from 'he';
+import * as he from 'he';
 import { URL } from 'node:url';
 
 import { createBrowser as defaultCreateBrowser } from './browser.js';

package/src/transport.js

@@ -63,12 +63,12 @@ export function fetchTransport({ url, method = 'GET', headers, signal }) {
  */
 export async function captureResponse(
   url,
-  { transport = fetchTransport, signal, headers = {} } = {}
+  { transport = fetchTransport, signal, headers = {}, method = 'GET' } = {}
 ) {
   if (!url) {
     throw new Error('Missing URL parameter');
   }
-  const request = { url, method: 'GET', headers, signal };
+  const request = { url, method, headers, signal };
   try {
     const response = await transport(request);
     if (

package/tests/ci/dependency-freshness.test.js

@@ -0,0 +1,74 @@
+import { jest } from '@jest/globals';
+import { checkVersions } from '../../../scripts/check-dependency-freshness.mjs';
+
+const latest = async () => '2.0.0';
+const dependencies = [{ name: 'example', current: '1.0.0' }];
+const blockers = {
+  example: {
+    issue: 'https://github.com/example/repo/issues/1',
+    reason: 'Upstream API breaks packaged consumers',
+  },
+};
+
+test('freshness rejects old releases, missing packages and undocumented exceptions', async () => {
+  for (const current of ['1.0.0', '(missing)']) {
+    await expect(
+      checkVersions([{ name: 'example', current }], {}, latest, jest.fn())
+    ).rejects.toThrow('behind 2.0.0');
+  }
+});
+test('freshness permits only documented open issues', async () => {
+  await expect(
+    checkVersions(dependencies, blockers, latest, async () => ({
+      state: 'open',
+    }))
+  ).resolves.toBeUndefined();
+  await expect(
+    checkVersions(dependencies, blockers, latest, async () => ({
+      state: 'closed',
+    }))
+  ).rejects.toThrow('is closed');
+  await expect(
+    checkVersions(
+      dependencies,
+      { example: { ...blockers.example, reason: '' } },
+      latest,
+      jest.fn()
+    )
+  ).rejects.toThrow('document an open issue');
+  await expect(
+    checkVersions(
+      dependencies,
+      { example: { ...blockers.example, reason: '   ' } },
+      latest,
+      jest.fn()
+    )
+  ).rejects.toThrow('document an open issue');
+});
+test('registry and issue failures fail the check rather than treating dependencies as fresh', async () => {
+  await expect(
+    checkVersions(
+      dependencies,
+      {},
+      async () => {
+        throw new Error('registry down');
+      },
+      jest.fn()
+    )
+  ).rejects.toThrow('registry down');
+  await expect(
+    checkVersions(dependencies, blockers, latest, async () => {
+      throw new Error('issue unavailable');
+    })
+  ).rejects.toThrow('issue unavailable');
+});
+test('current releases need no exception', async () => {
+  const getIssue = jest.fn();
+  await checkVersions(
+    [{ name: 'example', current: '2.0.0' }],
+    {},
+    latest,
+    getIssue
+  );
+  expect(getIssue).not.toHaveBeenCalled();
+});

package/tests/ci/docker-install.test.js

@@ -0,0 +1,31 @@
+import { readFileSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+
+test.each(['js', 'rust'])(
+  '%s Docker build fails when every browser package installation attempt fails',
+  (implementation) => {
+    const dockerfile = readFileSync(
+      new globalThis.URL(
+        `../../../${implementation}/Dockerfile`,
+        import.meta.url
+      ),
+      'utf8'
+    );
+    const loop = dockerfile.match(/for i in 1 2 3; do[\s\S]*?done/)[0];
+    // Execute only the finite retry loop; stub installation and waiting.
+    const result = spawnSync(
+      'bash',
+      [
+        '-c',
+        `
+      apt-get() { printf '%s\\n' "$1"; [ "$1" = update ]; }
+      sleep() { return 0; }
+      ${loop}
+    `,
+      ],
+      { encoding: 'utf8' }
+    );
+    expect(result.stdout.match(/^install$/gm)).toHaveLength(3);
+    expect(result.status).toBe(1);
+  }
+);

package/tests/ci/workflow-policy.test.js

@@ -61,3 +61,19 @@ describe('workflow policy', () => {
     );
   });
 });
+
+test('release workflows require dependency health gates', () => {
+  const js = readWorkflow('.github/workflows/js.yml');
+  expect(js).toContain('npm audit --package-lock-only --audit-level=moderate');
+  expect(js).toContain('check-dependency-freshness.mjs npm');
+  expect(js).toContain('needs: [lint, test, dependency-health]');
+  const rust = readWorkflow('.github/workflows/rust.yml');
+  expect(rust).toContain('check-dependency-freshness.mjs cargo');
+  expect(rust).toContain('cargo update --dry-run');
+  expect(rust).toContain('needs: [lint, test, build, dependency-health]');
+  for (const workflow of [js, rust]) {
+    expect(
+      workflow.match(/needs\.dependency-health\.result == 'success'/g)
+    ).toHaveLength(2);
+  }
+});

package/tests/e2e/process.test.js

@@ -191,7 +191,8 @@ async function expectTextResponse(res) {
     text = await res.text();
   } catch (error) {
     throw new Error(
-      `${error.message}\nServer output:\n${serverOutput || '(empty)'}`
+      `${error.message}\nServer output:\n${serverOutput || '(empty)'}`,
+      { cause: error }
     );
   }
   expectResponseStatus(res.status, text);

package/tests/helpers/nock.js

@@ -0,0 +1,10 @@
+import nock from 'nock';
+
+// Nock 15 interceptors survive Jest VM contexts unless explicitly restored.
+// https://github.com/nock/nock#memory-issues-with-jest
+afterAll(() => {
+  nock.cleanAll();
+  nock.restore();
+});
+
+export default nock;

package/tests/helpers/search-fixture.cjs

@@ -1,4 +1,4 @@
-const nock = require('nock');
+const { default: nock } = require('nock');
 
 nock('https://en.wikipedia.org')
   .persist()

package/tests/helpers/temporary-directory.js

@@ -0,0 +1,12 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+export async function withTemporaryDirectory(prefix, operation) {
+  const directory = await mkdtemp(path.join(tmpdir(), prefix));
+  try {
+    return await operation(directory);
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+}

package/tests/integration/api-endpoints.test.js

@@ -8,7 +8,7 @@
 
 import { jest } from '@jest/globals';
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import unzipper from 'unzipper';
 import { app } from '../../src/index.js';
 import { isKreuzbergAvailable } from '../../src/kreuzberg.js';

package/tests/integration/browser-packaged-path.test.js

@@ -0,0 +1,45 @@
+import { mkdtemp, rm, symlink } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { chromium } from 'playwright';
+import { createBrowser } from '../../src/browser.js';
+
+// Only deterministic local HTML is used. Neither engine may select its own cache.
+test.each(['puppeteer', 'playwright'])(
+  '%s renders using a Chromium executable at an explicit temporary path',
+  async (engine) => {
+    const directory = await mkdtemp(path.join(tmpdir(), 'packaged-browser-'));
+    const packagedPath = path.join(
+      directory,
+      process.platform === 'win32' ? 'chrome.exe' : 'chromium'
+    );
+    let browser;
+    let page;
+    try {
+      await symlink(chromium.executablePath(), packagedPath);
+      browser = await createBrowser(engine, {
+        executablePath: packagedPath,
+        userDataDir: path.join(directory, 'profile'),
+      });
+      page = await browser.newPage();
+      await page.goto(
+        'data:text/html,<main id="result"></main><script>document.getElementById("result").textContent="Packaged Chromium rendered"</script>'
+      );
+      expect(await page.content()).toContain('Packaged Chromium rendered');
+      const executable =
+        engine === 'puppeteer'
+          ? browser.rawBrowser.process().spawnfile
+          : packagedPath;
+      expect(executable).toBe(packagedPath);
+    } finally {
+      if (page) {
+        await page.close();
+      }
+      if (browser) {
+        await browser.close();
+      }
+      await rm(directory, { recursive: true, force: true });
+    }
+  },
+  30000
+);

package/tests/integration/deepwiki-rendered-page.mock.test.js

@@ -1,5 +1,5 @@
 import { jest } from '@jest/globals';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import request from 'supertest';
 import { URL, fileURLToPath } from 'url';
 

package/tests/integration/formalai-contract.test.js

@@ -6,7 +6,7 @@
  */
 
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import unzipper from 'unzipper';
 import { app, SEARCH_PROVIDERS } from '../../src/index.js';
 

package/tests/integration/google-drive-image.test.js

@@ -1,5 +1,5 @@
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import unzipper from 'unzipper';
 import { app } from '../../src/index.js';
 

package/tests/integration/search-endpoint.test.js

@@ -5,7 +5,7 @@
  */
 
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import { app } from '../../src/index.js';
 
 const WIKI_JSON = {

package/tests/mock/index.test.js

@@ -1,5 +1,5 @@
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import { jest } from '@jest/globals';
 
 let app;

package/tests/unit/browser-launch-options.test.js

@@ -0,0 +1,42 @@
+import { jest } from '@jest/globals';
+import path from 'node:path';
+import { withTemporaryDirectory } from '../helpers/temporary-directory.js';
+
+const initialPage = { close: jest.fn(), bringToFront: jest.fn() };
+const browser = { pages: jest.fn(() => [initialPage]), close: jest.fn() };
+const launch = jest.fn(async () => browser);
+const launchPersistentContext = jest.fn(async () => browser);
+jest.unstable_mockModule('puppeteer', () => ({ default: { launch } }));
+jest.unstable_mockModule('playwright', () => ({
+  chromium: { launchPersistentContext },
+}));
+const { createBrowser } = await import('../../src/browser.js');
+
+beforeEach(() => {
+  jest.clearAllMocks();
+});
+
+test.each(['puppeteer', 'playwright'])(
+  '%s forwards packaged executablePath and channel to the engine',
+  (engine) =>
+    withTemporaryDirectory('launch-options-', async (directory) => {
+      const options = {
+        executablePath: path.join(directory, 'packaged-chromium'),
+        channel: 'chrome',
+        userDataDir: directory,
+      };
+      const handle = await createBrowser(engine, options);
+      try {
+        const observed =
+          engine === 'puppeteer'
+            ? launch.mock.calls[0][0]
+            : launchPersistentContext.mock.calls[0][1];
+        expect(observed).toMatchObject({
+          executablePath: options.executablePath,
+          channel: options.channel,
+        });
+      } finally {
+        await handle.close();
+      }
+    })
+);

package/tests/unit/cache.test.js

@@ -0,0 +1,182 @@
+import { jest } from '@jest/globals';
+import { mkdtemp, readFile, rm, writeFile, readdir } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { CaptureStore, CachedTransport } from '../../src/cache.js';
+import { captureResponse } from '../../src/transport.js';
+
+const request = {
+  url: 'https://example.test/source',
+  method: 'GET',
+  headers: { accept: 'application/octet-stream' },
+};
+const receipt = {
+  body: Buffer.from([0, 255, 65]),
+  finalUrl: 'https://example.test/final',
+  status: 206,
+  headers: { etag: 'v1' },
+  diagnostics: { outcome: 'response' },
+};
+let directory, store;
+beforeEach(async () => {
+  directory = await mkdtemp(path.join(tmpdir(), 'capture-cache-'));
+  store = new CaptureStore(directory);
+});
+afterEach(async () => {
+  await rm(directory, { recursive: true, force: true });
+});
+
+test('exact receipts survive cache hits, offline stale replay and online refresh', async () => {
+  let time = 1000;
+  const inner = jest.fn(async () => receipt);
+  const options = { ttl: 100, now: () => time };
+  const online = new CachedTransport(inner, store, options);
+  const first = await online.capture(request);
+  expect(first).toMatchObject({ cached: false, stale: false, receipt });
+  expect(
+    await captureResponse(request.url, {
+      transport: online.execute,
+      headers: request.headers,
+    })
+  ).toEqual(receipt);
+  expect(inner).toHaveBeenCalledTimes(1);
+  const forbidden = jest.fn(() => {
+    throw new Error('offline network access');
+  });
+  const offline = new CachedTransport(forbidden, store, {
+    ...options,
+    offline: true,
+  });
+  expect(await offline.capture(request)).toMatchObject({
+    cached: true,
+    stale: false,
+    receipt,
+  });
+  time = 1100;
+  expect(await offline.capture(request)).toMatchObject({
+    cached: true,
+    stale: true,
+    receipt,
+  });
+  expect(forbidden).not.toHaveBeenCalled();
+  expect(await online.capture(request)).toMatchObject({
+    cached: false,
+    stale: false,
+  });
+  expect(inner).toHaveBeenCalledTimes(2);
+  expect(await store.list()).toHaveLength(1);
+  expect(await store.prune(1200, 100)).toBe(1);
+  await expect(offline.capture(request)).rejects.toMatchObject({
+    kind: 'offline_cache_miss',
+  });
+});
+
+test('portable fixture imports/exports, request isolation and body deduplication', async () => {
+  const fixture = await readFile(
+    new globalThis.URL(
+      '../../../rust/tests/fixtures/capture-v1.json',
+      import.meta.url
+    ),
+    'utf8'
+  );
+  await store.import(fixture);
+  expect((await store.load(request)).receipt).toEqual(receipt);
+  expect(JSON.parse(await store.export(request))).toEqual(JSON.parse(fixture));
+  for (const different of [
+    { ...request, method: 'POST' },
+    { ...request, url: `${request.url}?query=1` },
+    { ...request, headers: { authorization: 'different' } },
+  ]) {
+    expect(await store.load(different)).toBeNull();
+  }
+  await store.store(
+    { ...request, url: `${request.url}/duplicate` },
+    receipt,
+    2000
+  );
+  expect(await readdir(path.join(directory, 'bodies'))).toHaveLength(1);
+  expect(await store.prune(1100, 100)).toBe(1);
+  expect(await readdir(path.join(directory, 'bodies'))).toHaveLength(1);
+  await expect(
+    store.import(fixture.replace('255', '254'))
+  ).rejects.toMatchObject({ kind: 'cache' });
+  await expect(
+    store.import(fixture.replace('"version": 1', '"version": 999'))
+  ).rejects.toMatchObject({ kind: 'cache' });
+});
+
+test('corrupt bodies are rejected and failed transports are never cached', async () => {
+  const stored = await store.store(request, receipt, 1000);
+  await writeFile(path.join(directory, 'bodies', stored.sha256), 'corrupt');
+  await expect(store.load(request)).rejects.toMatchObject({ kind: 'cache' });
+  const failure = new Error('transport failed');
+  const cache = new CachedTransport(async () => {
+    throw failure;
+  }, store);
+  await expect(
+    cache.capture({ ...request, url: 'https://other.test' })
+  ).rejects.toThrow('transport failed');
+  expect(await store.list()).toHaveLength(1);
+});
+
+test('Unicode header ordering matches the shared Rust replay format', async () => {
+  const fixture = await readFile(
+    new globalThis.URL(
+      '../../../rust/tests/fixtures/capture-v1-unicode.json',
+      import.meta.url
+    ),
+    'utf8'
+  );
+  const replay = JSON.parse(fixture);
+  await store.import(fixture);
+  expect((await store.load(replay.request)).receipt).toEqual(receipt);
+  expect(JSON.parse(await store.export(replay.request))).toEqual(replay);
+});
+
+test('failed online refresh retains the expired capture for offline replay', async () => {
+  await store.store(request, receipt, 800);
+  const cache = new CachedTransport(
+    async () => {
+      throw new Error('offline upstream');
+    },
+    store,
+    { ttl: 100, now: () => 1000 }
+  );
+  await expect(cache.capture(request)).rejects.toThrow('offline upstream');
+  const offline = new CachedTransport(null, store, {
+    offline: true,
+    ttl: 100,
+    now: () => 1000,
+  });
+  expect(await offline.capture(request)).toMatchObject({
+    stale: true,
+    receipt,
+  });
+  for (const replay of ['{', 'null', '{}']) {
+    await expect(store.import(replay)).rejects.toMatchObject({ kind: 'cache' });
+  }
+});
+
+test('headers are sorted without mutating the caller; TTL zero always refreshes', async () => {
+  const a = { ...request, headers: { z: '1', a: '2' } };
+  const b = { ...request, headers: { a: '2', z: '1' } };
+  await store.store(a, receipt, 1000);
+  expect((await store.load(b)).receipt).toEqual(receipt);
+  const inner = jest.fn(async () => receipt);
+  const cache = new CachedTransport(inner, store, { ttl: 0, now: () => 1000 });
+  await cache.capture(a);
+  await cache.capture(a);
+  expect(inner).toHaveBeenCalledTimes(2);
+});
+
+test('caller mutation during a fetch cannot change the stored request identity', async () => {
+  const mutable = { ...request, headers: { ...request.headers } };
+  const cache = new CachedTransport(async () => {
+    mutable.url = 'https://mutated.test';
+    mutable.headers.accept = 'text/plain';
+    return receipt;
+  }, store);
+  await cache.capture(mutable);
+  expect((await store.load(request)).receipt).toEqual(receipt);
+  expect(await store.load(mutable)).toBeNull();
+});

package/tests/unit/fetch-html.test.js

@@ -1,4 +1,4 @@
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import { jest } from '@jest/globals';
 import { fetchHtml, fetchHtmlReceipt } from '../../src/lib.js';
 

package/tests/unit/gdocs-export-headers.test.js

@@ -1,4 +1,4 @@
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import { fetchGoogleDocAsMarkdown } from '../../src/gdocs.js';
 
 afterEach(() => {

package/tests/unit/gdocs.test.js

@@ -28,7 +28,7 @@ import fs from 'fs';
 import os from 'os';
 import path from 'path';
 import { fileURLToPath } from 'url';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 
 jest.setTimeout(30000);
 

package/tests/unit/github.test.js

@@ -1,4 +1,4 @@
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import {
   fetchGithubRepositorySnapshot,
   formatGithubRepositoryMarkdown,

package/tests/xpaste/xpaste-screenshot-markdown.mock.test.js

@@ -6,7 +6,7 @@
  */
 
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import fs from 'fs';
 import path from 'path';
 import { fileURLToPath } from 'url';

package/tests/xpaste/xpaste.e2e.test.js

@@ -1,5 +1,5 @@
 import getPort from 'get-port';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import { app } from '../../src/index.js';
 
 describe('xpaste.pro HTTP e2e', () => {

package/tests/xpaste/xpaste.mock.test.js

@@ -1,5 +1,5 @@
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import { jest } from '@jest/globals';
 import unzipper from 'unzipper';