@link-assistant/web-capture 1.11.2 → 2.0.0

package/.changeset/config.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://unpkg.com/@changesets/config@3.1.1/schema.json",
+  "$schema": "../node_modules/@changesets/config/schema.json",
   "changelog": "@changesets/cli/changelog",
   "commit": false,
   "fixed": [],

package/.dockerignore

@@ -0,0 +1,6 @@
+node_modules
+reports
+coverage
+*.log
+.captures
+capture-replay.json

package/.jscpd.json

@@ -1,8 +1,6 @@
 {
-  "threshold": 0,
   "minTokens": 30,
   "minLines": 5,
-  "skipComments": true,
   "ignore": [
     "**/node_modules/**",
     "**/build/**",
@@ -12,9 +10,10 @@
     "**/.changeset/**",
     "**/package-lock.json",
     "**/pnpm-lock.yaml",
-    "**/yarn.lock"
+    "**/yarn.lock",
+    "**/reports/**"
   ],
-  "format": "console",
   "reporters": ["console", "html"],
-  "output": "./reports/jscpd"
+  "output": "./reports/jscpd",
+  "mode": "weak"
 }

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/web-capture
 
+## 2.0.0
+
+### Major Changes
+
+- b055a30: Require Node.js 26.10 or newer and update Playwright to 1.64. Use current Trixie browser images, native Jest ESM and reproducibly locked release tooling. Add locked Python examples and weekly dependency/security maintenance across all ecosystems.
+
+## 1.12.0
+
+### Minor Changes
+
+- a6c74ba: Add a SHA-256 disk capture cache with TTL, offline stale replay, portable receipts, listing and pruning. Upgrade Puppeteer to 25 and forward packaged browser paths through browser-commander 0.10.0. Update other direct dependencies and require audit/freshness checks in CI. Node.js 22.22.1 or later is now required. Bundled consumers must externalize Playwright and Playwright Core.
+
 ## 1.11.2
 
 ### Patch Changes

package/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:22-bullseye
+FROM node:26.10.0-trixie
 
 WORKDIR /app
 
@@ -9,15 +9,15 @@ RUN apt-get update && \
       apt-get install -y \
         chromium \
         fonts-liberation \
-        libasound2 \
-        libatk-bridge2.0-0 \
-        libatk1.0-0 \
-        libatspi2.0-0 \
-        libcups2 \
+        libasound2t64 \
+        libatk-bridge2.0-0t64 \
+        libatk1.0-0t64 \
+        libatspi2.0-0t64 \
+        libcups2t64 \
         libdbus-1-3 \
         libdrm2 \
         libgbm1 \
-        libgtk-3-0 \
+        libgtk-3-0t64 \
         libnspr4 \
         libnss3 \
         libxcomposite1 \
@@ -27,12 +27,14 @@ RUN apt-get update && \
         libxshmfence1 \
         xdg-utils \
         --no-install-recommends && break || \
-      { echo "apt-get install attempt $i failed, retrying..."; sleep 5; }; \
+      { if [ "$i" -eq 3 ]; then exit 1; fi; \
+        echo "apt-get install attempt $i failed, retrying..."; \
+        apt-get update && sleep 5; }; \
     done && \
     rm -rf /var/lib/apt/lists/*
 
 # Set Puppeteer environment variables
-ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
+ENV PUPPETEER_SKIP_DOWNLOAD=true
 ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
 ENV PUPPETEER_ARGS="--no-sandbox --disable-setuid-sandbox --disable-dev-shm-usage"
 
@@ -40,8 +42,8 @@ ENV PUPPETEER_ARGS="--no-sandbox --disable-setuid-sandbox --disable-dev-shm-usag
 ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
 ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium
 
-COPY package.json yarn.lock ./
-RUN yarn install --frozen-lockfile
+COPY package.json package-lock.json ./
+RUN npm install --global npm@12.2.0 && npm ci
 
 COPY . .
 

package/README.md

@@ -19,6 +19,17 @@ A CLI and microservice to fetch URLs and render them as:
 
 ## Installation
 
+Node.js 26.10.0 or newer is required. CI uses 26.11.1 and the latest published
+Node Docker image uses 26.10.0 on Debian Trixie. Use `npm ci`
+for a locked repository install; Docker and CI share `package-lock.json` and its
+security overrides.
+
+Bundled applications must externalize `playwright` and `playwright-core` and
+ship them as production packages. Playwright does not support bundling its
+internal modules. Packaged Chromium is supported through
+`createBrowser(engine, { executablePath: packagedPath, channel: 'chrome' })`
+for both engines. See [the upstream bundling discussion](https://github.com/microsoft/playwright/issues/33031).
+
 ### From npm
 
 ```bash
@@ -518,14 +529,14 @@ with environment variables:
 
 ```bash
 # Download the Wikipedia page (markdown + image) in every supported engine
-WIKIPEDIA_INTEGRATION=true npm test -- --testPathPattern="wikipedia-download"
+WIKIPEDIA_INTEGRATION=true npm test -- --testPathPatterns="wikipedia-download"
 
 # Download a GitHub repository page as compact txt/markdown, original HTML, and screenshots
-GITHUB_REPOSITORY_INTEGRATION=true npm test -- --testPathPattern="github-readme"
+GITHUB_REPOSITORY_INTEGRATION=true npm test -- --testPathPatterns="github-readme"
 
 # Habr articles and public Google Docs live suites
-HABR_INTEGRATION=true npm test -- --testPathPattern="habr-article"
-GDOCS_INTEGRATION=true npm test -- --testPathPattern="gdocs-public-doc"
+HABR_INTEGRATION=true npm test -- --testPathPatterns="habr-article"
+GDOCS_INTEGRATION=true npm test -- --testPathPatterns="gdocs-public-doc"
 ```
 
 ## Built With
@@ -598,3 +609,11 @@ JSON response does not duplicate the full provider body.
 ## License
 
 [Unlicense](../LICENSE) — This is free and unencumbered software released into the public domain. You are free to copy, modify, publish, use, compile, sell, or distribute this software for any purpose, commercial or non-commercial, and by any means. See [https://unlicense.org](https://unlicense.org) for details.
+
+## Capture cache
+
+Both packages provide `CaptureStore` and `CachedTransport` over caller-owned transports. Captures keep the exact body bytes, final URL, status, response headers and diagnostics. Request keys include the exact URL, method and every request header in sorted order. Identical bodies share a SHA-256 file. The default TTL is 60 days; expiry is measured from fetch completion, in Unix milliseconds.
+
+Online mode reuses fresh captures and refetches expired entries. Offline mode never invokes the transport: it returns cached captures with `cached: true` and a `stale` flag, or an `offline_cache_miss` error. Use `capture` to read these flags; the transport adapter returns the original receipt unchanged. Transport errors are not stored. Corrupt or unsupported records produce cache errors instead of returning unverified bytes.
+
+`list` returns request metadata; `prune(now, ttl)` removes expired request records and unreferenced body files. `export(request)` and `import(json)` exchange version 1 JSON replay fixtures with byte arrays, shared between Rust and JavaScript. See [the cache format and examples](../docs/capture-cache.md).

package/bin/web-capture.js

@@ -1213,8 +1213,9 @@ async function captureUrl(url, options) {
               baseDelay: 500,
             });
             if (resp.ok) {
-              const buffer = await resp.buffer();
-              archive.append(buffer, { name: localPath });
+              archive.append(Buffer.from(await resp.arrayBuffer()), {
+                name: localPath,
+              });
             }
           } catch {
             /* skip failed image downloads */

package/examples/js/capture_cache.js

@@ -0,0 +1,12 @@
+import { CaptureStore, CachedTransport } from '../../src/cache.js';
+import { fetchTransport } from '../../src/transport.js';
+
+const store = new CaptureStore('.captures');
+const request = { url: 'https://example.com', method: 'GET', headers: {} };
+const online = new CachedTransport(fetchTransport, store);
+const first = await online.capture(request);
+const offline = new CachedTransport(fetchTransport, store, { offline: true });
+const replay = await offline.capture(request);
+console.log(replay.sha256, replay.cached, replay.stale);
+console.log('Exact replay:', first.receipt.body.equals(replay.receipt.body));
+console.log('Portable fixture:', await store.export(request));

package/examples/js/playwright_example.js

@@ -16,7 +16,7 @@ console.log('Fetching screenshot using Playwright engine...');
 const imageResponse = await fetch(
   `${baseUrl}/image?url=${encodeURIComponent(targetUrl)}&engine=playwright`
 );
-const imageBuffer = await imageResponse.buffer();
+const imageBuffer = Buffer.from(await imageResponse.arrayBuffer());
 fs.writeFileSync('output/playwright_screenshot.png', imageBuffer);
 console.log('Screenshot saved to output/playwright_screenshot.png');
 

package/examples/python/capture_client.py

@@ -0,0 +1,20 @@
+"""Shared request policy for the capture-service examples."""
+import os
+from pathlib import Path
+import sys
+import requests
+
+
+def capture(kind, url=None, **options):
+    if url is None:
+        url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
+    endpoint = os.environ.get('WEB_CAPTURE_URL', 'http://localhost:3000').rstrip('/')
+    response = requests.get(f'{endpoint}/{kind}', params={'url': url, **options}, timeout=30)
+    response.raise_for_status()
+    return response
+
+
+def download_text(kind, filename):
+    output = Path(__file__).with_name(filename)
+    output.write_text(capture(kind).text, encoding='utf-8')
+    print(f'{kind.title()} saved to {output}')

package/examples/python/html_download.py

@@ -1,15 +1,3 @@
-import sys
-import requests
-import os
+from capture_client import download_text
 
-url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
-endpoint = f'http://localhost:3000/html?url={url}'
-
-response = requests.get(endpoint)
-response.raise_for_status()
-
-output_path = os.path.join(os.path.dirname(__file__), 'downloaded.html')
-with open(output_path, 'w', encoding='utf-8') as f:
-    f.write(response.text)
-
-print(f'HTML saved to {output_path}') 
+download_text('html', 'downloaded.html')

package/examples/python/image_download.py

@@ -1,11 +1,7 @@
-import sys
-import requests
+from capture_client import capture
 import os
 
-url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
-endpoint = f'http://localhost:3000/image?url={url}'
-
-response = requests.get(endpoint)
+response = capture('image')
 print('Status:', response.status_code)
 print('Content-Type:', response.headers.get('content-type'))
 
@@ -17,4 +13,4 @@ if response.content[:8] == png_signature:
         f.write(response.content)
     print(f'Image saved to {output_path} (valid PNG)')
 else:
-    print('Response is not a valid PNG! First bytes:', response.content[:16]) 
+    print('Response is not a valid PNG! First bytes:', response.content[:16])

package/examples/python/markdown_download.py

@@ -1,15 +1,3 @@
-import sys
-import requests
-import os
+from capture_client import download_text
 
-url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
-endpoint = f'http://localhost:3000/markdown?url={url}'
-
-response = requests.get(endpoint)
-response.raise_for_status()
-
-output_path = os.path.join(os.path.dirname(__file__), 'downloaded.md')
-with open(output_path, 'w', encoding='utf-8') as f:
-    f.write(response.text)
-
-print(f'Markdown saved to {output_path}') 
+download_text('markdown', 'downloaded.md')

package/examples/python/playwright_example.py

@@ -1,20 +1,20 @@
-import requests
+import sys
+from capture_client import capture
 import os
 
-base_url = 'http://localhost:3000'
-target_url = 'https://example.com'
+target_url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
 
 # Create output directory if it doesn't exist
 os.makedirs('output', exist_ok=True)
 
 print('Fetching HTML using Playwright engine...')
-html_response = requests.get(f'{base_url}/html', params={'url': target_url, 'engine': 'playwright'})
-with open('output/playwright_html.html', 'w') as f:
+html_response = capture('html', target_url, engine='playwright')
+with open('output/playwright_html.html', 'w', encoding='utf-8') as f:
     f.write(html_response.text)
 print('HTML saved to output/playwright_html.html')
 
 print('Fetching screenshot using Playwright engine...')
-image_response = requests.get(f'{base_url}/image', params={'url': target_url, 'engine': 'playwright'})
+image_response = capture('image', target_url, engine='playwright')
 with open('output/playwright_screenshot.png', 'wb') as f:
     f.write(image_response.content)
 print('Screenshot saved to output/playwright_screenshot.png')

package/jest.config.mjs

@@ -1,8 +1,5 @@
 export default {
-  transform: {
-    '^.+\\.js$': ['babel-jest', { configFile: './babel.config.cjs' }],
-  },
-  transformIgnorePatterns: ['node_modules/(?!(turndown)/)'],
+  transform: {},
   moduleNameMapper: {
     '^(\\.{1,2}/.*)\\.js$': '$1',
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.11.2",
+  "version": "2.0.0",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {
@@ -8,22 +8,22 @@
   },
   "scripts": {
     "start": "docker compose up -d",
-    "dev": "nodemon bin/web-capture.js -- --serve",
+    "dev": "node --watch bin/web-capture.js --serve",
     "test": "node --experimental-vm-modules ./node_modules/.bin/jest",
     "test:watch": "node --experimental-vm-modules ./node_modules/.bin/jest --watch",
-    "test:e2e:process": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e.process.test.js",
-    "test:e2e:docker": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e.docker.test.js",
-    "test:e2e": "yarn test:e2e:process && yarn test:e2e:docker",
-    "test:all": "yarn build && yarn test && yarn test:e2e",
-    "build": "docker compose down && docker compose build && yarn start",
-    "examples:python": "python3 examples/python/html_download.py && python3 examples/python/markdown_download.py && python3 examples/python/image_download.py",
+    "test:e2e:process": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e/process.test.js",
+    "test:e2e:docker": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e/docker.test.js",
+    "test:e2e": "npm run test:e2e:process && npm run test:e2e:docker",
+    "test:all": "npm run build && npm test && npm run test:e2e",
+    "build": "docker compose down && docker compose build && npm start",
+    "examples:python": "uv run --project .. --locked python examples/python/html_download.py && uv run --project .. --locked python examples/python/markdown_download.py && uv run --project .. --locked python examples/python/image_download.py",
     "examples:javascript": "node examples/js/html_download.js && node examples/js/markdown_download.js && node examples/js/image_download.js",
-    "examples": "yarn build && yarn examples:javascript && yarn examples:python",
+    "examples": "npm run build && npm run examples:javascript && npm run examples:python",
     "lint": "eslint .",
     "lint:fix": "eslint . --fix",
     "format": "prettier --write .",
     "format:check": "prettier --check .",
-    "check:duplication": "jscpd .",
+    "check:duplication": "jscpd . --baseline-from-ref origin/main --fail-on-new-clones --fail-on-empty",
     "check": "npm run lint && npm run format:check && npm run check:duplication",
     "prepare": "husky || true",
     "changeset": "changeset",
@@ -32,42 +32,40 @@
     "changeset:status": "changeset status --since=origin/main"
   },
   "dependencies": {
-    "@kreuzberg/html-to-markdown-node": "^3.5.5",
+    "@kreuzberg/html-to-markdown-node": "^3.7.2",
     "archiver": "^8.0.0",
-    "browser-commander": "^0.8.0",
-    "cheerio": "^1.0.0",
-    "docx": "^9.6.1",
-    "express": "^4.18.2",
-    "he": "^1.2.0",
-    "iconv-lite": "^0.6.3",
+    "browser-commander": "0.10.0",
+    "cheerio": "^1.2.0",
+    "docx": "^9.9.0",
+    "express": "^5.2.1",
+    "he": "^2.0.0",
+    "iconv-lite": "^0.7.3",
     "lino-arguments": "^0.3.0",
     "log-lazy": "^1.0.4",
-    "node-fetch": "^2.7.0",
-    "playwright": "^1.49.0",
-    "puppeteer": "^24.8.2",
-    "puppeteer-core": "^24.40.0",
-    "turndown": "^7.1.1",
+    "node-fetch": "^3.3.2",
+    "playwright": "^1.64.0",
+    "puppeteer": "^25.12.0",
+    "puppeteer-core": "^25.12.0",
+    "turndown": "^7.2.4",
     "turndown-plugin-gfm": "^1.0.2"
   },
   "devDependencies": {
-    "@babel/core": "^7.24.0",
-    "@babel/preset-env": "^7.24.0",
-    "@changesets/cli": "^2.29.7",
-    "@types/archiver": "^7.0.0",
-    "babel-jest": "^29.7.0",
-    "eslint": "^9.38.0",
+    "@changesets/cli": "^3.0.3",
+    "@types/archiver": "^8.0.0",
+    "eslint": "^10.12.0",
     "eslint-config-prettier": "^10.1.8",
-    "eslint-plugin-prettier": "^5.5.4",
-    "get-port": "^7.1.0",
+    "eslint-plugin-prettier": "^5.5.6",
+    "get-port": "^7.2.0",
     "husky": "^9.1.7",
-    "jest": "^29.7.0",
-    "jscpd": "^4.0.5",
-    "lint-staged": "^16.2.6",
-    "markdown-it": "^14.1.1",
-    "nock": "^13.5.4",
-    "prettier": "^3.6.2",
-    "supertest": "^6.3.4",
-    "unzipper": "^0.12.3"
+    "jest": "^30.5.2",
+    "jscpd": "^5.4.0",
+    "lint-staged": "^17.6.0",
+    "markdown-it": "^15.0.2",
+    "nock": "^15.0.1",
+    "prettier": "^3.9.9",
+    "supertest": "^7.3.1",
+    "unzipper": "^0.12.5",
+    "@eslint/js": "^10.0.1"
   },
   "type": "module",
   "keywords": [
@@ -82,7 +80,7 @@
     "url": "https://github.com/link-assistant/web-capture"
   },
   "engines": {
-    "node": ">=22.0.0 <23.0.0"
+    "node": ">=26.10.0"
   },
   "lint-staged": {
     "*.{js,mjs,cjs}": [
@@ -94,5 +92,22 @@
       "prettier --write",
       "prettier --check"
     ]
+  },
+  "dependencyBlockers": {
+    "browser-commander": {
+      "issue": "https://github.com/link-assistant/web-capture/issues/160",
+      "reason": "Keep the first compatible packaged-browser release; later versions add native better-sqlite3 dependencies."
+    }
+  },
+  "overrides": {
+    "@istanbuljs/load-nyc-config": {
+      "js-yaml": "^4.1.1"
+    }
+  },
+  "packageManager": "npm@12.2.0",
+  "allowScripts": {
+    "@parcel/watcher": true,
+    "puppeteer": true,
+    "unrs-resolver": true
   }
 }

package/src/archive.js

@@ -200,7 +200,7 @@ export async function archiveHandler(req, res) {
             baseDelay: 500,
           });
           if (imgResp.ok) {
-            const buffer = await imgResp.buffer();
+            const buffer = Buffer.from(await imgResp.arrayBuffer());
             archive.append(buffer, { name: localPath });
           }
         } catch {

package/src/binary-download.js

@@ -0,0 +1,14 @@
+import fetch from 'node-fetch';
+import { retry } from './retry.js';
+
+// Shared image-download policy; keep HTTP errors distinct from network retries.
+export async function downloadBinary(url) {
+  const response = await retry(() => fetch(url), {
+    retries: 3,
+    baseDelay: 1000,
+  });
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status}`);
+  }
+  return Buffer.from(await response.arrayBuffer());
+}