@link-assistant/web-capture 2.0.0 → 2.0.1

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @link-assistant/web-capture
 
+## 2.0.1
+
+### Patch Changes
+
+- 724c1a9: Fix CI/CD false positives, false negatives, and warnings (#177):
+
+  - Restore the `browser-commander` 0.10.0 pin required by open blocker #160 and repair the npm lock so `npm ci` installs again after the Dependabot update.
+  - Use Node's built-in `fetch` instead of `node-fetch`, removing the deprecated `node-domexception` package from the install. Transport diagnostics now include the underlying network error (for example `fetch failed: connect ECONNREFUSED ...`) and no longer report server-side network failures as possible CORS blocks.
+  - Load the `docx` library only when a DOCX is generated. Its bundled browser polyfills read `globalThis.localStorage` on import, so on Node 25+ every process that imported the server printed "ExperimentalWarning: localStorage is not available because --localstorage-file was not provided".
+  - Publish only the runtime files (`bin/`, `src/`, `README.md`, `CHANGELOG.md`) instead of the tests and fixtures, normalize `bin` and `repository` so `npm publish` no longer auto-corrects them, and install the pre-commit hook from the repository root instead of printing ".git can't be found" on every install.
+
 ## 2.0.0
 
 ### Major Changes

package/bin/web-capture.js

@@ -1110,11 +1110,9 @@ async function captureUrl(url, options) {
         await import('../src/lib.js');
       const { retry } = await import('../src/retry.js');
       const cheerio = await import('cheerio');
-      const nodeFetch = await import('node-fetch');
 
+      // retry() defaults: 3 retries from a 1000ms base delay.
       const html = await retry(() => fetchHtml(absoluteUrl), {
-        retries: 3,
-        baseDelay: 1000,
         onRetry: (err, attempt) => {
           console.error(`Retry ${attempt} fetching page: ${err.message}`);
         },
@@ -1207,8 +1205,7 @@ async function captureUrl(url, options) {
       if (imageMap.size > 0) {
         for (const [imgUrl, localPath] of imageMap) {
           try {
-            const fetchFn = nodeFetch.default || nodeFetch;
-            const resp = await retry(() => fetchFn(imgUrl), {
+            const resp = await retry(() => fetch(imgUrl), {
               retries: 2,
               baseDelay: 500,
             });

package/package.json

@@ -1,11 +1,17 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {
-    "web-capture": "./bin/web-capture.js"
+    "web-capture": "bin/web-capture.js"
   },
+  "files": [
+    "bin/",
+    "src/",
+    "CHANGELOG.md",
+    "README.md"
+  ],
   "scripts": {
     "start": "docker compose up -d",
     "dev": "node --watch bin/web-capture.js --serve",
@@ -19,13 +25,13 @@
     "examples:python": "uv run --project .. --locked python examples/python/html_download.py && uv run --project .. --locked python examples/python/markdown_download.py && uv run --project .. --locked python examples/python/image_download.py",
     "examples:javascript": "node examples/js/html_download.js && node examples/js/markdown_download.js && node examples/js/image_download.js",
     "examples": "npm run build && npm run examples:javascript && npm run examples:python",
-    "lint": "eslint .",
+    "lint": "node ../scripts/lint.mjs",
     "lint:fix": "eslint . --fix",
     "format": "prettier --write .",
     "format:check": "prettier --check .",
     "check:duplication": "jscpd . --baseline-from-ref origin/main --fail-on-new-clones --fail-on-empty",
     "check": "npm run lint && npm run format:check && npm run check:duplication",
-    "prepare": "husky || true",
+    "prepare": "node ../scripts/install-git-hooks.mjs",
     "changeset": "changeset",
     "changeset:version": "node ../scripts/changeset-version.mjs",
     "changeset:publish": "changeset publish",
@@ -42,15 +48,15 @@
     "iconv-lite": "^0.7.3",
     "lino-arguments": "^0.3.0",
     "log-lazy": "^1.0.4",
-    "node-fetch": "^3.3.2",
     "playwright": "^1.64.0",
-    "puppeteer": "^25.12.0",
-    "puppeteer-core": "^25.12.0",
+    "puppeteer": "^25.13.0",
+    "puppeteer-core": "^25.13.0",
     "turndown": "^7.2.4",
     "turndown-plugin-gfm": "^1.0.2"
   },
   "devDependencies": {
     "@changesets/cli": "^3.0.3",
+    "@eslint/js": "^10.0.1",
     "@types/archiver": "^8.0.0",
     "eslint": "^10.12.0",
     "eslint-config-prettier": "^10.1.8",
@@ -64,8 +70,7 @@
     "nock": "^15.0.1",
     "prettier": "^3.9.9",
     "supertest": "^7.3.1",
-    "unzipper": "^0.12.5",
-    "@eslint/js": "^10.0.1"
+    "unzipper": "^0.12.5"
   },
   "type": "module",
   "keywords": [
@@ -77,14 +82,14 @@
   "license": "Unlicense",
   "repository": {
     "type": "git",
-    "url": "https://github.com/link-assistant/web-capture"
+    "url": "git+https://github.com/link-assistant/web-capture.git"
   },
   "engines": {
     "node": ">=26.10.0"
   },
   "lint-staged": {
     "*.{js,mjs,cjs}": [
-      "eslint --fix --max-warnings 0",
+      "eslint --fix --no-warn-ignored",
       "prettier --write",
       "prettier --check"
     ],
@@ -102,6 +107,12 @@
   "overrides": {
     "@istanbuljs/load-nyc-config": {
       "js-yaml": "^4.1.1"
+    },
+    "cheerio": {
+      "encoding-sniffer": "^1.0.2"
+    },
+    "babel-plugin-istanbul": {
+      "test-exclude": "^8.0.0"
     }
   },
   "packageManager": "npm@12.2.0",

package/src/archive.js

@@ -16,7 +16,6 @@
  */
 
 import { ZipArchive } from 'archiver';
-import fetch from 'node-fetch';
 import * as cheerio from 'cheerio';
 import { URL } from 'url';
 import {

package/src/binary-download.js

@@ -1,4 +1,3 @@
-import fetch from 'node-fetch';
 import { retry } from './retry.js';
 
 // Shared image-download policy; keep HTTP errors distinct from network retries.

package/src/docx.js

@@ -9,18 +9,8 @@
  *   engine - 'puppeteer' or 'playwright' (only used for browser rendering fallback)
  */
 
-import fetch from 'node-fetch';
 import * as cheerio from 'cheerio';
 import { URL } from 'url';
-import {
-  Document,
-  Packer,
-  Paragraph,
-  TextRun,
-  HeadingLevel,
-  ImageRun,
-  ExternalHyperlink,
-} from 'docx';
 import { fetchHtml, convertRelativeUrls } from './lib.js';
 
 export async function docxHandler(req, res) {
@@ -35,6 +25,19 @@ export async function docxHandler(req, res) {
     const absHtml = convertRelativeUrls(html, absoluteUrl);
     const $ = cheerio.load(absHtml);
 
+    // Loaded on demand: docx bundles a browser util-deprecate that reads
+    // globalThis.localStorage on import, which makes Node >=25 print an
+    // ExperimentalWarning for every process that merely imports the app.
+    const {
+      Document,
+      Packer,
+      Paragraph,
+      TextRun,
+      HeadingLevel,
+      ImageRun,
+      ExternalHyperlink,
+    } = await import('docx');
+
     // Remove unwanted elements
     $('style, script, noscript, nav, footer, header').remove();
 

package/src/fetch.js

@@ -1,4 +1,3 @@
-import fetch from 'node-fetch';
 import { convertGoogleDriveUrl } from './lib.js';
 import { copyProxyResponseHeaders } from './proxy-headers.js';
 

package/src/gdocs-images.js

@@ -5,7 +5,6 @@
 // the archive is self-contained. This module downloads the images and
 // rewrites the captured output accordingly.
 
-import fetch from 'node-fetch';
 import { URL } from 'url';
 
 const DEFAULT_USER_AGENT =

package/src/gdocs.js

@@ -7,7 +7,6 @@
 //   https://docs.googleapis.com/v1/documents/{DOCUMENT_ID}
 // Browser-model capture loads /edit and extracts DOCS_modelChunk data.
 
-import fetch from 'node-fetch';
 import * as he from 'he';
 import * as cheerio from 'cheerio';
 import { convertHtmlToMarkdown } from './lib.js';

package/src/github.js

@@ -1,4 +1,3 @@
-import fetch from 'node-fetch';
 import { URL } from 'node:url';
 
 const GITHUB_API_BASE = 'https://api.github.com';

package/src/lib.js

@@ -1,6 +1,5 @@
 /* eslint-disable no-useless-escape */
 // Common logic for the web-capture microservice
-import fetch from 'node-fetch';
 import * as cheerio from 'cheerio';
 import TurndownService from 'turndown';
 import iconv from 'iconv-lite';

package/src/markdown.js

@@ -18,7 +18,6 @@ import { applyImageMode } from './extract-images.js';
 import { createBrowser, getBrowserEngine } from './browser.js';
 import { retry } from './retry.js';
 import { ZipArchive } from 'archiver';
-import fetch from 'node-fetch';
 import * as cheerio from 'cheerio';
 
 const INLINE_MARKDOWN_LINE_LIMIT = 1500;

package/src/shared-dialog.js

@@ -8,7 +8,6 @@
  * unsupported diagnostic instead of guessed content.
  */
 
-import fetch from 'node-fetch';
 import * as he from 'he';
 import { URL } from 'node:url';
 

package/src/stream.js

@@ -1,4 +1,3 @@
-import fetch from 'node-fetch';
 import { convertGoogleDriveUrl } from './lib.js';
 import { copyProxyResponseHeaders } from './proxy-headers.js';
 

package/src/transport.js

@@ -1,5 +1,3 @@
-import fetch from 'node-fetch';
-
 export const RECEIPT_HEADERS = [
   'cache-control',
   'content-encoding',
@@ -22,22 +20,42 @@ function selectedHeaders(headers) {
   return selected;
 }
 
+const TIMEOUT_CAUSE_CODES = new Set([
+  'ETIMEDOUT',
+  'UND_ERR_CONNECT_TIMEOUT',
+  'UND_ERR_HEADERS_TIMEOUT',
+  'UND_ERR_BODY_TIMEOUT',
+]);
+
 function classifyTransportError(error, signal) {
   if (signal?.aborted || error?.name === 'AbortError') {
     return 'cancelled';
   }
-  if (error?.name === 'TimeoutError' || error?.type === 'request-timeout') {
+  if (
+    error?.name === 'TimeoutError' ||
+    TIMEOUT_CAUSE_CODES.has(error?.cause?.code)
+  ) {
     return 'timeout';
   }
-  if (error instanceof TypeError) {
+  // Node's built-in fetch rejects every network failure with
+  // TypeError('fetch failed') and the real reason in `cause`. Browsers give a
+  // cause-less TypeError for CORS and network errors alike.
+  if (error?.name === 'TypeError' && !error.cause) {
     return 'cors_or_transport';
   }
   return 'transport';
 }
 
+/** Message of `error` including its `cause`, e.g. "fetch failed: getaddrinfo ENOTFOUND host". */
+export function describeTransportError(error) {
+  const message = error?.message || String(error);
+  const cause = error?.cause?.message;
+  return cause && !message.includes(cause) ? `${message}: ${cause}` : message;
+}
+
 export class CaptureTransportError extends Error {
   constructor(error, url, signal) {
-    super(error?.message || String(error));
+    super(describeTransportError(error));
     this.name = 'CaptureTransportError';
     this.cause = error;
     this.diagnostics = {

package/src/txt.js

@@ -1,4 +1,3 @@
-import fetch from 'node-fetch';
 import { URL } from 'node:url';
 import {
   fetchHtml,

package/.changeset/README.md

@@ -1,8 +0,0 @@
-# Changesets
-
-Hello and welcome! This folder has been automatically generated by `@changesets/cli`, a build tool that works
-with multi-package repos, or single-package repos to help you version and publish your code. You can
-find the full documentation for it [in our repository](https://github.com/changesets/changesets)
-
-We have a quick list of common questions to get you started engaging with this project in
-[our documentation](https://github.com/changesets/changesets/blob/main/docs/common-questions.md)

package/.changeset/config.json

@@ -1,11 +0,0 @@
-{
-  "$schema": "../node_modules/@changesets/config/schema.json",
-  "changelog": "@changesets/cli/changelog",
-  "commit": false,
-  "fixed": [],
-  "linked": [],
-  "access": "public",
-  "baseBranch": "main",
-  "updateInternalDependencies": "patch",
-  "ignore": []
-}

package/.dockerignore

@@ -1,6 +0,0 @@
-node_modules
-reports
-coverage
-*.log
-.captures
-capture-replay.json

package/.jscpd.json

@@ -1,19 +0,0 @@
-{
-  "minTokens": 30,
-  "minLines": 5,
-  "ignore": [
-    "**/node_modules/**",
-    "**/build/**",
-    "**/dist/**",
-    "**/*.min.js",
-    "**/coverage/**",
-    "**/.changeset/**",
-    "**/package-lock.json",
-    "**/pnpm-lock.yaml",
-    "**/yarn.lock",
-    "**/reports/**"
-  ],
-  "reporters": ["console", "html"],
-  "output": "./reports/jscpd",
-  "mode": "weak"
-}

package/.lenv

@@ -1,8 +0,0 @@
-# web-capture default configuration
-# This file uses Links Notation format (key: value)
-
-# Server port
-PORT: 3000
-
-# Browser engine (puppeteer or playwright)
-BROWSER_ENGINE: puppeteer

package/.prettierignore

@@ -1,8 +0,0 @@
-node_modules
-coverage
-dist
-*.min.js
-package-lock.json
-.eslintcache
-CLAUDE.md
-tests/fixtures/deepwiki/*.md

package/.prettierrc

@@ -1,10 +0,0 @@
-{
-  "semi": true,
-  "trailingComma": "es5",
-  "singleQuote": true,
-  "printWidth": 80,
-  "tabWidth": 2,
-  "useTabs": false,
-  "arrowParens": "always",
-  "endOfLine": "lf"
-}

package/Dockerfile

@@ -1,52 +0,0 @@
-FROM node:26.10.0-trixie
-
-WORKDIR /app
-
-# Install dependencies for headless Chrome (used by both Puppeteer and Playwright)
-# Retry loop handles transient network errors from Debian mirrors in CI
-RUN apt-get update && \
-    for i in 1 2 3; do \
-      apt-get install -y \
-        chromium \
-        fonts-liberation \
-        libasound2t64 \
-        libatk-bridge2.0-0t64 \
-        libatk1.0-0t64 \
-        libatspi2.0-0t64 \
-        libcups2t64 \
-        libdbus-1-3 \
-        libdrm2 \
-        libgbm1 \
-        libgtk-3-0t64 \
-        libnspr4 \
-        libnss3 \
-        libxcomposite1 \
-        libxdamage1 \
-        libxfixes3 \
-        libxrandr2 \
-        libxshmfence1 \
-        xdg-utils \
-        --no-install-recommends && break || \
-      { if [ "$i" -eq 3 ]; then exit 1; fi; \
-        echo "apt-get install attempt $i failed, retrying..."; \
-        apt-get update && sleep 5; }; \
-    done && \
-    rm -rf /var/lib/apt/lists/*
-
-# Set Puppeteer environment variables
-ENV PUPPETEER_SKIP_DOWNLOAD=true
-ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
-ENV PUPPETEER_ARGS="--no-sandbox --disable-setuid-sandbox --disable-dev-shm-usage"
-
-# Set Playwright environment variables
-ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
-ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium
-
-COPY package.json package-lock.json ./
-RUN npm install --global npm@12.2.0 && npm ci
-
-COPY . .
-
-EXPOSE 3000
-
-ENTRYPOINT ["node", "bin/web-capture.js", "--serve"]

package/docker-compose.yml

@@ -1,11 +0,0 @@
-services:
-  web-capture:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    ports:
-      - '3000:3000'
-    extra_hosts:
-      - 'host.docker.internal:host-gateway'
-    environment:
-      - PORT=3000

package/eslint.config.js

@@ -1,112 +0,0 @@
-import js from '@eslint/js';
-import prettierConfig from 'eslint-config-prettier';
-import prettierPlugin from 'eslint-plugin-prettier';
-
-export default [
-  js.configs.recommended,
-  prettierConfig,
-  {
-    files: ['**/*.js', '**/*.mjs'],
-    plugins: {
-      prettier: prettierPlugin,
-    },
-    languageOptions: {
-      ecmaVersion: 'latest',
-      sourceType: 'module',
-      globals: {
-        // Node.js globals
-        console: 'readonly',
-        process: 'readonly',
-        Buffer: 'readonly',
-        __dirname: 'readonly',
-        __filename: 'readonly',
-        global: 'readonly',
-        setTimeout: 'readonly',
-        clearTimeout: 'readonly',
-        setInterval: 'readonly',
-        clearInterval: 'readonly',
-        // Node.js 18+ globals
-        fetch: 'readonly',
-        // Runtime-specific globals
-        Bun: 'readonly',
-        Deno: 'readonly',
-      },
-    },
-    rules: {
-      // Prettier integration
-      'prettier/prettier': 'error',
-
-      // Code quality rules
-      'no-unused-vars': 'error',
-      'no-console': 'off', // Allow console in this project
-      'no-debugger': 'error',
-
-      // Best practices
-      eqeqeq: ['error', 'always'],
-      curly: ['error', 'all'],
-      'no-var': 'error',
-      'prefer-const': 'error',
-      'prefer-arrow-callback': 'error',
-      'no-duplicate-imports': 'error',
-
-      // ES6+ features
-      'arrow-body-style': ['error', 'as-needed'],
-      'object-shorthand': ['error', 'always'],
-      'prefer-template': 'error',
-
-      // Async/await
-      'no-async-promise-executor': 'error',
-      'require-await': 'warn',
-
-      // Comments and documentation
-      'spaced-comment': ['error', 'always', { markers: ['/'] }],
-
-      // Complexity rules - reasonable thresholds for maintainability
-      complexity: ['warn', 15], // Cyclomatic complexity - allow more complex logic than strict 8
-      'max-depth': ['warn', 5], // Maximum nesting depth - slightly more lenient than strict 4
-      'max-lines-per-function': [
-        'warn',
-        {
-          max: 150, // More reasonable than strict 50 lines per function
-          skipBlankLines: true,
-          skipComments: true,
-        },
-      ],
-      'max-params': ['warn', 6], // Maximum function parameters - slightly more lenient than strict 5
-      'max-statements': ['warn', 60], // Maximum statements per function - reasonable limit for orchestration functions
-      'max-lines': ['error', 1500], // Maximum lines per file - counts all lines including blank lines and comments
-    },
-  },
-  {
-    // Test files have different requirements
-    files: ['tests/**/*.js', '**/*.test.js', '**/*.test.mjs'],
-    languageOptions: {
-      globals: {
-        // Jest globals
-        describe: 'readonly',
-        it: 'readonly',
-        test: 'readonly',
-        expect: 'readonly',
-        beforeAll: 'readonly',
-        afterAll: 'readonly',
-        beforeEach: 'readonly',
-        afterEach: 'readonly',
-        jest: 'readonly',
-      },
-    },
-    rules: {
-      'require-await': 'off', // Async functions without await are common in tests
-      'max-lines-per-function': 'off', // Test files can have long test functions
-      'max-statements': 'off', // Test files can have many statements
-    },
-  },
-  {
-    ignores: [
-      'node_modules/**',
-      'coverage/**',
-      'dist/**',
-      '*.min.js',
-      '.eslintcache',
-    ],
-  },
-];

package/examples/js/capture_cache.js

@@ -1,12 +0,0 @@
-import { CaptureStore, CachedTransport } from '../../src/cache.js';
-import { fetchTransport } from '../../src/transport.js';
-
-const store = new CaptureStore('.captures');
-const request = { url: 'https://example.com', method: 'GET', headers: {} };
-const online = new CachedTransport(fetchTransport, store);
-const first = await online.capture(request);
-const offline = new CachedTransport(fetchTransport, store, { offline: true });
-const replay = await offline.capture(request);
-console.log(replay.sha256, replay.cached, replay.stale);
-console.log('Exact replay:', first.receipt.body.equals(replay.receipt.body));
-console.log('Portable fixture:', await store.export(request));

package/examples/js/engine_comparison.js

@@ -1,39 +0,0 @@
-import fetch from 'node-fetch';
-import fs from 'fs';
-
-const baseUrl = 'http://localhost:3000';
-const targetUrl = 'https://example.com';
-
-console.log('Comparing Puppeteer and Playwright engines...\n');
-
-// Fetch using Puppeteer
-console.log('1. Fetching HTML using Puppeteer (default)...');
-const startPuppeteer = Date.now();
-const puppeteerResponse = await fetch(
-  `${baseUrl}/html?url=${encodeURIComponent(targetUrl)}`
-);
-const puppeteerHtml = await puppeteerResponse.text();
-const puppeteerTime = Date.now() - startPuppeteer;
-fs.writeFileSync('output/puppeteer_comparison.html', puppeteerHtml);
-console.log(`   Time: ${puppeteerTime}ms`);
-console.log(`   Size: ${puppeteerHtml.length} bytes`);
-
-// Fetch using Playwright
-console.log('\n2. Fetching HTML using Playwright...');
-const startPlaywright = Date.now();
-const playwrightResponse = await fetch(
-  `${baseUrl}/html?url=${encodeURIComponent(targetUrl)}&engine=playwright`
-);
-const playwrightHtml = await playwrightResponse.text();
-const playwrightTime = Date.now() - startPlaywright;
-fs.writeFileSync('output/playwright_comparison.html', playwrightHtml);
-console.log(`   Time: ${playwrightTime}ms`);
-console.log(`   Size: ${playwrightHtml.length} bytes`);
-
-// Summary
-console.log('\n=== Summary ===');
-console.log(`Puppeteer: ${puppeteerTime}ms, ${puppeteerHtml.length} bytes`);
-console.log(`Playwright: ${playwrightTime}ms, ${playwrightHtml.length} bytes`);
-console.log(`Difference: ${Math.abs(puppeteerTime - playwrightTime)}ms`);
-
-console.log('\nResults saved to output/ directory.');

package/examples/js/html_download.js

@@ -1,23 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const url = process.argv[2] || 'https://example.com';
-const endpoint = `http://localhost:3000/html?url=${encodeURIComponent(url)}`;
-
-fetch(endpoint)
-  .then((res) => {
-    if (!res.ok) {
-      throw new Error(`HTTP error! status: ${res.status}`);
-    }
-    return res.text();
-  })
-  .then((text) => {
-    const outputPath = path.join(__dirname, 'downloaded.html');
-    fs.writeFileSync(outputPath, text, 'utf-8');
-    console.log(`HTML saved to ${outputPath}`);
-  })
-  .catch((err) => {
-    console.error('Error:', err);
-  });