@link-assistant/web-capture 1.11.2 → 2.0.0

package/tests/ci/workflow-policy.test.js

@@ -40,7 +40,7 @@ describe('workflow policy', () => {
     (workflowFile) => {
       const workflow = readWorkflow(workflowFile);
 
-      expect(workflow).toContain('uses: actions/checkout@v6');
+      expect(workflow).toContain('uses: actions/checkout@v7.0.1');
       expect(workflow).not.toContain('uses: actions/checkout@v4');
     }
   );
@@ -48,7 +48,7 @@ describe('workflow policy', () => {
   test('Rust workflow uses the current cache action', () => {
     const workflow = readWorkflow('.github/workflows/rust.yml');
 
-    expect(workflow).toContain('uses: actions/cache@v5');
+    expect(workflow).toContain('uses: actions/cache@v6.1.0');
     expect(workflow).not.toContain('uses: actions/cache@v4');
   });
 
@@ -61,3 +61,19 @@ describe('workflow policy', () => {
     );
   });
 });
+
+test('release workflows require dependency health gates', () => {
+  const js = readWorkflow('.github/workflows/js.yml');
+  expect(js).toContain('npm audit --package-lock-only --audit-level=moderate');
+  expect(js).toContain('check-dependency-freshness.mjs npm');
+  expect(js).toContain('needs: [lint, test, dependency-health]');
+  const rust = readWorkflow('.github/workflows/rust.yml');
+  expect(rust).toContain('check-dependency-freshness.mjs cargo');
+  expect(rust).toContain('cargo update --dry-run');
+  expect(rust).toContain('needs: [lint, test, build, dependency-health]');
+  for (const workflow of [js, rust]) {
+    expect(
+      workflow.match(/needs\.dependency-health\.result == 'success'/g)
+    ).toHaveLength(2);
+  }
+});

package/tests/e2e/process.test.js

@@ -191,7 +191,8 @@ async function expectTextResponse(res) {
     text = await res.text();
   } catch (error) {
     throw new Error(
-      `${error.message}\nServer output:\n${serverOutput || '(empty)'}`
+      `${error.message}\nServer output:\n${serverOutput || '(empty)'}`,
+      { cause: error }
     );
   }
   expectResponseStatus(res.status, text);

package/tests/helpers/nock.js

@@ -0,0 +1,10 @@
+import nock from 'nock';
+
+// Nock 15 interceptors survive Jest VM contexts unless explicitly restored.
+// https://github.com/nock/nock#memory-issues-with-jest
+afterAll(() => {
+  nock.cleanAll();
+  nock.restore();
+});
+
+export default nock;

package/tests/helpers/screenshot.js

@@ -0,0 +1,10 @@
+import { expect } from '@jest/globals';
+
+// PNG magic number (89 50 4E 47 0D 0A 1A 0A).
+const PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
+
+export function expectPngScreenshot(screenshot, minimumLength = 1000) {
+  expect(screenshot).toBeInstanceOf(Uint8Array);
+  expect(screenshot.length).toBeGreaterThan(minimumLength);
+  expect(Buffer.from(screenshot).subarray(0, 8)).toEqual(PNG_SIGNATURE);
+}

package/tests/helpers/search-fixture.cjs

@@ -1,4 +1,4 @@
-const nock = require('nock');
+const { default: nock } = require('nock');
 
 nock('https://en.wikipedia.org')
   .persist()

package/tests/helpers/temporary-directory.js

@@ -0,0 +1,12 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+
+export async function withTemporaryDirectory(prefix, operation) {
+  const directory = await mkdtemp(path.join(tmpdir(), prefix));
+  try {
+    return await operation(directory);
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+}

package/tests/integration/api-endpoints.test.js

@@ -8,7 +8,7 @@
 
 import { jest } from '@jest/globals';
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import unzipper from 'unzipper';
 import { app } from '../../src/index.js';
 import { isKreuzbergAvailable } from '../../src/kreuzberg.js';

package/tests/integration/browser-engines.test.js

@@ -1,11 +1,12 @@
 import { createBrowser } from '../../src/browser.js';
+import { expectPngScreenshot } from '../helpers/screenshot.js';
 
 describe('Browser Engine Integration Tests', () => {
-  describe('Puppeteer Engine', () => {
+  describe.each(['puppeteer', 'playwright'])('%s Engine', (engine) => {
     let browser;
 
     beforeEach(async () => {
-      browser = await createBrowser('puppeteer');
+      browser = await createBrowser(engine);
     });
 
     afterEach(async () => {
@@ -32,56 +33,7 @@ describe('Browser Engine Integration Tests', () => {
         timeout: 30000,
       });
       const screenshot = await page.screenshot({ type: 'png' });
-      expect(screenshot).toBeInstanceOf(Buffer);
-      expect(screenshot.length).toBeGreaterThan(100);
-    }, 60000);
-
-    it('can set custom headers and user agent', async () => {
-      const page = await browser.newPage();
-      await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-US' });
-      await page.setUserAgent('Test User Agent');
-      await page.goto('https://example.com', {
-        waitUntil: 'networkidle0',
-        timeout: 30000,
-      });
-      const content = await page.content();
-      expect(content).toBeTruthy();
-    }, 60000);
-  });
-
-  describe('Playwright Engine', () => {
-    let browser;
-
-    beforeEach(async () => {
-      browser = await createBrowser('playwright');
-    });
-
-    afterEach(async () => {
-      if (browser) {
-        await browser.close();
-      }
-    });
-
-    it('can navigate to a page and get content', async () => {
-      const page = await browser.newPage();
-      await page.goto('https://example.com', {
-        waitUntil: 'networkidle0',
-        timeout: 30000,
-      });
-      const content = await page.content();
-      expect(content).toContain('Example Domain');
-    }, 60000);
-
-    it('can take a screenshot', async () => {
-      const page = await browser.newPage();
-      await page.setViewport({ width: 1280, height: 800 });
-      await page.goto('https://example.com', {
-        waitUntil: 'networkidle0',
-        timeout: 30000,
-      });
-      const screenshot = await page.screenshot({ type: 'png' });
-      expect(screenshot).toBeInstanceOf(Buffer);
-      expect(screenshot.length).toBeGreaterThan(100);
+      expectPngScreenshot(screenshot, 100);
     }, 60000);
 
     it('can set custom headers and user agent', async () => {

package/tests/integration/browser-packaged-path.test.js

@@ -0,0 +1,45 @@
+import { mkdtemp, rm, symlink } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { chromium } from 'playwright';
+import { createBrowser } from '../../src/browser.js';
+
+// Only deterministic local HTML is used. Neither engine may select its own cache.
+test.each(['puppeteer', 'playwright'])(
+  '%s renders using a Chromium executable at an explicit temporary path',
+  async (engine) => {
+    const directory = await mkdtemp(path.join(tmpdir(), 'packaged-browser-'));
+    const packagedPath = path.join(
+      directory,
+      process.platform === 'win32' ? 'chrome.exe' : 'chromium'
+    );
+    let browser;
+    let page;
+    try {
+      await symlink(chromium.executablePath(), packagedPath);
+      browser = await createBrowser(engine, {
+        executablePath: packagedPath,
+        userDataDir: path.join(directory, 'profile'),
+      });
+      page = await browser.newPage();
+      await page.goto(
+        'data:text/html,<main id="result"></main><script>document.getElementById("result").textContent="Packaged Chromium rendered"</script>'
+      );
+      expect(await page.content()).toContain('Packaged Chromium rendered');
+      const executable =
+        engine === 'puppeteer'
+          ? browser.rawBrowser.process().spawnfile
+          : packagedPath;
+      expect(executable).toBe(packagedPath);
+    } finally {
+      if (page) {
+        await page.close();
+      }
+      if (browser) {
+        await browser.close();
+      }
+      await rm(directory, { recursive: true, force: true });
+    }
+  },
+  30000
+);

package/tests/integration/deepwiki-rendered-page.mock.test.js

@@ -1,5 +1,5 @@
 import { jest } from '@jest/globals';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import request from 'supertest';
 import { URL, fileURLToPath } from 'url';
 

package/tests/integration/formalai-contract.test.js

@@ -6,7 +6,7 @@
  */
 
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import unzipper from 'unzipper';
 import { app, SEARCH_PROVIDERS } from '../../src/index.js';
 

package/tests/integration/github-readme.test.js

@@ -12,6 +12,7 @@ import { jest } from '@jest/globals';
 import { createBrowser } from '../../src/browser.js';
 import { fetchHtml } from '../../src/lib.js';
 import { retry } from '../../src/retry.js';
+import { expectPngScreenshot } from '../helpers/screenshot.js';
 import {
   fetchGithubRepositorySnapshot,
   formatGithubRepositoryMarkdown,
@@ -22,9 +23,6 @@ import {
 const GITHUB_REPOSITORY_URL =
   process.env.GITHUB_REPOSITORY_URL ||
   'https://github.com/link-assistant/web-capture';
-const PNG_SIGNATURE = Buffer.from([
-  0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
-]);
 
 const SKIP_LIVE =
   !process.env.GITHUB_REPOSITORY_INTEGRATION ||
@@ -128,9 +126,7 @@ describe.each(['puppeteer', 'playwright'])(
 
         const screenshot = await page.screenshot({ type: 'png' });
 
-        expect(screenshot).toBeInstanceOf(Buffer);
-        expect(screenshot.length).toBeGreaterThan(1000);
-        expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
+        expectPngScreenshot(screenshot);
       });
     });
   }

package/tests/integration/google-drive-image.test.js

@@ -1,5 +1,5 @@
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import unzipper from 'unzipper';
 import { app } from '../../src/index.js';
 

package/tests/integration/search-endpoint.test.js

@@ -5,7 +5,7 @@
  */
 
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import { app } from '../../src/index.js';
 
 const WIKI_JSON = {

package/tests/integration/wikipedia-download.test.js

@@ -19,14 +19,10 @@ import { jest } from '@jest/globals';
 import { createBrowser } from '../../src/browser.js';
 import { convertHtmlToMarkdown } from '../../src/lib.js';
 import { retry } from '../../src/retry.js';
+import { expectPngScreenshot } from '../helpers/screenshot.js';
 
 const WIKIPEDIA_URL = 'https://en.wikipedia.org/wiki/Wikipedia';
 
-// PNG magic number (89 50 4E 47 0D 0A 1A 0A).
-const PNG_SIGNATURE = Buffer.from([
-  0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
-]);
-
 const SKIP_LIVE =
   !process.env.WIKIPEDIA_INTEGRATION ||
   process.env.WIKIPEDIA_INTEGRATION === 'false';
@@ -99,10 +95,7 @@ describe.each(['puppeteer', 'playwright'])(
       it('downloads the Wikipedia page as an image (PNG screenshot)', async () => {
         const screenshot = await page.screenshot({ type: 'png' });
 
-        expect(screenshot).toBeInstanceOf(Buffer);
-        expect(screenshot.length).toBeGreaterThan(1000);
-        // Verify the PNG signature so we know it is a real image.
-        expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
+        expectPngScreenshot(screenshot);
       });
     });
   }
@@ -135,8 +128,7 @@ describeIfLive('Wikipedia Page Download (engine parity)', () => {
       expect(markdown).toContain('Wikipedia');
       expect(markdown.length).toBeGreaterThan(500);
       // Image captured from both engines.
-      expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
-      expect(screenshot.length).toBeGreaterThan(1000);
+      expectPngScreenshot(screenshot);
     }
   }, 180000);
 });

package/tests/mock/index.test.js

@@ -1,5 +1,5 @@
 import request from 'supertest';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import { jest } from '@jest/globals';
 
 let app;

package/tests/unit/browser-launch-options.test.js

@@ -0,0 +1,42 @@
+import { jest } from '@jest/globals';
+import path from 'node:path';
+import { withTemporaryDirectory } from '../helpers/temporary-directory.js';
+
+const initialPage = { close: jest.fn(), bringToFront: jest.fn() };
+const browser = { pages: jest.fn(() => [initialPage]), close: jest.fn() };
+const launch = jest.fn(async () => browser);
+const launchPersistentContext = jest.fn(async () => browser);
+jest.unstable_mockModule('puppeteer', () => ({ default: { launch } }));
+jest.unstable_mockModule('playwright', () => ({
+  chromium: { launchPersistentContext },
+}));
+const { createBrowser } = await import('../../src/browser.js');
+
+beforeEach(() => {
+  jest.clearAllMocks();
+});
+
+test.each(['puppeteer', 'playwright'])(
+  '%s forwards packaged executablePath and channel to the engine',
+  (engine) =>
+    withTemporaryDirectory('launch-options-', async (directory) => {
+      const options = {
+        executablePath: path.join(directory, 'packaged-chromium'),
+        channel: 'chrome',
+        userDataDir: directory,
+      };
+      const handle = await createBrowser(engine, options);
+      try {
+        const observed =
+          engine === 'puppeteer'
+            ? launch.mock.calls[0][0]
+            : launchPersistentContext.mock.calls[0][1];
+        expect(observed).toMatchObject({
+          executablePath: options.executablePath,
+          channel: options.channel,
+        });
+      } finally {
+        await handle.close();
+      }
+    })
+);

package/tests/unit/cache.test.js

@@ -0,0 +1,182 @@
+import { jest } from '@jest/globals';
+import { mkdtemp, readFile, rm, writeFile, readdir } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { CaptureStore, CachedTransport } from '../../src/cache.js';
+import { captureResponse } from '../../src/transport.js';
+
+const request = {
+  url: 'https://example.test/source',
+  method: 'GET',
+  headers: { accept: 'application/octet-stream' },
+};
+const receipt = {
+  body: Buffer.from([0, 255, 65]),
+  finalUrl: 'https://example.test/final',
+  status: 206,
+  headers: { etag: 'v1' },
+  diagnostics: { outcome: 'response' },
+};
+let directory, store;
+beforeEach(async () => {
+  directory = await mkdtemp(path.join(tmpdir(), 'capture-cache-'));
+  store = new CaptureStore(directory);
+});
+afterEach(async () => {
+  await rm(directory, { recursive: true, force: true });
+});
+
+test('exact receipts survive cache hits, offline stale replay and online refresh', async () => {
+  let time = 1000;
+  const inner = jest.fn(async () => receipt);
+  const options = { ttl: 100, now: () => time };
+  const online = new CachedTransport(inner, store, options);
+  const first = await online.capture(request);
+  expect(first).toMatchObject({ cached: false, stale: false, receipt });
+  expect(
+    await captureResponse(request.url, {
+      transport: online.execute,
+      headers: request.headers,
+    })
+  ).toEqual(receipt);
+  expect(inner).toHaveBeenCalledTimes(1);
+  const forbidden = jest.fn(() => {
+    throw new Error('offline network access');
+  });
+  const offline = new CachedTransport(forbidden, store, {
+    ...options,
+    offline: true,
+  });
+  expect(await offline.capture(request)).toMatchObject({
+    cached: true,
+    stale: false,
+    receipt,
+  });
+  time = 1100;
+  expect(await offline.capture(request)).toMatchObject({
+    cached: true,
+    stale: true,
+    receipt,
+  });
+  expect(forbidden).not.toHaveBeenCalled();
+  expect(await online.capture(request)).toMatchObject({
+    cached: false,
+    stale: false,
+  });
+  expect(inner).toHaveBeenCalledTimes(2);
+  expect(await store.list()).toHaveLength(1);
+  expect(await store.prune(1200, 100)).toBe(1);
+  await expect(offline.capture(request)).rejects.toMatchObject({
+    kind: 'offline_cache_miss',
+  });
+});
+
+test('portable fixture imports/exports, request isolation and body deduplication', async () => {
+  const fixture = await readFile(
+    new globalThis.URL(
+      '../../../rust/tests/fixtures/capture-v1.json',
+      import.meta.url
+    ),
+    'utf8'
+  );
+  await store.import(fixture);
+  expect((await store.load(request)).receipt).toEqual(receipt);
+  expect(JSON.parse(await store.export(request))).toEqual(JSON.parse(fixture));
+  for (const different of [
+    { ...request, method: 'POST' },
+    { ...request, url: `${request.url}?query=1` },
+    { ...request, headers: { authorization: 'different' } },
+  ]) {
+    expect(await store.load(different)).toBeNull();
+  }
+  await store.store(
+    { ...request, url: `${request.url}/duplicate` },
+    receipt,
+    2000
+  );
+  expect(await readdir(path.join(directory, 'bodies'))).toHaveLength(1);
+  expect(await store.prune(1100, 100)).toBe(1);
+  expect(await readdir(path.join(directory, 'bodies'))).toHaveLength(1);
+  await expect(
+    store.import(fixture.replace('255', '254'))
+  ).rejects.toMatchObject({ kind: 'cache' });
+  await expect(
+    store.import(fixture.replace('"version": 1', '"version": 999'))
+  ).rejects.toMatchObject({ kind: 'cache' });
+});
+
+test('corrupt bodies are rejected and failed transports are never cached', async () => {
+  const stored = await store.store(request, receipt, 1000);
+  await writeFile(path.join(directory, 'bodies', stored.sha256), 'corrupt');
+  await expect(store.load(request)).rejects.toMatchObject({ kind: 'cache' });
+  const failure = new Error('transport failed');
+  const cache = new CachedTransport(async () => {
+    throw failure;
+  }, store);
+  await expect(
+    cache.capture({ ...request, url: 'https://other.test' })
+  ).rejects.toThrow('transport failed');
+  expect(await store.list()).toHaveLength(1);
+});
+
+test('Unicode header ordering matches the shared Rust replay format', async () => {
+  const fixture = await readFile(
+    new globalThis.URL(
+      '../../../rust/tests/fixtures/capture-v1-unicode.json',
+      import.meta.url
+    ),
+    'utf8'
+  );
+  const replay = JSON.parse(fixture);
+  await store.import(fixture);
+  expect((await store.load(replay.request)).receipt).toEqual(receipt);
+  expect(JSON.parse(await store.export(replay.request))).toEqual(replay);
+});
+
+test('failed online refresh retains the expired capture for offline replay', async () => {
+  await store.store(request, receipt, 800);
+  const cache = new CachedTransport(
+    async () => {
+      throw new Error('offline upstream');
+    },
+    store,
+    { ttl: 100, now: () => 1000 }
+  );
+  await expect(cache.capture(request)).rejects.toThrow('offline upstream');
+  const offline = new CachedTransport(null, store, {
+    offline: true,
+    ttl: 100,
+    now: () => 1000,
+  });
+  expect(await offline.capture(request)).toMatchObject({
+    stale: true,
+    receipt,
+  });
+  for (const replay of ['{', 'null', '{}']) {
+    await expect(store.import(replay)).rejects.toMatchObject({ kind: 'cache' });
+  }
+});
+
+test('headers are sorted without mutating the caller; TTL zero always refreshes', async () => {
+  const a = { ...request, headers: { z: '1', a: '2' } };
+  const b = { ...request, headers: { a: '2', z: '1' } };
+  await store.store(a, receipt, 1000);
+  expect((await store.load(b)).receipt).toEqual(receipt);
+  const inner = jest.fn(async () => receipt);
+  const cache = new CachedTransport(inner, store, { ttl: 0, now: () => 1000 });
+  await cache.capture(a);
+  await cache.capture(a);
+  expect(inner).toHaveBeenCalledTimes(2);
+});
+
+test('caller mutation during a fetch cannot change the stored request identity', async () => {
+  const mutable = { ...request, headers: { ...request.headers } };
+  const cache = new CachedTransport(async () => {
+    mutable.url = 'https://mutated.test';
+    mutable.headers.accept = 'text/plain';
+    return receipt;
+  }, store);
+  await cache.capture(mutable);
+  expect((await store.load(request)).receipt).toEqual(receipt);
+  expect(await store.load(mutable)).toBeNull();
+});

package/tests/unit/fetch-html.test.js

@@ -1,4 +1,4 @@
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import { jest } from '@jest/globals';
 import { fetchHtml, fetchHtmlReceipt } from '../../src/lib.js';
 

package/tests/unit/gdocs-export-headers.test.js

@@ -1,4 +1,4 @@
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import { fetchGoogleDocAsMarkdown } from '../../src/gdocs.js';
 
 afterEach(() => {

package/tests/unit/gdocs.test.js

@@ -28,7 +28,7 @@ import fs from 'fs';
 import os from 'os';
 import path from 'path';
 import { fileURLToPath } from 'url';
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 
 jest.setTimeout(30000);
 

package/tests/unit/github.test.js

@@ -1,4 +1,4 @@
-import nock from 'nock';
+import nock from '../helpers/nock.js';
 import {
   fetchGithubRepositorySnapshot,
   formatGithubRepositoryMarkdown,

package/tests/unit/publish-verification.test.js

@@ -1,23 +1,20 @@
 import { jest } from '@jest/globals';
 
-const originalFetch = global.fetch;
+jest.unstable_mockModule('execa', () => ({
+  $: () => async () => ({ exitCode: 0, stdout: '', stderr: '' }),
+}));
+jest.unstable_mockModule('lino-arguments', () => ({
+  makeConfig: () => ({ shouldPull: false, jsRoot: 'js' }),
+}));
 
 beforeAll(() => {
-  global.fetch = jest.fn().mockResolvedValue({
-    text: async () => `
-      ({
-        use: async (name) => {
-          if (name === 'command-stream') return { $: () => ({ run: async () => ({ code: 0, stdout: '', stderr: '' }) }) };
-          if (name === 'lino-arguments') return { makeConfig: () => ({ shouldPull: false, jsRoot: 'js' }) };
-          throw new Error('Unexpected dynamic import: ' + name);
-        }
-      })
-    `,
-  });
+  jest
+    .spyOn(global, 'fetch')
+    .mockRejectedValue(new Error('unexpected remote code download'));
 });
 
 afterAll(() => {
-  global.fetch = originalFetch;
+  global.fetch.mockRestore();
 });
 
 describe('publish verification', () => {
@@ -28,17 +25,17 @@ describe('publish verification', () => {
     const runVerify = jest
       .fn()
       .mockResolvedValueOnce({
-        code: 1,
+        exitCode: 1,
         stdout: '',
         stderr: 'npm error code E404',
       })
       .mockResolvedValueOnce({
-        code: 1,
+        exitCode: 1,
         stdout: '',
         stderr: 'npm error code E404',
       })
       .mockResolvedValueOnce({
-        code: 0,
+        exitCode: 0,
         stdout: '1.7.1\n',
         stderr: '',
       });