@link-assistant/web-capture 1.11.2 → 2.0.0

package/src/cache.js

@@ -0,0 +1,313 @@
+import { createHash, randomUUID } from 'node:crypto';
+import {
+  mkdir,
+  readFile,
+  writeFile,
+  rename,
+  unlink,
+  readdir,
+} from 'node:fs/promises';
+import path from 'node:path';
+import { captureResponse } from './transport.js';
+
+const VERSION = 1;
+const queues = new Map();
+const digest = (bytes) => createHash('sha256').update(bytes).digest('hex');
+
+export class CaptureCacheError extends Error {
+  constructor(message, kind = 'cache', sourceUrl = '') {
+    super(message);
+    this.name = 'CaptureCacheError';
+    this.kind = kind;
+    this.sourceUrl = sourceUrl;
+  }
+}
+
+export function captureRequestKey({ url, method = 'GET', headers = {} }) {
+  if (
+    typeof url !== 'string' ||
+    typeof method !== 'string' ||
+    !headers ||
+    typeof headers !== 'object' ||
+    Array.isArray(headers) ||
+    Object.values(headers).some((value) => typeof value !== 'string')
+  ) {
+    throw new CaptureCacheError('Invalid capture request');
+  }
+  return digest(
+    JSON.stringify([
+      VERSION,
+      url,
+      method,
+      Object.entries(headers).sort(([a], [b]) =>
+        Buffer.compare(Buffer.from(a), Buffer.from(b))
+      ),
+    ])
+  );
+}
+
+function validate(record) {
+  if (
+    !record?.request ||
+    record.version !== VERSION ||
+    !/^[a-f0-9]{64}$/.test(record.sha256) ||
+    record.requestKey !== captureRequestKey(record.request) ||
+    !Number.isSafeInteger(record.fetchedAt) ||
+    record.fetchedAt < 0 ||
+    !record.receipt?.finalUrl ||
+    !record.receipt?.diagnostics
+  ) {
+    throw new CaptureCacheError('Unsupported or invalid capture record');
+  }
+}
+
+function capture(record, cached = true) {
+  return {
+    receipt: { ...record.receipt, body: Buffer.from(record.receipt.body) },
+    requestKey: record.requestKey,
+    fetchedAt: record.fetchedAt,
+    sha256: record.sha256,
+    cached,
+    stale: false,
+  };
+}
+
+/** Shared disk format; operations are serialized within this Node process. */
+export class CaptureStore {
+  constructor(directory) {
+    this.directory = path.resolve(directory);
+  }
+
+  async locked(operation) {
+    const previous = queues.get(this.directory) || Promise.resolve();
+    const next = previous.catch(() => {}).then(operation);
+    queues.set(this.directory, next);
+    try {
+      return await next;
+    } catch (error) {
+      if (error instanceof CaptureCacheError) {
+        throw error;
+      }
+      throw new CaptureCacheError(error.message);
+    } finally {
+      if (queues.get(this.directory) === next) {
+        queues.delete(this.directory);
+      }
+    }
+  }
+  recordPath(key) {
+    return path.join(this.directory, 'requests', `${key}.json`);
+  }
+  bodyPath(sha256) {
+    return path.join(this.directory, 'bodies', sha256);
+  }
+
+  async read(request) {
+    let record;
+    try {
+      record = JSON.parse(
+        await readFile(this.recordPath(captureRequestKey(request)), 'utf8')
+      );
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        return null;
+      }
+      throw error;
+    }
+    validate(record);
+    if (record.requestKey !== captureRequestKey(request)) {
+      throw new CaptureCacheError('Request does not match capture');
+    }
+    record.receipt.body = await readFile(this.bodyPath(record.sha256));
+    if (digest(record.receipt.body) !== record.sha256) {
+      throw new CaptureCacheError('Capture SHA-256 mismatch');
+    }
+    return record;
+  }
+  load(request) {
+    return this.locked(async () => {
+      const record = await this.read(request);
+      return record ? capture(record) : null;
+    });
+  }
+  async atomicWrite(target, bytes) {
+    await mkdir(path.dirname(target), { recursive: true, mode: 0o700 });
+    const temporary = path.join(path.dirname(target), `.tmp-${randomUUID()}`);
+    try {
+      await writeFile(temporary, bytes, { flag: 'wx', mode: 0o600 });
+      await rename(temporary, target);
+    } finally {
+      await unlink(temporary).catch((error) => {
+        if (error.code !== 'ENOENT') {
+          throw error;
+        }
+      });
+    }
+  }
+  store(request, receipt, fetchedAt) {
+    return this.locked(async () => {
+      const record = {
+        version: VERSION,
+        requestKey: captureRequestKey(request),
+        request: {
+          url: request.url,
+          method: request.method ?? 'GET',
+          headers: { ...request.headers },
+        },
+        fetchedAt,
+        sha256: digest(receipt.body),
+        receipt: { ...receipt, body: [] },
+      };
+      validate(record);
+      await this.atomicWrite(this.bodyPath(record.sha256), receipt.body);
+      await this.atomicWrite(
+        this.recordPath(record.requestKey),
+        JSON.stringify(record, null, 2)
+      );
+      record.receipt.body = receipt.body;
+      return capture(record, false);
+    });
+  }
+  export(request) {
+    return this.locked(async () => {
+      const record = await this.read(request);
+      if (!record) {
+        throw new CaptureCacheError('No capture to export');
+      }
+      record.receipt.body = [...record.receipt.body];
+      return JSON.stringify(record, null, 2);
+    });
+  }
+  async import(replay) {
+    let record;
+    try {
+      record = JSON.parse(replay);
+    } catch (error) {
+      throw new CaptureCacheError(`Invalid replay JSON: ${error.message}`);
+    }
+    validate(record);
+    if (
+      !Array.isArray(record.receipt.body) ||
+      record.receipt.body.some(
+        (byte) => !Number.isInteger(byte) || byte < 0 || byte > 255
+      )
+    ) {
+      throw new CaptureCacheError('Invalid replay bytes');
+    }
+    const body = Buffer.from(record.receipt.body);
+    if (digest(body) !== record.sha256) {
+      throw new CaptureCacheError('Replay SHA-256 mismatch');
+    }
+    return await this.store(
+      record.request,
+      { ...record.receipt, body },
+      record.fetchedAt
+    );
+  }
+  async records() {
+    let names;
+    try {
+      names = await readdir(path.join(this.directory, 'requests'));
+    } catch (error) {
+      if (error.code === 'ENOENT') {
+        return [];
+      }
+      throw error;
+    }
+    const result = [];
+    for (const name of names.filter((name) => name.endsWith('.json')).sort()) {
+      const record = JSON.parse(
+        await readFile(path.join(this.directory, 'requests', name), 'utf8')
+      );
+      validate(record);
+      if (name !== `${record.requestKey}.json`) {
+        throw new CaptureCacheError('Capture filename mismatch');
+      }
+      result.push(record);
+    }
+    return result;
+  }
+  list() {
+    return this.locked(() => this.records());
+  }
+  prune(now, ttl) {
+    return this.locked(async () => {
+      const records = await this.records();
+      const live = new Set();
+      let removed = 0;
+      for (const record of records) {
+        if (Math.max(0, now - record.fetchedAt) >= ttl) {
+          await unlink(this.recordPath(record.requestKey));
+          removed++;
+        } else {
+          live.add(record.sha256);
+        }
+      }
+      let bodies = [];
+      try {
+        bodies = await readdir(path.join(this.directory, 'bodies'));
+      } catch (error) {
+        if (error.code !== 'ENOENT') {
+          throw error;
+        }
+      }
+      for (const body of bodies) {
+        if (!live.has(body)) {
+          await unlink(this.bodyPath(body));
+        }
+      }
+      return removed;
+    });
+  }
+}
+
+/** Decorate any injectable transport; `execute` is safe to pass as a callback. */
+export class CachedTransport {
+  constructor(
+    inner,
+    store,
+    { ttl = 60 * 24 * 60 * 60 * 1000, offline = false, now = Date.now } = {}
+  ) {
+    if (!Number.isFinite(ttl) || ttl < 0) {
+      throw new TypeError('TTL must be non-negative milliseconds');
+    }
+    this.inner = inner;
+    this.store = store;
+    this.ttl = ttl;
+    this.offline = offline;
+    this.now = now;
+    this.execute = this.execute.bind(this);
+  }
+  async capture(request) {
+    request.signal?.throwIfAborted();
+    const identity = {
+      url: request.url,
+      method: request.method ?? 'GET',
+      headers: { ...request.headers },
+    };
+    const saved = await this.store.load(identity);
+    if (saved) {
+      saved.stale = Math.max(0, this.now() - saved.fetchedAt) >= this.ttl;
+      if (!saved.stale || this.offline) {
+        return saved;
+      }
+    }
+    if (this.offline) {
+      throw new CaptureCacheError(
+        'No cached capture in offline mode',
+        'offline_cache_miss',
+        identity.url
+      );
+    }
+    const receipt = await captureResponse(identity.url, {
+      ...identity,
+      headers: { ...identity.headers },
+      signal: request.signal,
+      transport: this.inner,
+    });
+    return this.store.store(identity, receipt, this.now());
+  }
+  async execute(request) {
+    return (await this.capture(request)).receipt;
+  }
+}

package/src/docx.js

@@ -75,7 +75,7 @@ export async function docxHandler(req, res) {
             const imgUrl = new URL(src, absoluteUrl).href;
             const imgResp = await fetch(imgUrl);
             if (imgResp.ok) {
-              const buffer = await imgResp.buffer();
+              const buffer = Buffer.from(await imgResp.arrayBuffer());
               const contentType = imgResp.headers.get('content-type') || '';
               children.push(
                 new Paragraph({

package/src/fetch.js

@@ -14,7 +14,7 @@ export async function fetchHandler(req, res) {
     copyProxyResponseHeaders(response.headers, res);
 
     // Get the response body as buffer and send it
-    const buffer = await response.buffer();
+    const buffer = Buffer.from(await response.arrayBuffer());
     res.send(buffer);
   } catch (err) {
     console.error('Fetch error:', err);

package/src/figures.js

@@ -10,10 +10,9 @@
  * @module figures
  */
 
-import fetch from 'node-fetch';
 import * as cheerio from 'cheerio';
 import { URL } from 'url';
-import { retry } from './retry.js';
+import { downloadBinary } from './binary-download.js';
 import { createBrowser, getBrowserEngine } from './browser.js';
 import { dismissPopups, scrollToLoadContent } from './popups.js';
 
@@ -86,16 +85,7 @@ export async function downloadFigures(figures, options = {}) {
     const filename = `figure-${figure.figureNum}.${ext}`;
 
     try {
-      const resp = await retry(() => fetch(figure.src), {
-        retries: 3,
-        baseDelay: 1000,
-      });
-
-      if (!resp.ok) {
-        throw new Error(`HTTP ${resp.status}`);
-      }
-
-      const buffer = await resp.buffer();
+      const buffer = await downloadBinary(figure.src);
 
       if (options.onProgress) {
         options.onProgress(figure.figureNum, 'downloaded');

package/src/gdocs-model-wait.js

@@ -12,7 +12,6 @@ export async function waitForGoogleDocsModelQuiescence(
   const started = Date.now();
   let lastFingerprint = null;
   let stableSince = null;
-  let lastModelData = null;
   let lastStableForMs = 0;
   let pollCount = 0;
 
@@ -20,7 +19,6 @@ export async function waitForGoogleDocsModelQuiescence(
     const modelData = await readGoogleDocsModelData(page);
     pollCount += 1;
     const fingerprint = googleDocsModelFingerprint(modelData);
-    lastModelData = modelData;
 
     if (
       fingerprint.chunks > 0 &&
@@ -56,8 +54,8 @@ export async function waitForGoogleDocsModelQuiescence(
 
     const elapsedMs = Date.now() - started;
     if (elapsedMs >= maxWaitMs) {
-      const lastChunks = lastModelData?.chunks?.length || 0;
-      const lastCidUrls = Object.keys(lastModelData?.cidUrlMap || {}).length;
+      const lastChunks = modelData?.chunks?.length || 0;
+      const lastCidUrls = Object.keys(modelData?.cidUrlMap || {}).length;
       throw googleDocsBrowserModelUnavailableError(
         `Google Docs DOCS_modelChunk stream did not quiesce within ${maxWaitMs} ms for document ${documentId} (last chunks=${lastChunks}, payload_bytes=${fingerprint.payloadBytes}, cid_urls=${lastCidUrls}, poll_count=${pollCount}, stable_for_ms=${lastStableForMs})`
       );

package/src/gdocs.js

@@ -8,7 +8,7 @@
 // Browser-model capture loads /edit and extracts DOCS_modelChunk data.
 
 import fetch from 'node-fetch';
-import he from 'he';
+import * as he from 'he';
 import * as cheerio from 'cheerio';
 import { convertHtmlToMarkdown } from './lib.js';
 import { createBrowser as defaultCreateBrowser } from './browser.js';

package/src/index.js

@@ -97,3 +97,10 @@ export {
   parseSharedDialog,
   sharedDialogHandler,
 } from './shared-dialog.js';
+
+export {
+  CaptureStore,
+  CachedTransport,
+  CaptureCacheError,
+  captureRequestKey,
+} from './cache.js';

package/src/lib.js

@@ -6,7 +6,7 @@ import TurndownService from 'turndown';
 import iconv from 'iconv-lite';
 import { URL, URLSearchParams } from 'url';
 import turndownPluginGfm from 'turndown-plugin-gfm';
-import he from 'he';
+import * as he from 'he';
 import { isFormulaImage, isMathElement, extractFormula } from './latex.js';
 import { extractMetadata } from './metadata.js';
 import { postProcessMarkdown } from './postprocess.js';
@@ -151,7 +151,7 @@ export async function fetchGoogleDriveImage(url) {
   }
 
   return {
-    buffer: await response.buffer(),
+    buffer: Buffer.from(await response.arrayBuffer()),
     contentType,
     downloadUrl,
     filename: `google-drive-${fileId}.${imageExtensionForContentType(contentType)}`,

package/src/localize-images.js

@@ -13,9 +13,8 @@
  * @module localize-images
  */
 
-import fetch from 'node-fetch';
 import { URL } from 'url';
-import { retry } from './retry.js';
+import { downloadBinary } from './binary-download.js';
 
 /**
  * Extract image references from markdown text.
@@ -165,16 +164,7 @@ export async function localizeImages(markdownText, options = {}) {
     }
 
     try {
-      const resp = await retry(() => fetch(image.url), {
-        retries: 3,
-        baseDelay: 1000,
-      });
-
-      if (!resp.ok) {
-        throw new Error(`HTTP ${resp.status}`);
-      }
-
-      const buffer = await resp.buffer();
+      const buffer = await downloadBinary(image.url);
       downloadedCount++;
 
       replacements.push({

package/src/search.js

@@ -20,7 +20,7 @@
  */
 
 import * as cheerio from 'cheerio';
-import he from 'he';
+import * as he from 'he';
 import { URL } from 'url';
 import { captureResponse, fetchTransport } from './transport.js';
 

package/src/shared-dialog.js

@@ -9,7 +9,7 @@
  */
 
 import fetch from 'node-fetch';
-import he from 'he';
+import * as he from 'he';
 import { URL } from 'node:url';
 
 import { createBrowser as defaultCreateBrowser } from './browser.js';

package/src/transport.js

@@ -63,12 +63,12 @@ export function fetchTransport({ url, method = 'GET', headers, signal }) {
  */
 export async function captureResponse(
   url,
-  { transport = fetchTransport, signal, headers = {} } = {}
+  { transport = fetchTransport, signal, headers = {}, method = 'GET' } = {}
 ) {
   if (!url) {
     throw new Error('Missing URL parameter');
   }
-  const request = { url, method: 'GET', headers, signal };
+  const request = { url, method, headers, signal };
   try {
     const response = await transport(request);
     if (

package/tests/ci/dependency-freshness.test.js

@@ -0,0 +1,74 @@
+import { jest } from '@jest/globals';
+import { checkVersions } from '../../../scripts/check-dependency-freshness.mjs';
+
+const latest = async () => '2.0.0';
+const dependencies = [{ name: 'example', current: '1.0.0' }];
+const blockers = {
+  example: {
+    issue: 'https://github.com/example/repo/issues/1',
+    reason: 'Upstream API breaks packaged consumers',
+  },
+};
+
+test('freshness rejects old releases, missing packages and undocumented exceptions', async () => {
+  for (const current of ['1.0.0', '(missing)']) {
+    await expect(
+      checkVersions([{ name: 'example', current }], {}, latest, jest.fn())
+    ).rejects.toThrow('behind 2.0.0');
+  }
+});
+test('freshness permits only documented open issues', async () => {
+  await expect(
+    checkVersions(dependencies, blockers, latest, async () => ({
+      state: 'open',
+    }))
+  ).resolves.toBeUndefined();
+  await expect(
+    checkVersions(dependencies, blockers, latest, async () => ({
+      state: 'closed',
+    }))
+  ).rejects.toThrow('is closed');
+  await expect(
+    checkVersions(
+      dependencies,
+      { example: { ...blockers.example, reason: '' } },
+      latest,
+      jest.fn()
+    )
+  ).rejects.toThrow('document an open issue');
+  await expect(
+    checkVersions(
+      dependencies,
+      { example: { ...blockers.example, reason: '   ' } },
+      latest,
+      jest.fn()
+    )
+  ).rejects.toThrow('document an open issue');
+});
+test('registry and issue failures fail the check rather than treating dependencies as fresh', async () => {
+  await expect(
+    checkVersions(
+      dependencies,
+      {},
+      async () => {
+        throw new Error('registry down');
+      },
+      jest.fn()
+    )
+  ).rejects.toThrow('registry down');
+  await expect(
+    checkVersions(dependencies, blockers, latest, async () => {
+      throw new Error('issue unavailable');
+    })
+  ).rejects.toThrow('issue unavailable');
+});
+test('current releases need no exception', async () => {
+  const getIssue = jest.fn();
+  await checkVersions(
+    [{ name: 'example', current: '2.0.0' }],
+    {},
+    latest,
+    getIssue
+  );
+  expect(getIssue).not.toHaveBeenCalled();
+});

package/tests/ci/detect-code-changes.test.js

@@ -39,7 +39,10 @@ function commitAll(repoPath, message) {
   git(repoPath, ['commit', '-m', message]);
 }
 
-function createMergedFeatureRepo() {
+function createMergedFeatureRepo({
+  featureCode = true,
+  baseCode = false,
+} = {}) {
   const repoPath = mkdtempSync(join(tmpdir(), 'web-capture-detect-'));
 
   initializeRepo(repoPath);
@@ -48,13 +51,19 @@ function createMergedFeatureRepo() {
   commitAll(repoPath, 'initial commit');
 
   git(repoPath, ['checkout', '-b', 'feature']);
-  writeTrackedFile(repoPath, 'rust/src/lib.rs', 'pub fn capture() {}\n');
-  commitAll(repoPath, 'add rust code');
+  if (featureCode) {
+    writeTrackedFile(repoPath, 'rust/src/lib.rs', 'pub fn capture() {}\n');
+    commitAll(repoPath, 'add rust code');
+  }
 
   writeTrackedFile(repoPath, 'docs/notes.md', '# docs only tail commit\n');
   commitAll(repoPath, 'document rust change');
 
   git(repoPath, ['checkout', 'main']);
+  if (baseCode) {
+    writeTrackedFile(repoPath, 'rust/src/lib.rs', 'pub fn base_capture() {}\n');
+    commitAll(repoPath, 'base branch code change');
+  }
   git(repoPath, ['merge', '--no-ff', 'feature', '-m', 'merge feature']);
 
   return repoPath;
@@ -123,28 +132,32 @@ describe('detect-code-changes', () => {
     }
   });
 
-  test('detects code introduced by a real merge commit pushed to main', () => {
-    repoPath = createMergedFeatureRepo();
-
-    const outputs = runDetector(repoPath, 'push');
-
-    expect(outputs['rust-code-changed']).toBe('true');
-    expect(outputs['rust-changed']).toBe('true');
-    expect(outputs['any-rust-code-changed']).toBe('true');
-    expect(outputs['any-code-changed']).toBe('true');
-    expect(outputs['docs-changed']).toBe('true');
-  });
-
-  test('keeps pull request merge commits scoped to the PR head commit', () => {
-    repoPath = createMergedFeatureRepo();
+  test.each(['push', 'pull_request'])(
+    'detects code before a docs-only final commit for %s',
+    (eventName) => {
+      repoPath = createMergedFeatureRepo();
+
+      expect(runDetector(repoPath, eventName)).toMatchObject({
+        'rust-code-changed': 'true',
+        'rust-changed': 'true',
+        'any-rust-code-changed': 'true',
+        'any-code-changed': 'true',
+        'docs-changed': 'true',
+      });
+    }
+  );
 
-    const outputs = runDetector(repoPath, 'pull_request');
+  test.each([
+    ['docs-only', { featureCode: false }],
+    ['base branch code only', { featureCode: false, baseCode: true }],
+  ])('keeps code checks disabled for %s PRs', (_name, options) => {
+    repoPath = createMergedFeatureRepo(options);
 
-    expect(outputs['rust-code-changed']).toBe('false');
-    expect(outputs['rust-changed']).toBe('false');
-    expect(outputs['any-rust-code-changed']).toBe('false');
-    expect(outputs['any-code-changed']).toBe('false');
-    expect(outputs['docs-changed']).toBe('true');
+    expect(runDetector(repoPath, 'pull_request')).toMatchObject({
+      'rust-code-changed': 'false',
+      'any-code-changed': 'false',
+      'docs-changed': 'true',
+    });
   });
 
   test('detects files when a push only has one commit', () => {

package/tests/ci/docker-install.test.js

@@ -0,0 +1,31 @@
+import { readFileSync } from 'node:fs';
+import { spawnSync } from 'node:child_process';
+
+test.each(['js', 'rust'])(
+  '%s Docker build fails when every browser package installation attempt fails',
+  (implementation) => {
+    const dockerfile = readFileSync(
+      new globalThis.URL(
+        `../../../${implementation}/Dockerfile`,
+        import.meta.url
+      ),
+      'utf8'
+    );
+    const loop = dockerfile.match(/for i in 1 2 3; do[\s\S]*?done/)[0];
+    // Execute only the finite retry loop; stub installation and waiting.
+    const result = spawnSync(
+      'bash',
+      [
+        '-c',
+        `
+      apt-get() { printf '%s\\n' "$1"; [ "$1" = update ]; }
+      sleep() { return 0; }
+      ${loop}
+    `,
+      ],
+      { encoding: 'utf8' }
+    );
+    expect(result.stdout.match(/^install$/gm)).toHaveLength(3);
+    expect(result.status).toBe(1);
+  }
+);