@link-assistant/web-capture 1.10.12 → 1.11.1

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/web-capture
 
+## 1.11.1
+
+### Patch Changes
+
+- Auto-release unreleased changes
+
+## 1.11.0
+
+### Minor Changes
+
+- 66d8144: Add caller-owned transports, AbortSignal cancellation, and exact-byte response receipts to capture and search library APIs.
+
 ## 1.10.12
 
 ### Patch Changes

package/README.md

@@ -568,6 +568,33 @@ const buffer = await page.screenshot({ type: 'png', fullPage: true });
 await browser.close();
 ```
 
+### Exact response receipts and cancellation
+
+Library callers can retain response provenance without decoding the source
+through a text API. A receipt contains the exact `Buffer` body, final URL, HTTP
+status, selected cache/content headers, and structured diagnostics:
+
+```javascript
+import { fetchHtmlReceipt, search } from '@link-assistant/web-capture';
+
+const controller = new AbortController();
+const receipt = await fetchHtmlReceipt('https://example.com', {
+  signal: controller.signal,
+  // transport: ({ url, method, headers, signal }) => myTransport(...),
+});
+
+const result = await search({
+  query: 'formal methods',
+  signal: controller.signal,
+});
+console.log(receipt.status, result.receipt.body, result.receipt.finalUrl);
+```
+
+`transport` may return either a Fetch-compatible `Response` or a materialized
+receipt. The search receipt is available to library callers as
+`result.receipt`; it is intentionally non-enumerable so the stable `/search`
+JSON response does not duplicate the full provider body.
+
 ## License
 
 [Unlicense](../LICENSE) — This is free and unencumbered software released into the public domain. You are free to copy, modify, publish, use, compile, sell, or distribute this software for any purpose, commercial or non-commercial, and by any means. See [https://unlicense.org](https://unlicense.org) for details.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.10.12",
+  "version": "1.11.1",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/index.js

@@ -80,6 +80,13 @@ export {
   formatSearchAsMarkdown,
   SEARCH_PROVIDERS,
 } from './search.js';
+export {
+  captureResponse,
+  fetchTransport,
+  CaptureTransportError,
+  RECEIPT_HEADERS,
+} from './transport.js';
+export { fetchHtml, fetchHtmlReceipt } from './lib.js';
 export {
   captureSharedDialog,
   formatSharedDialogAsDemoMemory,

package/src/lib.js

@@ -11,6 +11,7 @@ import { isFormulaImage, isMathElement, extractFormula } from './latex.js';
 import { extractMetadata } from './metadata.js';
 import { postProcessMarkdown } from './postprocess.js';
 import { retry } from './retry.js';
+import { captureResponse, fetchTransport } from './transport.js';
 
 const STACKPRINTER_RETRIES = 3;
 const STACKPRINTER_RETRY_BASE_DELAY_MS = 1000;
@@ -19,7 +20,7 @@ const HTML_FETCH_HEADERS = {
   'accept-encoding': 'identity',
 };
 
-export async function fetchHtml(url) {
+export async function fetchHtml(url, options = {}) {
   if (!url) {
     throw new Error('Missing URL parameter');
   }
@@ -33,8 +34,16 @@ export async function fetchHtml(url) {
     return await fetchStackPrinterHtml(stackPrinterUrl);
   }
 
-  const response = await fetchHtmlResponse(url);
-  return response.text();
+  const receipt = await fetchHtmlReceipt(url, options);
+  return receipt.body.toString('utf8');
+}
+
+/** Return an undecoded, exact-byte receipt for a document fetch. */
+export function fetchHtmlReceipt(url, options = {}) {
+  return captureResponse(url, {
+    ...options,
+    headers: { ...HTML_FETCH_HEADERS, ...options.headers },
+  });
 }
 
 export function getGoogleDriveFileId(url) {
@@ -216,9 +225,7 @@ async function fetchStackPrinterHtml(url) {
 }
 
 function fetchHtmlResponse(url) {
-  return fetch(url, {
-    headers: HTML_FETCH_HEADERS,
-  });
+  return fetchTransport({ url, headers: HTML_FETCH_HEADERS });
 }
 
 function isStackPrinterTransientError(html) {

package/src/search.js

@@ -19,10 +19,10 @@
  * @module search
  */
 
-import fetch from 'node-fetch';
 import * as cheerio from 'cheerio';
 import he from 'he';
 import { URL } from 'url';
+import { captureResponse, fetchTransport } from './transport.js';
 
 /** Providers understood by the search contract. */
 export const SEARCH_PROVIDERS = [
@@ -310,7 +310,9 @@ export function formatSearchAsMarkdown(result) {
  * @param {string} [options.provider=wikipedia] - Provider id
  * @param {number} [options.limit=10] - Max results to return
  * @param {string} [options.captureMode=fetch] - Reported capture mode
- * @param {Function} [options.fetchImpl=fetch] - Fetch implementation (injectable for tests)
+ * @param {Function} [options.transport] - Receipt transport (injectable)
+ * @param {AbortSignal} [options.signal] - Caller cancellation signal
+ * @param {Function} [options.fetchImpl] - Legacy Fetch-compatible injection
  * @param {Function} [options.now] - Clock returning an ISO timestamp (injectable for tests)
  * @returns {Promise<Object>} Normalized search result object
  */
@@ -319,7 +321,9 @@ export async function search({
   provider = DEFAULT_PROVIDER,
   limit = DEFAULT_LIMIT,
   captureMode = 'fetch',
-  fetchImpl = fetch,
+  transport,
+  signal,
+  fetchImpl,
   now = () => new Date().toISOString(),
 } = {}) {
   if (!query || !String(query).trim()) {
@@ -342,7 +346,14 @@ export async function search({
 
   let results = [];
   try {
-    const response = await fetchImpl(sourceUrl, {
+    const effectiveTransport = transport
+      ? transport
+      : fetchImpl
+        ? ({ url, ...init }) => fetchImpl(url, init)
+        : fetchTransport;
+    const receipt = await captureResponse(sourceUrl, {
+      transport: effectiveTransport,
+      signal,
       headers: {
         'User-Agent': USER_AGENT,
         Accept:
@@ -352,16 +363,33 @@ export async function search({
         'Accept-Language': 'en-US,en;q=0.9',
       },
     });
-    diagnostics.status = response.status;
-    const body = await response.text();
+    diagnostics.status = receipt.status;
+    const body = receipt.body.toString('utf8');
     const parsed = parseSearchResults(provider, body, { limit });
     results = parsed.results;
     diagnostics.blockedByCaptcha = parsed.blockedByCaptcha;
+    const result = {
+      query,
+      provider,
+      captureMode,
+      capturedAt,
+      results,
+      diagnostics,
+    };
+    // Library callers can bind parsed results to the exact receipt without
+    // inflating the stable HTTP /search JSON representation.
+    Object.defineProperty(result, 'receipt', {
+      value: receipt,
+      enumerable: false,
+    });
+    return result;
   } catch (err) {
     // A network/transport failure from a server context is reported as an
     // error status; browser CORS failures surface the same way and are flagged.
     diagnostics.status = diagnostics.status || 0;
     diagnostics.error = err.message;
+    diagnostics.errorKind = err.diagnostics?.errorKind || 'transport';
+    diagnostics.blockedByCors = diagnostics.errorKind === 'cors_or_transport';
   }
 
   return {

package/src/transport.js

@@ -0,0 +1,108 @@
+import fetch from 'node-fetch';
+
+export const RECEIPT_HEADERS = [
+  'cache-control',
+  'content-encoding',
+  'content-length',
+  'content-type',
+  'etag',
+  'last-modified',
+  'location',
+];
+
+function selectedHeaders(headers) {
+  const selected = {};
+  for (const name of RECEIPT_HEADERS) {
+    const value =
+      typeof headers?.get === 'function' ? headers.get(name) : headers?.[name];
+    if (value !== null && value !== undefined) {
+      selected[name] = String(value);
+    }
+  }
+  return selected;
+}
+
+function classifyTransportError(error, signal) {
+  if (signal?.aborted || error?.name === 'AbortError') {
+    return 'cancelled';
+  }
+  if (error?.name === 'TimeoutError' || error?.type === 'request-timeout') {
+    return 'timeout';
+  }
+  if (error instanceof TypeError) {
+    return 'cors_or_transport';
+  }
+  return 'transport';
+}
+
+export class CaptureTransportError extends Error {
+  constructor(error, url, signal) {
+    super(error?.message || String(error));
+    this.name = 'CaptureTransportError';
+    this.cause = error;
+    this.diagnostics = {
+      outcome: 'error',
+      errorKind: classifyTransportError(error, signal),
+      sourceUrl: url,
+      error: this.message,
+    };
+  }
+}
+
+/** Default caller-replaceable transport. Dropping/aborting `signal` cancels fetch. */
+export function fetchTransport({ url, method = 'GET', headers, signal }) {
+  return fetch(url, { method, headers, signal });
+}
+
+/**
+ * Capture an HTTP response without decoding its body.
+ *
+ * A custom transport receives `{url, method, headers, signal}` and may return
+ * either a Fetch `Response` or an already materialized receipt. The returned
+ * body is a Buffer containing the exact response bytes.
+ */
+export async function captureResponse(
+  url,
+  { transport = fetchTransport, signal, headers = {} } = {}
+) {
+  if (!url) {
+    throw new Error('Missing URL parameter');
+  }
+  const request = { url, method: 'GET', headers, signal };
+  try {
+    const response = await transport(request);
+    if (
+      response?.body !== null &&
+      response?.body !== undefined &&
+      response?.finalUrl &&
+      response?.diagnostics
+    ) {
+      return { ...response, body: Buffer.from(response.body) };
+    }
+    if (
+      !response ||
+      (typeof response.arrayBuffer !== 'function' &&
+        typeof response.text !== 'function')
+    ) {
+      throw new TypeError(
+        'Transport must return a Response or response receipt'
+      );
+    }
+    const body =
+      typeof response.arrayBuffer === 'function'
+        ? Buffer.from(await response.arrayBuffer())
+        : Buffer.from(await response.text());
+    return {
+      body,
+      finalUrl: response.url || url,
+      status: response.status,
+      headers: selectedHeaders(response.headers),
+      diagnostics: { outcome: 'response' },
+    };
+  } catch (error) {
+    if (error instanceof CaptureTransportError) {
+      throw error;
+    }
+    throw new CaptureTransportError(error, url, signal);
+  }
+}

package/tests/unit/fetch-html.test.js

@@ -1,5 +1,6 @@
 import nock from 'nock';
-import { fetchHtml } from '../../src/lib.js';
+import { jest } from '@jest/globals';
+import { fetchHtml, fetchHtmlReceipt } from '../../src/lib.js';
 
 afterEach(() => {
   nock.cleanAll();
@@ -21,4 +22,33 @@ describe('fetchHtml', () => {
       'ok'
     );
   });
+
+  it('returns an exact-byte receipt through an injectable transport', async () => {
+    const body = Buffer.from([0x00, 0xff, 0x41]);
+    const transport = jest.fn(async ({ url, signal }) => ({
+      body,
+      finalUrl: `${url}/final`,
+      status: 206,
+      headers: { 'content-type': 'application/octet-stream', etag: 'v1' },
+      diagnostics: { outcome: 'response' },
+      signal,
+    }));
+    const controller = new globalThis.AbortController();
+
+    const receipt = await fetchHtmlReceipt('https://example.com/bytes', {
+      transport,
+      signal: controller.signal,
+    });
+
+    expect(Buffer.from(receipt.body)).toEqual(body);
+    expect(receipt).toMatchObject({
+      finalUrl: 'https://example.com/bytes/final',
+      status: 206,
+      headers: { 'content-type': 'application/octet-stream', etag: 'v1' },
+      diagnostics: { outcome: 'response' },
+    });
+    expect(transport).toHaveBeenCalledWith(
+      expect.objectContaining({ signal: controller.signal })
+    );
+  });
 });

package/tests/unit/search.test.js

@@ -4,6 +4,7 @@
  * Parsers are pure functions tested against fixtures; the orchestrating
  * `search()` is exercised with an injected fetch so no network is required.
  */
+import { jest } from '@jest/globals';
 
 import {
   search,
@@ -75,6 +76,25 @@ describe('search module (#130)', () => {
     });
   });
 
+  it('keeps URL construction and response parsing transport-independent', () => {
+    const sourceUrl = buildSearchUrl('wikipedia', 'formal ai', 1);
+    const { results } = parseSearchResults('wikipedia', WIKI_JSON, {
+      limit: 1,
+    });
+
+    expect(sourceUrl).toBe(
+      'https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal%20ai&limit=1'
+    );
+    expect(results).toEqual([
+      {
+        rank: 1,
+        title: 'Formal methods',
+        url: 'https://en.wikipedia.org/wiki/Formal_methods',
+        snippet: 'the study of formal',
+      },
+    ]);
+  });
+
   describe('parseSearchResults', () => {
     it('normalizes Wikipedia REST JSON and strips markup', () => {
       const { results } = parseSearchResults('wikipedia', WIKI_JSON, {
@@ -205,6 +225,57 @@ describe('search module (#130)', () => {
       expect(result.diagnostics.error).toBe('network down');
     });
 
+    it('returns the source receipt and forwards caller cancellation', async () => {
+      const body = Buffer.from(WIKI_JSON);
+      const controller = new globalThis.AbortController();
+      const transport = jest.fn(async ({ url, signal }) => ({
+        body,
+        finalUrl: `${url}&redirected=true`,
+        status: 200,
+        headers: { 'content-type': 'application/json', etag: 'search-v1' },
+        diagnostics: { outcome: 'response' },
+        signal,
+      }));
+
+      const result = await search({
+        query: 'formal-ai',
+        transport,
+        signal: controller.signal,
+        now: fixedNow,
+      });
+
+      expect(Buffer.from(result.receipt.body)).toEqual(body);
+      expect(result.receipt.finalUrl).toContain('redirected=true');
+      expect(result.receipt.headers.etag).toBe('search-v1');
+      expect(transport).toHaveBeenCalledWith(
+        expect.objectContaining({ signal: controller.signal })
+      );
+    });
+
+    it('classifies AbortSignal cancellation in structured diagnostics', async () => {
+      const controller = new globalThis.AbortController();
+      controller.abort();
+      const transport = async ({ signal }) => {
+        expect(signal).toBe(controller.signal);
+        const error = new Error('cancelled by caller');
+        error.name = 'AbortError';
+        throw error;
+      };
+
+      const result = await search({
+        query: 'formal-ai',
+        transport,
+        signal: controller.signal,
+        now: fixedNow,
+      });
+
+      expect(result.diagnostics).toMatchObject({
+        status: 0,
+        errorKind: 'cancelled',
+        error: 'cancelled by caller',
+      });
+    });
+
     it('rejects an empty query', async () => {
       await expect(search({ query: '   ' })).rejects.toThrow(/Missing .query/);
     });