@link-assistant/web-capture 1.10.6 → 1.10.8

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # @link-assistant/web-capture
 
+## 1.10.8
+
+### Patch Changes
+
+- 8cee5e4: Keep JS/Rust kreuzberg parity for the html-to-markdown 3.6 image dimensions
+  change (issue #137). Export `normalizeStructuredKeys` and add regression tests
+  asserting inline image `dimensions` are exposed as `{ width, height }`, mirroring
+  the Rust `inline_image_to_json` fix.
+
+## 1.10.7
+
+### Patch Changes
+
+- 5311a23: Document the FormalAI-compatible capture/search contract and add smoke tests for stable HTTP and CLI response shapes.
+
 ## 1.10.6
 
 ### Patch Changes

package/README.md

@@ -127,6 +127,8 @@ parsing. For very large or slow documents, tune the wait with
 ## API Endpoints (Server Mode)
 
 Start the server with `web-capture --serve` and use the endpoints below.
+FormalAI integration should use the stable HTTP/CLI contract documented in
+[`../docs/formalai-contract.md`](../docs/formalai-contract.md).
 
 ### HTML Endpoint
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.10.6",
+  "version": "1.10.8",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/kreuzberg.js

@@ -39,7 +39,19 @@ function toSnakeCase(key) {
   return key.replace(/[A-Z]/g, (match) => `_${match.toLowerCase()}`);
 }
 
-function normalizeStructuredKeys(value) {
+/**
+ * Recursively normalize object keys to snake_case.
+ *
+ * Used to give the structured `metadata`, `tables`, and `images` payloads a
+ * stable shape regardless of the casing the native binding emits. In
+ * particular, inline image `dimensions` are exposed as `{ width, height }` —
+ * mirroring the Rust `inline_image_to_json` mapping after the
+ * html-to-markdown 3.6 `ImageDimensions` change (see issue #137).
+ *
+ * @param {*} value - Value to normalize
+ * @returns {*} The value with all object keys converted to snake_case
+ */
+export function normalizeStructuredKeys(value) {
   if (Array.isArray(value)) {
     return value.map(normalizeStructuredKeys);
   }

package/tests/helpers/search-fixture.cjs

@@ -0,0 +1,17 @@
+const nock = require('nock');
+
+nock('https://en.wikipedia.org')
+  .persist()
+  .get('/w/rest.php/v1/search/page')
+  .query(true)
+  .reply(200, {
+    pages: [
+      {
+        id: 1,
+        key: 'Formal_methods',
+        title: 'Formal methods',
+        excerpt: 'the study of <b>formal</b> methods',
+        description: 'rigorous techniques',
+      },
+    ],
+  });

package/tests/integration/formalai-contract.test.js

@@ -0,0 +1,214 @@
+/**
+ * Smoke tests for the FormalAI-facing HTTP contract (issue #135).
+ *
+ * These tests intentionally assert the stable response shapes documented in
+ * docs/formalai-contract.md rather than endpoint internals.
+ */
+
+import request from 'supertest';
+import nock from 'nock';
+import unzipper from 'unzipper';
+import { app, SEARCH_PROVIDERS } from '../../src/index.js';
+
+const HTML = `<!doctype html>
+<html>
+  <head><title>FormalAI fixture</title></head>
+  <body><h1>FormalAI Fixture</h1><p>Stable HTTP shape.</p></body>
+</html>`;
+
+const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+
+const WIKI_JSON = {
+  pages: [
+    {
+      id: 1,
+      key: 'Formal_methods',
+      title: 'Formal methods',
+      excerpt: 'the study of <b>formal</b> methods',
+      description: 'rigorous techniques',
+    },
+  ],
+};
+
+function parseBinary(res, callback) {
+  const chunks = [];
+  res.on('data', (chunk) => chunks.push(chunk));
+  res.on('end', () => callback(null, Buffer.concat(chunks)));
+}
+
+afterEach(() => {
+  nock.cleanAll();
+});
+
+describe('FormalAI HTTP contract (#135)', () => {
+  it('returns stable text artifact shapes for /html, /txt, and /markdown', async () => {
+    nock('https://formalai.example').get('/page').reply(200, HTML, {
+      'content-type': 'text/html; charset=utf-8',
+    });
+    const html = await request(app)
+      .get('/html')
+      .query({ url: 'https://formalai.example/page' })
+      .expect(200)
+      .expect('Content-Type', /text\/html/);
+    expect(html.text).toContain('<h1>FormalAI Fixture</h1>');
+
+    nock('https://formalai.example')
+      .get('/text')
+      .reply(200, 'FormalAI plain text', {
+        'content-type': 'text/plain; charset=utf-8',
+      });
+    const text = await request(app)
+      .get('/txt')
+      .query({ url: 'https://formalai.example/text' })
+      .expect(200)
+      .expect('Content-Type', /text\/plain/);
+    expect(text.headers['content-disposition']).toContain('.txt');
+    expect(text.text).toBe('FormalAI plain text');
+
+    nock('https://formalai.example').get('/markdown').reply(200, HTML, {
+      'content-type': 'text/html; charset=utf-8',
+    });
+    const markdown = await request(app)
+      .get('/markdown')
+      .query({ url: 'https://formalai.example/markdown' })
+      .expect(200)
+      .expect('Content-Type', /text\/markdown/);
+    expect(markdown.text).toContain('FormalAI Fixture');
+    expect(markdown.text).toContain('Stable HTTP shape.');
+  });
+
+  it('preserves upstream status and content for /fetch and /stream', async () => {
+    nock('https://formalai.example')
+      .get('/fetch-source')
+      .reply(203, 'fetch body', {
+        'content-type': 'text/plain; charset=utf-8',
+      });
+    const fetched = await request(app)
+      .get('/fetch')
+      .query({ url: 'https://formalai.example/fetch-source' })
+      .expect(203)
+      .expect('Content-Type', /text\/plain/);
+    expect(fetched.text).toBe('fetch body');
+
+    nock('https://formalai.example')
+      .get('/stream-source')
+      .reply(206, 'stream body', {
+        'content-type': 'text/plain; charset=utf-8',
+      });
+    const streamed = await request(app)
+      .get('/stream')
+      .query({ url: 'https://formalai.example/stream-source' })
+      .expect(206)
+      .expect('Content-Type', /text\/plain/);
+    expect(streamed.text).toBe('stream body');
+  });
+
+  it('returns stable binary artifact shapes for /image and /archive', async () => {
+    nock('https://drive.google.com')
+      .get('/uc')
+      .query({ export: 'download', id: 'formalaiimage' })
+      .reply(200, PNG, { 'content-type': 'image/png' });
+    const image = await request(app)
+      .get('/image')
+      .query({
+        url: 'https://drive.google.com/file/d/formalaiimage/view',
+      })
+      .buffer(true)
+      .parse(parseBinary)
+      .expect(200)
+      .expect('Content-Type', /image\/png/);
+    expect(image.headers['content-disposition']).toContain(
+      'google-drive-formalaiimage.png'
+    );
+    expect(image.body.subarray(0, 8)).toEqual(PNG);
+
+    nock('https://formalai.example').get('/archive').reply(200, HTML, {
+      'content-type': 'text/html; charset=utf-8',
+    });
+    const archive = await request(app)
+      .get('/archive')
+      .query({
+        url: 'https://formalai.example/archive',
+        localImages: 'false',
+      })
+      .buffer(true)
+      .parse(parseBinary)
+      .expect(200)
+      .expect('Content-Type', /application\/zip/);
+    expect(archive.body[0]).toBe(0x50);
+    expect(archive.body[1]).toBe(0x4b);
+    const zip = await unzipper.Open.buffer(archive.body);
+    expect(zip.files.map((file) => file.path)).toEqual(
+      expect.arrayContaining(['document.md', 'document.html'])
+    );
+  });
+
+  it('returns normalized /search JSON diagnostics and the documented provider catalog', async () => {
+    expect(SEARCH_PROVIDERS).toEqual([
+      'wikipedia',
+      'duckduckgo',
+      'google',
+      'bing',
+      'brave',
+    ]);
+
+    nock('https://en.wikipedia.org')
+      .get('/w/rest.php/v1/search/page')
+      .query(true)
+      .reply(200, WIKI_JSON);
+
+    const res = await request(app)
+      .get('/search')
+      .query({ q: 'formal-ai', provider: 'wikipedia', limit: 1 })
+      .expect(200)
+      .expect('Content-Type', /application\/json/);
+
+    expect(res.body).toMatchObject({
+      query: 'formal-ai',
+      provider: 'wikipedia',
+      captureMode: 'fetch',
+      results: [
+        {
+          rank: 1,
+          title: 'Formal methods',
+          url: 'https://en.wikipedia.org/wiki/Formal_methods',
+          snippet: 'the study of formal methods',
+        },
+      ],
+      diagnostics: {
+        status: 200,
+        blockedByCors: false,
+        blockedByCaptcha: false,
+      },
+    });
+    expect(res.body.capturedAt).toEqual(expect.any(String));
+    expect(res.body.diagnostics.sourceUrl).toContain('en.wikipedia.org');
+  });
+
+  it('keeps normalized /search diagnostics when provider capture fails', async () => {
+    nock('https://en.wikipedia.org')
+      .get('/w/rest.php/v1/search/page')
+      .query(true)
+      .replyWithError(new Error('provider offline'));
+
+    const res = await request(app)
+      .get('/search')
+      .query({ q: 'formal-ai', provider: 'wikipedia', limit: 1 })
+      .expect(200)
+      .expect('Content-Type', /application\/json/);
+
+    expect(res.body).toMatchObject({
+      query: 'formal-ai',
+      provider: 'wikipedia',
+      captureMode: 'fetch',
+      results: [],
+      diagnostics: {
+        status: 0,
+        blockedByCors: false,
+        blockedByCaptcha: false,
+      },
+    });
+    expect(res.body.diagnostics.sourceUrl).toContain('en.wikipedia.org');
+    expect(res.body.diagnostics.error).toContain('provider offline');
+  });
+});

package/tests/unit/cli.test.js

@@ -1,7 +1,7 @@
 // Unit tests for CLI argument parsing and functionality
 import { spawn } from 'child_process';
 import http from 'node:http';
-import { mkdtempSync, rmSync, writeFileSync } from 'fs';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
 import { fileURLToPath } from 'url';
 import { dirname, join, resolve } from 'path';
 import packageJson from '../../package.json' with { type: 'json' };
@@ -9,11 +9,12 @@ import packageJson from '../../package.json' with { type: 'json' };
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
 const cliPath = resolve(__dirname, '../../bin/web-capture.js');
+const searchFixturePath = resolve(__dirname, '../helpers/search-fixture.cjs');
 
 // Helper function to run CLI and capture output
 function runCli(args, options = {}) {
   return new Promise((resolve, reject) => {
-    const proc = spawn('node', [cliPath, ...args], {
+    const proc = spawn(process.execPath, [cliPath, ...args], {
       cwd: options.cwd || dirname(cliPath),
       env: { ...process.env, ...options.env },
     });
@@ -36,13 +37,14 @@ function runCli(args, options = {}) {
   });
 }
 
-function startFixtureServer() {
+function startFixtureServer({
+  body = '<!doctype html><html><body><h1>Issue 68</h1><p>Captured from positional URL.</p></body></html>',
+  contentType = 'text/html; charset=utf-8',
+} = {}) {
   return new Promise((resolve, reject) => {
     const server = http.createServer((req, res) => {
-      res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
-      res.end(
-        '<!doctype html><html><body><h1>Issue 68</h1><p>Captured from positional URL.</p></body></html>'
-      );
+      res.writeHead(200, { 'Content-Type': contentType });
+      res.end(body);
     });
 
     server.on('error', reject);
@@ -255,4 +257,99 @@ describe('CLI', () => {
       expect(result.stdout).toContain('-o');
     });
   });
+
+  describe('FormalAI-compatible CLI contract', () => {
+    test('writes HTML, Markdown, and text captures to stdout with -o -', async () => {
+      const { server, url } = await startFixtureServer({
+        body: '<!doctype html><html><body><h1>FormalAI Fixture</h1><p>Stable CLI shape.</p></body></html>',
+      });
+
+      try {
+        const html = await runCli([url, '--format', 'html', '--output', '-']);
+        expect(html.code).toBe(0);
+        expect(html.stdout).toContain('<h1>FormalAI Fixture</h1>');
+
+        const markdown = await runCli([
+          url,
+          '--format',
+          'markdown',
+          '--output',
+          '-',
+        ]);
+        expect(markdown.code).toBe(0);
+        expect(markdown.stdout).toContain('FormalAI Fixture');
+        expect(markdown.stdout).toContain('Stable CLI shape.');
+
+        const text = await runCli([url, '--format', 'txt', '--output', '-']);
+        expect(text.code).toBe(0);
+        expect(text.stdout).toContain('FormalAI Fixture');
+      } finally {
+        await stopFixtureServer(server);
+      }
+    }, 20000);
+
+    test('writes archive captures as ZIP files for binary CLI consumers', async () => {
+      const outputDir = mkdtempSync(join(process.cwd(), 'formalai-cli-'));
+      const outputPath = join(outputDir, 'capture.zip');
+      const { server, url } = await startFixtureServer({
+        body: '<!doctype html><html><body><h1>FormalAI Archive</h1></body></html>',
+      });
+
+      try {
+        const result = await runCli([
+          url,
+          '--archive',
+          'zip',
+          '--output',
+          outputPath,
+        ]);
+
+        expect(result.code).toBe(0);
+        const zipBytes = readFileSync(outputPath);
+        expect(zipBytes.length).toBeGreaterThan(50);
+        expect(zipBytes[0]).toBe(0x50);
+        expect(zipBytes[1]).toBe(0x4b);
+      } finally {
+        await stopFixtureServer(server);
+        rmSync(outputDir, { recursive: true, force: true });
+      }
+    }, 20000);
+
+    test('emits normalized search JSON from the search subcommand', async () => {
+      const nodeOptions = [
+        process.env.NODE_OPTIONS,
+        `--require=${searchFixturePath}`,
+      ]
+        .filter(Boolean)
+        .join(' ');
+      const result = await runCli(
+        ['search', 'formal-ai', '--provider', 'wikipedia', '--limit', '1'],
+        {
+          env: { NODE_OPTIONS: nodeOptions },
+        }
+      );
+
+      expect(result.code).toBe(0);
+      const body = JSON.parse(result.stdout);
+      expect(body).toMatchObject({
+        query: 'formal-ai',
+        provider: 'wikipedia',
+        captureMode: 'fetch',
+      });
+      expect(body.results).toEqual([
+        {
+          rank: 1,
+          title: 'Formal methods',
+          url: 'https://en.wikipedia.org/wiki/Formal_methods',
+          snippet: 'the study of formal methods',
+        },
+      ]);
+      expect(body.diagnostics).toMatchObject({
+        status: 200,
+        blockedByCors: false,
+        blockedByCaptcha: false,
+      });
+      expect(body.diagnostics.sourceUrl).toContain('en.wikipedia.org');
+    }, 20000);
+  });
 });

package/tests/unit/kreuzberg.test.js

@@ -1,6 +1,7 @@
 import {
   convertWithKreuzberg,
   isKreuzbergAvailable,
+  normalizeStructuredKeys,
 } from '../../src/kreuzberg.js';
 
 let available;
@@ -207,3 +208,42 @@ describe('kreuzberg html-to-markdown integration', () => {
     expect(result.content).toContain('Item 3');
   });
 });
+
+describe('kreuzberg structured key normalization (issue #137 parity)', () => {
+  // Mirrors the Rust `inline_image_to_json` regression tests: after the
+  // html-to-markdown 3.6 change, inline image dimensions are a structured
+  // `{ width, height }` value rather than a tuple. The normalization pipeline
+  // that backs `result.images` must preserve those `width`/`height` keys.
+  it('preserves width/height keys on inline image dimensions', () => {
+    const images = normalizeStructuredKeys([
+      {
+        format: 'png',
+        filename: 'pixel.png',
+        description: 'one pixel',
+        dimensions: { width: 800, height: 600 },
+        source: 'ImgDataUri',
+      },
+    ]);
+
+    expect(images).toHaveLength(1);
+    expect(images[0].dimensions).toEqual({ width: 800, height: 600 });
+    expect(images[0].format).toBe('png');
+    expect(images[0].filename).toBe('pixel.png');
+  });
+
+  it('leaves images without dimensions untouched', () => {
+    const images = normalizeStructuredKeys([
+      { format: 'svg', source: 'SvgElement', dimensions: null },
+    ]);
+
+    expect(images[0].dimensions).toBeNull();
+  });
+
+  it('snake_cases nested structured keys', () => {
+    const normalized = normalizeStructuredKeys({
+      openGraph: { siteName: 'Example' },
+    });
+
+    expect(normalized).toEqual({ open_graph: { site_name: 'Example' } });
+  });
+});