@link-assistant/web-capture 1.9.0 → 1.10.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @link-assistant/web-capture
 
+## 1.10.0
+
+### Minor Changes
+
+- b892585: Add kreuzberg html-to-markdown as high-performance alternative converter with structured results
+
 ## 1.9.0
 
 ### Minor Changes

package/README.md

@@ -141,11 +141,13 @@ GET /markdown?url=<URL>
 
 Converts the HTML content of the specified URL to Markdown format. By default, original remote image URLs are preserved and base64 data URIs are stripped (clean single-file output). Use `keepOriginalLinks=false` to strip all images, or `embedImages=true` to keep base64 images inline.
 
-| Parameter           | Required | Description                                | Default |
-| ------------------- | -------- | ------------------------------------------ | ------- |
-| `url`               | Yes      | URL to fetch                               | -       |
-| `embedImages`       | No       | Keep base64 images inline (`true`/`false`) | `false` |
-| `keepOriginalLinks` | No       | Keep original remote URLs, strip base64    | `true`  |
+| Parameter           | Required | Description                                                               | Default  |
+| ------------------- | -------- | ------------------------------------------------------------------------- | -------- |
+| `url`               | Yes      | URL to fetch                                                              | -        |
+| `converter`         | No       | Converter: `turndown` or `kreuzberg`                                      | turndown |
+| `format`            | No       | Response format: `text` or `json` (`json` requires `converter=kreuzberg`) | text     |
+| `embedImages`       | No       | Keep base64 images inline (`true`/`false`)                                | `false`  |
+| `keepOriginalLinks` | No       | Keep original remote URLs, strip base64                                   | `true`   |
 
 ### Image Endpoint
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.9.0",
+  "version": "1.10.0",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {
@@ -32,6 +32,7 @@
     "changeset:status": "changeset status --since=origin/main"
   },
   "dependencies": {
+    "@kreuzberg/html-to-markdown-node": "^3.5.5",
     "archiver": "^7.0.1",
     "browser-commander": "^0.8.0",
     "cheerio": "^1.0.0",

package/src/kreuzberg.js

@@ -0,0 +1,128 @@
+/**
+ * Kreuzberg html-to-markdown integration module.
+ *
+ * Provides high-performance HTML to Markdown conversion using the
+ * @kreuzberg/html-to-markdown-node library (Rust-powered, 150-280 MB/s).
+ *
+ * This converter is available as an alternative to the default Turndown-based
+ * converter, selectable via the `converter=kreuzberg` query parameter.
+ *
+ * @module kreuzberg
+ * @see https://github.com/kreuzberg-dev/html-to-markdown
+ */
+
+import { convertRelativeUrls } from './lib.js';
+
+let _initPromise = null;
+let _convert = null;
+
+async function ensureLoaded() {
+  if (_convert) {
+    return _convert;
+  }
+  if (_initPromise) {
+    return _initPromise;
+  }
+  _initPromise = import('@kreuzberg/html-to-markdown-node')
+    .then((mod) => {
+      _convert = mod.convert || mod.default?.convert || null;
+      return _convert;
+    })
+    .catch(() => {
+      _convert = null;
+      return null;
+    });
+  return await _initPromise;
+}
+
+function toSnakeCase(key) {
+  return key.replace(/[A-Z]/g, (match) => `_${match.toLowerCase()}`);
+}
+
+function normalizeStructuredKeys(value) {
+  if (Array.isArray(value)) {
+    return value.map(normalizeStructuredKeys);
+  }
+  if (!value || typeof value !== 'object') {
+    return value;
+  }
+
+  return Object.fromEntries(
+    Object.entries(value).map(([key, entry]) => [
+      toSnakeCase(key),
+      normalizeStructuredKeys(entry),
+    ])
+  );
+}
+
+/**
+ * Check if the kreuzberg converter is available.
+ *
+ * @returns {Promise<boolean>} Whether the converter is available
+ */
+export async function isKreuzbergAvailable() {
+  const fn = await ensureLoaded();
+  return fn !== null;
+}
+
+/**
+ * Convert HTML to Markdown using the kreuzberg html-to-markdown library.
+ *
+ * Returns a structured result with content, metadata, tables, images, and warnings.
+ *
+ * @param {string} html - HTML content to convert
+ * @param {Object} [options] - Conversion options
+ * @param {string} [options.baseUrl] - Base URL for resolving relative URLs
+ * @param {string} [options.headingStyle='Atx'] - Heading style ('Atx' or 'Setext')
+ * @param {string} [options.bulletListMarker] - Bullet character ('-', '*', '+')
+ * @param {string} [options.codeBlockStyle] - Code block style ('Fenced' or 'Indented')
+ * @returns {Promise<Object>} Structured conversion result
+ * @returns {string} result.content - The converted markdown content
+ * @returns {Object|null} result.metadata - Extracted metadata (title, links, headings, images, etc.)
+ * @returns {Array} result.tables - Extracted table data
+ * @returns {Array} result.images - Extracted image data
+ * @returns {Array} result.warnings - Non-fatal conversion warnings
+ * @throws {Error} If the kreuzberg converter is not available
+ */
+export async function convertWithKreuzberg(html, options = {}) {
+  const convert = await ensureLoaded();
+  if (!convert) {
+    throw new Error(
+      'Kreuzberg html-to-markdown is not installed. ' +
+        'Run: npm install @kreuzberg/html-to-markdown-node'
+    );
+  }
+
+  const convertOptions = {
+    headingStyle: 'Atx',
+    codeBlockStyle: 'Backticks',
+    ...options,
+  };
+  delete convertOptions.baseUrl;
+
+  const processedHtml = options.baseUrl
+    ? convertRelativeUrls(html, options.baseUrl)
+    : html;
+  const result = convert(processedHtml, convertOptions);
+
+  // Parse the metadata JSON string into an object
+  let metadata = null;
+  if (result.metadata) {
+    try {
+      metadata =
+        typeof result.metadata === 'string'
+          ? JSON.parse(result.metadata)
+          : result.metadata;
+    } catch {
+      metadata = null;
+    }
+  }
+
+  return {
+    content: result.content || '',
+    metadata: normalizeStructuredKeys(metadata),
+    tables: normalizeStructuredKeys(result.tables || []),
+    images: normalizeStructuredKeys(result.images || []),
+    warnings: result.warnings || [],
+  };
+}

package/src/lib.js

@@ -411,6 +411,20 @@ function selectedHtml($, selector) {
   return $selected.length ? $.html($selected) : null;
 }
 
+export function scopeHtmlForMarkdown(html, options = {}) {
+  const { contentSelector, bodySelector } = options;
+  const $ = cheerio.load(html);
+  const bodyHtml = selectedHtml($, bodySelector);
+  const contentHtml = selectedHtml($, contentSelector);
+  if (!bodyHtml && !contentHtml) {
+    return html;
+  }
+
+  const titleSelector = contentSelector ? `${contentSelector} h1, h1` : 'h1';
+  const titleHtml = bodyHtml ? selectedHtml($, titleSelector) : null;
+  return [titleHtml, bodyHtml || contentHtml].filter(Boolean).join('\n');
+}
+
 // Convert relative URLs to absolute URLs in HTML content
 export function convertRelativeUrls(html, baseUrl) {
   const base = new URL(baseUrl);
@@ -581,15 +595,8 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     metadata = extractMetadata($);
   }
 
-  const bodyHtml = selectedHtml($, bodySelector);
-  const contentHtml = selectedHtml($, contentSelector);
-  if (bodyHtml || contentHtml) {
-    const titleSelector = contentSelector ? `${contentSelector} h1, h1` : 'h1';
-    const titleHtml = bodyHtml ? selectedHtml($, titleSelector) : null;
-    $ = cheerio.load(
-      [titleHtml, bodyHtml || contentHtml].filter(Boolean).join('\n')
-    );
-  }
+  html = scopeHtmlForMarkdown(html, { contentSelector, bodySelector });
+  $ = cheerio.load(html);
 
   // Remove unwanted elements
   $('style, script, noscript').remove();

package/src/markdown.js

@@ -1,4 +1,9 @@
-import { fetchHtml, convertHtmlToMarkdownEnhanced } from './lib.js';
+import {
+  fetchHtml,
+  convertHtmlToMarkdownEnhanced,
+  scopeHtmlForMarkdown,
+} from './lib.js';
+import { convertWithKreuzberg, isKreuzbergAvailable } from './kreuzberg.js';
 import { applyImageMode } from './extract-images.js';
 
 export async function markdownHandler(req, res) {
@@ -6,9 +11,52 @@ export async function markdownHandler(req, res) {
   if (!url) {
     return res.status(400).send('Missing `url` parameter');
   }
+
+  const converter = (req.query.converter || 'turndown').toLowerCase();
+  const format = (req.query.format || 'text').toLowerCase();
   const embedImages = req.query.embedImages === 'true';
+
+  if (!['turndown', 'kreuzberg'].includes(converter)) {
+    return res.status(400).send('Unsupported `converter` parameter');
+  }
+  if (!['text', 'json'].includes(format)) {
+    return res.status(400).send('Unsupported `format` parameter');
+  }
+  if (format === 'json' && converter !== 'kreuzberg') {
+    return res
+      .status(400)
+      .send('`format=json` is only supported with `converter=kreuzberg`');
+  }
+
   try {
     const html = await fetchHtml(url);
+
+    if (converter === 'kreuzberg') {
+      const available = await isKreuzbergAvailable();
+      if (!available) {
+        return res
+          .status(501)
+          .send(
+            'Kreuzberg converter not available. Install @kreuzberg/html-to-markdown-node.'
+          );
+      }
+
+      const scopedHtml = scopeHtmlForMarkdown(html, {
+        contentSelector: req.query.contentSelector,
+        bodySelector: req.query.bodySelector,
+      });
+      const result = await convertWithKreuzberg(scopedHtml, { baseUrl: url });
+      const imageResult = await applyImageMode(result.content, {
+        mode: embedImages ? 'embed' : 'default',
+      });
+      result.content = imageResult.markdown;
+
+      if (format === 'json') {
+        return res.json(result);
+      }
+      return res.type('text/markdown').send(result.content);
+    }
+
     const { markdown } = convertHtmlToMarkdownEnhanced(html, url, {
       contentSelector: req.query.contentSelector,
       bodySelector: req.query.bodySelector,

package/tests/integration/api-endpoints.test.js

@@ -11,6 +11,7 @@ import request from 'supertest';
 import nock from 'nock';
 import unzipper from 'unzipper';
 import { app } from '../../src/index.js';
+import { isKreuzbergAvailable } from '../../src/kreuzberg.js';
 
 jest.setTimeout(60000);
 
@@ -68,6 +69,68 @@ describe('API Endpoint Tests', () => {
       expect(res.text).toContain('Test Page');
       expect(res.text).toContain('Hello world');
     });
+
+    it('rejects unsupported converter names', async () => {
+      await request(app)
+        .get('/markdown')
+        .query({
+          url: 'https://example.com/md-test',
+          converter: 'unknown',
+        })
+        .expect(400);
+    });
+
+    it('rejects JSON format without the kreuzberg converter', async () => {
+      await request(app)
+        .get('/markdown')
+        .query({
+          url: 'https://example.com/md-test',
+          format: 'json',
+        })
+        .expect(400);
+    });
+
+    it('returns structured kreuzberg JSON with selectors and absolute links', async () => {
+      if (!(await isKreuzbergAvailable())) {
+        return;
+      }
+
+      nock('https://example.com')
+        .get('/kreuzberg-json')
+        .reply(
+          200,
+          `<!DOCTYPE html><html><body>
+            <main>
+              <h1>Article Title</h1>
+              <nav>Navigation</nav>
+              <article><p>Wanted body with <a href="/about">relative link</a>.</p></article>
+            </main>
+          </body></html>`,
+          { 'content-type': 'text/html' }
+        );
+
+      const res = await request(app)
+        .get('/markdown')
+        .query({
+          url: 'https://example.com/kreuzberg-json',
+          converter: 'kreuzberg',
+          format: 'json',
+          contentSelector: 'main',
+          bodySelector: 'article',
+        })
+        .expect(200);
+
+      expect(res.headers['content-type']).toMatch(/application\/json/);
+      expect(res.body).toHaveProperty('content');
+      expect(res.body).toHaveProperty('metadata');
+      expect(res.body).toHaveProperty('tables');
+      expect(res.body).toHaveProperty('images');
+      expect(res.body).toHaveProperty('warnings');
+      expect(res.body.content).toContain('Article Title');
+      expect(res.body.content).toContain('Wanted body');
+      expect(res.body.content).toContain('https://example.com/about');
+      expect(res.body.content).not.toContain('Navigation');
+    });
   });
 
   describe('GET /archive', () => {

package/tests/unit/kreuzberg.test.js

@@ -0,0 +1,209 @@
+import {
+  convertWithKreuzberg,
+  isKreuzbergAvailable,
+} from '../../src/kreuzberg.js';
+
+let available;
+
+beforeAll(async () => {
+  available = await isKreuzbergAvailable();
+});
+
+function skipIfUnavailable() {
+  if (!available) {
+    return 'kreuzberg native binding not installed';
+  }
+  return false;
+}
+
+describe('kreuzberg html-to-markdown integration', () => {
+  it('reports availability correctly', async () => {
+    expect(typeof available).toBe('boolean');
+  });
+
+  it('converts basic HTML to Markdown', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const result = await convertWithKreuzberg(
+      '<h1>Hello World</h1><p>This is a test.</p>'
+    );
+    expect(result.content).toContain('# Hello World');
+    expect(result.content).toContain('This is a test.');
+  });
+
+  it('returns structured result with all fields', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const result = await convertWithKreuzberg('<p>Test</p>');
+    expect(result).toHaveProperty('content');
+    expect(result).toHaveProperty('metadata');
+    expect(result).toHaveProperty('tables');
+    expect(result).toHaveProperty('images');
+    expect(result).toHaveProperty('warnings');
+  });
+
+  it('extracts metadata from HTML', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = `<html>
+      <head>
+        <title>Test Page</title>
+        <meta property="og:description" content="A test page">
+      </head>
+      <body><h1>Hello</h1></body>
+    </html>`;
+    const result = await convertWithKreuzberg(html);
+    expect(result.metadata).toBeTruthy();
+    expect(result.metadata.document.title).toBe('Test Page');
+    expect(result.metadata.document.open_graph.description).toBe('A test page');
+  });
+
+  it('extracts headings from metadata', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<h1>First</h1><h2>Second</h2><h3>Third</h3>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.metadata).toBeTruthy();
+    expect(result.metadata.headers.length).toBe(3);
+    expect(result.metadata.headers[0].text).toBe('First');
+    expect(result.metadata.headers[0].level).toBe(1);
+  });
+
+  it('extracts links from metadata', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html =
+      '<a href="https://example.com">Example</a><a href="mailto:test@test.com">Email</a>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.metadata).toBeTruthy();
+    expect(result.metadata.links.length).toBeGreaterThanOrEqual(1);
+    const externalLink = result.metadata.links.find(
+      (l) => l.href === 'https://example.com'
+    );
+    expect(externalLink).toBeTruthy();
+    expect(externalLink.text).toBe('Example');
+  });
+
+  it('converts tables to Markdown', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = `<table>
+      <thead><tr><th>Name</th><th>Value</th></tr></thead>
+      <tbody><tr><td>A</td><td>1</td></tr><tr><td>B</td><td>2</td></tr></tbody>
+    </table>`;
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('| Name | Value |');
+    expect(result.content).toMatch(/\| A\s+\| 1\s+\|/);
+    expect(result.content).toMatch(/\| B\s+\| 2\s+\|/);
+  });
+
+  it('converts links correctly', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<a href="https://example.com">Click here</a>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('[Click here](https://example.com)');
+  });
+
+  it('resolves relative links with baseUrl', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<a href="/about">About</a>';
+    const result = await convertWithKreuzberg(html, {
+      baseUrl: 'https://example.com/docs/page',
+    });
+    expect(result.content).toContain('[About](https://example.com/about)');
+  });
+
+  it('converts bold and italic correctly', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<p>This is <strong>bold</strong> and <em>italic</em></p>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('**bold**');
+    expect(result.content).toContain('*italic*');
+  });
+
+  it('converts code blocks correctly', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<pre><code class="language-js">const x = 1;</code></pre>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('```');
+    expect(result.content).toContain('const x = 1;');
+  });
+
+  it('handles empty HTML gracefully', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const result = await convertWithKreuzberg('');
+    expect(result.content).toBeDefined();
+  });
+
+  it('removes script tags from output', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<p>Content</p><script>alert("xss")</script>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('Content');
+    expect(result.content).not.toContain('alert');
+    expect(result.content).not.toContain('script');
+  });
+
+  it('removes style tags from output', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<style>body { color: red; }</style><p>Content</p>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('Content');
+    expect(result.content).not.toContain('color: red');
+  });
+
+  it('handles GFM strikethrough', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<p>This is <del>deleted</del> text</p>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('~~deleted~~');
+  });
+
+  it('handles lists correctly', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<ul><li>Item 1</li><li>Item 2</li><li>Item 3</li></ul>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('Item 1');
+    expect(result.content).toContain('Item 2');
+    expect(result.content).toContain('Item 3');
+  });
+});