@link-assistant/web-capture 1.8.1 → 1.10.0

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @link-assistant/web-capture
 
+## 1.10.0
+
+### Minor Changes
+
+- b892585: Add kreuzberg html-to-markdown as high-performance alternative converter with structured results
+
+## 1.9.0
+
+### Minor Changes
+
+- 3f01d90: Add a structured search-provider capture contract (issue #130). New
+  `search({ query, provider, limit, captureMode })` library API, `GET /search?q=&provider=&format=json|markdown`
+  HTTP endpoint, and `web-capture search "<query>" --provider <provider> --format json|markdown`
+  CLI subcommand. Supported providers: `wikipedia` (default, CORS-friendly REST
+  API), `duckduckgo`, `google`, `bing`, and `brave` (HTML parsed server-side).
+  Results are normalized to a single machine-readable shape —
+  `{ query, provider, captureMode, capturedAt, results: [{ rank, title, url, snippet }], diagnostics: { status, blockedByCors, blockedByCaptcha, sourceUrl } }`
+  — and blocked or CAPTCHA-gated pages are surfaced through `diagnostics`
+  instead of failing silently. The same contract is mirrored in the Rust crate
+  so both implementations stay aligned.
+
 ## 1.8.1
 
 ### Patch Changes

package/README.md

@@ -88,6 +88,9 @@ web-capture https://example.com --extract-images --images-dir assets -o page.md
 # Disable specific features
 web-capture https://example.com --no-extract-latex --no-post-process -o page.md
 
+# Structured search-provider capture (JSON by default)
+web-capture search "formal methods" --provider wikipedia
+
 # Start as API server
 web-capture --serve
 
@@ -138,11 +141,13 @@ GET /markdown?url=<URL>
 
 Converts the HTML content of the specified URL to Markdown format. By default, original remote image URLs are preserved and base64 data URIs are stripped (clean single-file output). Use `keepOriginalLinks=false` to strip all images, or `embedImages=true` to keep base64 images inline.
 
-| Parameter           | Required | Description                                | Default |
-| ------------------- | -------- | ------------------------------------------ | ------- |
-| `url`               | Yes      | URL to fetch                               | -       |
-| `embedImages`       | No       | Keep base64 images inline (`true`/`false`) | `false` |
-| `keepOriginalLinks` | No       | Keep original remote URLs, strip base64    | `true`  |
+| Parameter           | Required | Description                                                               | Default  |
+| ------------------- | -------- | ------------------------------------------------------------------------- | -------- |
+| `url`               | Yes      | URL to fetch                                                              | -        |
+| `converter`         | No       | Converter: `turndown` or `kreuzberg`                                      | turndown |
+| `format`            | No       | Response format: `text` or `json` (`json` requires `converter=kreuzberg`) | text     |
+| `embedImages`       | No       | Keep base64 images inline (`true`/`false`)                                | `false`  |
+| `keepOriginalLinks` | No       | Keep original remote URLs, strip base64                                   | `true`   |
 
 ### Image Endpoint
 
@@ -232,6 +237,50 @@ GET /stream?url=<URL>
 
 Proxy fetch and streaming proxy endpoints.
 
+### Search Endpoint
+
+```
+GET /search?q=<QUERY>&provider=<PROVIDER>&format=json|markdown
+```
+
+Captures structured results from a search provider and returns them in a
+normalized, machine-readable shape. `wikipedia` (the default) uses the
+CORS-friendly REST API; the HTML engines (`duckduckgo`, `google`, `bing`,
+`brave`) are parsed server-side. Blocked or CAPTCHA-gated pages are reported
+through `diagnostics` instead of failing.
+
+| Parameter  | Required | Description                                          | Default     |
+| ---------- | -------- | ---------------------------------------------------- | ----------- |
+| `q`        | Yes      | Search query (`query` is accepted as an alias)       | -           |
+| `provider` | No       | `wikipedia`, `duckduckgo`, `google`, `bing`, `brave` | `wikipedia` |
+| `limit`    | No       | Maximum number of results                            | `10`        |
+| `format`   | No       | Response format: `json` or `markdown`                | `json`      |
+
+Example response (`format=json`):
+
+```json
+{
+  "query": "formal methods",
+  "provider": "wikipedia",
+  "captureMode": "fetch",
+  "capturedAt": "2026-05-18T20:30:00.000Z",
+  "results": [
+    {
+      "rank": 1,
+      "title": "Formal methods",
+      "url": "https://en.wikipedia.org/wiki/Formal_methods",
+      "snippet": "mathematically rigorous techniques for the specification..."
+    }
+  ],
+  "diagnostics": {
+    "status": 200,
+    "blockedByCors": false,
+    "blockedByCaptcha": false,
+    "sourceUrl": "https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal%20methods&limit=10"
+  }
+}
+```
+
 ## CLI Reference
 
 ### Server Mode
@@ -291,6 +340,45 @@ web-capture <url> [options]
 | `docx`            | Word document with embedded images       |
 | `archive` / `zip` | ZIP archive with markdown + local images |
 
+### Search Mode
+
+```bash
+web-capture search "<query>" [options]
+```
+
+Captures structured search-provider results. Output defaults to JSON; pass
+`--format markdown` for a human-readable document.
+
+```bash
+# Search Wikipedia (default provider), JSON output
+web-capture search "formal methods"
+
+# Search DuckDuckGo, limit to 5 results
+web-capture search "formal methods" --provider duckduckgo --limit 5
+
+# Render the results as Markdown
+web-capture search "formal methods" --format markdown
+```
+
+| Option       | Short | Description                                          | Default     |
+| ------------ | ----- | ---------------------------------------------------- | ----------- |
+| `--provider` |       | `wikipedia`, `duckduckgo`, `google`, `bing`, `brave` | `wikipedia` |
+| `--limit`    |       | Maximum number of results                            | `10`        |
+| `--format`   | `-f`  | Output format: `json` or `markdown`                  | `json`      |
+
+The library API exposes the same contract:
+
+```js
+import { search } from '@link-assistant/web-capture';
+
+const result = await search({
+  query: 'formal methods',
+  provider: 'wikipedia',
+  limit: 5,
+});
+console.log(result.results[0].title);
+```
+
 ## Configuration
 
 Configuration values are resolved with the following priority (highest to lowest):

package/bin/web-capture.js

@@ -40,6 +40,26 @@ const config = makeConfig({
           description: 'URL to capture',
         })
       )
+      .command(
+        'search <query>',
+        'Capture structured search-provider results',
+        (yargs) =>
+          yargs.positional('query', {
+            type: 'string',
+            description: 'Search query',
+          })
+      )
+      .option('provider', {
+        type: 'string',
+        description:
+          'Search provider: wikipedia, duckduckgo, google, bing, brave',
+        default: 'wikipedia',
+      })
+      .option('limit', {
+        type: 'number',
+        description: 'Maximum number of search results',
+        default: 10,
+      })
       .option('serve', {
         alias: 's',
         type: 'boolean',
@@ -1234,10 +1254,44 @@ async function captureUrl(url, options) {
   }
 }
 
+async function runSearch(query) {
+  const { search, formatSearchAsMarkdown } = await import('../src/search.js');
+  // `--format` defaults to `markdown` for capture mode, but search results are
+  // primarily structured JSON. Only honor the format flag when the user passed
+  // it explicitly; otherwise default search output to JSON.
+  const formatPassedExplicitly = process.argv.some(
+    (arg) => arg === '-f' || arg === '--format' || arg.startsWith('--format=')
+  );
+  const format = (
+    formatPassedExplicitly ? config.format : 'json'
+  ).toLowerCase();
+  const result = await search({
+    query,
+    provider: config.provider,
+    limit: config.limit,
+  });
+  if (format === 'markdown' || format === 'md') {
+    console.log(formatSearchAsMarkdown(result));
+  } else {
+    console.log(JSON.stringify(result, null, 2));
+  }
+}
+
 async function main() {
   const url = getUrlArgument();
+  // The `query` key is only present when the `search <query>` command matched.
+  const isSearch = Object.prototype.hasOwnProperty.call(config, 'query');
 
-  if (config.serve) {
+  if (isSearch) {
+    // Search mode
+    const query = typeof config.query === 'string' ? config.query : '';
+    if (!query.trim()) {
+      console.error('Error: Missing search query');
+      console.error('Usage: web-capture search "<query>" [--provider <p>]');
+      process.exit(1);
+    }
+    await runSearch(query);
+  } else if (config.serve) {
     // Server mode
     await startServer(config.port);
   } else if (url) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.8.1",
+  "version": "1.10.0",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {
@@ -32,6 +32,7 @@
     "changeset:status": "changeset status --since=origin/main"
   },
   "dependencies": {
+    "@kreuzberg/html-to-markdown-node": "^3.5.5",
     "archiver": "^7.0.1",
     "browser-commander": "^0.8.0",
     "cheerio": "^1.0.0",

package/src/index.js

@@ -11,6 +11,7 @@ import { docxHandler } from './docx.js';
 import { animationHandler } from './animation.js';
 import { figuresHandler } from './figures.js';
 import { themedImageHandler } from './themed-image.js';
+import { searchHandler } from './search.js';
 
 const app = express();
 const port = process.env.PORT || 3000;
@@ -27,6 +28,7 @@ app.get('/docx', docxHandler);
 app.get('/animation', animationHandler);
 app.get('/figures', figuresHandler);
 app.get('/themed-image', themedImageHandler);
+app.get('/search', searchHandler);
 
 // Start the server if this is the main module
 const isMainModule =
@@ -66,3 +68,11 @@ if (isMainModule) {
 }
 
 export { app };
+export {
+  search,
+  searchHandler,
+  parseSearchResults,
+  buildSearchUrl,
+  formatSearchAsMarkdown,
+  SEARCH_PROVIDERS,
+} from './search.js';

package/src/kreuzberg.js

@@ -0,0 +1,128 @@
+/**
+ * Kreuzberg html-to-markdown integration module.
+ *
+ * Provides high-performance HTML to Markdown conversion using the
+ * @kreuzberg/html-to-markdown-node library (Rust-powered, 150-280 MB/s).
+ *
+ * This converter is available as an alternative to the default Turndown-based
+ * converter, selectable via the `converter=kreuzberg` query parameter.
+ *
+ * @module kreuzberg
+ * @see https://github.com/kreuzberg-dev/html-to-markdown
+ */
+
+import { convertRelativeUrls } from './lib.js';
+
+let _initPromise = null;
+let _convert = null;
+
+async function ensureLoaded() {
+  if (_convert) {
+    return _convert;
+  }
+  if (_initPromise) {
+    return _initPromise;
+  }
+  _initPromise = import('@kreuzberg/html-to-markdown-node')
+    .then((mod) => {
+      _convert = mod.convert || mod.default?.convert || null;
+      return _convert;
+    })
+    .catch(() => {
+      _convert = null;
+      return null;
+    });
+  return await _initPromise;
+}
+
+function toSnakeCase(key) {
+  return key.replace(/[A-Z]/g, (match) => `_${match.toLowerCase()}`);
+}
+
+function normalizeStructuredKeys(value) {
+  if (Array.isArray(value)) {
+    return value.map(normalizeStructuredKeys);
+  }
+  if (!value || typeof value !== 'object') {
+    return value;
+  }
+
+  return Object.fromEntries(
+    Object.entries(value).map(([key, entry]) => [
+      toSnakeCase(key),
+      normalizeStructuredKeys(entry),
+    ])
+  );
+}
+
+/**
+ * Check if the kreuzberg converter is available.
+ *
+ * @returns {Promise<boolean>} Whether the converter is available
+ */
+export async function isKreuzbergAvailable() {
+  const fn = await ensureLoaded();
+  return fn !== null;
+}
+
+/**
+ * Convert HTML to Markdown using the kreuzberg html-to-markdown library.
+ *
+ * Returns a structured result with content, metadata, tables, images, and warnings.
+ *
+ * @param {string} html - HTML content to convert
+ * @param {Object} [options] - Conversion options
+ * @param {string} [options.baseUrl] - Base URL for resolving relative URLs
+ * @param {string} [options.headingStyle='Atx'] - Heading style ('Atx' or 'Setext')
+ * @param {string} [options.bulletListMarker] - Bullet character ('-', '*', '+')
+ * @param {string} [options.codeBlockStyle] - Code block style ('Fenced' or 'Indented')
+ * @returns {Promise<Object>} Structured conversion result
+ * @returns {string} result.content - The converted markdown content
+ * @returns {Object|null} result.metadata - Extracted metadata (title, links, headings, images, etc.)
+ * @returns {Array} result.tables - Extracted table data
+ * @returns {Array} result.images - Extracted image data
+ * @returns {Array} result.warnings - Non-fatal conversion warnings
+ * @throws {Error} If the kreuzberg converter is not available
+ */
+export async function convertWithKreuzberg(html, options = {}) {
+  const convert = await ensureLoaded();
+  if (!convert) {
+    throw new Error(
+      'Kreuzberg html-to-markdown is not installed. ' +
+        'Run: npm install @kreuzberg/html-to-markdown-node'
+    );
+  }
+
+  const convertOptions = {
+    headingStyle: 'Atx',
+    codeBlockStyle: 'Backticks',
+    ...options,
+  };
+  delete convertOptions.baseUrl;
+
+  const processedHtml = options.baseUrl
+    ? convertRelativeUrls(html, options.baseUrl)
+    : html;
+  const result = convert(processedHtml, convertOptions);
+
+  // Parse the metadata JSON string into an object
+  let metadata = null;
+  if (result.metadata) {
+    try {
+      metadata =
+        typeof result.metadata === 'string'
+          ? JSON.parse(result.metadata)
+          : result.metadata;
+    } catch {
+      metadata = null;
+    }
+  }
+
+  return {
+    content: result.content || '',
+    metadata: normalizeStructuredKeys(metadata),
+    tables: normalizeStructuredKeys(result.tables || []),
+    images: normalizeStructuredKeys(result.images || []),
+    warnings: result.warnings || [],
+  };
+}

package/src/lib.js

@@ -411,6 +411,20 @@ function selectedHtml($, selector) {
   return $selected.length ? $.html($selected) : null;
 }
 
+export function scopeHtmlForMarkdown(html, options = {}) {
+  const { contentSelector, bodySelector } = options;
+  const $ = cheerio.load(html);
+  const bodyHtml = selectedHtml($, bodySelector);
+  const contentHtml = selectedHtml($, contentSelector);
+  if (!bodyHtml && !contentHtml) {
+    return html;
+  }
+
+  const titleSelector = contentSelector ? `${contentSelector} h1, h1` : 'h1';
+  const titleHtml = bodyHtml ? selectedHtml($, titleSelector) : null;
+  return [titleHtml, bodyHtml || contentHtml].filter(Boolean).join('\n');
+}
+
 // Convert relative URLs to absolute URLs in HTML content
 export function convertRelativeUrls(html, baseUrl) {
   const base = new URL(baseUrl);
@@ -581,15 +595,8 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     metadata = extractMetadata($);
   }
 
-  const bodyHtml = selectedHtml($, bodySelector);
-  const contentHtml = selectedHtml($, contentSelector);
-  if (bodyHtml || contentHtml) {
-    const titleSelector = contentSelector ? `${contentSelector} h1, h1` : 'h1';
-    const titleHtml = bodyHtml ? selectedHtml($, titleSelector) : null;
-    $ = cheerio.load(
-      [titleHtml, bodyHtml || contentHtml].filter(Boolean).join('\n')
-    );
-  }
+  html = scopeHtmlForMarkdown(html, { contentSelector, bodySelector });
+  $ = cheerio.load(html);
 
   // Remove unwanted elements
   $('style, script, noscript').remove();

package/src/markdown.js

@@ -1,4 +1,9 @@
-import { fetchHtml, convertHtmlToMarkdownEnhanced } from './lib.js';
+import {
+  fetchHtml,
+  convertHtmlToMarkdownEnhanced,
+  scopeHtmlForMarkdown,
+} from './lib.js';
+import { convertWithKreuzberg, isKreuzbergAvailable } from './kreuzberg.js';
 import { applyImageMode } from './extract-images.js';
 
 export async function markdownHandler(req, res) {
@@ -6,9 +11,52 @@ export async function markdownHandler(req, res) {
   if (!url) {
     return res.status(400).send('Missing `url` parameter');
   }
+
+  const converter = (req.query.converter || 'turndown').toLowerCase();
+  const format = (req.query.format || 'text').toLowerCase();
   const embedImages = req.query.embedImages === 'true';
+
+  if (!['turndown', 'kreuzberg'].includes(converter)) {
+    return res.status(400).send('Unsupported `converter` parameter');
+  }
+  if (!['text', 'json'].includes(format)) {
+    return res.status(400).send('Unsupported `format` parameter');
+  }
+  if (format === 'json' && converter !== 'kreuzberg') {
+    return res
+      .status(400)
+      .send('`format=json` is only supported with `converter=kreuzberg`');
+  }
+
   try {
     const html = await fetchHtml(url);
+
+    if (converter === 'kreuzberg') {
+      const available = await isKreuzbergAvailable();
+      if (!available) {
+        return res
+          .status(501)
+          .send(
+            'Kreuzberg converter not available. Install @kreuzberg/html-to-markdown-node.'
+          );
+      }
+
+      const scopedHtml = scopeHtmlForMarkdown(html, {
+        contentSelector: req.query.contentSelector,
+        bodySelector: req.query.bodySelector,
+      });
+      const result = await convertWithKreuzberg(scopedHtml, { baseUrl: url });
+      const imageResult = await applyImageMode(result.content, {
+        mode: embedImages ? 'embed' : 'default',
+      });
+      result.content = imageResult.markdown;
+
+      if (format === 'json') {
+        return res.json(result);
+      }
+      return res.type('text/markdown').send(result.content);
+    }
+
     const { markdown } = convertHtmlToMarkdownEnhanced(html, url, {
       contentSelector: req.query.contentSelector,
       bodySelector: req.query.bodySelector,