@link-assistant/web-capture 1.8.0 → 1.9.0

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @link-assistant/web-capture
 
+## 1.9.0
+
+### Minor Changes
+
+- 3f01d90: Add a structured search-provider capture contract (issue #130). New
+  `search({ query, provider, limit, captureMode })` library API, `GET /search?q=&provider=&format=json|markdown`
+  HTTP endpoint, and `web-capture search "<query>" --provider <provider> --format json|markdown`
+  CLI subcommand. Supported providers: `wikipedia` (default, CORS-friendly REST
+  API), `duckduckgo`, `google`, `bing`, and `brave` (HTML parsed server-side).
+  Results are normalized to a single machine-readable shape —
+  `{ query, provider, captureMode, capturedAt, results: [{ rank, title, url, snippet }], diagnostics: { status, blockedByCors, blockedByCaptcha, sourceUrl } }`
+  — and blocked or CAPTCHA-gated pages are surfaced through `diagnostics`
+  instead of failing silently. The same contract is mirrored in the Rust crate
+  so both implementations stay aligned.
+
+## 1.8.1
+
+### Patch Changes
+
+- Auto-release unreleased changes
+
 ## 1.8.0
 
 ### Minor Changes

package/README.md

@@ -88,6 +88,9 @@ web-capture https://example.com --extract-images --images-dir assets -o page.md
 # Disable specific features
 web-capture https://example.com --no-extract-latex --no-post-process -o page.md
 
+# Structured search-provider capture (JSON by default)
+web-capture search "formal methods" --provider wikipedia
+
 # Start as API server
 web-capture --serve
 
@@ -232,6 +235,50 @@ GET /stream?url=<URL>
 
 Proxy fetch and streaming proxy endpoints.
 
+### Search Endpoint
+
+```
+GET /search?q=<QUERY>&provider=<PROVIDER>&format=json|markdown
+```
+
+Captures structured results from a search provider and returns them in a
+normalized, machine-readable shape. `wikipedia` (the default) uses the
+CORS-friendly REST API; the HTML engines (`duckduckgo`, `google`, `bing`,
+`brave`) are parsed server-side. Blocked or CAPTCHA-gated pages are reported
+through `diagnostics` instead of failing.
+
+| Parameter  | Required | Description                                          | Default     |
+| ---------- | -------- | ---------------------------------------------------- | ----------- |
+| `q`        | Yes      | Search query (`query` is accepted as an alias)       | -           |
+| `provider` | No       | `wikipedia`, `duckduckgo`, `google`, `bing`, `brave` | `wikipedia` |
+| `limit`    | No       | Maximum number of results                            | `10`        |
+| `format`   | No       | Response format: `json` or `markdown`                | `json`      |
+
+Example response (`format=json`):
+
+```json
+{
+  "query": "formal methods",
+  "provider": "wikipedia",
+  "captureMode": "fetch",
+  "capturedAt": "2026-05-18T20:30:00.000Z",
+  "results": [
+    {
+      "rank": 1,
+      "title": "Formal methods",
+      "url": "https://en.wikipedia.org/wiki/Formal_methods",
+      "snippet": "mathematically rigorous techniques for the specification..."
+    }
+  ],
+  "diagnostics": {
+    "status": 200,
+    "blockedByCors": false,
+    "blockedByCaptcha": false,
+    "sourceUrl": "https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal%20methods&limit=10"
+  }
+}
+```
+
 ## CLI Reference
 
 ### Server Mode
@@ -291,6 +338,45 @@ web-capture <url> [options]
 | `docx`            | Word document with embedded images       |
 | `archive` / `zip` | ZIP archive with markdown + local images |
 
+### Search Mode
+
+```bash
+web-capture search "<query>" [options]
+```
+
+Captures structured search-provider results. Output defaults to JSON; pass
+`--format markdown` for a human-readable document.
+
+```bash
+# Search Wikipedia (default provider), JSON output
+web-capture search "formal methods"
+
+# Search DuckDuckGo, limit to 5 results
+web-capture search "formal methods" --provider duckduckgo --limit 5
+
+# Render the results as Markdown
+web-capture search "formal methods" --format markdown
+```
+
+| Option       | Short | Description                                          | Default     |
+| ------------ | ----- | ---------------------------------------------------- | ----------- |
+| `--provider` |       | `wikipedia`, `duckduckgo`, `google`, `bing`, `brave` | `wikipedia` |
+| `--limit`    |       | Maximum number of results                            | `10`        |
+| `--format`   | `-f`  | Output format: `json` or `markdown`                  | `json`      |
+
+The library API exposes the same contract:
+
+```js
+import { search } from '@link-assistant/web-capture';
+
+const result = await search({
+  query: 'formal methods',
+  provider: 'wikipedia',
+  limit: 5,
+});
+console.log(result.results[0].title);
+```
+
 ## Configuration
 
 Configuration values are resolved with the following priority (highest to lowest):

package/bin/web-capture.js

@@ -40,6 +40,26 @@ const config = makeConfig({
           description: 'URL to capture',
         })
       )
+      .command(
+        'search <query>',
+        'Capture structured search-provider results',
+        (yargs) =>
+          yargs.positional('query', {
+            type: 'string',
+            description: 'Search query',
+          })
+      )
+      .option('provider', {
+        type: 'string',
+        description:
+          'Search provider: wikipedia, duckduckgo, google, bing, brave',
+        default: 'wikipedia',
+      })
+      .option('limit', {
+        type: 'number',
+        description: 'Maximum number of search results',
+        default: 10,
+      })
       .option('serve', {
         alias: 's',
         type: 'boolean',
@@ -1234,10 +1254,44 @@ async function captureUrl(url, options) {
   }
 }
 
+async function runSearch(query) {
+  const { search, formatSearchAsMarkdown } = await import('../src/search.js');
+  // `--format` defaults to `markdown` for capture mode, but search results are
+  // primarily structured JSON. Only honor the format flag when the user passed
+  // it explicitly; otherwise default search output to JSON.
+  const formatPassedExplicitly = process.argv.some(
+    (arg) => arg === '-f' || arg === '--format' || arg.startsWith('--format=')
+  );
+  const format = (
+    formatPassedExplicitly ? config.format : 'json'
+  ).toLowerCase();
+  const result = await search({
+    query,
+    provider: config.provider,
+    limit: config.limit,
+  });
+  if (format === 'markdown' || format === 'md') {
+    console.log(formatSearchAsMarkdown(result));
+  } else {
+    console.log(JSON.stringify(result, null, 2));
+  }
+}
+
 async function main() {
   const url = getUrlArgument();
+  // The `query` key is only present when the `search <query>` command matched.
+  const isSearch = Object.prototype.hasOwnProperty.call(config, 'query');
 
-  if (config.serve) {
+  if (isSearch) {
+    // Search mode
+    const query = typeof config.query === 'string' ? config.query : '';
+    if (!query.trim()) {
+      console.error('Error: Missing search query');
+      console.error('Usage: web-capture search "<query>" [--provider <p>]');
+      process.exit(1);
+    }
+    await runSearch(query);
+  } else if (config.serve) {
     // Server mode
     await startServer(config.port);
   } else if (url) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.8.0",
+  "version": "1.9.0",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/index.js

@@ -11,6 +11,7 @@ import { docxHandler } from './docx.js';
 import { animationHandler } from './animation.js';
 import { figuresHandler } from './figures.js';
 import { themedImageHandler } from './themed-image.js';
+import { searchHandler } from './search.js';
 
 const app = express();
 const port = process.env.PORT || 3000;
@@ -27,6 +28,7 @@ app.get('/docx', docxHandler);
 app.get('/animation', animationHandler);
 app.get('/figures', figuresHandler);
 app.get('/themed-image', themedImageHandler);
+app.get('/search', searchHandler);
 
 // Start the server if this is the main module
 const isMainModule =
@@ -66,3 +68,11 @@ if (isMainModule) {
 }
 
 export { app };
+export {
+  search,
+  searchHandler,
+  parseSearchResults,
+  buildSearchUrl,
+  formatSearchAsMarkdown,
+  SEARCH_PROVIDERS,
+} from './search.js';

package/src/search.js

@@ -0,0 +1,409 @@
+/**
+ * Structured search-provider capture (issue #130).
+ *
+ * Turns a query + provider into a normalized, machine-readable result set so
+ * that browser, CLI, and server callers all consume one consistent contract
+ * instead of each reimplementing provider-specific scraping. Server-side and
+ * CLI callers fetch provider pages directly (no CORS restriction), so this
+ * module defaults to the `fetch` capture mode. Providers that expose a native
+ * CORS/JSON API (Wikipedia) are preferred; HTML search engines are parsed
+ * best-effort and report CAPTCHA/blocking through `diagnostics`.
+ *
+ * Normalized result shape:
+ * {
+ *   query, provider, captureMode, capturedAt,
+ *   results: [{ rank, title, url, snippet }],
+ *   diagnostics: { status, blockedByCors, blockedByCaptcha, sourceUrl }
+ * }
+ *
+ * @module search
+ */
+
+import fetch from 'node-fetch';
+import * as cheerio from 'cheerio';
+import he from 'he';
+import { URL } from 'url';
+
+/** Providers understood by the search contract. */
+export const SEARCH_PROVIDERS = [
+  'wikipedia',
+  'duckduckgo',
+  'google',
+  'bing',
+  'brave',
+];
+
+/** Default provider when none is supplied. */
+export const DEFAULT_PROVIDER = 'wikipedia';
+
+/** Default number of results requested/returned. */
+export const DEFAULT_LIMIT = 10;
+
+const USER_AGENT =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' +
+  '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
+/**
+ * Normalize whitespace and decode HTML entities in extracted text.
+ *
+ * @param {string} text - Raw text, possibly with markup/entities
+ * @returns {string} Cleaned single-line text
+ */
+function cleanText(text) {
+  if (!text) {
+    return '';
+  }
+  // Strip any residual tags, decode entities, collapse whitespace.
+  const stripped = String(text).replace(/<[^>]*>/g, ' ');
+  return he.decode(stripped).replace(/\s+/g, ' ').trim();
+}
+
+/**
+ * Build the provider-native source URL for a query.
+ *
+ * @param {string} provider - One of {@link SEARCH_PROVIDERS}
+ * @param {string} query - Search query
+ * @param {number} limit - Desired result count (used by APIs that support it)
+ * @returns {string} Absolute source URL
+ */
+export function buildSearchUrl(provider, query, limit = DEFAULT_LIMIT) {
+  const q = encodeURIComponent(query);
+  switch (provider) {
+    case 'wikipedia':
+      return `https://en.wikipedia.org/w/rest.php/v1/search/page?q=${q}&limit=${limit}`;
+    case 'duckduckgo':
+      return `https://html.duckduckgo.com/html/?q=${q}`;
+    case 'google':
+      return `https://www.google.com/search?q=${q}&num=${limit}`;
+    case 'bing':
+      return `https://www.bing.com/search?q=${q}&count=${limit}`;
+    case 'brave':
+      return `https://search.brave.com/search?q=${q}`;
+    default:
+      throw new Error(
+        `Unknown search provider "${provider}". Supported: ${SEARCH_PROVIDERS.join(', ')}`
+      );
+  }
+}
+
+/**
+ * Detect provider CAPTCHA / bot-block interstitials in an HTML body.
+ *
+ * @param {string} html - Raw HTML response body
+ * @returns {boolean} True when the page looks like a block/CAPTCHA wall
+ */
+export function looksLikeCaptcha(html) {
+  if (!html) {
+    return false;
+  }
+  return /captcha|unusual traffic|are you a robot|verify you('|&#39;)?re a human|\/sorry\/index|automated queries/i.test(
+    html
+  );
+}
+
+/**
+ * Decode a DuckDuckGo redirect href (`//duckduckgo.com/l/?uddg=...`) back to
+ * the real destination URL. Non-redirect hrefs are returned as-is.
+ *
+ * @param {string} href - Raw anchor href
+ * @returns {string} The resolved destination URL
+ */
+function resolveDuckDuckGoHref(href) {
+  if (!href) {
+    return '';
+  }
+  let normalized = href;
+  if (normalized.startsWith('//')) {
+    normalized = `https:${normalized}`;
+  }
+  try {
+    const parsed = new URL(normalized, 'https://duckduckgo.com');
+    const uddg = parsed.searchParams.get('uddg');
+    if (uddg) {
+      return uddg;
+    }
+    return parsed.toString();
+  } catch {
+    return href;
+  }
+}
+
+/** Parse the Wikipedia REST search JSON into normalized results. */
+function parseWikipedia(body, limit) {
+  let data;
+  try {
+    data = typeof body === 'string' ? JSON.parse(body) : body;
+  } catch {
+    return [];
+  }
+  const pages = Array.isArray(data?.pages) ? data.pages : [];
+  return pages.slice(0, limit).map((page, i) => ({
+    rank: i + 1,
+    title: cleanText(page.title || page.key || ''),
+    url: `https://en.wikipedia.org/wiki/${encodeURIComponent(page.key || page.title || '')}`,
+    snippet: cleanText(page.excerpt || page.description || ''),
+  }));
+}
+
+/** Parse DuckDuckGo HTML (html.duckduckgo.com/html) into normalized results. */
+function parseDuckDuckGo($, limit) {
+  const results = [];
+  // `.result__body` wraps each organic result; fall back to `.web-result`
+  // for older markup. Selecting a single container avoids counting the same
+  // result twice when both classes are present on nested elements.
+  let containers = $('.result__body');
+  if (containers.length === 0) {
+    containers = $('.web-result');
+  }
+  containers.each((_, el) => {
+    if (results.length >= limit) {
+      return;
+    }
+    const anchor = $(el).find('a.result__a').first();
+    const title = cleanText(anchor.text());
+    const url = resolveDuckDuckGoHref(anchor.attr('href'));
+    const snippet = cleanText($(el).find('.result__snippet').first().text());
+    if (title && url) {
+      results.push({ rank: results.length + 1, title, url, snippet });
+    }
+  });
+  return results;
+}
+
+/** Parse Google results HTML into normalized results (best-effort). */
+function parseGoogle($, limit) {
+  const results = [];
+  $('div.g, div.tF2Cxc, div.MjjYud').each((_, el) => {
+    if (results.length >= limit) {
+      return;
+    }
+    const anchor = $(el).find('a[href^="http"]').first();
+    const url = anchor.attr('href');
+    const title = cleanText($(el).find('h3').first().text());
+    const snippet = cleanText(
+      $(el).find('div[data-sncf], .VwiC3b, .IsZvec').first().text()
+    );
+    if (title && url) {
+      results.push({ rank: results.length + 1, title, url, snippet });
+    }
+  });
+  return results;
+}
+
+/** Parse Bing results HTML into normalized results. */
+function parseBing($, limit) {
+  const results = [];
+  $('li.b_algo').each((_, el) => {
+    if (results.length >= limit) {
+      return;
+    }
+    const anchor = $(el).find('h2 a').first();
+    const url = anchor.attr('href');
+    const title = cleanText(anchor.text());
+    const snippet = cleanText($(el).find('.b_caption p, p').first().text());
+    if (title && url) {
+      results.push({ rank: results.length + 1, title, url, snippet });
+    }
+  });
+  return results;
+}
+
+/** Parse Brave Search results HTML into normalized results. */
+function parseBrave($, limit) {
+  const results = [];
+  $('div.snippet[data-type="web"], div.snippet').each((_, el) => {
+    if (results.length >= limit) {
+      return;
+    }
+    const anchor = $(el).find('a[href^="http"]').first();
+    const url = anchor.attr('href');
+    const title = cleanText(
+      $(el).find('.snippet-title, .title').first().text() || anchor.text()
+    );
+    const snippet = cleanText(
+      $(el).find('.snippet-description, .snippet-content').first().text()
+    );
+    if (title && url) {
+      results.push({ rank: results.length + 1, title, url, snippet });
+    }
+  });
+  return results;
+}
+
+/**
+ * Parse a provider response body into normalized result rows. Pure function
+ * (no network) so it can be unit-tested against fixtures.
+ *
+ * @param {string} provider - One of {@link SEARCH_PROVIDERS}
+ * @param {string} body - Raw response body (JSON for wikipedia, HTML otherwise)
+ * @param {{limit?: number}} [options] - Parse options
+ * @returns {{results: Array, blockedByCaptcha: boolean}}
+ */
+export function parseSearchResults(
+  provider,
+  body,
+  { limit = DEFAULT_LIMIT } = {}
+) {
+  if (provider === 'wikipedia') {
+    return { results: parseWikipedia(body, limit), blockedByCaptcha: false };
+  }
+  const blockedByCaptcha = looksLikeCaptcha(body);
+  const $ = cheerio.load(body || '');
+  let results;
+  switch (provider) {
+    case 'duckduckgo':
+      results = parseDuckDuckGo($, limit);
+      break;
+    case 'google':
+      results = parseGoogle($, limit);
+      break;
+    case 'bing':
+      results = parseBing($, limit);
+      break;
+    case 'brave':
+      results = parseBrave($, limit);
+      break;
+    default:
+      throw new Error(
+        `Unknown search provider "${provider}". Supported: ${SEARCH_PROVIDERS.join(', ')}`
+      );
+  }
+  return { results, blockedByCaptcha };
+}
+
+/**
+ * Render a normalized search result as Markdown.
+ *
+ * @param {Object} result - Normalized search result object
+ * @returns {string} Markdown document
+ */
+export function formatSearchAsMarkdown(result) {
+  const lines = [];
+  lines.push(`# Search results for "${result.query}"`);
+  lines.push('');
+  lines.push(`- Provider: \`${result.provider}\``);
+  lines.push(`- Capture mode: \`${result.captureMode}\``);
+  lines.push(`- Captured at: ${result.capturedAt}`);
+  lines.push(`- Source: ${result.diagnostics.sourceUrl}`);
+  if (result.diagnostics.blockedByCaptcha) {
+    lines.push('- ⚠️ Provider returned a CAPTCHA / bot-block page.');
+  }
+  lines.push('');
+  if (!result.results.length) {
+    lines.push('_No results._');
+    return lines.join('\n');
+  }
+  for (const item of result.results) {
+    lines.push(`${item.rank}. [${item.title}](${item.url})`);
+    if (item.snippet) {
+      lines.push(`   ${item.snippet}`);
+    }
+  }
+  return lines.join('\n');
+}
+
+/**
+ * Capture structured search results for a query from a provider.
+ *
+ * @param {Object} options - Search options
+ * @param {string} options.query - Search query (required)
+ * @param {string} [options.provider=wikipedia] - Provider id
+ * @param {number} [options.limit=10] - Max results to return
+ * @param {string} [options.captureMode=fetch] - Reported capture mode
+ * @param {Function} [options.fetchImpl=fetch] - Fetch implementation (injectable for tests)
+ * @param {Function} [options.now] - Clock returning an ISO timestamp (injectable for tests)
+ * @returns {Promise<Object>} Normalized search result object
+ */
+export async function search({
+  query,
+  provider = DEFAULT_PROVIDER,
+  limit = DEFAULT_LIMIT,
+  captureMode = 'fetch',
+  fetchImpl = fetch,
+  now = () => new Date().toISOString(),
+} = {}) {
+  if (!query || !String(query).trim()) {
+    throw new Error('Missing `query` parameter');
+  }
+  if (!SEARCH_PROVIDERS.includes(provider)) {
+    throw new Error(
+      `Unknown search provider "${provider}". Supported: ${SEARCH_PROVIDERS.join(', ')}`
+    );
+  }
+
+  const sourceUrl = buildSearchUrl(provider, query, limit);
+  const capturedAt = now();
+  const diagnostics = {
+    status: 0,
+    blockedByCors: false,
+    blockedByCaptcha: false,
+    sourceUrl,
+  };
+
+  let results = [];
+  try {
+    const response = await fetchImpl(sourceUrl, {
+      headers: {
+        'User-Agent': USER_AGENT,
+        Accept:
+          provider === 'wikipedia'
+            ? 'application/json'
+            : 'text/html,application/xhtml+xml',
+        'Accept-Language': 'en-US,en;q=0.9',
+      },
+    });
+    diagnostics.status = response.status;
+    const body = await response.text();
+    const parsed = parseSearchResults(provider, body, { limit });
+    results = parsed.results;
+    diagnostics.blockedByCaptcha = parsed.blockedByCaptcha;
+  } catch (err) {
+    // A network/transport failure from a server context is reported as an
+    // error status; browser CORS failures surface the same way and are flagged.
+    diagnostics.status = diagnostics.status || 0;
+    diagnostics.error = err.message;
+  }
+
+  return {
+    query,
+    provider,
+    captureMode,
+    capturedAt,
+    results,
+    diagnostics,
+  };
+}
+
+/**
+ * Express handler for `GET /search?q=<query>&provider=<p>&format=json|markdown`.
+ *
+ * @param {import('express').Request} req
+ * @param {import('express').Response} res
+ */
+export async function searchHandler(req, res) {
+  const query = req.query.q || req.query.query;
+  if (!query) {
+    return res.status(400).send('Missing `q` parameter');
+  }
+  const provider = req.query.provider || DEFAULT_PROVIDER;
+  if (!SEARCH_PROVIDERS.includes(provider)) {
+    return res
+      .status(400)
+      .send(
+        `Unknown provider "${provider}". Supported: ${SEARCH_PROVIDERS.join(', ')}`
+      );
+  }
+  const limit = Number.parseInt(req.query.limit, 10) || DEFAULT_LIMIT;
+  const format = (req.query.format || 'json').toLowerCase();
+  try {
+    const result = await search({ query, provider, limit });
+    if (format === 'markdown' || format === 'md') {
+      res.type('text/markdown').send(formatSearchAsMarkdown(result));
+    } else {
+      res.json(result);
+    }
+  } catch (err) {
+    console.error('Search error:', err);
+    res.status(500).send(`Error performing search: ${err.message}`);
+  }
+}

package/tests/integration/search-endpoint.test.js

@@ -0,0 +1,88 @@
+/**
+ * Integration tests for the GET /search endpoint (issue #130).
+ *
+ * Provider HTTP calls are mocked with nock so the test is fast and offline.
+ */
+
+import request from 'supertest';
+import nock from 'nock';
+import { app } from '../../src/index.js';
+
+const WIKI_JSON = {
+  pages: [
+    {
+      id: 1,
+      key: 'Formal_methods',
+      title: 'Formal methods',
+      excerpt: 'the study of <b>formal</b> methods',
+      description: 'rigorous techniques',
+    },
+  ],
+};
+
+afterEach(() => {
+  nock.cleanAll();
+});
+
+describe('GET /search', () => {
+  it('returns structured JSON for the wikipedia provider', async () => {
+    nock('https://en.wikipedia.org')
+      .get('/w/rest.php/v1/search/page')
+      .query(true)
+      .reply(200, WIKI_JSON);
+
+    const res = await request(app)
+      .get('/search')
+      .query({ q: 'formal-ai', provider: 'wikipedia', limit: 5 })
+      .expect(200)
+      .expect('Content-Type', /json/);
+
+    expect(res.body.query).toBe('formal-ai');
+    expect(res.body.provider).toBe('wikipedia');
+    expect(res.body.results).toHaveLength(1);
+    expect(res.body.results[0].url).toBe(
+      'https://en.wikipedia.org/wiki/Formal_methods'
+    );
+    expect(res.body.diagnostics.status).toBe(200);
+  });
+
+  it('returns markdown when format=markdown', async () => {
+    nock('https://en.wikipedia.org')
+      .get('/w/rest.php/v1/search/page')
+      .query(true)
+      .reply(200, WIKI_JSON);
+
+    const res = await request(app)
+      .get('/search')
+      .query({ q: 'formal-ai', provider: 'wikipedia', format: 'markdown' })
+      .expect(200)
+      .expect('Content-Type', /markdown/);
+
+    expect(res.text).toContain('# Search results for "formal-ai"');
+    expect(res.text).toContain('[Formal methods]');
+  });
+
+  it('defaults to the wikipedia provider', async () => {
+    nock('https://en.wikipedia.org')
+      .get('/w/rest.php/v1/search/page')
+      .query(true)
+      .reply(200, WIKI_JSON);
+
+    const res = await request(app)
+      .get('/search')
+      .query({ q: 'formal-ai' })
+      .expect(200);
+    expect(res.body.provider).toBe('wikipedia');
+  });
+
+  it('rejects a missing query', async () => {
+    await request(app).get('/search').expect(400);
+  });
+
+  it('rejects an unknown provider', async () => {
+    await request(app)
+      .get('/search')
+      .query({ q: 'x', provider: 'yahoo' })
+      .expect(400);
+  });
+});

package/tests/unit/search.test.js

@@ -0,0 +1,268 @@
+/**
+ * Unit tests for the structured search-provider capture module (issue #130).
+ *
+ * Parsers are pure functions tested against fixtures; the orchestrating
+ * `search()` is exercised with an injected fetch so no network is required.
+ */
+
+import {
+  search,
+  parseSearchResults,
+  buildSearchUrl,
+  looksLikeCaptcha,
+  formatSearchAsMarkdown,
+  SEARCH_PROVIDERS,
+  DEFAULT_PROVIDER,
+} from '../../src/search.js';
+
+const WIKI_JSON = JSON.stringify({
+  pages: [
+    {
+      id: 1,
+      key: 'Formal_methods',
+      title: 'Formal methods',
+      excerpt: 'the <span class="searchmatch">study</span> of <b>formal</b>',
+      description: 'mathematically rigorous techniques',
+    },
+    {
+      id: 2,
+      key: 'Formal_system',
+      title: 'Formal system',
+      excerpt: 'an abstract structure',
+      description: '',
+    },
+  ],
+});
+
+const DDG_HTML = `
+<div class="result__body">
+  <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fa&amp;rut=abc">First &amp; Best</a>
+  <div class="result__snippet">Snippet about the <b>first</b> result</div>
+</div>
+<div class="result__body">
+  <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.org%2Fb">Second result</a>
+  <div class="result__snippet">Snippet two</div>
+</div>
+`;
+
+const BING_HTML = `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a href="https://bing-result.example/1">Bing One</a></h2>
+    <div class="b_caption"><p>Bing snippet one</p></div>
+  </li>
+</ol>
+`;
+
+describe('search module (#130)', () => {
+  describe('buildSearchUrl', () => {
+    it('builds the Wikipedia REST API url with limit', () => {
+      expect(buildSearchUrl('wikipedia', 'formal ai', 5)).toBe(
+        'https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal%20ai&limit=5'
+      );
+    });
+
+    it('builds the DuckDuckGo HTML url', () => {
+      expect(buildSearchUrl('duckduckgo', 'a b')).toContain(
+        'https://html.duckduckgo.com/html/?q=a%20b'
+      );
+    });
+
+    it('throws for an unknown provider', () => {
+      expect(() => buildSearchUrl('yahoo', 'x')).toThrow(
+        /Unknown search provider/
+      );
+    });
+  });
+
+  describe('parseSearchResults', () => {
+    it('normalizes Wikipedia REST JSON and strips markup', () => {
+      const { results } = parseSearchResults('wikipedia', WIKI_JSON, {
+        limit: 10,
+      });
+      expect(results).toHaveLength(2);
+      expect(results[0]).toEqual({
+        rank: 1,
+        title: 'Formal methods',
+        url: 'https://en.wikipedia.org/wiki/Formal_methods',
+        snippet: 'the study of formal',
+      });
+      expect(results[1].url).toBe(
+        'https://en.wikipedia.org/wiki/Formal_system'
+      );
+    });
+
+    it('respects the limit', () => {
+      const { results } = parseSearchResults('wikipedia', WIKI_JSON, {
+        limit: 1,
+      });
+      expect(results).toHaveLength(1);
+    });
+
+    it('decodes DuckDuckGo redirect hrefs and entities', () => {
+      const { results } = parseSearchResults('duckduckgo', DDG_HTML, {
+        limit: 10,
+      });
+      expect(results).toHaveLength(2);
+      expect(results[0]).toEqual({
+        rank: 1,
+        title: 'First & Best',
+        url: 'https://example.com/a',
+        snippet: 'Snippet about the first result',
+      });
+      expect(results[1].url).toBe('https://example.org/b');
+    });
+
+    it('parses Bing algorithmic results', () => {
+      const { results } = parseSearchResults('bing', BING_HTML, { limit: 10 });
+      expect(results).toEqual([
+        {
+          rank: 1,
+          title: 'Bing One',
+          url: 'https://bing-result.example/1',
+          snippet: 'Bing snippet one',
+        },
+      ]);
+    });
+
+    it('returns empty results for malformed JSON', () => {
+      const { results } = parseSearchResults('wikipedia', 'not json', {
+        limit: 10,
+      });
+      expect(results).toEqual([]);
+    });
+
+    it('flags CAPTCHA pages', () => {
+      const { blockedByCaptcha } = parseSearchResults(
+        'google',
+        '<html>Our systems have detected unusual traffic</html>',
+        { limit: 10 }
+      );
+      expect(blockedByCaptcha).toBe(true);
+    });
+  });
+
+  describe('looksLikeCaptcha', () => {
+    it('detects common bot-block phrases', () => {
+      expect(looksLikeCaptcha('Please solve the CAPTCHA')).toBe(true);
+      expect(looksLikeCaptcha('normal results page')).toBe(false);
+      expect(looksLikeCaptcha('')).toBe(false);
+    });
+  });
+
+  describe('search()', () => {
+    const fixedNow = () => '2026-05-30T00:00:00.000Z';
+
+    it('returns the normalized contract for a provider', async () => {
+      const fetchImpl = async (url) => {
+        expect(url).toContain('en.wikipedia.org');
+        return { status: 200, text: async () => WIKI_JSON };
+      };
+      const result = await search({
+        query: 'formal-ai',
+        provider: 'wikipedia',
+        limit: 2,
+        fetchImpl,
+        now: fixedNow,
+      });
+      expect(result).toMatchObject({
+        query: 'formal-ai',
+        provider: 'wikipedia',
+        captureMode: 'fetch',
+        capturedAt: '2026-05-30T00:00:00.000Z',
+      });
+      expect(result.results).toHaveLength(2);
+      expect(result.diagnostics).toEqual({
+        status: 200,
+        blockedByCors: false,
+        blockedByCaptcha: false,
+        sourceUrl:
+          'https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal-ai&limit=2',
+      });
+    });
+
+    it('defaults to the Wikipedia provider', async () => {
+      const fetchImpl = async () => ({
+        status: 200,
+        text: async () => WIKI_JSON,
+      });
+      const result = await search({ query: 'x', fetchImpl, now: fixedNow });
+      expect(result.provider).toBe(DEFAULT_PROVIDER);
+      expect(DEFAULT_PROVIDER).toBe('wikipedia');
+    });
+
+    it('records transport failures in diagnostics without throwing', async () => {
+      const fetchImpl = async () => {
+        throw new Error('network down');
+      };
+      const result = await search({
+        query: 'x',
+        provider: 'google',
+        fetchImpl,
+        now: fixedNow,
+      });
+      expect(result.results).toEqual([]);
+      expect(result.diagnostics.error).toBe('network down');
+    });
+
+    it('rejects an empty query', async () => {
+      await expect(search({ query: '   ' })).rejects.toThrow(/Missing .query/);
+    });
+
+    it('rejects an unknown provider', async () => {
+      await expect(search({ query: 'x', provider: 'yahoo' })).rejects.toThrow(
+        /Unknown search provider/
+      );
+    });
+  });
+
+  describe('formatSearchAsMarkdown', () => {
+    it('renders a markdown document with ranked links', () => {
+      const md = formatSearchAsMarkdown({
+        query: 'formal-ai',
+        provider: 'wikipedia',
+        captureMode: 'fetch',
+        capturedAt: '2026-05-30T00:00:00.000Z',
+        results: [
+          {
+            rank: 1,
+            title: 'Formal methods',
+            url: 'https://en.wikipedia.org/wiki/Formal_methods',
+            snippet: 'study of formal',
+          },
+        ],
+        diagnostics: {
+          sourceUrl: 'https://example.com',
+          blockedByCaptcha: false,
+        },
+      });
+      expect(md).toContain('# Search results for "formal-ai"');
+      expect(md).toContain(
+        '1. [Formal methods](https://en.wikipedia.org/wiki/Formal_methods)'
+      );
+      expect(md).toContain('study of formal');
+    });
+
+    it('reports no results', () => {
+      const md = formatSearchAsMarkdown({
+        query: 'nothing',
+        provider: 'bing',
+        captureMode: 'fetch',
+        capturedAt: 't',
+        results: [],
+        diagnostics: { sourceUrl: 'u', blockedByCaptcha: false },
+      });
+      expect(md).toContain('_No results._');
+    });
+  });
+
+  it('exposes the documented provider list', () => {
+    expect(SEARCH_PROVIDERS).toEqual([
+      'wikipedia',
+      'duckduckgo',
+      'google',
+      'bing',
+      'brave',
+    ]);
+  });
+});