@link-assistant/web-capture 1.8.1 → 1.10.0

package/tests/unit/kreuzberg.test.js

@@ -0,0 +1,209 @@
+import {
+  convertWithKreuzberg,
+  isKreuzbergAvailable,
+} from '../../src/kreuzberg.js';
+
+let available;
+
+beforeAll(async () => {
+  available = await isKreuzbergAvailable();
+});
+
+function skipIfUnavailable() {
+  if (!available) {
+    return 'kreuzberg native binding not installed';
+  }
+  return false;
+}
+
+describe('kreuzberg html-to-markdown integration', () => {
+  it('reports availability correctly', async () => {
+    expect(typeof available).toBe('boolean');
+  });
+
+  it('converts basic HTML to Markdown', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const result = await convertWithKreuzberg(
+      '<h1>Hello World</h1><p>This is a test.</p>'
+    );
+    expect(result.content).toContain('# Hello World');
+    expect(result.content).toContain('This is a test.');
+  });
+
+  it('returns structured result with all fields', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const result = await convertWithKreuzberg('<p>Test</p>');
+    expect(result).toHaveProperty('content');
+    expect(result).toHaveProperty('metadata');
+    expect(result).toHaveProperty('tables');
+    expect(result).toHaveProperty('images');
+    expect(result).toHaveProperty('warnings');
+  });
+
+  it('extracts metadata from HTML', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = `<html>
+      <head>
+        <title>Test Page</title>
+        <meta property="og:description" content="A test page">
+      </head>
+      <body><h1>Hello</h1></body>
+    </html>`;
+    const result = await convertWithKreuzberg(html);
+    expect(result.metadata).toBeTruthy();
+    expect(result.metadata.document.title).toBe('Test Page');
+    expect(result.metadata.document.open_graph.description).toBe('A test page');
+  });
+
+  it('extracts headings from metadata', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<h1>First</h1><h2>Second</h2><h3>Third</h3>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.metadata).toBeTruthy();
+    expect(result.metadata.headers.length).toBe(3);
+    expect(result.metadata.headers[0].text).toBe('First');
+    expect(result.metadata.headers[0].level).toBe(1);
+  });
+
+  it('extracts links from metadata', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html =
+      '<a href="https://example.com">Example</a><a href="mailto:test@test.com">Email</a>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.metadata).toBeTruthy();
+    expect(result.metadata.links.length).toBeGreaterThanOrEqual(1);
+    const externalLink = result.metadata.links.find(
+      (l) => l.href === 'https://example.com'
+    );
+    expect(externalLink).toBeTruthy();
+    expect(externalLink.text).toBe('Example');
+  });
+
+  it('converts tables to Markdown', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = `<table>
+      <thead><tr><th>Name</th><th>Value</th></tr></thead>
+      <tbody><tr><td>A</td><td>1</td></tr><tr><td>B</td><td>2</td></tr></tbody>
+    </table>`;
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('| Name | Value |');
+    expect(result.content).toMatch(/\| A\s+\| 1\s+\|/);
+    expect(result.content).toMatch(/\| B\s+\| 2\s+\|/);
+  });
+
+  it('converts links correctly', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<a href="https://example.com">Click here</a>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('[Click here](https://example.com)');
+  });
+
+  it('resolves relative links with baseUrl', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<a href="/about">About</a>';
+    const result = await convertWithKreuzberg(html, {
+      baseUrl: 'https://example.com/docs/page',
+    });
+    expect(result.content).toContain('[About](https://example.com/about)');
+  });
+
+  it('converts bold and italic correctly', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<p>This is <strong>bold</strong> and <em>italic</em></p>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('**bold**');
+    expect(result.content).toContain('*italic*');
+  });
+
+  it('converts code blocks correctly', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<pre><code class="language-js">const x = 1;</code></pre>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('```');
+    expect(result.content).toContain('const x = 1;');
+  });
+
+  it('handles empty HTML gracefully', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const result = await convertWithKreuzberg('');
+    expect(result.content).toBeDefined();
+  });
+
+  it('removes script tags from output', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<p>Content</p><script>alert("xss")</script>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('Content');
+    expect(result.content).not.toContain('alert');
+    expect(result.content).not.toContain('script');
+  });
+
+  it('removes style tags from output', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<style>body { color: red; }</style><p>Content</p>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('Content');
+    expect(result.content).not.toContain('color: red');
+  });
+
+  it('handles GFM strikethrough', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<p>This is <del>deleted</del> text</p>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('~~deleted~~');
+  });
+
+  it('handles lists correctly', async () => {
+    const skip = skipIfUnavailable();
+    if (skip) {
+      return;
+    }
+    const html = '<ul><li>Item 1</li><li>Item 2</li><li>Item 3</li></ul>';
+    const result = await convertWithKreuzberg(html);
+    expect(result.content).toContain('Item 1');
+    expect(result.content).toContain('Item 2');
+    expect(result.content).toContain('Item 3');
+  });
+});

package/tests/unit/search.test.js

@@ -0,0 +1,268 @@
+/**
+ * Unit tests for the structured search-provider capture module (issue #130).
+ *
+ * Parsers are pure functions tested against fixtures; the orchestrating
+ * `search()` is exercised with an injected fetch so no network is required.
+ */
+
+import {
+  search,
+  parseSearchResults,
+  buildSearchUrl,
+  looksLikeCaptcha,
+  formatSearchAsMarkdown,
+  SEARCH_PROVIDERS,
+  DEFAULT_PROVIDER,
+} from '../../src/search.js';
+
+const WIKI_JSON = JSON.stringify({
+  pages: [
+    {
+      id: 1,
+      key: 'Formal_methods',
+      title: 'Formal methods',
+      excerpt: 'the <span class="searchmatch">study</span> of <b>formal</b>',
+      description: 'mathematically rigorous techniques',
+    },
+    {
+      id: 2,
+      key: 'Formal_system',
+      title: 'Formal system',
+      excerpt: 'an abstract structure',
+      description: '',
+    },
+  ],
+});
+
+const DDG_HTML = `
+<div class="result__body">
+  <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fa&amp;rut=abc">First &amp; Best</a>
+  <div class="result__snippet">Snippet about the <b>first</b> result</div>
+</div>
+<div class="result__body">
+  <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.org%2Fb">Second result</a>
+  <div class="result__snippet">Snippet two</div>
+</div>
+`;
+
+const BING_HTML = `
+<ol id="b_results">
+  <li class="b_algo">
+    <h2><a href="https://bing-result.example/1">Bing One</a></h2>
+    <div class="b_caption"><p>Bing snippet one</p></div>
+  </li>
+</ol>
+`;
+
+describe('search module (#130)', () => {
+  describe('buildSearchUrl', () => {
+    it('builds the Wikipedia REST API url with limit', () => {
+      expect(buildSearchUrl('wikipedia', 'formal ai', 5)).toBe(
+        'https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal%20ai&limit=5'
+      );
+    });
+
+    it('builds the DuckDuckGo HTML url', () => {
+      expect(buildSearchUrl('duckduckgo', 'a b')).toContain(
+        'https://html.duckduckgo.com/html/?q=a%20b'
+      );
+    });
+
+    it('throws for an unknown provider', () => {
+      expect(() => buildSearchUrl('yahoo', 'x')).toThrow(
+        /Unknown search provider/
+      );
+    });
+  });
+
+  describe('parseSearchResults', () => {
+    it('normalizes Wikipedia REST JSON and strips markup', () => {
+      const { results } = parseSearchResults('wikipedia', WIKI_JSON, {
+        limit: 10,
+      });
+      expect(results).toHaveLength(2);
+      expect(results[0]).toEqual({
+        rank: 1,
+        title: 'Formal methods',
+        url: 'https://en.wikipedia.org/wiki/Formal_methods',
+        snippet: 'the study of formal',
+      });
+      expect(results[1].url).toBe(
+        'https://en.wikipedia.org/wiki/Formal_system'
+      );
+    });
+
+    it('respects the limit', () => {
+      const { results } = parseSearchResults('wikipedia', WIKI_JSON, {
+        limit: 1,
+      });
+      expect(results).toHaveLength(1);
+    });
+
+    it('decodes DuckDuckGo redirect hrefs and entities', () => {
+      const { results } = parseSearchResults('duckduckgo', DDG_HTML, {
+        limit: 10,
+      });
+      expect(results).toHaveLength(2);
+      expect(results[0]).toEqual({
+        rank: 1,
+        title: 'First & Best',
+        url: 'https://example.com/a',
+        snippet: 'Snippet about the first result',
+      });
+      expect(results[1].url).toBe('https://example.org/b');
+    });
+
+    it('parses Bing algorithmic results', () => {
+      const { results } = parseSearchResults('bing', BING_HTML, { limit: 10 });
+      expect(results).toEqual([
+        {
+          rank: 1,
+          title: 'Bing One',
+          url: 'https://bing-result.example/1',
+          snippet: 'Bing snippet one',
+        },
+      ]);
+    });
+
+    it('returns empty results for malformed JSON', () => {
+      const { results } = parseSearchResults('wikipedia', 'not json', {
+        limit: 10,
+      });
+      expect(results).toEqual([]);
+    });
+
+    it('flags CAPTCHA pages', () => {
+      const { blockedByCaptcha } = parseSearchResults(
+        'google',
+        '<html>Our systems have detected unusual traffic</html>',
+        { limit: 10 }
+      );
+      expect(blockedByCaptcha).toBe(true);
+    });
+  });
+
+  describe('looksLikeCaptcha', () => {
+    it('detects common bot-block phrases', () => {
+      expect(looksLikeCaptcha('Please solve the CAPTCHA')).toBe(true);
+      expect(looksLikeCaptcha('normal results page')).toBe(false);
+      expect(looksLikeCaptcha('')).toBe(false);
+    });
+  });
+
+  describe('search()', () => {
+    const fixedNow = () => '2026-05-30T00:00:00.000Z';
+
+    it('returns the normalized contract for a provider', async () => {
+      const fetchImpl = async (url) => {
+        expect(url).toContain('en.wikipedia.org');
+        return { status: 200, text: async () => WIKI_JSON };
+      };
+      const result = await search({
+        query: 'formal-ai',
+        provider: 'wikipedia',
+        limit: 2,
+        fetchImpl,
+        now: fixedNow,
+      });
+      expect(result).toMatchObject({
+        query: 'formal-ai',
+        provider: 'wikipedia',
+        captureMode: 'fetch',
+        capturedAt: '2026-05-30T00:00:00.000Z',
+      });
+      expect(result.results).toHaveLength(2);
+      expect(result.diagnostics).toEqual({
+        status: 200,
+        blockedByCors: false,
+        blockedByCaptcha: false,
+        sourceUrl:
+          'https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal-ai&limit=2',
+      });
+    });
+
+    it('defaults to the Wikipedia provider', async () => {
+      const fetchImpl = async () => ({
+        status: 200,
+        text: async () => WIKI_JSON,
+      });
+      const result = await search({ query: 'x', fetchImpl, now: fixedNow });
+      expect(result.provider).toBe(DEFAULT_PROVIDER);
+      expect(DEFAULT_PROVIDER).toBe('wikipedia');
+    });
+
+    it('records transport failures in diagnostics without throwing', async () => {
+      const fetchImpl = async () => {
+        throw new Error('network down');
+      };
+      const result = await search({
+        query: 'x',
+        provider: 'google',
+        fetchImpl,
+        now: fixedNow,
+      });
+      expect(result.results).toEqual([]);
+      expect(result.diagnostics.error).toBe('network down');
+    });
+
+    it('rejects an empty query', async () => {
+      await expect(search({ query: '   ' })).rejects.toThrow(/Missing .query/);
+    });
+
+    it('rejects an unknown provider', async () => {
+      await expect(search({ query: 'x', provider: 'yahoo' })).rejects.toThrow(
+        /Unknown search provider/
+      );
+    });
+  });
+
+  describe('formatSearchAsMarkdown', () => {
+    it('renders a markdown document with ranked links', () => {
+      const md = formatSearchAsMarkdown({
+        query: 'formal-ai',
+        provider: 'wikipedia',
+        captureMode: 'fetch',
+        capturedAt: '2026-05-30T00:00:00.000Z',
+        results: [
+          {
+            rank: 1,
+            title: 'Formal methods',
+            url: 'https://en.wikipedia.org/wiki/Formal_methods',
+            snippet: 'study of formal',
+          },
+        ],
+        diagnostics: {
+          sourceUrl: 'https://example.com',
+          blockedByCaptcha: false,
+        },
+      });
+      expect(md).toContain('# Search results for "formal-ai"');
+      expect(md).toContain(
+        '1. [Formal methods](https://en.wikipedia.org/wiki/Formal_methods)'
+      );
+      expect(md).toContain('study of formal');
+    });
+
+    it('reports no results', () => {
+      const md = formatSearchAsMarkdown({
+        query: 'nothing',
+        provider: 'bing',
+        captureMode: 'fetch',
+        capturedAt: 't',
+        results: [],
+        diagnostics: { sourceUrl: 'u', blockedByCaptcha: false },
+      });
+      expect(md).toContain('_No results._');
+    });
+  });
+
+  it('exposes the documented provider list', () => {
+    expect(SEARCH_PROVIDERS).toEqual([
+      'wikipedia',
+      'duckduckgo',
+      'google',
+      'bing',
+      'brave',
+    ]);
+  });
+});