@link-assistant/web-capture 1.12.0 → 2.0.1

package/tests/unit/search.test.js

@@ -1,339 +0,0 @@
-/**
- * Unit tests for the structured search-provider capture module (issue #130).
- *
- * Parsers are pure functions tested against fixtures; the orchestrating
- * `search()` is exercised with an injected fetch so no network is required.
- */
-import { jest } from '@jest/globals';
-
-import {
-  search,
-  parseSearchResults,
-  buildSearchUrl,
-  looksLikeCaptcha,
-  formatSearchAsMarkdown,
-  SEARCH_PROVIDERS,
-  DEFAULT_PROVIDER,
-} from '../../src/search.js';
-
-const WIKI_JSON = JSON.stringify({
-  pages: [
-    {
-      id: 1,
-      key: 'Formal_methods',
-      title: 'Formal methods',
-      excerpt: 'the <span class="searchmatch">study</span> of <b>formal</b>',
-      description: 'mathematically rigorous techniques',
-    },
-    {
-      id: 2,
-      key: 'Formal_system',
-      title: 'Formal system',
-      excerpt: 'an abstract structure',
-      description: '',
-    },
-  ],
-});
-
-const DDG_HTML = `
-<div class="result__body">
-  <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.com%2Fa&amp;rut=abc">First &amp; Best</a>
-  <div class="result__snippet">Snippet about the <b>first</b> result</div>
-</div>
-<div class="result__body">
-  <a class="result__a" href="//duckduckgo.com/l/?uddg=https%3A%2F%2Fexample.org%2Fb">Second result</a>
-  <div class="result__snippet">Snippet two</div>
-</div>
-`;
-
-const BING_HTML = `
-<ol id="b_results">
-  <li class="b_algo">
-    <h2><a href="https://bing-result.example/1">Bing One</a></h2>
-    <div class="b_caption"><p>Bing snippet one</p></div>
-  </li>
-</ol>
-`;
-
-describe('search module (#130)', () => {
-  describe('buildSearchUrl', () => {
-    it('builds the Wikipedia REST API url with limit', () => {
-      expect(buildSearchUrl('wikipedia', 'formal ai', 5)).toBe(
-        'https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal%20ai&limit=5'
-      );
-    });
-
-    it('builds the DuckDuckGo HTML url', () => {
-      expect(buildSearchUrl('duckduckgo', 'a b')).toContain(
-        'https://html.duckduckgo.com/html/?q=a%20b'
-      );
-    });
-
-    it('throws for an unknown provider', () => {
-      expect(() => buildSearchUrl('yahoo', 'x')).toThrow(
-        /Unknown search provider/
-      );
-    });
-  });
-
-  it('keeps URL construction and response parsing transport-independent', () => {
-    const sourceUrl = buildSearchUrl('wikipedia', 'formal ai', 1);
-    const { results } = parseSearchResults('wikipedia', WIKI_JSON, {
-      limit: 1,
-    });
-
-    expect(sourceUrl).toBe(
-      'https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal%20ai&limit=1'
-    );
-    expect(results).toEqual([
-      {
-        rank: 1,
-        title: 'Formal methods',
-        url: 'https://en.wikipedia.org/wiki/Formal_methods',
-        snippet: 'the study of formal',
-      },
-    ]);
-  });
-
-  describe('parseSearchResults', () => {
-    it('normalizes Wikipedia REST JSON and strips markup', () => {
-      const { results } = parseSearchResults('wikipedia', WIKI_JSON, {
-        limit: 10,
-      });
-      expect(results).toHaveLength(2);
-      expect(results[0]).toEqual({
-        rank: 1,
-        title: 'Formal methods',
-        url: 'https://en.wikipedia.org/wiki/Formal_methods',
-        snippet: 'the study of formal',
-      });
-      expect(results[1].url).toBe(
-        'https://en.wikipedia.org/wiki/Formal_system'
-      );
-    });
-
-    it('respects the limit', () => {
-      const { results } = parseSearchResults('wikipedia', WIKI_JSON, {
-        limit: 1,
-      });
-      expect(results).toHaveLength(1);
-    });
-
-    it('decodes DuckDuckGo redirect hrefs and entities', () => {
-      const { results } = parseSearchResults('duckduckgo', DDG_HTML, {
-        limit: 10,
-      });
-      expect(results).toHaveLength(2);
-      expect(results[0]).toEqual({
-        rank: 1,
-        title: 'First & Best',
-        url: 'https://example.com/a',
-        snippet: 'Snippet about the first result',
-      });
-      expect(results[1].url).toBe('https://example.org/b');
-    });
-
-    it('parses Bing algorithmic results', () => {
-      const { results } = parseSearchResults('bing', BING_HTML, { limit: 10 });
-      expect(results).toEqual([
-        {
-          rank: 1,
-          title: 'Bing One',
-          url: 'https://bing-result.example/1',
-          snippet: 'Bing snippet one',
-        },
-      ]);
-    });
-
-    it('returns empty results for malformed JSON', () => {
-      const { results } = parseSearchResults('wikipedia', 'not json', {
-        limit: 10,
-      });
-      expect(results).toEqual([]);
-    });
-
-    it('flags CAPTCHA pages', () => {
-      const { blockedByCaptcha } = parseSearchResults(
-        'google',
-        '<html>Our systems have detected unusual traffic</html>',
-        { limit: 10 }
-      );
-      expect(blockedByCaptcha).toBe(true);
-    });
-  });
-
-  describe('looksLikeCaptcha', () => {
-    it('detects common bot-block phrases', () => {
-      expect(looksLikeCaptcha('Please solve the CAPTCHA')).toBe(true);
-      expect(looksLikeCaptcha('normal results page')).toBe(false);
-      expect(looksLikeCaptcha('')).toBe(false);
-    });
-  });
-
-  describe('search()', () => {
-    const fixedNow = () => '2026-05-30T00:00:00.000Z';
-
-    it('returns the normalized contract for a provider', async () => {
-      const fetchImpl = async (url) => {
-        expect(url).toContain('en.wikipedia.org');
-        return { status: 200, text: async () => WIKI_JSON };
-      };
-      const result = await search({
-        query: 'formal-ai',
-        provider: 'wikipedia',
-        limit: 2,
-        fetchImpl,
-        now: fixedNow,
-      });
-      expect(result).toMatchObject({
-        query: 'formal-ai',
-        provider: 'wikipedia',
-        captureMode: 'fetch',
-        capturedAt: '2026-05-30T00:00:00.000Z',
-      });
-      expect(result.results).toHaveLength(2);
-      expect(result.diagnostics).toEqual({
-        status: 200,
-        blockedByCors: false,
-        blockedByCaptcha: false,
-        sourceUrl:
-          'https://en.wikipedia.org/w/rest.php/v1/search/page?q=formal-ai&limit=2',
-      });
-    });
-
-    it('defaults to the Wikipedia provider', async () => {
-      const fetchImpl = async () => ({
-        status: 200,
-        text: async () => WIKI_JSON,
-      });
-      const result = await search({ query: 'x', fetchImpl, now: fixedNow });
-      expect(result.provider).toBe(DEFAULT_PROVIDER);
-      expect(DEFAULT_PROVIDER).toBe('wikipedia');
-    });
-
-    it('records transport failures in diagnostics without throwing', async () => {
-      const fetchImpl = async () => {
-        throw new Error('network down');
-      };
-      const result = await search({
-        query: 'x',
-        provider: 'google',
-        fetchImpl,
-        now: fixedNow,
-      });
-      expect(result.results).toEqual([]);
-      expect(result.diagnostics.error).toBe('network down');
-    });
-
-    it('returns the source receipt and forwards caller cancellation', async () => {
-      const body = Buffer.from(WIKI_JSON);
-      const controller = new globalThis.AbortController();
-      const transport = jest.fn(async ({ url, signal }) => ({
-        body,
-        finalUrl: `${url}&redirected=true`,
-        status: 200,
-        headers: { 'content-type': 'application/json', etag: 'search-v1' },
-        diagnostics: { outcome: 'response' },
-        signal,
-      }));
-
-      const result = await search({
-        query: 'formal-ai',
-        transport,
-        signal: controller.signal,
-        now: fixedNow,
-      });
-
-      expect(Buffer.from(result.receipt.body)).toEqual(body);
-      expect(result.receipt.finalUrl).toContain('redirected=true');
-      expect(result.receipt.headers.etag).toBe('search-v1');
-      expect(transport).toHaveBeenCalledWith(
-        expect.objectContaining({ signal: controller.signal })
-      );
-    });
-
-    it('classifies AbortSignal cancellation in structured diagnostics', async () => {
-      const controller = new globalThis.AbortController();
-      controller.abort();
-      const transport = async ({ signal }) => {
-        expect(signal).toBe(controller.signal);
-        const error = new Error('cancelled by caller');
-        error.name = 'AbortError';
-        throw error;
-      };
-
-      const result = await search({
-        query: 'formal-ai',
-        transport,
-        signal: controller.signal,
-        now: fixedNow,
-      });
-
-      expect(result.diagnostics).toMatchObject({
-        status: 0,
-        errorKind: 'cancelled',
-        error: 'cancelled by caller',
-      });
-    });
-
-    it('rejects an empty query', async () => {
-      await expect(search({ query: '   ' })).rejects.toThrow(/Missing .query/);
-    });
-
-    it('rejects an unknown provider', async () => {
-      await expect(search({ query: 'x', provider: 'yahoo' })).rejects.toThrow(
-        /Unknown search provider/
-      );
-    });
-  });
-
-  describe('formatSearchAsMarkdown', () => {
-    it('renders a markdown document with ranked links', () => {
-      const md = formatSearchAsMarkdown({
-        query: 'formal-ai',
-        provider: 'wikipedia',
-        captureMode: 'fetch',
-        capturedAt: '2026-05-30T00:00:00.000Z',
-        results: [
-          {
-            rank: 1,
-            title: 'Formal methods',
-            url: 'https://en.wikipedia.org/wiki/Formal_methods',
-            snippet: 'study of formal',
-          },
-        ],
-        diagnostics: {
-          sourceUrl: 'https://example.com',
-          blockedByCaptcha: false,
-        },
-      });
-      expect(md).toContain('# Search results for "formal-ai"');
-      expect(md).toContain(
-        '1. [Formal methods](https://en.wikipedia.org/wiki/Formal_methods)'
-      );
-      expect(md).toContain('study of formal');
-    });
-
-    it('reports no results', () => {
-      const md = formatSearchAsMarkdown({
-        query: 'nothing',
-        provider: 'bing',
-        captureMode: 'fetch',
-        capturedAt: 't',
-        results: [],
-        diagnostics: { sourceUrl: 'u', blockedByCaptcha: false },
-      });
-      expect(md).toContain('_No results._');
-    });
-  });
-
-  it('exposes the documented provider list', () => {
-    expect(SEARCH_PROVIDERS).toEqual([
-      'wikipedia',
-      'duckduckgo',
-      'google',
-      'bing',
-      'brave',
-    ]);
-  });
-});

package/tests/unit/shared-dialog.test.js

@@ -1,173 +0,0 @@
-import { readFileSync } from 'node:fs';
-import { dirname, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-import { jest } from '@jest/globals';
-
-import {
-  captureSharedDialog,
-  formatSharedDialogAsDemoMemory,
-  formatSharedDialogAsMarkdown,
-  formatSharedDialogAsMetaLanguage,
-  parseSharedDialog,
-} from '../../src/shared-dialog.js';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = dirname(__filename);
-const repoRoot = resolve(__dirname, '../../..');
-
-const CHATGPT_SHARE_URL =
-  'https://chatgpt.com/share/6a3825b9-8de4-83ee-9c24-52fd1eb38d24';
-const GOOGLE_AI_MODE_URL = 'https://share.google/aimode/VG0HhpnAXrBkC0QgP';
-
-function readCaseStudyFixture(name) {
-  return readFileSync(
-    resolve(repoRoot, 'docs/case-studies/issue-141/raw-data', name),
-    'utf-8'
-  );
-}
-
-describe('shared-dialog module (#141)', () => {
-  it('extracts the four visible ChatGPT shared-dialog turns', () => {
-    const capture = parseSharedDialog(
-      readCaseStudyFixture('chatgpt-share-6a3825b9.html'),
-      {
-        sourceUrl: CHATGPT_SHARE_URL,
-        captureMethod: 'static_http',
-      }
-    );
-
-    expect(capture).toMatchObject({
-      provider: 'chatgpt',
-      sourceUrl: CHATGPT_SHARE_URL,
-      captureMethod: 'static_http',
-      status: 'ok',
-      conversationId: '6a3825b9-8de4-83ee-9c24-52fd1eb38d24',
-      title: 'Infinite loop script',
-    });
-    expect(capture.turns).toHaveLength(4);
-    expect(capture.turns.map((turn) => turn.role)).toEqual([
-      'user',
-      'assistant',
-      'user',
-      'assistant',
-    ]);
-    expect(capture.turns[0]).toMatchObject({
-      visibility: 'visible',
-      sourceEvidence: [
-        {
-          sourceUrl: CHATGPT_SHARE_URL,
-          kind: 'chatgpt_linear_conversation',
-        },
-      ],
-    });
-    expect(capture.turns[0].content).toContain('make a loop of that');
-    expect(capture.turns[1].content).toContain(
-      'while true; do sleep 30m && hive-cleanup -f; done'
-    );
-    expect(capture.turns[3].content).toContain(
-      'screen -dmS auto-cleanup bash -c'
-    );
-  });
-
-  it('formats ChatGPT captures as demo_memory', () => {
-    const capture = parseSharedDialog(
-      readCaseStudyFixture('chatgpt-share-6a3825b9.html'),
-      {
-        sourceUrl: CHATGPT_SHARE_URL,
-        captureMethod: 'static_http',
-      }
-    );
-
-    const memory = formatSharedDialogAsDemoMemory(capture, {
-      demoLabel: 'issue-552-chatgpt-share',
-    });
-
-    expect(memory).toContain('demo_memory');
-    expect(memory.match(/\n {2}event "/g)).toHaveLength(4);
-    expect(memory).toContain('role "user"');
-    expect(memory).toContain('role "assistant"');
-    expect(memory).toContain(
-      'conversationId "6a3825b9-8de4-83ee-9c24-52fd1eb38d24"'
-    );
-    expect(memory).toContain('conversationTitle "Infinite loop script"');
-    expect(memory).toContain(`evidence "${CHATGPT_SHARE_URL}"`);
-    expect(memory).toContain(
-      "screen -dmS auto-cleanup bash -c 'while true; do sleep 30m && hive-cleanup -f; done'"
-    );
-  });
-
-  it('formats captures as shared-dialog meta-language and Markdown', () => {
-    const capture = parseSharedDialog(
-      readCaseStudyFixture('chatgpt-share-6a3825b9.html'),
-      {
-        sourceUrl: CHATGPT_SHARE_URL,
-        captureMethod: 'static_http',
-      }
-    );
-
-    const metaLanguage = formatSharedDialogAsMetaLanguage(capture);
-    expect(metaLanguage).toContain('shared_dialog_capture');
-    expect(metaLanguage).toContain('provider "chatgpt"');
-    expect(metaLanguage).toContain(
-      'turn "0c9f0151-b5a1-402f-afc3-6bd34a0d01d2"'
-    );
-
-    const markdown = formatSharedDialogAsMarkdown(capture);
-    expect(markdown).toContain('# Infinite loop script');
-    expect(markdown).toContain('**User**');
-    expect(markdown).toContain('**Assistant**');
-  });
-
-  it('returns a structured Google AI Mode interstitial diagnostic', () => {
-    const capture = parseSharedDialog(
-      readCaseStudyFixture('google-ai-mode-VG0HhpnAXrBkC0QgP.html'),
-      {
-        sourceUrl: GOOGLE_AI_MODE_URL,
-        captureMethod: 'static_http',
-      }
-    );
-
-    expect(capture).toMatchObject({
-      provider: 'google_ai_mode',
-      sourceUrl: GOOGLE_AI_MODE_URL,
-      captureMethod: 'static_http',
-      status: 'unsupported',
-      turns: [],
-      diagnostics: {
-        unsupportedReason: 'provider_challenge_interstitial',
-      },
-    });
-  });
-
-  it('uses browser-rendered DOM when static Google AI Mode capture is blocked', async () => {
-    const googleHtml = readCaseStudyFixture(
-      'google-ai-mode-VG0HhpnAXrBkC0QgP.html'
-    );
-    const renderHtml = jest.fn(async () => googleHtml);
-    const capture = await captureSharedDialog({
-      url: GOOGLE_AI_MODE_URL,
-      capture: 'browser',
-      fetchImpl: async () => ({
-        ok: true,
-        status: 200,
-        text: async () => googleHtml,
-      }),
-      renderHtml,
-      now: () => '2026-06-25T00:00:00.000Z',
-    });
-
-    expect(renderHtml).toHaveBeenCalledWith(GOOGLE_AI_MODE_URL, {
-      engine: 'puppeteer',
-    });
-    expect(capture).toMatchObject({
-      provider: 'google_ai_mode',
-      captureMethod: 'browser',
-      status: 'unsupported',
-      capturedAt: '2026-06-25T00:00:00.000Z',
-      diagnostics: {
-        unsupportedReason: 'provider_challenge_interstitial',
-      },
-    });
-  });
-});

package/tests/unit/stackoverflow.test.js

@@ -1,32 +0,0 @@
-import {
-  isStackOverflowQuestionUrl,
-  stackPrinterUrlForQuestion,
-} from '../../src/lib.js';
-
-describe('Stack Overflow URL helpers', () => {
-  it('recognizes Stack Overflow question URLs', () => {
-    expect(
-      isStackOverflowQuestionUrl(
-        'https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-local-commits-in-git'
-      )
-    ).toBe(true);
-    expect(
-      isStackOverflowQuestionUrl(
-        'https://stackoverflow.com/questions/tagged/git'
-      )
-    ).toBe(false);
-    expect(
-      isStackOverflowQuestionUrl('https://serverfault.com/questions/927358')
-    ).toBe(false);
-  });
-
-  it('builds a StackPrinter URL for direct captures', () => {
-    const stackPrinterUrl = stackPrinterUrlForQuestion(
-      'https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-local-commits-in-git'
-    );
-
-    expect(stackPrinterUrl).toBe(
-      'https://stackprinter.appspot.com/export?question=927358&service=stackoverflow&language=en&hideAnswers=false&showAll=true&width=640'
-    );
-  });
-});

package/tests/unit/stream.test.js

@@ -1,85 +0,0 @@
-import express from 'express';
-import http from 'http';
-import request from 'supertest';
-import { streamHandler } from '../../src/stream.js';
-
-describe('streamHandler', () => {
-  it('streams upstream content with proxy-safe framing', async () => {
-    const body = '<!DOCTYPE html><html><body>Example Domain</body></html>';
-    let upstreamHeaders;
-    const { server, url } = await startUpstreamServer((req, res) => {
-      upstreamHeaders = req.headers;
-      res.writeHead(200, {
-        'Content-Type': 'text/html; charset=utf-8',
-        'Content-Length': Buffer.byteLength(body),
-      });
-      res.end(body);
-    });
-
-    try {
-      const res = await request(testApp())
-        .get('/stream')
-        .query({ url })
-        .expect(200);
-
-      expect(upstreamHeaders['accept-encoding']).toBe('identity');
-      expect(res.headers['content-type']).toBe('text/html; charset=utf-8');
-      expect(res.headers['content-length']).toBe(
-        String(Buffer.byteLength(body))
-      );
-      expect(res.text).toContain('Example Domain');
-    } finally {
-      await closeServer(server);
-    }
-  });
-
-  it('ends the downstream response when upstream closes after body bytes', async () => {
-    const body = '<!DOCTYPE html><html><body>Example Domain</body></html>';
-    const { server, url } = await startUpstreamServer((req, res) => {
-      res.writeHead(200, {
-        'Content-Type': 'text/html; charset=utf-8',
-      });
-      res.write(body);
-      res.socket.end();
-    });
-
-    try {
-      const res = await request(testApp())
-        .get('/stream')
-        .query({ url })
-        .expect(200);
-
-      expect(res.text).toContain('Example Domain');
-    } finally {
-      await closeServer(server);
-    }
-  });
-});
-
-function testApp() {
-  const app = express();
-  app.get('/stream', streamHandler);
-  return app;
-}
-
-async function startUpstreamServer(handler) {
-  const server = http.createServer(handler);
-  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
-  const { port } = server.address();
-  return {
-    server,
-    url: `http://127.0.0.1:${port}`,
-  };
-}
-
-async function closeServer(server) {
-  await new Promise((resolve, reject) => {
-    server.close((error) => {
-      if (error) {
-        reject(error);
-      } else {
-        resolve();
-      }
-    });
-  });
-}