@link-assistant/web-capture 1.12.0 → 2.0.1

package/tests/integration/browser-engines.test.js

@@ -1,128 +0,0 @@
-import { createBrowser } from '../../src/browser.js';
-
-describe('Browser Engine Integration Tests', () => {
-  describe('Puppeteer Engine', () => {
-    let browser;
-
-    beforeEach(async () => {
-      browser = await createBrowser('puppeteer');
-    });
-
-    afterEach(async () => {
-      if (browser) {
-        await browser.close();
-      }
-    });
-
-    it('can navigate to a page and get content', async () => {
-      const page = await browser.newPage();
-      await page.goto('https://example.com', {
-        waitUntil: 'networkidle0',
-        timeout: 30000,
-      });
-      const content = await page.content();
-      expect(content).toContain('Example Domain');
-    }, 60000);
-
-    it('can take a screenshot', async () => {
-      const page = await browser.newPage();
-      await page.setViewport({ width: 1280, height: 800 });
-      await page.goto('https://example.com', {
-        waitUntil: 'networkidle0',
-        timeout: 30000,
-      });
-      const screenshot = await page.screenshot({ type: 'png' });
-      expect(screenshot).toBeInstanceOf(Buffer);
-      expect(screenshot.length).toBeGreaterThan(100);
-    }, 60000);
-
-    it('can set custom headers and user agent', async () => {
-      const page = await browser.newPage();
-      await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-US' });
-      await page.setUserAgent('Test User Agent');
-      await page.goto('https://example.com', {
-        waitUntil: 'networkidle0',
-        timeout: 30000,
-      });
-      const content = await page.content();
-      expect(content).toBeTruthy();
-    }, 60000);
-  });
-
-  describe('Playwright Engine', () => {
-    let browser;
-
-    beforeEach(async () => {
-      browser = await createBrowser('playwright');
-    });
-
-    afterEach(async () => {
-      if (browser) {
-        await browser.close();
-      }
-    });
-
-    it('can navigate to a page and get content', async () => {
-      const page = await browser.newPage();
-      await page.goto('https://example.com', {
-        waitUntil: 'networkidle0',
-        timeout: 30000,
-      });
-      const content = await page.content();
-      expect(content).toContain('Example Domain');
-    }, 60000);
-
-    it('can take a screenshot', async () => {
-      const page = await browser.newPage();
-      await page.setViewport({ width: 1280, height: 800 });
-      await page.goto('https://example.com', {
-        waitUntil: 'networkidle0',
-        timeout: 30000,
-      });
-      const screenshot = await page.screenshot({ type: 'png' });
-      expect(screenshot).toBeInstanceOf(Buffer);
-      expect(screenshot.length).toBeGreaterThan(100);
-    }, 60000);
-
-    it('can set custom headers and user agent', async () => {
-      const page = await browser.newPage();
-      await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-US' });
-      await page.setUserAgent('Test User Agent');
-      await page.goto('https://example.com', {
-        waitUntil: 'networkidle0',
-        timeout: 30000,
-      });
-      const content = await page.content();
-      expect(content).toBeTruthy();
-    }, 60000);
-  });
-
-  describe('Engine Parity', () => {
-    it('both engines produce similar content for the same page', async () => {
-      const puppeteerBrowser = await createBrowser('puppeteer');
-      const playwrightBrowser = await createBrowser('playwright');
-
-      const puppeteerPage = await puppeteerBrowser.newPage();
-      const playwrightPage = await playwrightBrowser.newPage();
-
-      await puppeteerPage.goto('https://example.com', {
-        waitUntil: 'networkidle0',
-        timeout: 30000,
-      });
-      await playwrightPage.goto('https://example.com', {
-        waitUntil: 'networkidle0',
-        timeout: 30000,
-      });
-
-      const puppeteerContent = await puppeteerPage.content();
-      const playwrightContent = await playwrightPage.content();
-
-      // Both should contain the main content
-      expect(puppeteerContent).toContain('Example Domain');
-      expect(playwrightContent).toContain('Example Domain');
-
-      await puppeteerBrowser.close();
-      await playwrightBrowser.close();
-    }, 60000);
-  });
-});

package/tests/integration/browser-packaged-path.test.js

@@ -1,45 +0,0 @@
-import { mkdtemp, rm, symlink } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import path from 'node:path';
-import { chromium } from 'playwright';
-import { createBrowser } from '../../src/browser.js';
-
-// Only deterministic local HTML is used. Neither engine may select its own cache.
-test.each(['puppeteer', 'playwright'])(
-  '%s renders using a Chromium executable at an explicit temporary path',
-  async (engine) => {
-    const directory = await mkdtemp(path.join(tmpdir(), 'packaged-browser-'));
-    const packagedPath = path.join(
-      directory,
-      process.platform === 'win32' ? 'chrome.exe' : 'chromium'
-    );
-    let browser;
-    let page;
-    try {
-      await symlink(chromium.executablePath(), packagedPath);
-      browser = await createBrowser(engine, {
-        executablePath: packagedPath,
-        userDataDir: path.join(directory, 'profile'),
-      });
-      page = await browser.newPage();
-      await page.goto(
-        'data:text/html,<main id="result"></main><script>document.getElementById("result").textContent="Packaged Chromium rendered"</script>'
-      );
-      expect(await page.content()).toContain('Packaged Chromium rendered');
-      const executable =
-        engine === 'puppeteer'
-          ? browser.rawBrowser.process().spawnfile
-          : packagedPath;
-      expect(executable).toBe(packagedPath);
-    } finally {
-      if (page) {
-        await page.close();
-      }
-      if (browser) {
-        await browser.close();
-      }
-      await rm(directory, { recursive: true, force: true });
-    }
-  },
-  30000
-);

package/tests/integration/deepwiki-download.test.js

@@ -1,144 +0,0 @@
-/**
- * Integration test for converting the DeepWiki search page from issue #17.
- *
- * The page is JavaScript-rendered, so this suite verifies the /markdown
- * endpoint captures rendered content instead of the initial Next.js shell.
- *
- * Set DEEPWIKI_INTEGRATION=true to run it. The suite is skipped by default so
- * local/offline test runs remain deterministic.
- */
-
-import { jest } from '@jest/globals';
-import fs from 'fs';
-import path from 'path';
-import request from 'supertest';
-import { fileURLToPath } from 'url';
-import { app } from '../../src/index.js';
-import { retry } from '../../src/retry.js';
-
-const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const DEEPWIKI_URL =
-  'https://deepwiki.com/search/-57-4-23-57_0e4aa687-7a9d-4591-8c6f-67c4b2d732f6';
-const FIXTURE_DIR = path.join(__dirname, '..', 'fixtures', 'deepwiki');
-const INLINE_MARKDOWN_LINE_LIMIT = 1500;
-const PNG_SIGNATURE = Buffer.from([
-  0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
-]);
-const ORDERED_CONTENT_MARKERS = [
-  'Search | DeepWiki',
-  'deep-assistant/hive-mind',
-  'расскажи пожалуйста на английском',
-  'Deep\n\nThought Process',
-  '# Hive Mind: A Comprehensive Overview of an AI Agent System for Software Development',
-  '## Executive Summary',
-  '## I. Foundational Philosophy: Human-AI Collaboration Model',
-  '## II. Architectural Layers: The Three-Tier Design',
-  '## III. Original Ideas and Innovations',
-  'Innovation 1: The Task Clarification System',
-  'Innovation 2: Multi-Dimensional Feedback Detection',
-  'Innovation 3: Thinking Depth Control',
-  '## IV. The Workflow Logic: How Everything Connects',
-  '## XI. Practical Implications: What This Enables',
-];
-
-const SKIP_LIVE =
-  !process.env.DEEPWIKI_INTEGRATION ||
-  process.env.DEEPWIKI_INTEGRATION === 'false';
-const describeIfLive = SKIP_LIVE ? describe.skip : describe;
-
-jest.setTimeout(180000);
-
-function readFixtureChunks() {
-  return ['index.md', 'part-1.md'].map((filename) =>
-    fs.readFileSync(path.join(FIXTURE_DIR, filename), 'utf8')
-  );
-}
-
-function splitMarkdown(markdown) {
-  const lines = markdown.split('\n');
-  const chunks = [];
-  for (
-    let index = 0;
-    index < lines.length;
-    index += INLINE_MARKDOWN_LINE_LIMIT
-  ) {
-    chunks.push(
-      lines.slice(index, index + INLINE_MARKDOWN_LINE_LIMIT).join('\n')
-    );
-  }
-  return chunks;
-}
-
-function expectMarkersInOrder(markdown, markers) {
-  let previousIndex = -1;
-  for (const marker of markers) {
-    const index = markdown.indexOf(marker);
-    expect(index).toBeGreaterThan(previousIndex);
-    previousIndex = index;
-  }
-}
-
-async function getDeepwikiMarkdown() {
-  const response = await retry(
-    () =>
-      request(app)
-        .get('/markdown')
-        .query({ url: DEEPWIKI_URL, engine: 'playwright' })
-        .expect(200),
-    {
-      retries: 2,
-      baseDelay: 3000,
-      onRetry: (err, attempt, delay) => {
-        console.log(
-          `Retry ${attempt} for DeepWiki markdown after ${delay}ms: ${err.message}`
-        );
-      },
-    }
-  );
-
-  expect(response.headers['content-type']).toMatch(/text\/markdown/);
-  return response.text;
-}
-
-describe('DeepWiki committed reference fixtures (issue #17)', () => {
-  it('stores markdown split into inline-sized reference documents', () => {
-    const fixtureChunks = readFixtureChunks();
-    const fixtureMarkdown = fixtureChunks.join('\n');
-
-    expect(fixtureChunks).toHaveLength(2);
-    expect(fixtureChunks[0].split('\n').length).toBeLessThanOrEqual(
-      INLINE_MARKDOWN_LINE_LIMIT
-    );
-    expect(fixtureChunks[1].split('\n').length).toBeGreaterThan(0);
-    expect(fixtureMarkdown.split('\n').length).toBeGreaterThan(
-      INLINE_MARKDOWN_LINE_LIMIT
-    );
-    expectMarkersInOrder(fixtureMarkdown, ORDERED_CONTENT_MARKERS);
-  });
-
-  it('keeps the full-page screenshot fixture as a PNG visual reference', () => {
-    const screenshot = fs.readFileSync(
-      path.join(FIXTURE_DIR, 'deepwiki-full-page.png')
-    );
-
-    expect(screenshot.subarray(0, PNG_SIGNATURE.length)).toEqual(PNG_SIGNATURE);
-    expect(screenshot.length).toBeGreaterThan(100000);
-  });
-});
-
-describeIfLive('DeepWiki search page markdown capture (issue #17)', () => {
-  it('matches the committed markdown fixture split from the rendered page', async () => {
-    const markdown = await getDeepwikiMarkdown();
-    const fixtureChunks = readFixtureChunks();
-    const fixtureMarkdown = fixtureChunks.join('\n');
-
-    expect(markdown).not.toMatch(/<html/i);
-    expect(markdown).not.toContain('__NEXT_DATA__');
-    expect(markdown.split('\n').length).toBeGreaterThan(
-      INLINE_MARKDOWN_LINE_LIMIT
-    );
-    expectMarkersInOrder(markdown, ORDERED_CONTENT_MARKERS);
-    expectMarkersInOrder(fixtureMarkdown, ORDERED_CONTENT_MARKERS);
-    expect(splitMarkdown(markdown)).toEqual(fixtureChunks);
-  }, 180000);
-});

package/tests/integration/deepwiki-rendered-page.mock.test.js

@@ -1,121 +0,0 @@
-import { jest } from '@jest/globals';
-import nock from '../helpers/nock.js';
-import request from 'supertest';
-import { URL, fileURLToPath } from 'url';
-
-const DEEPWIKI_URL =
-  'https://deepwiki.com/search/-57-4-23-57_0e4aa687-7a9d-4591-8c6f-67c4b2d732f6';
-const deepwikiShellHtml = `<!doctype html>
-<html lang="en">
-  <head>
-    <title>Search | DeepWiki</title>
-    <script src="/_next/static/chunks/app.js"></script>
-    <script>self.__next_f = []; self.__next_f.push([1, "shell"])</script>
-  </head>
-  <body><div id="__next"></div></body>
-</html>`;
-const renderedDeepwikiHtml = `<!doctype html>
-<html lang="en">
-  <head><title>Search | DeepWiki</title></head>
-  <body>
-    <main>
-      <p>deep-assistant/hive-mind</p>
-      <p>расскажи пожалуйста на английском</p>
-      <h1>Hive Mind: A Comprehensive Overview of an AI Agent System for Software Development</h1>
-      <h2>Executive Summary</h2>
-      <p>Deep analysis of the issue-solver workflow.</p>
-      <h2>Thought Process</h2>
-      <pre><code>const clarifyPrompt = \`Task: "\${taskDescription}"\`;
-await solver.run(clarifyPrompt);</code></pre>
-    </main>
-  </body>
-</html>`;
-
-const fakePage = {
-  type: 'playwright',
-  rawPage: {
-    evaluate: jest
-      .fn()
-      .mockRejectedValueOnce(
-        new Error(
-          'Execution context was destroyed, most likely because of a navigation'
-        )
-      )
-      .mockResolvedValue(
-        renderedDeepwikiHtml.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ')
-          .length
-      ),
-  },
-  setViewport: jest.fn(async () => {}),
-  setExtraHTTPHeaders: jest.fn(async () => {}),
-  setUserAgent: jest.fn(async () => {}),
-  goto: jest.fn(async () => {}),
-  waitForTimeout: jest.fn(
-    (ms) => new Promise((resolve) => setTimeout(resolve, ms))
-  ),
-  content: jest.fn(async () => renderedDeepwikiHtml),
-};
-const fakeBrowser = {
-  newPage: jest.fn(async () => fakePage),
-  close: jest.fn(async () => {}),
-};
-const createBrowser = jest.fn(async () => fakeBrowser);
-const browserModulePath = fileURLToPath(
-  new URL('../../src/browser.js', import.meta.url)
-);
-
-jest.unstable_mockModule(browserModulePath, () => ({
-  createBrowser,
-  getBrowserEngine: (req) => req.query.engine || 'puppeteer',
-}));
-
-const { app } = await import('../../src/index.js');
-
-describe('DeepWiki rendered markdown endpoint behavior', () => {
-  beforeEach(() => {
-    nock.cleanAll();
-    jest.clearAllMocks();
-  });
-
-  afterEach(() => {
-    nock.cleanAll();
-  });
-
-  it('converts browser-rendered content when the initial fetch is a Next.js shell', async () => {
-    nock('https://deepwiki.com')
-      .get('/search/-57-4-23-57_0e4aa687-7a9d-4591-8c6f-67c4b2d732f6')
-      .reply(200, deepwikiShellHtml);
-
-    const response = await request(app)
-      .get('/markdown')
-      .query({ url: DEEPWIKI_URL, engine: 'playwright' })
-      .expect(200);
-
-    expect(createBrowser).toHaveBeenCalledWith(
-      'playwright',
-      expect.objectContaining({
-        args: expect.arrayContaining([
-          expect.stringContaining('--user-agent='),
-          '--window-size=1280,800',
-          '--disable-blink-features=AutomationControlled',
-        ]),
-      })
-    );
-    expect(fakePage.goto).toHaveBeenCalledWith(
-      DEEPWIKI_URL,
-      expect.objectContaining({ waitUntil: 'networkidle0' })
-    );
-    expect(fakeBrowser.close).toHaveBeenCalledTimes(1);
-    expect(response.headers['content-type']).toMatch(/text\/markdown/);
-    expect(response.text).toContain('Search | DeepWiki');
-    expect(response.text).toContain('deep-assistant/hive-mind');
-    expect(response.text).toContain('расскажи пожалуйста на английском');
-    expect(response.text).toContain(
-      '# Hive Mind: A Comprehensive Overview of an AI Agent System for Software Development'
-    );
-    expect(response.text).toContain('## Thought Process');
-    expect(response.text).toContain('```');
-    expect(response.text).toContain('const clarifyPrompt');
-    expect(response.text).not.toContain('__next_f');
-  });
-});

package/tests/integration/formalai-contract.test.js

@@ -1,214 +0,0 @@
-/**
- * Smoke tests for the FormalAI-facing HTTP contract (issue #135).
- *
- * These tests intentionally assert the stable response shapes documented in
- * docs/formalai-contract.md rather than endpoint internals.
- */
-
-import request from 'supertest';
-import nock from '../helpers/nock.js';
-import unzipper from 'unzipper';
-import { app, SEARCH_PROVIDERS } from '../../src/index.js';
-
-const HTML = `<!doctype html>
-<html>
-  <head><title>FormalAI fixture</title></head>
-  <body><h1>FormalAI Fixture</h1><p>Stable HTTP shape.</p></body>
-</html>`;
-
-const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
-
-const WIKI_JSON = {
-  pages: [
-    {
-      id: 1,
-      key: 'Formal_methods',
-      title: 'Formal methods',
-      excerpt: 'the study of <b>formal</b> methods',
-      description: 'rigorous techniques',
-    },
-  ],
-};
-
-function parseBinary(res, callback) {
-  const chunks = [];
-  res.on('data', (chunk) => chunks.push(chunk));
-  res.on('end', () => callback(null, Buffer.concat(chunks)));
-}
-
-afterEach(() => {
-  nock.cleanAll();
-});
-
-describe('FormalAI HTTP contract (#135)', () => {
-  it('returns stable text artifact shapes for /html, /txt, and /markdown', async () => {
-    nock('https://formalai.example').get('/page').reply(200, HTML, {
-      'content-type': 'text/html; charset=utf-8',
-    });
-    const html = await request(app)
-      .get('/html')
-      .query({ url: 'https://formalai.example/page' })
-      .expect(200)
-      .expect('Content-Type', /text\/html/);
-    expect(html.text).toContain('<h1>FormalAI Fixture</h1>');
-
-    nock('https://formalai.example')
-      .get('/text')
-      .reply(200, 'FormalAI plain text', {
-        'content-type': 'text/plain; charset=utf-8',
-      });
-    const text = await request(app)
-      .get('/txt')
-      .query({ url: 'https://formalai.example/text' })
-      .expect(200)
-      .expect('Content-Type', /text\/plain/);
-    expect(text.headers['content-disposition']).toContain('.txt');
-    expect(text.text).toBe('FormalAI plain text');
-
-    nock('https://formalai.example').get('/markdown').reply(200, HTML, {
-      'content-type': 'text/html; charset=utf-8',
-    });
-    const markdown = await request(app)
-      .get('/markdown')
-      .query({ url: 'https://formalai.example/markdown' })
-      .expect(200)
-      .expect('Content-Type', /text\/markdown/);
-    expect(markdown.text).toContain('FormalAI Fixture');
-    expect(markdown.text).toContain('Stable HTTP shape.');
-  });
-
-  it('preserves upstream status and content for /fetch and /stream', async () => {
-    nock('https://formalai.example')
-      .get('/fetch-source')
-      .reply(203, 'fetch body', {
-        'content-type': 'text/plain; charset=utf-8',
-      });
-    const fetched = await request(app)
-      .get('/fetch')
-      .query({ url: 'https://formalai.example/fetch-source' })
-      .expect(203)
-      .expect('Content-Type', /text\/plain/);
-    expect(fetched.text).toBe('fetch body');
-
-    nock('https://formalai.example')
-      .get('/stream-source')
-      .reply(206, 'stream body', {
-        'content-type': 'text/plain; charset=utf-8',
-      });
-    const streamed = await request(app)
-      .get('/stream')
-      .query({ url: 'https://formalai.example/stream-source' })
-      .expect(206)
-      .expect('Content-Type', /text\/plain/);
-    expect(streamed.text).toBe('stream body');
-  });
-
-  it('returns stable binary artifact shapes for /image and /archive', async () => {
-    nock('https://drive.google.com')
-      .get('/uc')
-      .query({ export: 'download', id: 'formalaiimage' })
-      .reply(200, PNG, { 'content-type': 'image/png' });
-    const image = await request(app)
-      .get('/image')
-      .query({
-        url: 'https://drive.google.com/file/d/formalaiimage/view',
-      })
-      .buffer(true)
-      .parse(parseBinary)
-      .expect(200)
-      .expect('Content-Type', /image\/png/);
-    expect(image.headers['content-disposition']).toContain(
-      'google-drive-formalaiimage.png'
-    );
-    expect(image.body.subarray(0, 8)).toEqual(PNG);
-
-    nock('https://formalai.example').get('/archive').reply(200, HTML, {
-      'content-type': 'text/html; charset=utf-8',
-    });
-    const archive = await request(app)
-      .get('/archive')
-      .query({
-        url: 'https://formalai.example/archive',
-        localImages: 'false',
-      })
-      .buffer(true)
-      .parse(parseBinary)
-      .expect(200)
-      .expect('Content-Type', /application\/zip/);
-    expect(archive.body[0]).toBe(0x50);
-    expect(archive.body[1]).toBe(0x4b);
-    const zip = await unzipper.Open.buffer(archive.body);
-    expect(zip.files.map((file) => file.path)).toEqual(
-      expect.arrayContaining(['document.md', 'document.html'])
-    );
-  });
-
-  it('returns normalized /search JSON diagnostics and the documented provider catalog', async () => {
-    expect(SEARCH_PROVIDERS).toEqual([
-      'wikipedia',
-      'duckduckgo',
-      'google',
-      'bing',
-      'brave',
-    ]);
-
-    nock('https://en.wikipedia.org')
-      .get('/w/rest.php/v1/search/page')
-      .query(true)
-      .reply(200, WIKI_JSON);
-
-    const res = await request(app)
-      .get('/search')
-      .query({ q: 'formal-ai', provider: 'wikipedia', limit: 1 })
-      .expect(200)
-      .expect('Content-Type', /application\/json/);
-
-    expect(res.body).toMatchObject({
-      query: 'formal-ai',
-      provider: 'wikipedia',
-      captureMode: 'fetch',
-      results: [
-        {
-          rank: 1,
-          title: 'Formal methods',
-          url: 'https://en.wikipedia.org/wiki/Formal_methods',
-          snippet: 'the study of formal methods',
-        },
-      ],
-      diagnostics: {
-        status: 200,
-        blockedByCors: false,
-        blockedByCaptcha: false,
-      },
-    });
-    expect(res.body.capturedAt).toEqual(expect.any(String));
-    expect(res.body.diagnostics.sourceUrl).toContain('en.wikipedia.org');
-  });
-
-  it('keeps normalized /search diagnostics when provider capture fails', async () => {
-    nock('https://en.wikipedia.org')
-      .get('/w/rest.php/v1/search/page')
-      .query(true)
-      .replyWithError(new Error('provider offline'));
-
-    const res = await request(app)
-      .get('/search')
-      .query({ q: 'formal-ai', provider: 'wikipedia', limit: 1 })
-      .expect(200)
-      .expect('Content-Type', /application\/json/);
-
-    expect(res.body).toMatchObject({
-      query: 'formal-ai',
-      provider: 'wikipedia',
-      captureMode: 'fetch',
-      results: [],
-      diagnostics: {
-        status: 0,
-        blockedByCors: false,
-        blockedByCaptcha: false,
-      },
-    });
-    expect(res.body.diagnostics.sourceUrl).toContain('en.wikipedia.org');
-    expect(res.body.diagnostics.error).toContain('provider offline');
-  });
-});