@link-assistant/web-capture 2.0.0 → 2.0.1

package/tests/integration/habr-article.test.js

@@ -1,337 +0,0 @@
-/**
- * Integration tests for Habr article downloads.
- *
- * Tests that the web-capture service can successfully download real-world content
- * from Habr.com using both Puppeteer and Playwright engines across all three
- * archived articles from link-foundation/meta-theory.
- *
- * These tests hit live servers. Set HABR_INTEGRATION=true to run them.
- * In CI they are enabled via the workflow environment variable.
- *
- * Tests use retry with exponential backoff to handle transient network failures.
- * Browser instances are shared within describe blocks to reduce startup overhead.
- *
- * Covers:
- * - Markdown conversion (fast, no browser needed)
- * - PNG and JPEG screenshots (light and dark themes)
- * - Full-page captures
- * - Viewport configuration
- * - Engine comparison
- * - Archive endpoint with documentFormat option (html/markdown)
- */
-
-import { jest } from '@jest/globals';
-import request from 'supertest';
-import { app } from '../../src/index.js';
-import { createBrowser } from '../../src/browser.js';
-import { fetchHtml, convertHtmlToMarkdown } from '../../src/lib.js';
-import { dismissPopups, scrollToLoadContent } from '../../src/popups.js';
-import { retry } from '../../src/retry.js';
-import { getAllArticles, HABR_ARTICLES } from '../fixtures/habr-articles.js';
-
-const SKIP_LIVE =
-  !process.env.HABR_INTEGRATION || process.env.HABR_INTEGRATION === 'false';
-const describeIfLive = SKIP_LIVE ? describe.skip : describe;
-
-// These tests hit live servers and may take a while
-jest.setTimeout(120000);
-
-// Helper: navigate to URL with retry and exponential backoff
-// Uses 'domcontentloaded' instead of 'networkidle0' because Habr pages have
-// persistent ad/tracking network activity that prevents networkidle0 from firing.
-async function navigateWithRetry(page, url) {
-  await retry(
-    async () => {
-      await page.goto(url, {
-        waitUntil: 'domcontentloaded',
-        timeout: 60000,
-      });
-    },
-    {
-      retries: 3,
-      baseDelay: 2000,
-      onRetry: (err, attempt, delay) => {
-        console.log(
-          `Navigation retry ${attempt} for ${url} after ${delay}ms: ${err.message}`
-        );
-      },
-    }
-  );
-  // Wait for page content and images to load after DOM is ready
-  await new Promise((r) => setTimeout(r, 5000));
-}
-
-describe('Habr Article Download Tests', () => {
-  // Use article 0.0.2 as the primary test article (English, most recent)
-  const primaryArticle = HABR_ARTICLES['0.0.2'];
-
-  describeIfLive('Markdown Download (fetch only)', () => {
-    it('downloads article 0.0.2 as markdown', async () => {
-      const html = await retry(() => fetchHtml(primaryArticle.url), {
-        retries: 3,
-        baseDelay: 2000,
-      });
-      expect(html.length).toBeGreaterThan(1000);
-
-      const markdown = convertHtmlToMarkdown(html, primaryArticle.url);
-      expect(markdown.length).toBeGreaterThan(100);
-      // Should contain markdown headings
-      expect(markdown).toMatch(/^#{1,3}\s/m);
-      // Should contain links
-      expect(markdown).toMatch(/\[.*?\]\(.*?\)/);
-    });
-
-    it('downloads all 3 articles as markdown', async () => {
-      for (const article of getAllArticles()) {
-        const html = await retry(() => fetchHtml(article.url), {
-          retries: 3,
-          baseDelay: 2000,
-        });
-        expect(html.length).toBeGreaterThan(1000);
-
-        const markdown = convertHtmlToMarkdown(html, article.url);
-        expect(markdown.length).toBeGreaterThan(100);
-        expect(markdown).toMatch(/^#{1,3}\s/m);
-      }
-    });
-  });
-
-  describeIfLive('Screenshots with Puppeteer', () => {
-    let browser;
-    let page;
-
-    beforeAll(async () => {
-      browser = await createBrowser('puppeteer');
-      page = await browser.newPage();
-      await page.setViewport({ width: 1280, height: 800 });
-      await navigateWithRetry(page, primaryArticle.url);
-      await dismissPopups(page);
-    });
-
-    afterAll(async () => {
-      if (browser) {
-        await browser.close();
-      }
-    });
-
-    it('captures article as PNG screenshot', async () => {
-      const buffer = await page.screenshot({ type: 'png' });
-      expect(buffer).toBeTruthy();
-      expect(buffer.length).toBeGreaterThan(10000);
-      // PNG signature: 137 80 78 71
-      expect(buffer[0]).toBe(137);
-      expect(buffer[1]).toBe(80);
-      expect(buffer[2]).toBe(78);
-      expect(buffer[3]).toBe(71);
-    });
-
-    it('captures article as JPEG screenshot', async () => {
-      const buffer = await page.screenshot({ type: 'jpeg', quality: 80 });
-      expect(buffer).toBeTruthy();
-      expect(buffer.length).toBeGreaterThan(5000);
-      // JPEG signature: 0xFF 0xD8 0xFF
-      expect(buffer[0]).toBe(0xff);
-      expect(buffer[1]).toBe(0xd8);
-      expect(buffer[2]).toBe(0xff);
-    });
-
-    it('captures full page screenshot', async () => {
-      await scrollToLoadContent(page);
-
-      const viewportBuffer = await page.screenshot({ type: 'png' });
-      const fullPageBuffer = await page.screenshot({
-        type: 'png',
-        fullPage: true,
-      });
-
-      // Full page should be significantly larger than viewport-only
-      expect(fullPageBuffer.length).toBeGreaterThan(viewportBuffer.length);
-    });
-
-    it('captures with custom viewport width (1920px)', async () => {
-      await page.setViewport({ width: 1920, height: 1080 });
-      const buffer = await page.screenshot({ type: 'png' });
-      expect(buffer).toBeTruthy();
-      expect(buffer.length).toBeGreaterThan(10000);
-      // Restore original viewport
-      await page.setViewport({ width: 1280, height: 800 });
-    });
-  });
-
-  describeIfLive('Screenshots with Playwright', () => {
-    let browser;
-    let page;
-
-    beforeAll(async () => {
-      browser = await createBrowser('playwright');
-      page = await browser.newPage();
-      await page.setViewport({ width: 1280, height: 800 });
-      await navigateWithRetry(page, primaryArticle.url);
-      await dismissPopups(page);
-    });
-
-    afterAll(async () => {
-      if (browser) {
-        await browser.close();
-      }
-    });
-
-    it('captures article as PNG screenshot', async () => {
-      const buffer = await page.screenshot({ type: 'png' });
-      expect(buffer).toBeTruthy();
-      expect(buffer.length).toBeGreaterThan(10000);
-      expect(buffer[0]).toBe(137);
-    });
-
-    it('captures article as JPEG screenshot', async () => {
-      const buffer = await page.screenshot({ type: 'jpeg', quality: 80 });
-      expect(buffer).toBeTruthy();
-      expect(buffer.length).toBeGreaterThan(5000);
-      expect(buffer[0]).toBe(0xff);
-      expect(buffer[1]).toBe(0xd8);
-    });
-  });
-
-  describeIfLive('Theme Support (Playwright)', () => {
-    it.each(['light', 'dark'])(
-      'captures %s theme screenshot',
-      async (theme) => {
-        const browser = await createBrowser('playwright', {
-          colorScheme: theme,
-        });
-        try {
-          const page = await browser.newPage();
-          await page.setViewport({ width: 1280, height: 800 });
-          await navigateWithRetry(page, primaryArticle.url);
-          await scrollToLoadContent(page);
-          await dismissPopups(page);
-
-          const buffer = await page.screenshot({
-            type: 'png',
-            fullPage: true,
-          });
-          expect(buffer).toBeTruthy();
-          expect(buffer.length).toBeGreaterThan(50000);
-        } finally {
-          await browser.close();
-        }
-      }
-    );
-  });
-
-  describeIfLive('All Three Articles (Puppeteer)', () => {
-    let browser;
-
-    beforeAll(async () => {
-      browser = await createBrowser('puppeteer');
-    });
-
-    afterAll(async () => {
-      if (browser) {
-        await browser.close();
-      }
-    });
-
-    it.each(getAllArticles().map((a) => [a.version, a.url]))(
-      'captures article %s as PNG screenshot',
-      async (version, url) => {
-        const page = await browser.newPage();
-        try {
-          await page.setViewport({ width: 1280, height: 800 });
-          await navigateWithRetry(page, url);
-          await dismissPopups(page);
-
-          const buffer = await page.screenshot({ type: 'png' });
-          expect(buffer.length).toBeGreaterThan(10000);
-          expect(buffer[0]).toBe(137);
-        } finally {
-          await page.close();
-        }
-      }
-    );
-  });
-
-  describeIfLive('Popup Dismissal (Playwright)', () => {
-    it('dismisses popups before capture', async () => {
-      const browser = await createBrowser('playwright');
-      try {
-        const page = await browser.newPage();
-        await page.setViewport({ width: 1280, height: 800 });
-        await navigateWithRetry(page, primaryArticle.url);
-
-        await dismissPopups(page);
-
-        const buffer = await page.screenshot({ type: 'png' });
-        expect(buffer.length).toBeGreaterThan(10000);
-      } finally {
-        await browser.close();
-      }
-    });
-  });
-
-  describeIfLive('Archive Endpoint (live)', () => {
-    it('creates markdown archive of article', async () => {
-      const res = await retry(
-        async () => {
-          const r = await request(app)
-            .get('/archive')
-            .query({
-              url: primaryArticle.url,
-              localImages: 'false',
-              documentFormat: 'markdown',
-            })
-            .buffer(true)
-            .parse((res, callback) => {
-              const chunks = [];
-              res.on('data', (chunk) => chunks.push(chunk));
-              res.on('end', () => callback(null, Buffer.concat(chunks)));
-            })
-            .timeout({ response: 30000 });
-          if (r.status !== 200) {
-            throw new Error(`Archive returned status ${r.status}`);
-          }
-          return r;
-        },
-        { retries: 2, baseDelay: 3000 }
-      );
-
-      expect(res.body).toBeInstanceOf(Buffer);
-      expect(res.body.length).toBeGreaterThan(50);
-      // ZIP signature: PK (0x50 0x4B)
-      expect(res.body[0]).toBe(0x50);
-      expect(res.body[1]).toBe(0x4b);
-    });
-
-    it('creates HTML archive of article', async () => {
-      const res = await retry(
-        async () => {
-          const r = await request(app)
-            .get('/archive')
-            .query({
-              url: primaryArticle.url,
-              localImages: 'false',
-              documentFormat: 'html',
-            })
-            .buffer(true)
-            .parse((res, callback) => {
-              const chunks = [];
-              res.on('data', (chunk) => chunks.push(chunk));
-              res.on('end', () => callback(null, Buffer.concat(chunks)));
-            })
-            .timeout({ response: 30000 });
-          if (r.status !== 200) {
-            throw new Error(`Archive returned status ${r.status}`);
-          }
-          return r;
-        },
-        { retries: 2, baseDelay: 3000 }
-      );
-
-      expect(res.body).toBeInstanceOf(Buffer);
-      expect(res.body.length).toBeGreaterThan(50);
-      // ZIP signature
-      expect(res.body[0]).toBe(0x50);
-      expect(res.body[1]).toBe(0x4b);
-    });
-  });
-});

package/tests/integration/search-endpoint.test.js

@@ -1,88 +0,0 @@
-/**
- * Integration tests for the GET /search endpoint (issue #130).
- *
- * Provider HTTP calls are mocked with nock so the test is fast and offline.
- */
-
-import request from 'supertest';
-import nock from '../helpers/nock.js';
-import { app } from '../../src/index.js';
-
-const WIKI_JSON = {
-  pages: [
-    {
-      id: 1,
-      key: 'Formal_methods',
-      title: 'Formal methods',
-      excerpt: 'the study of <b>formal</b> methods',
-      description: 'rigorous techniques',
-    },
-  ],
-};
-
-afterEach(() => {
-  nock.cleanAll();
-});
-
-describe('GET /search', () => {
-  it('returns structured JSON for the wikipedia provider', async () => {
-    nock('https://en.wikipedia.org')
-      .get('/w/rest.php/v1/search/page')
-      .query(true)
-      .reply(200, WIKI_JSON);
-
-    const res = await request(app)
-      .get('/search')
-      .query({ q: 'formal-ai', provider: 'wikipedia', limit: 5 })
-      .expect(200)
-      .expect('Content-Type', /json/);
-
-    expect(res.body.query).toBe('formal-ai');
-    expect(res.body.provider).toBe('wikipedia');
-    expect(res.body.results).toHaveLength(1);
-    expect(res.body.results[0].url).toBe(
-      'https://en.wikipedia.org/wiki/Formal_methods'
-    );
-    expect(res.body.diagnostics.status).toBe(200);
-  });
-
-  it('returns markdown when format=markdown', async () => {
-    nock('https://en.wikipedia.org')
-      .get('/w/rest.php/v1/search/page')
-      .query(true)
-      .reply(200, WIKI_JSON);
-
-    const res = await request(app)
-      .get('/search')
-      .query({ q: 'formal-ai', provider: 'wikipedia', format: 'markdown' })
-      .expect(200)
-      .expect('Content-Type', /markdown/);
-
-    expect(res.text).toContain('# Search results for "formal-ai"');
-    expect(res.text).toContain('[Formal methods]');
-  });
-
-  it('defaults to the wikipedia provider', async () => {
-    nock('https://en.wikipedia.org')
-      .get('/w/rest.php/v1/search/page')
-      .query(true)
-      .reply(200, WIKI_JSON);
-
-    const res = await request(app)
-      .get('/search')
-      .query({ q: 'formal-ai' })
-      .expect(200);
-    expect(res.body.provider).toBe('wikipedia');
-  });
-
-  it('rejects a missing query', async () => {
-    await request(app).get('/search').expect(400);
-  });
-
-  it('rejects an unknown provider', async () => {
-    await request(app)
-      .get('/search')
-      .query({ q: 'x', provider: 'yahoo' })
-      .expect(400);
-  });
-});

package/tests/integration/stackoverflow-download.test.js

@@ -1,173 +0,0 @@
-/**
- * Integration tests for downloading the StackOverflow page from issue #11.
- *
- * These tests hit the live StackOverflow page:
- * https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-local-commits-in-git
- *
- * Set STACKOVERFLOW_INTEGRATION=true to run them. The suite is skipped by
- * default so offline/local test runs stay deterministic.
- */
-
-import { jest } from '@jest/globals';
-import request from 'supertest';
-import unzipper from 'unzipper';
-import { app } from '../../src/index.js';
-import { retry } from '../../src/retry.js';
-
-const STACKOVERFLOW_URL =
-  'https://stackoverflow.com/questions/927358/how-do-i-undo-the-most-recent-local-commits-in-git';
-const STACKOVERFLOW_TITLE =
-  'How do I undo the most recent local commits in Git';
-
-const PNG_SIGNATURE = Buffer.from([
-  0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
-]);
-const ZIP_SIGNATURE = Buffer.from([0x50, 0x4b]);
-const PDF_SIGNATURE = Buffer.from([0x25, 0x50, 0x44, 0x46]);
-
-const SKIP_LIVE =
-  !process.env.STACKOVERFLOW_INTEGRATION ||
-  process.env.STACKOVERFLOW_INTEGRATION === 'false';
-const describeIfLive = SKIP_LIVE ? describe.skip : describe;
-
-jest.setTimeout(180000);
-
-function parseBinary(res, callback) {
-  const chunks = [];
-  res.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
-  res.on('end', () => callback(null, Buffer.concat(chunks)));
-}
-
-async function getText(path, query = {}) {
-  return await retry(
-    () =>
-      request(app)
-        .get(path)
-        .query({ url: STACKOVERFLOW_URL, ...query })
-        .expect(200),
-    {
-      retries: 2,
-      baseDelay: 2000,
-      onRetry: (err, attempt, delay) => {
-        console.log(
-          `Retry ${attempt} for ${path} after ${delay}ms: ${err.message}`
-        );
-      },
-    }
-  );
-}
-
-async function getBinary(path, query = {}) {
-  return await retry(
-    () =>
-      request(app)
-        .get(path)
-        .query({ url: STACKOVERFLOW_URL, ...query })
-        .buffer(true)
-        .parse(parseBinary)
-        .expect(200),
-    {
-      retries: 2,
-      baseDelay: 2000,
-      onRetry: (err, attempt, delay) => {
-        console.log(
-          `Retry ${attempt} for ${path} after ${delay}ms: ${err.message}`
-        );
-      },
-    }
-  );
-}
-
-function expectStackOverflowContent(text) {
-  expect(text).toContain(STACKOVERFLOW_TITLE);
-  expect(text).toMatch(/Stack Overflow/i);
-  expect(text).toMatch(/\bgit\b/i);
-}
-
-function expectBufferWithSignature(buffer, signature) {
-  expect(Buffer.isBuffer(buffer)).toBe(true);
-  expect(buffer.length).toBeGreaterThan(1000);
-  expect(buffer.subarray(0, signature.length)).toEqual(signature);
-}
-
-describeIfLive('StackOverflow page downloads (JavaScript)', () => {
-  it('downloads the page as Markdown and plain text', async () => {
-    const markdown = await getText('/markdown');
-    expect(markdown.headers['content-type']).toMatch(/text\/markdown/);
-    expectStackOverflowContent(markdown.text);
-    expect(markdown.text).not.toMatch(/<html/i);
-
-    const text = await getText('/txt');
-    expect(text.headers['content-type']).toMatch(/text\/plain/);
-    expect(text.headers['content-disposition']).toContain('stackoverflow-com');
-    expectStackOverflowContent(text.text);
-  });
-
-  it('downloads Markdown and HTML archives with the expected files', async () => {
-    const markdownArchive = await getBinary('/archive', {
-      documentFormat: 'markdown',
-      localImages: 'false',
-    });
-    expect(markdownArchive.headers['content-type']).toMatch(/application\/zip/);
-    expectBufferWithSignature(markdownArchive.body, ZIP_SIGNATURE);
-
-    const markdownZip = await unzipper.Open.buffer(markdownArchive.body);
-    const markdownEntries = new Map(
-      markdownZip.files.map((file) => [file.path, file])
-    );
-    expect(markdownEntries.has('document.md')).toBe(true);
-    expect(markdownEntries.has('document.html')).toBe(true);
-    expectStackOverflowContent(
-      (await markdownEntries.get('document.md').buffer()).toString('utf8')
-    );
-
-    const htmlArchive = await getBinary('/archive', {
-      documentFormat: 'html',
-      localImages: 'false',
-    });
-    expect(htmlArchive.headers['content-type']).toMatch(/application\/zip/);
-    expectBufferWithSignature(htmlArchive.body, ZIP_SIGNATURE);
-
-    const htmlZip = await unzipper.Open.buffer(htmlArchive.body);
-    const htmlEntries = new Map(htmlZip.files.map((file) => [file.path, file]));
-    expect(htmlEntries.has('document.html')).toBe(true);
-    expectStackOverflowContent(
-      (await htmlEntries.get('document.html').buffer()).toString('utf8')
-    );
-  });
-
-  it('downloads the page as DOCX and PDF', async () => {
-    const docx = await getBinary('/docx');
-    expect(docx.headers['content-type']).toMatch(
-      /application\/vnd\.openxmlformats-officedocument\.wordprocessingml\.document/
-    );
-    expectBufferWithSignature(docx.body, ZIP_SIGNATURE);
-
-    const pdf = await getBinary('/pdf');
-    expect(pdf.headers['content-type']).toMatch(/application\/pdf/);
-    expectBufferWithSignature(pdf.body, PDF_SIGNATURE);
-  }, 180000);
-});
-
-describe.each(['puppeteer', 'playwright'])(
-  'StackOverflow browser downloads (%s engine)',
-  (engine) => {
-    describeIfLive(`${engine}`, () => {
-      it('downloads rendered HTML for the page', async () => {
-        const html = await getText('/html', { engine });
-        expect(html.headers['content-type']).toMatch(/text\/html/);
-        expectStackOverflowContent(html.text);
-        expect(html.text).toMatch(/<html/i);
-      }, 120000);
-
-      it('downloads a PNG screenshot for the page', async () => {
-        const image = await getBinary('/image', {
-          engine,
-          dismissPopups: 'true',
-        });
-        expect(image.headers['content-type']).toMatch(/image\/png/);
-        expectBufferWithSignature(image.body, PNG_SIGNATURE);
-      }, 120000);
-    });
-  }
-);