@link-assistant/web-capture 1.12.0 → 2.0.1

package/tests/integration/wikipedia-download.test.js

@@ -1,142 +0,0 @@
-/**
- * Integration tests for downloading the Wikipedia page (issue #8).
- *
- * Verifies that web-capture can actually download the Wikipedia article about
- * Wikipedia itself (https://en.wikipedia.org/wiki/Wikipedia) as both Markdown
- * and an image (PNG screenshot) using every supported browser engine
- * (Puppeteer and Playwright).
- *
- * These tests hit the live Wikipedia servers. Set WIKIPEDIA_INTEGRATION=true to
- * run them. In CI they are enabled via the workflow environment variable. When
- * the variable is unset the suite is skipped so default/offline runs stay
- * deterministic.
- *
- * Navigation uses retry with exponential backoff to absorb transient network
- * failures, mirroring the Habr integration suite.
- */
-
-import { jest } from '@jest/globals';
-import { createBrowser } from '../../src/browser.js';
-import { convertHtmlToMarkdown } from '../../src/lib.js';
-import { retry } from '../../src/retry.js';
-
-const WIKIPEDIA_URL = 'https://en.wikipedia.org/wiki/Wikipedia';
-
-// PNG magic number (89 50 4E 47 0D 0A 1A 0A).
-const PNG_SIGNATURE = Buffer.from([
-  0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
-]);
-
-const SKIP_LIVE =
-  !process.env.WIKIPEDIA_INTEGRATION ||
-  process.env.WIKIPEDIA_INTEGRATION === 'false';
-const describeIfLive = SKIP_LIVE ? describe.skip : describe;
-
-// These tests hit live servers and may take a while.
-jest.setTimeout(120000);
-
-// Navigate with retry + exponential backoff, then give the page a moment to
-// settle so late-loading content and images are present before we capture.
-async function navigateWithRetry(page, url) {
-  await retry(
-    async () => {
-      await page.goto(url, {
-        waitUntil: 'networkidle0',
-        timeout: 60000,
-      });
-    },
-    {
-      retries: 3,
-      baseDelay: 2000,
-      onRetry: (err, attempt, delay) => {
-        console.log(
-          `Navigation retry ${attempt} for ${url} after ${delay}ms: ${err.message}`
-        );
-      },
-    }
-  );
-  await new Promise((resolve) => setTimeout(resolve, 5000));
-}
-
-// Run the full matrix (Markdown + image) once per supported engine.
-describe.each(['puppeteer', 'playwright'])(
-  'Wikipedia Page Download (%s engine)',
-  (engine) => {
-    describeIfLive(`${engine}`, () => {
-      let browser;
-      let page;
-
-      beforeAll(async () => {
-        browser = await createBrowser(engine);
-        page = await browser.newPage();
-        await page.setViewport({ width: 1280, height: 800 });
-        await navigateWithRetry(page, WIKIPEDIA_URL);
-      });
-
-      afterAll(async () => {
-        if (browser) {
-          await browser.close();
-        }
-      });
-
-      it('downloads the Wikipedia page as Markdown', async () => {
-        const html = await page.content();
-        expect(html).toMatch(/<html/i);
-        expect(html.length).toBeGreaterThan(1000);
-
-        const markdown = convertHtmlToMarkdown(html, WIKIPEDIA_URL);
-
-        // Markdown should carry the page's real content...
-        expect(markdown).toContain('Wikipedia');
-        expect(markdown.length).toBeGreaterThan(500);
-        // ...and should contain markdown structure (headings and/or links)...
-        expect(markdown).toMatch(/(^#{1,6}\s)|(\[.*?\]\(.*?\))/m);
-        // ...but no longer be raw HTML document scaffolding.
-        expect(markdown).not.toMatch(/<html/i);
-        expect(markdown).not.toMatch(/<head[\s>]/i);
-      });
-
-      it('downloads the Wikipedia page as an image (PNG screenshot)', async () => {
-        const screenshot = await page.screenshot({ type: 'png' });
-
-        expect(screenshot).toBeInstanceOf(Buffer);
-        expect(screenshot.length).toBeGreaterThan(1000);
-        // Verify the PNG signature so we know it is a real image.
-        expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
-      });
-    });
-  }
-);
-
-describeIfLive('Wikipedia Page Download (engine parity)', () => {
-  it('both engines download Markdown and an image for the Wikipedia page', async () => {
-    const engines = ['puppeteer', 'playwright'];
-    const results = [];
-
-    for (const engine of engines) {
-      const browser = await createBrowser(engine);
-      try {
-        const page = await browser.newPage();
-        await page.setViewport({ width: 1280, height: 800 });
-        await navigateWithRetry(page, WIKIPEDIA_URL);
-
-        const html = await page.content();
-        const markdown = convertHtmlToMarkdown(html, WIKIPEDIA_URL);
-        const screenshot = await page.screenshot({ type: 'png' });
-
-        results.push({ engine, markdown, screenshot });
-      } finally {
-        await browser.close();
-      }
-    }
-
-    for (const { markdown, screenshot } of results) {
-      // Markdown captured from both engines.
-      expect(markdown).toContain('Wikipedia');
-      expect(markdown.length).toBeGreaterThan(500);
-      // Image captured from both engines.
-      expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
-      expect(screenshot.length).toBeGreaterThan(1000);
-    }
-  }, 180000);
-});

package/tests/jest.setup.mjs

@@ -1,7 +0,0 @@
-import { jest } from '@jest/globals';
-
-// Increase timeout for tests that might take longer
-jest.setTimeout(10000);
-
-// Suppress console.error during tests
-global.console.error = jest.fn();

package/tests/mock/index.test.js

@@ -1,225 +0,0 @@
-import request from 'supertest';
-import nock from '../helpers/nock.js';
-import { jest } from '@jest/globals';
-
-let app;
-
-beforeAll(async () => {
-  app = (await import('../../src/index.js')).app;
-});
-
-describe('Web Capture Microservice', () => {
-  beforeEach(() => {
-    nock.cleanAll();
-    jest.clearAllMocks();
-  });
-
-  describe('GET /html', () => {
-    const testUrl = 'https://example.com';
-    const testHtml = '<html><body><h1>Test Page</h1></body></html>';
-
-    it('should return HTML content when URL is provided', async () => {
-      nock(testUrl).get('/').reply(200, testHtml);
-
-      const response = await request(app).get('/html').query({ url: testUrl });
-
-      expect(response.status).toBe(200);
-      expect(response.type).toBe('text/html');
-      expect(response.text).toBe(testHtml);
-    });
-
-    it('should return 400 when URL is missing', async () => {
-      const response = await request(app).get('/html');
-
-      expect(response.status).toBe(400);
-      expect(response.text).toBe('Missing `url` parameter');
-    });
-
-    it('should return 500 when fetch fails', async () => {
-      nock(testUrl).get('/').replyWithError('Network error');
-
-      const response = await request(app).get('/html').query({ url: testUrl });
-
-      expect(response.status).toBe(500);
-      expect(response.text).toBe('Error fetching HTML');
-    });
-  });
-
-  describe('GET /markdown', () => {
-    const testUrl = 'https://example.com';
-    const testHtml =
-      '<html><body><h1>Test Page</h1><p>Some text</p></body></html>';
-    const expectedMarkdown = '# Test Page\n\nSome text';
-
-    it('should convert HTML to Markdown when URL is provided', async () => {
-      nock(testUrl).get('/').reply(200, testHtml);
-
-      const response = await request(app)
-        .get('/markdown')
-        .query({ url: testUrl });
-
-      expect(response.status).toBe(200);
-      expect(response.type).toBe('text/markdown');
-      expect(response.text).toBe(expectedMarkdown);
-    });
-
-    it('should remove CSS from the markdown output', async () => {
-      const htmlWithCss = `
-        <html>
-          <head>
-            <style>
-              body { background-color: #f0f0f2; }
-              div { width: 600px; }
-            </style>
-          </head>
-          <body>
-            <h1>Test Page</h1>
-            <p>Some text</p>
-          </body>
-        </html>
-      `;
-
-      nock(testUrl).get('/').reply(200, htmlWithCss);
-
-      const response = await request(app)
-        .get('/markdown')
-        .query({ url: testUrl });
-
-      expect(response.status).toBe(200);
-      expect(response.type).toBe('text/markdown');
-      expect(response.text).not.toContain('background-color');
-      expect(response.text).not.toContain('width: 600px');
-      expect(response.text).toContain('Test Page');
-      expect(response.text).toContain('Some text');
-    });
-
-    it('should return 400 when URL is missing', async () => {
-      const response = await request(app).get('/markdown');
-
-      expect(response.status).toBe(400);
-      expect(response.text).toBe('Missing `url` parameter');
-    });
-
-    it('should return 500 when fetch fails', async () => {
-      nock(testUrl).get('/').replyWithError('Network error');
-
-      const response = await request(app)
-        .get('/markdown')
-        .query({ url: testUrl });
-
-      expect(response.status).toBe(500);
-      expect(response.text).toBe('Error converting to Markdown');
-    });
-  });
-
-  describe('GET /image', () => {
-    const testUrl = 'https://example.com';
-    // Buffer for PNG image mock
-    // eslint-disable-next-line no-unused-vars
-    const mockBuffer = Buffer.from([
-      0x89,
-      0x50,
-      0x4e,
-      0x47,
-      0x0d,
-      0x0a,
-      0x1a,
-      0x0a, // PNG signature
-      0x00,
-      0x00,
-      0x00,
-      0x0d,
-      0x49,
-      0x48,
-      0x44,
-      0x52, // IHDR chunk
-      // ... (rest can be arbitrary for test)
-      0x00,
-      0x00,
-      0x00,
-      0x00,
-      0x49,
-      0x45,
-      0x4e,
-      0x44,
-      0xae,
-      0x42,
-      0x60,
-      0x82, // IEND chunk
-    ]);
-
-    it('should return PNG image when URL is provided', async () => {
-      const response = await request(app).get('/image').query({ url: testUrl });
-
-      expect(response.status).toBe(200);
-      expect(response.type).toBe('image/png');
-      const pngSignature = Buffer.from([
-        0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
-      ]);
-      expect(response.body.slice(0, 8)).toEqual(pngSignature);
-    }, 60000);
-
-    it('should return 400 when URL is missing', async () => {
-      const response = await request(app).get('/image');
-
-      expect(response.status).toBe(400);
-      expect(response.text).toBe('Missing `url` parameter');
-    });
-  });
-
-  describe('GET /stream', () => {
-    it('should stream content from the given URL', async () => {
-      const testUrl = 'https://example.com';
-      const testHtml = '<html><body><h1>Streamed Page</h1></body></html>';
-      nock(testUrl).get('/').reply(200, testHtml, {
-        'content-type': 'text/html',
-      });
-      const response = await request(app).get(`/stream?url=${testUrl}`);
-      expect(response.status).toBe(200);
-      expect(response.text).toMatch(/<html/i);
-    });
-
-    it('should return 400 when URL is missing', async () => {
-      const response = await request(app).get('/stream');
-      expect(response.status).toBe(400);
-      expect(response.text).toBe('Missing `url` parameter');
-    });
-
-    it('should return 500 when fetch fails', async () => {
-      const testUrl = 'https://nonexistent.example.com';
-      const response = await request(app).get(`/stream?url=${testUrl}`);
-      expect(response.status).toBe(500);
-      expect(response.text).toBe('Error proxying content');
-    });
-  });
-
-  describe('GET /fetch', () => {
-    const testUrl = 'https://example.com';
-    const testHtml = '<html><body><h1>Test Page</h1></body></html>';
-
-    it('should return content when URL is provided', async () => {
-      nock(testUrl).get('/').reply(200, testHtml);
-
-      const response = await request(app).get('/fetch').query({ url: testUrl });
-
-      expect(response.status).toBe(200);
-      expect(response.text).toBe(testHtml);
-    });
-
-    it('should return 400 when URL is missing', async () => {
-      const response = await request(app).get('/fetch');
-
-      expect(response.status).toBe(400);
-      expect(response.text).toBe('Missing `url` parameter');
-    });
-
-    it('should return 500 when fetch fails', async () => {
-      nock(testUrl).get('/').replyWithError('Network error');
-
-      const response = await request(app).get('/fetch').query({ url: testUrl });
-
-      expect(response.status).toBe(500);
-      expect(response.text).toBe('Error fetching content');
-    });
-  });
-});

package/tests/unit/animation.test.js

@@ -1,33 +0,0 @@
-import { compareFrames } from '../../src/animation.js';
-
-describe('animation module', () => {
-  describe('compareFrames', () => {
-    it('returns 1.0 for identical frames', () => {
-      const frame = Buffer.from([1, 2, 3, 4, 5]);
-      expect(compareFrames(frame, frame)).toBe(1.0);
-    });
-
-    it('returns 0 for completely different frames', () => {
-      const frame1 = Buffer.from([0, 0, 0, 0]);
-      const frame2 = Buffer.from([255, 255, 255, 255]);
-      expect(compareFrames(frame1, frame2)).toBe(0);
-    });
-
-    it('returns 0 for different sized frames', () => {
-      const frame1 = Buffer.from([1, 2, 3]);
-      const frame2 = Buffer.from([1, 2]);
-      expect(compareFrames(frame1, frame2)).toBe(0);
-    });
-
-    it('returns 0 for null frames', () => {
-      expect(compareFrames(null, Buffer.from([1]))).toBe(0);
-      expect(compareFrames(Buffer.from([1]), null)).toBe(0);
-    });
-
-    it('calculates partial similarity', () => {
-      const frame1 = Buffer.from([1, 2, 3, 4]);
-      const frame2 = Buffer.from([1, 2, 0, 0]);
-      expect(compareFrames(frame1, frame2)).toBe(0.5);
-    });
-  });
-});

package/tests/unit/batch.test.js

@@ -1,114 +0,0 @@
-import {
-  getArticle,
-  getAllVersions,
-  getAllArticles,
-  createConfigFromUrls,
-  validateConfig,
-} from '../../src/batch.js';
-
-describe('batch module', () => {
-  const sampleConfig = {
-    articles: {
-      '0.0.0': {
-        url: 'https://example.com/article-1',
-        title: 'First Article',
-        language: 'en',
-      },
-      '0.0.1': {
-        url: 'https://example.com/article-2',
-        title: 'Second Article',
-        language: 'ru',
-      },
-    },
-    defaults: {
-      archivePath: 'archive',
-      markdownFile: 'document.md',
-      imagesDir: 'images',
-      hasLocalImages: true,
-    },
-  };
-
-  describe('getArticle', () => {
-    it('returns article config by version', () => {
-      const article = getArticle(sampleConfig, '0.0.0');
-      expect(article.url).toBe('https://example.com/article-1');
-      expect(article.title).toBe('First Article');
-    });
-
-    it('merges defaults into article config', () => {
-      const article = getArticle(sampleConfig, '0.0.0');
-      expect(article.markdownFile).toBe('document.md');
-      expect(article.hasLocalImages).toBe(true);
-    });
-
-    it('throws for unknown version', () => {
-      expect(() => getArticle(sampleConfig, '9.9.9')).toThrow(
-        /Unknown article version/
-      );
-    });
-  });
-
-  describe('getAllVersions', () => {
-    it('returns all version keys', () => {
-      const versions = getAllVersions(sampleConfig);
-      expect(versions).toEqual(['0.0.0', '0.0.1']);
-    });
-  });
-
-  describe('getAllArticles', () => {
-    it('returns all articles with defaults merged', () => {
-      const articles = getAllArticles(sampleConfig);
-      expect(articles).toHaveLength(2);
-      expect(articles[0].markdownFile).toBe('document.md');
-      expect(articles[1].language).toBe('ru');
-    });
-  });
-
-  describe('createConfigFromUrls', () => {
-    it('creates config from URL list', () => {
-      const config = createConfigFromUrls([
-        'https://example.com/a',
-        'https://example.com/b',
-      ]);
-      expect(Object.keys(config.articles)).toHaveLength(2);
-      expect(config.articles['1'].url).toBe('https://example.com/a');
-      expect(config.articles['2'].url).toBe('https://example.com/b');
-    });
-
-    it('sets default archive paths', () => {
-      const config = createConfigFromUrls(['https://habr.com/article/123']);
-      expect(config.articles['1'].archivePath).toContain('habr-com');
-      expect(config.articles['1'].markdownFile).toBe('document.md');
-    });
-  });
-
-  describe('validateConfig', () => {
-    it('validates correct config', () => {
-      const result = validateConfig(sampleConfig);
-      expect(result.valid).toBe(true);
-      expect(result.errors).toHaveLength(0);
-    });
-
-    it('rejects config without articles', () => {
-      const result = validateConfig({});
-      expect(result.valid).toBe(false);
-      expect(result.errors[0]).toContain('articles');
-    });
-
-    it('rejects article without url', () => {
-      const result = validateConfig({
-        articles: { 1: { title: 'No URL' } },
-      });
-      expect(result.valid).toBe(false);
-      expect(result.errors[0]).toContain('missing required "url"');
-    });
-
-    it('rejects article with invalid url', () => {
-      const result = validateConfig({
-        articles: { 1: { url: 'not-a-url' } },
-      });
-      expect(result.valid).toBe(false);
-      expect(result.errors[0]).toContain('invalid URL');
-    });
-  });
-});

package/tests/unit/bold-coalesce-and-leakage.test.js

@@ -1,54 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import { fileURLToPath } from 'url';
-import {
-  normalizeGoogleDocsExportMarkdown,
-  preprocessGoogleDocsExportHtml,
-} from '../../src/gdocs.js';
-import { convertHtmlToMarkdown } from '../../src/lib.js';
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-const HTML = fs.readFileSync(
-  path.join(__dirname, '..', 'fixtures', 'bold-coalesce-and-leakage.html'),
-  'utf8'
-);
-
-describe('bold-run coalescing and block boundary handling (issue #120)', () => {
-  let md;
-  beforeAll(() => {
-    const preprocessed = preprocessGoogleDocsExportHtml(HTML);
-    md = normalizeGoogleDocsExportMarkdown(
-      convertHtmlToMarkdown(preprocessed.html)
-    );
-  });
-
-  it('A: coalesces adjacent bold spans into a single bold run', () => {
-    expect(md).toMatch(/\*\*13\.1 First subsection\*\*/);
-    expect(md).not.toMatch(/\*\*13\.1\*\*\s+\*\*First subsection\*\*/);
-  });
-
-  it('B: never emits an empty "****" pair between adjacent bold runs', () => {
-    expect(md).not.toMatch(/\*{4}/);
-  });
-
-  it('C: closes bold at block-level boundaries (<br>, <img>)', () => {
-    // No single bold run may contain an image. CommonMark bold cannot span
-    // a blank line (`\n\n`), so the inner content is restricted accordingly.
-    const strongRe = /\*\*((?:(?!\*\*|\n\n)[\s\S])+?)\*\*/g;
-    const imageRe = /!\[[^\]]*\]\([^)]+\)/;
-    for (const match of md.matchAll(strongRe)) {
-      expect(imageRe.test(match[1])).toBe(false);
-    }
-    expect(md).toMatch(/\*\*Caption A:\*\*/);
-    expect(md).toMatch(/\*\*Caption B:\*\*/);
-  });
-
-  it('D: every "**" opener has a matching closer (balanced asterisks)', () => {
-    const stripped = md
-      .replace(/`[^`]*`/g, '')
-      .replace(/!\[[^\]]*\]\([^)]*\)/g, '');
-    const count = (stripped.match(/\*\*/g) || []).length;
-    expect(count % 2).toBe(0);
-  });
-});

package/tests/unit/browser-launch-options.test.js

@@ -1,42 +0,0 @@
-import { jest } from '@jest/globals';
-import path from 'node:path';
-import { withTemporaryDirectory } from '../helpers/temporary-directory.js';
-
-const initialPage = { close: jest.fn(), bringToFront: jest.fn() };
-const browser = { pages: jest.fn(() => [initialPage]), close: jest.fn() };
-const launch = jest.fn(async () => browser);
-const launchPersistentContext = jest.fn(async () => browser);
-jest.unstable_mockModule('puppeteer', () => ({ default: { launch } }));
-jest.unstable_mockModule('playwright', () => ({
-  chromium: { launchPersistentContext },
-}));
-const { createBrowser } = await import('../../src/browser.js');
-
-beforeEach(() => {
-  jest.clearAllMocks();
-});
-
-test.each(['puppeteer', 'playwright'])(
-  '%s forwards packaged executablePath and channel to the engine',
-  (engine) =>
-    withTemporaryDirectory('launch-options-', async (directory) => {
-      const options = {
-        executablePath: path.join(directory, 'packaged-chromium'),
-        channel: 'chrome',
-        userDataDir: directory,
-      };
-      const handle = await createBrowser(engine, options);
-      try {
-        const observed =
-          engine === 'puppeteer'
-            ? launch.mock.calls[0][0]
-            : launchPersistentContext.mock.calls[0][1];
-        expect(observed).toMatchObject({
-          executablePath: options.executablePath,
-          channel: options.channel,
-        });
-      } finally {
-        await handle.close();
-      }
-    })
-);