@link-assistant/web-capture 1.1.2

package/tests/integration/browser-engines.test.js

@@ -0,0 +1,128 @@
+import { createBrowser } from '../../src/browser.js';
+
+describe('Browser Engine Integration Tests', () => {
+  describe('Puppeteer Engine', () => {
+    let browser;
+
+    beforeEach(async () => {
+      browser = await createBrowser('puppeteer');
+    });
+
+    afterEach(async () => {
+      if (browser) {
+        await browser.close();
+      }
+    });
+
+    it('can navigate to a page and get content', async () => {
+      const page = await browser.newPage();
+      await page.goto('https://example.com', {
+        waitUntil: 'networkidle0',
+        timeout: 30000,
+      });
+      const content = await page.content();
+      expect(content).toContain('Example Domain');
+    }, 60000);
+
+    it('can take a screenshot', async () => {
+      const page = await browser.newPage();
+      await page.setViewport({ width: 1280, height: 800 });
+      await page.goto('https://example.com', {
+        waitUntil: 'networkidle0',
+        timeout: 30000,
+      });
+      const screenshot = await page.screenshot({ type: 'png' });
+      expect(screenshot).toBeInstanceOf(Buffer);
+      expect(screenshot.length).toBeGreaterThan(100);
+    }, 60000);
+
+    it('can set custom headers and user agent', async () => {
+      const page = await browser.newPage();
+      await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-US' });
+      await page.setUserAgent('Test User Agent');
+      await page.goto('https://example.com', {
+        waitUntil: 'networkidle0',
+        timeout: 30000,
+      });
+      const content = await page.content();
+      expect(content).toBeTruthy();
+    }, 60000);
+  });
+
+  describe('Playwright Engine', () => {
+    let browser;
+
+    beforeEach(async () => {
+      browser = await createBrowser('playwright');
+    });
+
+    afterEach(async () => {
+      if (browser) {
+        await browser.close();
+      }
+    });
+
+    it('can navigate to a page and get content', async () => {
+      const page = await browser.newPage();
+      await page.goto('https://example.com', {
+        waitUntil: 'networkidle0',
+        timeout: 30000,
+      });
+      const content = await page.content();
+      expect(content).toContain('Example Domain');
+    }, 60000);
+
+    it('can take a screenshot', async () => {
+      const page = await browser.newPage();
+      await page.setViewport({ width: 1280, height: 800 });
+      await page.goto('https://example.com', {
+        waitUntil: 'networkidle0',
+        timeout: 30000,
+      });
+      const screenshot = await page.screenshot({ type: 'png' });
+      expect(screenshot).toBeInstanceOf(Buffer);
+      expect(screenshot.length).toBeGreaterThan(100);
+    }, 60000);
+
+    it('can set custom headers and user agent', async () => {
+      const page = await browser.newPage();
+      await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-US' });
+      await page.setUserAgent('Test User Agent');
+      await page.goto('https://example.com', {
+        waitUntil: 'networkidle0',
+        timeout: 30000,
+      });
+      const content = await page.content();
+      expect(content).toBeTruthy();
+    }, 60000);
+  });
+
+  describe('Engine Parity', () => {
+    it('both engines produce similar content for the same page', async () => {
+      const puppeteerBrowser = await createBrowser('puppeteer');
+      const playwrightBrowser = await createBrowser('playwright');
+
+      const puppeteerPage = await puppeteerBrowser.newPage();
+      const playwrightPage = await playwrightBrowser.newPage();
+
+      await puppeteerPage.goto('https://example.com', {
+        waitUntil: 'networkidle0',
+        timeout: 30000,
+      });
+      await playwrightPage.goto('https://example.com', {
+        waitUntil: 'networkidle0',
+        timeout: 30000,
+      });
+
+      const puppeteerContent = await puppeteerPage.content();
+      const playwrightContent = await playwrightPage.content();
+
+      // Both should contain the main content
+      expect(puppeteerContent).toContain('Example Domain');
+      expect(playwrightContent).toContain('Example Domain');
+
+      await puppeteerBrowser.close();
+      await playwrightBrowser.close();
+    }, 60000);
+  });
+});

package/tests/jest.setup.mjs

@@ -0,0 +1,7 @@
+import { jest } from '@jest/globals';
+
+// Increase timeout for tests that might take longer
+jest.setTimeout(10000);
+
+// Suppress console.error during tests
+global.console.error = jest.fn();

package/tests/mock/index.test.js

@@ -0,0 +1,223 @@
+import request from 'supertest';
+import nock from 'nock';
+import { jest } from '@jest/globals';
+
+let app;
+
+beforeAll(async () => {
+  app = (await import('../../src/index.js')).app;
+});
+
+describe('Web Capture Microservice', () => {
+  beforeEach(() => {
+    nock.cleanAll();
+    jest.clearAllMocks();
+  });
+
+  describe('GET /html', () => {
+    const testUrl = 'https://example.com';
+    const testHtml = '<html><body><h1>Test Page</h1></body></html>';
+
+    it('should return HTML content when URL is provided', async () => {
+      nock(testUrl).get('/').reply(200, testHtml);
+
+      const response = await request(app).get('/html').query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+      expect(response.type).toBe('text/html');
+      expect(response.text).toBe(testHtml);
+    });
+
+    it('should return 400 when URL is missing', async () => {
+      const response = await request(app).get('/html');
+
+      expect(response.status).toBe(400);
+      expect(response.text).toBe('Missing `url` parameter');
+    });
+
+    it('should return 500 when fetch fails', async () => {
+      nock(testUrl).get('/').replyWithError('Network error');
+
+      const response = await request(app).get('/html').query({ url: testUrl });
+
+      expect(response.status).toBe(500);
+      expect(response.text).toBe('Error fetching HTML');
+    });
+  });
+
+  describe('GET /markdown', () => {
+    const testUrl = 'https://example.com';
+    const testHtml =
+      '<html><body><h1>Test Page</h1><p>Some text</p></body></html>';
+    const expectedMarkdown = '# Test Page\n\nSome text';
+
+    it('should convert HTML to Markdown when URL is provided', async () => {
+      nock(testUrl).get('/').reply(200, testHtml);
+
+      const response = await request(app)
+        .get('/markdown')
+        .query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+      expect(response.type).toBe('text/markdown');
+      expect(response.text).toBe(expectedMarkdown);
+    });
+
+    it('should remove CSS from the markdown output', async () => {
+      const htmlWithCss = `
+        <html>
+          <head>
+            <style>
+              body { background-color: #f0f0f2; }
+              div { width: 600px; }
+            </style>
+          </head>
+          <body>
+            <h1>Test Page</h1>
+            <p>Some text</p>
+          </body>
+        </html>
+      `;
+
+      nock(testUrl).get('/').reply(200, htmlWithCss);
+
+      const response = await request(app)
+        .get('/markdown')
+        .query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+      expect(response.type).toBe('text/markdown');
+      expect(response.text).not.toContain('background-color');
+      expect(response.text).not.toContain('width: 600px');
+      expect(response.text).toContain('Test Page');
+      expect(response.text).toContain('Some text');
+    });
+
+    it('should return 400 when URL is missing', async () => {
+      const response = await request(app).get('/markdown');
+
+      expect(response.status).toBe(400);
+      expect(response.text).toBe('Missing `url` parameter');
+    });
+
+    it('should return 500 when fetch fails', async () => {
+      nock(testUrl).get('/').replyWithError('Network error');
+
+      const response = await request(app)
+        .get('/markdown')
+        .query({ url: testUrl });
+
+      expect(response.status).toBe(500);
+      expect(response.text).toBe('Error converting to Markdown');
+    });
+  });
+
+  describe('GET /image', () => {
+    const testUrl = 'https://example.com';
+    // Buffer for PNG image mock
+    // eslint-disable-next-line no-unused-vars
+    const mockBuffer = Buffer.from([
+      0x89,
+      0x50,
+      0x4e,
+      0x47,
+      0x0d,
+      0x0a,
+      0x1a,
+      0x0a, // PNG signature
+      0x00,
+      0x00,
+      0x00,
+      0x0d,
+      0x49,
+      0x48,
+      0x44,
+      0x52, // IHDR chunk
+      // ... (rest can be arbitrary for test)
+      0x00,
+      0x00,
+      0x00,
+      0x00,
+      0x49,
+      0x45,
+      0x4e,
+      0x44,
+      0xae,
+      0x42,
+      0x60,
+      0x82, // IEND chunk
+    ]);
+
+    it('should return PNG image when URL is provided', async () => {
+      const response = await request(app).get('/image').query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+      expect(response.type).toBe('image/png');
+      const pngSignature = Buffer.from([
+        0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+      ]);
+      expect(response.body.slice(0, 8)).toEqual(pngSignature);
+      // Skipping strict buffer match: Puppeteer output is not deterministic
+      // expect(response.body.equals(mockBuffer)).toBe(true);
+    });
+
+    it('should return 400 when URL is missing', async () => {
+      const response = await request(app).get('/image');
+
+      expect(response.status).toBe(400);
+      expect(response.text).toBe('Missing `url` parameter');
+    });
+  });
+
+  describe('GET /stream', () => {
+    it('should stream content from the given URL', async () => {
+      const testUrl = 'https://example.com';
+      const response = await request(app).get(`/stream?url=${testUrl}`);
+      expect(response.status).toBe(200);
+      expect(response.text).toMatch(/<html/i);
+    });
+
+    it('should return 400 when URL is missing', async () => {
+      const response = await request(app).get('/stream');
+      expect(response.status).toBe(400);
+      expect(response.text).toBe('Missing `url` parameter');
+    });
+
+    it('should return 500 when fetch fails', async () => {
+      const testUrl = 'https://nonexistent.example.com';
+      const response = await request(app).get(`/stream?url=${testUrl}`);
+      expect(response.status).toBe(500);
+      expect(response.text).toBe('Error proxying content');
+    });
+  });
+
+  describe('GET /fetch', () => {
+    const testUrl = 'https://example.com';
+    const testHtml = '<html><body><h1>Test Page</h1></body></html>';
+
+    it('should return content when URL is provided', async () => {
+      nock(testUrl).get('/').reply(200, testHtml);
+
+      const response = await request(app).get('/fetch').query({ url: testUrl });
+
+      expect(response.status).toBe(200);
+      expect(response.text).toBe(testHtml);
+    });
+
+    it('should return 400 when URL is missing', async () => {
+      const response = await request(app).get('/fetch');
+
+      expect(response.status).toBe(400);
+      expect(response.text).toBe('Missing `url` parameter');
+    });
+
+    it('should return 500 when fetch fails', async () => {
+      nock(testUrl).get('/').replyWithError('Network error');
+
+      const response = await request(app).get('/fetch').query({ url: testUrl });
+
+      expect(response.status).toBe(500);
+      expect(response.text).toBe('Error fetching content');
+    });
+  });
+});

package/tests/unit/browser.test.js

@@ -0,0 +1,136 @@
+import { createBrowser, getBrowserEngine } from '../../src/browser.js';
+
+describe('Browser Abstraction Layer', () => {
+  describe('getBrowserEngine', () => {
+    it('returns puppeteer by default', () => {
+      const req = { query: {} };
+      expect(getBrowserEngine(req)).toBe('puppeteer');
+    });
+
+    it('returns playwright when engine=playwright query param is provided', () => {
+      const req = { query: { engine: 'playwright' } };
+      expect(getBrowserEngine(req)).toBe('playwright');
+    });
+
+    it('returns playwright when engine=pw query param is provided', () => {
+      const req = { query: { engine: 'pw' } };
+      expect(getBrowserEngine(req)).toBe('playwright');
+    });
+
+    it('returns puppeteer when engine=puppeteer query param is provided', () => {
+      const req = { query: { engine: 'puppeteer' } };
+      expect(getBrowserEngine(req)).toBe('puppeteer');
+    });
+
+    it('returns puppeteer when engine=pptr query param is provided', () => {
+      const req = { query: { engine: 'pptr' } };
+      expect(getBrowserEngine(req)).toBe('puppeteer');
+    });
+
+    it('returns playwright when browser=playwright query param is provided', () => {
+      const req = { query: { browser: 'playwright' } };
+      expect(getBrowserEngine(req)).toBe('playwright');
+    });
+
+    it('is case insensitive', () => {
+      const req1 = { query: { engine: 'PLAYWRIGHT' } };
+      const req2 = { query: { engine: 'PlAyWrIgHt' } };
+      expect(getBrowserEngine(req1)).toBe('playwright');
+      expect(getBrowserEngine(req2)).toBe('playwright');
+    });
+
+    it('returns playwright from BROWSER_ENGINE env var', () => {
+      const originalEnv = process.env.BROWSER_ENGINE;
+      process.env.BROWSER_ENGINE = 'playwright';
+      const req = { query: {} };
+      expect(getBrowserEngine(req)).toBe('playwright');
+      if (originalEnv) {
+        process.env.BROWSER_ENGINE = originalEnv;
+      } else {
+        delete process.env.BROWSER_ENGINE;
+      }
+    });
+
+    it('prefers query param over env var', () => {
+      const originalEnv = process.env.BROWSER_ENGINE;
+      process.env.BROWSER_ENGINE = 'playwright';
+      const req = { query: { engine: 'puppeteer' } };
+      expect(getBrowserEngine(req)).toBe('puppeteer');
+      if (originalEnv) {
+        process.env.BROWSER_ENGINE = originalEnv;
+      } else {
+        delete process.env.BROWSER_ENGINE;
+      }
+    });
+  });
+
+  describe('createBrowser - Puppeteer', () => {
+    it('creates a puppeteer browser with type field', async () => {
+      const browser = await createBrowser('puppeteer');
+      expect(browser.type).toBe('puppeteer');
+      expect(browser.newPage).toBeDefined();
+      expect(browser.close).toBeDefined();
+      await browser.close();
+    });
+
+    it('can create and use a page', async () => {
+      const browser = await createBrowser('puppeteer');
+      const page = await browser.newPage();
+      expect(page.setViewport).toBeDefined();
+      expect(page.goto).toBeDefined();
+      expect(page.content).toBeDefined();
+      expect(page.screenshot).toBeDefined();
+      expect(page._type).toBe('puppeteer');
+      await browser.close();
+    });
+  });
+
+  describe('createBrowser - Playwright', () => {
+    it('creates a playwright browser with type field', async () => {
+      const browser = await createBrowser('playwright');
+      expect(browser.type).toBe('playwright');
+      expect(browser.newPage).toBeDefined();
+      expect(browser.close).toBeDefined();
+      await browser.close();
+    });
+
+    it('can create and use a page', async () => {
+      const browser = await createBrowser('playwright');
+      const page = await browser.newPage();
+      expect(page.setViewport).toBeDefined();
+      expect(page.goto).toBeDefined();
+      expect(page.content).toBeDefined();
+      expect(page.screenshot).toBeDefined();
+      expect(page._type).toBe('playwright');
+      await browser.close();
+    });
+  });
+
+  describe('Page Adapter Compatibility', () => {
+    it('both adapters support the same interface', async () => {
+      const puppeteerBrowser = await createBrowser('puppeteer');
+      const playwrightBrowser = await createBrowser('playwright');
+
+      const puppeteerPage = await puppeteerBrowser.newPage();
+      const playwrightPage = await playwrightBrowser.newPage();
+
+      // Check that both have the same methods
+      const methods = [
+        'setExtraHTTPHeaders',
+        'setUserAgent',
+        'setViewport',
+        'goto',
+        'content',
+        'screenshot',
+        'close',
+      ];
+      for (const method of methods) {
+        expect(puppeteerPage[method]).toBeDefined();
+        expect(playwrightPage[method]).toBeDefined();
+      }
+
+      await puppeteerBrowser.close();
+      await playwrightBrowser.close();
+    });
+  });
+});

package/tests/unit/cli.test.js

@@ -0,0 +1,151 @@
+// Unit tests for CLI argument parsing and functionality
+import { spawn } from 'child_process';
+import { fileURLToPath } from 'url';
+import { dirname, resolve } from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const cliPath = resolve(__dirname, '../../bin/web-capture.js');
+
+// Helper function to run CLI and capture output
+function runCli(args, options = {}) {
+  return new Promise((resolve, reject) => {
+    const proc = spawn('node', [cliPath, ...args], {
+      cwd: options.cwd || dirname(cliPath),
+      env: { ...process.env, ...options.env },
+    });
+
+    let stdout = '';
+    let stderr = '';
+
+    proc.stdout.on('data', (data) => {
+      stdout += data.toString();
+    });
+    proc.stderr.on('data', (data) => {
+      stderr += data.toString();
+    });
+
+    proc.on('close', (code) => {
+      resolve({ code, stdout, stderr });
+    });
+
+    proc.on('error', reject);
+  });
+}
+
+describe('CLI', () => {
+  describe('--help', () => {
+    test('shows help message with --help flag', async () => {
+      const result = await runCli(['--help']);
+      expect(result.code).toBe(0);
+      expect(result.stdout).toContain('web-capture');
+      expect(result.stdout).toContain('Usage:');
+      expect(result.stdout).toContain('--serve');
+      expect(result.stdout).toContain('--format');
+      expect(result.stdout).toContain('--output');
+      expect(result.stdout).toContain('--engine');
+    });
+
+    test('shows help message with -h flag', async () => {
+      const result = await runCli(['-h']);
+      expect(result.code).toBe(0);
+      expect(result.stdout).toContain('web-capture');
+      expect(result.stdout).toContain('Usage:');
+    });
+
+    test('shows error and exits with code 1 when no arguments provided', async () => {
+      const result = await runCli([]);
+      expect(result.code).toBe(1);
+      // With lino-arguments and yargs, error messages go to stderr
+      expect(result.stderr).toContain('Error');
+    });
+  });
+
+  describe('--version', () => {
+    test('shows version with --version flag', async () => {
+      const result = await runCli(['--version']);
+      expect(result.code).toBe(0);
+      // yargs outputs just the version number
+      expect(result.stdout).toMatch(/\d+\.\d+\.\d+/);
+    });
+
+    test('shows version with -v flag', async () => {
+      const result = await runCli(['-v']);
+      expect(result.code).toBe(0);
+      // yargs outputs just the version number
+      expect(result.stdout).toMatch(/\d+\.\d+\.\d+/);
+    });
+  });
+
+  describe('URL validation', () => {
+    test('rejects invalid URL', async () => {
+      const result = await runCli(['not-a-valid-url-without-dots']);
+      // Should fail with invalid URL or show help due to strict mode
+      expect(result.code).toBe(1);
+      // Either an error message or help output (from yargs strict mode)
+      expect(result.stdout.length + result.stderr.length).toBeGreaterThan(0);
+    }, 15000);
+  });
+
+  describe('format options', () => {
+    test('accepts --format html', async () => {
+      // Just test that the argument is accepted (actual fetch would fail without network)
+      const result = await runCli(['--help']);
+      expect(result.stdout).toContain('--format');
+      expect(result.stdout).toContain('html');
+    });
+
+    test('accepts --format markdown', async () => {
+      const result = await runCli(['--help']);
+      expect(result.stdout).toContain('markdown');
+    });
+
+    test('accepts --format image', async () => {
+      const result = await runCli(['--help']);
+      expect(result.stdout).toContain('image');
+    });
+  });
+
+  describe('engine options', () => {
+    test('help shows engine options', async () => {
+      const result = await runCli(['--help']);
+      expect(result.stdout).toContain('--engine');
+      expect(result.stdout).toContain('puppeteer');
+      expect(result.stdout).toContain('playwright');
+    });
+
+    test('accepts --engine option', async () => {
+      const result = await runCli(['--help']);
+      expect(result.stdout).toContain('--engine');
+    });
+  });
+
+  describe('server mode options', () => {
+    test('help shows --serve option', async () => {
+      const result = await runCli(['--help']);
+      expect(result.stdout).toContain('--serve');
+      expect(result.stdout).toContain('-s');
+    });
+
+    test('help shows --port option', async () => {
+      const result = await runCli(['--help']);
+      expect(result.stdout).toContain('--port');
+      expect(result.stdout).toContain('-p');
+    });
+
+    test('help shows API endpoints', async () => {
+      const result = await runCli(['--help']);
+      expect(result.stdout).toContain('/html');
+      expect(result.stdout).toContain('/markdown');
+      expect(result.stdout).toContain('/image');
+    });
+  });
+
+  describe('output options', () => {
+    test('help shows --output option', async () => {
+      const result = await runCli(['--help']);
+      expect(result.stdout).toContain('--output');
+      expect(result.stdout).toContain('-o');
+    });
+  });
+});

package/tests/unit/convertRelativeUrls.test.js

@@ -0,0 +1,34 @@
+import { convertRelativeUrls } from '../../src/lib.js';
+
+describe('convertRelativeUrls', () => {
+  const baseUrl = 'https://example.com/';
+
+  it('should NOT inject runtimeHook if there are no <script> tags', () => {
+    const html = `<!DOCTYPE html><html><head><title>Test</title></head><body><h1>Hello</h1><p>World</p></body></html>`;
+    const result = convertRelativeUrls(html, baseUrl);
+    // Should not inject the runtimeHook
+    expect(result).not.toMatch(/MutationObserver/);
+    expect(result).not.toMatch(/fixAllUrls/);
+    expect(result).not.toMatch(/runtime JS hook/);
+    // Should not add any <script> tag
+    expect(result.match(/<script/gi)).toBeNull();
+  });
+
+  it('should inject runtimeHook if there is a <script> tag', () => {
+    const html = `<!DOCTYPE html><html><head><title>Test</title><script src="foo.js"></script></head><body><h1>Hello</h1></body></html>`;
+    const result = convertRelativeUrls(html, baseUrl);
+    // Should inject the runtimeHook
+    expect(result).toMatch(/MutationObserver/);
+    expect(result).toMatch(/fixAllUrls/);
+    // Should add a <script> tag for the hook
+    expect((result.match(/<script/gi) || []).length).toBeGreaterThan(1);
+  });
+
+  it('should inject runtimeHook if there is an inline <script>', () => {
+    const html = `<!DOCTYPE html><html><head><title>Test</title><script>console.log('hi');</script></head><body></body></html>`;
+    const result = convertRelativeUrls(html, baseUrl);
+    expect(result).toMatch(/MutationObserver/);
+    expect(result).toMatch(/fixAllUrls/);
+    expect((result.match(/<script/gi) || []).length).toBeGreaterThan(1);
+  });
+});