@link-assistant/web-capture 1.1.2

package/src/lib.js

@@ -0,0 +1,330 @@
+/* eslint-disable no-useless-escape */
+// Common logic for the web-capture microservice
+import fetch from 'node-fetch';
+import * as cheerio from 'cheerio';
+import TurndownService from 'turndown';
+import iconv from 'iconv-lite';
+import { URL } from 'url';
+import turndownPluginGfm from 'turndown-plugin-gfm';
+
+export async function fetchHtml(url) {
+  if (!url) {
+    throw new Error('Missing URL parameter');
+  }
+  const response = await fetch(url);
+  return response.text();
+}
+
+export function convertHtmlToMarkdown(html, baseUrl) {
+  // Ensure all URLs are absolute before Markdown conversion
+  if (baseUrl) {
+    html = convertRelativeUrls(html, baseUrl);
+  }
+  // Load HTML into Cheerio
+  const $ = cheerio.load(html);
+
+  // Remove <style>, <script>, and <noscript> tags
+  $('style, script, noscript').remove();
+
+  // Remove inline event handlers (attributes starting with 'on')
+  $('*').each(function () {
+    const attribs = this.attribs || {};
+    Object.keys(attribs).forEach((attr) => {
+      if (attr.toLowerCase().startsWith('on')) {
+        $(this).removeAttr(attr);
+      }
+    });
+  });
+  // Remove javascript: links
+  $('a[href^="javascript:"]').remove();
+
+  // Remove inline style attributes
+  $('[style]').removeAttr('style');
+
+  // Remove empty headings (h1-h2) and headings with only whitespace
+  $('h1, h2, h3, h4, h5, h6').each(function () {
+    if (!$(this).text().trim()) {
+      $(this).remove();
+    }
+  });
+
+  // Remove <a> tags with no direct text content (including only whitespace or only child elements)
+  $('a').each(function () {
+    // Get all text nodes directly under this <a>
+    const directText = $(this)
+      .contents()
+      .filter(function () {
+        return this.type === 'text';
+      })
+      .text();
+    // If no direct text and all children are empty or whitespace, remove the <a>
+    if (!directText.trim()) {
+      // Also check if all children are empty or whitespace
+      let allChildrenEmpty = true;
+      $(this)
+        .children()
+        .each(function () {
+          if (
+            $(this).text().trim() ||
+            (this.tagName === 'img' && $(this).attr('alt'))
+          ) {
+            allChildrenEmpty = false;
+          }
+        });
+      if (allChildrenEmpty) {
+        $(this).remove();
+      }
+    }
+  });
+
+  // Remove <a> tags with only <img> as child and no alt text
+  $('a').each(function () {
+    const children = $(this).children();
+    if (
+      children.length === 1 &&
+      children[0].tagName === 'img' &&
+      !$(children[0]).attr('alt')
+    ) {
+      $(this).remove();
+    }
+  });
+
+  // Remove any leftover empty <a> tags
+  $('a').each(function () {
+    if (!$(this).text().trim()) {
+      $(this).remove();
+    }
+  });
+
+  // Remove any leftover empty elements (optional, for robustness)
+  $('[data-remove-empty]').each(function () {
+    if (!$(this).text().trim()) {
+      $(this).remove();
+    }
+  });
+
+  // Preprocess ARIA role-based tables to semantic tables with <thead> and <tbody>
+  $('[role="table"]').each(function () {
+    const $table = $(this);
+    const $newTable = $('<table></table>');
+    // Add caption if present
+    const label = $table.attr('aria-label');
+    const descId = $table.attr('aria-describedby');
+    if (label) {
+      $newTable.append(`<caption>${label}</caption>`);
+    } else if (descId && $(`#${descId}`).length) {
+      $newTable.append(`<caption>${$(`#${descId}`).text()}</caption>`);
+    }
+    const rowgroups = $table.find('> [role="rowgroup"]');
+    if (rowgroups.length > 0) {
+      // First rowgroup is header
+      const $thead = $('<thead></thead>');
+      const $tbody = $('<tbody></tbody>');
+      rowgroups.each(function (i) {
+        $(this)
+          .find('> [role="row"]')
+          .each(function () {
+            const $row = $(this);
+            const $tr = $('<tr></tr>');
+            $row.children('[role="columnheader"]').each(function () {
+              $tr.append($('<th></th>').text($(this).text()));
+            });
+            $row.children('[role="cell"]').each(function () {
+              $tr.append($('<td></td>').text($(this).text()));
+            });
+            if (i === 0 && $tr.children('th').length) {
+              $thead.append($tr);
+            } else {
+              $tbody.append($tr);
+            }
+          });
+      });
+      if ($thead.children().length) {
+        $newTable.append($thead);
+      }
+      if ($tbody.children().length) {
+        $newTable.append($tbody);
+      }
+    }
+    $table.replaceWith($newTable);
+  });
+
+  // Convert cleaned HTML to Markdown
+  const turndown = new TurndownService({
+    headingStyle: 'atx',
+    codeBlockStyle: 'fenced',
+    emDelimiter: '*',
+    bulletListMarker: '-',
+    strongDelimiter: '**',
+    linkStyle: 'inlined',
+    linkReferenceStyle: 'full',
+    hr: '---',
+    style: false,
+  });
+  turndown.use(turndownPluginGfm.gfm);
+  return turndown.turndown($.html());
+}
+
+// Convert relative URLs to absolute URLs in HTML content
+export function convertRelativeUrls(html, baseUrl) {
+  const base = new URL(baseUrl);
+
+  // Function to convert a single URL
+  const toAbsolute = (url) => {
+    if (
+      !url ||
+      url.startsWith('data:') ||
+      url.startsWith('blob:') ||
+      url.startsWith('javascript:')
+    ) {
+      return url;
+    }
+    try {
+      return new URL(url, base).href;
+    } catch {
+      return url;
+    }
+  };
+
+  // Convert URLs in various attributes
+  const attributes = [
+    { tag: 'a', attr: 'href' },
+    { tag: 'img', attr: 'src' },
+    { tag: 'script', attr: 'src' },
+    { tag: 'link', attr: 'href' },
+    { tag: 'form', attr: 'action' },
+    { tag: 'video', attr: 'src' },
+    { tag: 'audio', attr: 'src' },
+    { tag: 'source', attr: 'src' },
+    { tag: 'track', attr: 'src' },
+    { tag: 'embed', attr: 'src' },
+    { tag: 'object', attr: 'data' },
+    { tag: 'iframe', attr: 'src' },
+  ];
+
+  let result = html;
+
+  // Process each attribute type
+  for (const { tag, attr } of attributes) {
+    const regex = new RegExp(
+      `<${tag}[^>]*${attr}=["']([^"']+)["'][^>]*>`,
+      'gi'
+    );
+    result = result.replace(regex, (match, url) => {
+      const absoluteUrl = toAbsolute(url);
+      return match.replace(url, absoluteUrl);
+    });
+  }
+
+  // Also handle inline styles with url()
+  result = result.replace(/url\(['"]?([^'"()]+)['"]?\)/gi, (match, url) => {
+    const absoluteUrl = toAbsolute(url);
+    return `url("${absoluteUrl}")`;
+  });
+
+  // Only inject runtime JS hook if there is a <script> tag in the original HTML
+  if (/<script[\s>]/i.test(html)) {
+    const runtimeHook = `
+<script>(function() {
+  const baseUrl = '${base.href}';
+  function absolutifyUrl(url) {
+    if (!url) return url;
+    if (url.startsWith('http') || url.startsWith('data:') || url.startsWith('blob:')) return url;
+    try { return new URL(url, baseUrl).href; } catch { return url; }
+  }
+  function fixElementUrls(el) {
+    if (el.tagName === 'A' || el.tagName === 'LINK') {
+      el.href = absolutifyUrl(el.getAttribute('href'));
+    }
+    if (el.tagName === 'IMG' || el.tagName === 'SCRIPT' || el.tagName === 'IFRAME' || el.tagName === 'SOURCE') {
+      el.src = absolutifyUrl(el.getAttribute('src'));
+    }
+    if (el.hasAttribute('style')) {
+      // The escapes in the regex are intentional for matching CSS url() syntax
+      // eslint-disable-next-line no-useless-escape
+      const urlPattern = /url\(["']?([^'")]+)["']?\)/g;
+      el.setAttribute(
+        'style',
+        el.getAttribute('style').replace(urlPattern, (m, capturedUrl) => {
+          const absoluteUrl = absolutifyUrl(capturedUrl);
+          return "url('" + absoluteUrl + "')";
+        })
+      );
+    }
+  }
+  function fixAllUrls(root) {
+    root.querySelectorAll('*').forEach(fixElementUrls);
+    // The escapes in the regex are intentional for matching CSS url() syntax
+    // eslint-disable-next-line no-useless-escape
+    const urlPattern = /url\(["']?([^'")]+)["']?\)/g;
+    root.querySelectorAll('style').forEach(function (styleTag) {
+      styleTag.textContent = styleTag.textContent.replace(
+        urlPattern,
+        (m, capturedUrl) => {
+          const absoluteUrl = absolutifyUrl(capturedUrl);
+          return "url('" + absoluteUrl + "')";
+        }
+      );
+    });
+  }
+  fixAllUrls(document);
+  const observer = new MutationObserver(function(mutations) {
+    for (const mutation of mutations) {
+      mutation.addedNodes.forEach(function(node) {
+        if (node.nodeType === 1) {
+          fixElementUrls(node);
+          fixAllUrls(node);
+        }
+      });
+    }
+  });
+  observer.observe(document.body, { childList: true, subtree: true });
+})();</script>
+`;
+    result = result.replace(/<\/head>/i, `${runtimeHook}</head>`);
+  }
+
+  return result;
+}
+
+// Convert HTML content to UTF-8 if it's not already
+export function convertToUtf8(html) {
+  // First, try to detect the current encoding from meta tag
+  const charsetMatch = html.match(/<meta[^>]+charset=["']?([^"'>\s]+)/i);
+  const currentCharset = charsetMatch ? charsetMatch[1].toLowerCase() : 'utf-8';
+
+  // If it's already UTF-8, just ensure the meta tag is present
+  if (currentCharset === 'utf-8' || currentCharset === 'utf8') {
+    if (!charsetMatch) {
+      return html.replace(/<head[^>]*>/i, '$&<meta charset="utf-8">');
+    }
+    return html;
+  }
+
+  // Convert from detected charset to UTF-8
+  try {
+    // Convert the HTML string to a buffer using the detected charset
+    const buffer = iconv.encode(html, currentCharset);
+    // Decode the buffer to UTF-8
+    const utf8Html = iconv.decode(buffer, 'utf-8');
+
+    // Replace the charset meta tag with UTF-8
+    return utf8Html.replace(
+      /<meta[^>]+charset=["']?[^"'>\s]+["']?/i,
+      '<meta charset="utf-8">'
+    );
+  } catch (error) {
+    console.error('Error converting charset:', error);
+    // If conversion fails, return original HTML with UTF-8 meta tag
+    return html.replace(/<head[^>]*>/i, '$&<meta charset="utf-8">');
+  }
+}
+
+// Detect encoding and convert to UTF-8 for Puppeteer-rendered HTML
+export function ensureUtf8(html) {
+  // If no charset meta tag is present, inject one
+  if (!/<meta[^>]+charset/i.test(html)) {
+    html = html.replace(/<head[^>]*>/i, '$&<meta charset="utf-8">');
+  }
+  return html;
+}

package/src/markdown.js

@@ -0,0 +1,17 @@
+import { fetchHtml, convertHtmlToMarkdown } from './lib.js';
+
+export async function markdownHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+  try {
+    const html = await fetchHtml(url);
+    // Pass baseUrl to convertHtmlToMarkdown so all URLs are absolute
+    const markdown = convertHtmlToMarkdown(html, url);
+    res.type('text/markdown').send(markdown);
+  } catch (err) {
+    console.error(err);
+    res.status(500).send('Error converting to Markdown');
+  }
+}

package/src/stream.js

@@ -0,0 +1,50 @@
+import fetch from 'node-fetch';
+import { pipeline } from 'stream';
+
+export async function streamHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+  try {
+    const response = await fetch(url);
+    // Copy status and headers
+    res.status(response.status);
+
+    // Set default content type if not present
+    const contentType = response.headers.get('content-type') || 'text/plain';
+    res.setHeader('Content-Type', contentType);
+
+    // Copy other headers
+    for (const [key, value] of response.headers.entries()) {
+      if (
+        key.toLowerCase() !== 'transfer-encoding' &&
+        key.toLowerCase() !== 'content-encoding' &&
+        key.toLowerCase() !== 'content-length'
+      ) {
+        res.setHeader(key, value);
+      }
+    }
+
+    // Stream the response body
+    if (response.body) {
+      pipeline(response.body, res, (err) => {
+        if (err) {
+          console.error('Pipeline error in /stream:', err);
+          if (!res.headersSent) {
+            res.status(500);
+            res.end('Error proxying content');
+          }
+        }
+      });
+    } else {
+      res.end();
+    }
+  } catch (err) {
+    console.error('Stream error:', err);
+    if (!res.headersSent) {
+      res.status(500);
+      res.end('Error proxying content');
+    }
+  }
+}

package/tests/e2e/docker.test.js

@@ -0,0 +1,148 @@
+import fetch from 'node-fetch';
+import { exec } from 'child_process';
+import { promisify } from 'util';
+
+const execAsync = promisify(exec);
+const WAIT_FOR_READY = 10000; // ms
+const PORT = 3000; // Use the same port as in docker-compose.yml
+const baseUrl = `http://localhost:${PORT}`;
+
+const timings = {};
+
+async function isServiceRunning() {
+  try {
+    const res = await fetch(`${baseUrl}/html?url=https://example.com`);
+    return res.status === 200;
+  } catch {
+    return false;
+  }
+}
+
+beforeAll(async () => {
+  timings.start = Date.now();
+  console.log('Checking if Docker service is already running...');
+  const alreadyRunning = await isServiceRunning();
+  if (!alreadyRunning) {
+    console.log('Service not running, starting Docker container...');
+    try {
+      const dockerStart = Date.now();
+      // Start the Docker container using docker compose (v2 command)
+      const { stdout, stderr } = await execAsync('docker compose up -d');
+      timings.dockerStartup = Date.now() - dockerStart;
+      console.log('Docker compose output:', stdout);
+      if (stderr) {
+        console.error('Docker compose errors:', stderr);
+      }
+
+      // Wait for the service to be ready
+      const readyStart = Date.now();
+      await new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => {
+          reject(new Error('Docker service did not start in time'));
+        }, WAIT_FOR_READY);
+
+        const checkReady = async () => {
+          try {
+            console.log(`Checking if service is ready at ${baseUrl}...`);
+            const res = await fetch(`${baseUrl}/html?url=https://example.com`);
+            if (res.status === 200) {
+              console.log('Service is ready!');
+              clearTimeout(timeout);
+              resolve();
+            } else {
+              console.log(`Service not ready yet, status: ${res.status}`);
+              setTimeout(checkReady, 500);
+            }
+          } catch (err) {
+            console.log('Service not ready yet, error:', err.message);
+            setTimeout(checkReady, 500);
+          }
+        };
+        checkReady();
+      });
+      timings.serviceReady = Date.now() - readyStart;
+    } catch (error) {
+      console.error('Failed to start Docker container:', error);
+      throw error;
+    }
+    timings.beforeAll = Date.now() - timings.start;
+    console.log('Timing: Docker startup:', `${timings.dockerStartup}ms`);
+    console.log('Timing: Service readiness:', `${timings.serviceReady}ms`);
+    console.log('Timing: beforeAll total:', `${timings.beforeAll}ms`);
+  } else {
+    console.log('Docker service is already running. Skipping startup.');
+    timings.beforeAll = Date.now() - timings.start;
+    console.log(
+      'Timing: beforeAll (already running):',
+      `${timings.beforeAll}ms`
+    );
+  }
+}, 30000); // Reduced timeout for beforeAll
+
+describe('E2E (Docker): Web Capture Microservice', () => {
+  it('should return HTML from /html endpoint', async () => {
+    const htmlStart = Date.now();
+    const url = 'https://example.com';
+    const res = await fetch(`${baseUrl}/html?url=${encodeURIComponent(url)}`);
+    expect(res.status).toBe(200);
+    const text = await res.text();
+    expect(text).toMatch(/<html/i);
+    timings.html = Date.now() - htmlStart;
+    console.log('Timing: /html endpoint:', `${timings.html}ms`);
+  }, 20000);
+
+  it('should return Markdown from /markdown endpoint', async () => {
+    const mdStart = Date.now();
+    const url = 'https://example.com';
+    const res = await fetch(
+      `${baseUrl}/markdown?url=${encodeURIComponent(url)}`
+    );
+    expect(res.status).toBe(200);
+    const text = await res.text();
+    expect(text).toMatch(/example/i);
+    timings.markdown = Date.now() - mdStart;
+    console.log('Timing: /markdown endpoint:', `${timings.markdown}ms`);
+  }, 20000);
+
+  it('should return PNG from /image endpoint', async () => {
+    const pngStart = Date.now();
+    const url = 'https://example.com';
+    const res = await fetch(`${baseUrl}/image?url=${encodeURIComponent(url)}`);
+    expect(res.status).toBe(200);
+    expect(res.headers.get('content-type')).toMatch(/^image\/png/);
+    const buf = Buffer.from(await res.arrayBuffer());
+    // PNG signature check
+    const pngSignature = Buffer.from([
+      0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+    ]);
+    expect(buf.slice(0, 8)).toEqual(pngSignature);
+    expect(buf.length).toBeGreaterThan(100); // Should be a non-trivial PNG
+    timings.png = Date.now() - pngStart;
+    console.log('Timing: /image endpoint:', `${timings.png}ms`);
+  }, 60000);
+
+  it('should stream content from /stream endpoint', async () => {
+    const startTime = Date.now();
+    const url = 'https://example.com';
+    const res = await fetch(`${baseUrl}/stream?url=${encodeURIComponent(url)}`);
+    expect(res.status).toBe(200);
+    // Get the response as text
+    const text = await res.text();
+    expect(text).toMatch(/<html/i);
+    expect(text).toMatch(/Example Domain/i);
+    const endTime = Date.now();
+    console.log(`Timing: /stream endpoint: ${endTime - startTime}ms`);
+  }, 20000);
+
+  it('should return content from /fetch endpoint', async () => {
+    const startTime = Date.now();
+    const url = 'https://example.com';
+    const res = await fetch(`${baseUrl}/fetch?url=${encodeURIComponent(url)}`);
+    expect(res.status).toBe(200);
+    const text = await res.text();
+    expect(text).toMatch(/<html/i);
+    expect(text).toMatch(/Example Domain/i);
+    const endTime = Date.now();
+    console.log(`Timing: /fetch endpoint: ${endTime - startTime}ms`);
+  }, 20000);
+});

package/tests/e2e/process.test.js

@@ -0,0 +1,117 @@
+import { spawn } from 'child_process';
+import fetch from 'node-fetch';
+import getPort from 'get-port';
+import path from 'path';
+
+const WAIT_FOR_READY = 5000; // ms
+let serverProcess;
+let baseUrl;
+
+beforeAll(async () => {
+  const port = await getPort();
+  baseUrl = `http://localhost:${port}`;
+
+  serverProcess = spawn(
+    'node',
+    [path.resolve('bin/web-capture.js'), '--serve', '--port', port.toString()],
+    {
+      env: process.env,
+      stdio: ['ignore', 'pipe', 'pipe'],
+    }
+  );
+
+  // Wait for the server to be ready (simple delay or poll)
+  await new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject(new Error('Server did not start in time'));
+    }, WAIT_FOR_READY);
+
+    let serverStarted = false;
+    serverProcess.stdout.on('data', (data) => {
+      if (
+        data.toString().includes('listening') ||
+        data.toString().includes('Server running')
+      ) {
+        serverStarted = true;
+        clearTimeout(timeout);
+        resolve();
+      }
+    });
+
+    serverProcess.on('error', (error) => {
+      clearTimeout(timeout);
+      reject(error);
+    });
+
+    serverProcess.on('exit', (code, signal) => {
+      if (!serverStarted) {
+        clearTimeout(timeout);
+        reject(
+          new Error(`Server process exited with code ${code}, signal ${signal}`)
+        );
+      }
+    });
+
+    // Fallback: resolve after WAIT_FOR_READY
+    setTimeout(resolve, WAIT_FOR_READY);
+  });
+});
+
+afterAll(() => {
+  if (serverProcess) {
+    serverProcess.kill();
+  }
+});
+
+describe('E2E: Web Capture Microservice', () => {
+  it('should return HTML from /html endpoint', async () => {
+    const url = 'https://example.com';
+    const res = await fetch(`${baseUrl}/html?url=${encodeURIComponent(url)}`);
+    expect(res.status).toBe(200);
+    const text = await res.text();
+    expect(text).toMatch(/<html/i);
+  });
+
+  it('should return Markdown from /markdown endpoint', async () => {
+    const url = 'https://example.com';
+    const res = await fetch(
+      `${baseUrl}/markdown?url=${encodeURIComponent(url)}`
+    );
+    expect(res.status).toBe(200);
+    const text = await res.text();
+    expect(text).toMatch(/example/i);
+  });
+
+  it('should return PNG from /image endpoint', async () => {
+    const url = 'https://example.com';
+    const res = await fetch(`${baseUrl}/image?url=${encodeURIComponent(url)}`);
+    expect(res.status).toBe(200);
+    expect(res.headers.get('content-type')).toMatch(/^image\/png/);
+    const buf = Buffer.from(await res.arrayBuffer());
+    // PNG signature check
+    const pngSignature = Buffer.from([
+      0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
+    ]);
+    expect(buf.slice(0, 8)).toEqual(pngSignature);
+    expect(buf.length).toBeGreaterThan(100); // Should be a non-trivial PNG
+  }, 60000);
+
+  it('should stream content from /stream endpoint', async () => {
+    const url = 'https://example.com';
+    const res = await fetch(`${baseUrl}/stream?url=${encodeURIComponent(url)}`);
+    expect(res.status).toBe(200);
+    // Get the response as text
+    const text = await res.text();
+    expect(text).toMatch(/<html/i);
+    expect(text).toMatch(/Example Domain/i);
+  }, 20000);
+
+  it('should return content from /fetch endpoint', async () => {
+    const url = 'https://example.com';
+    const res = await fetch(`${baseUrl}/fetch?url=${encodeURIComponent(url)}`);
+    expect(res.status).toBe(200);
+    const text = await res.text();
+    expect(text).toMatch(/<html/i);
+    expect(text).toMatch(/Example Domain/i);
+  }, 20000);
+});