@link-assistant/web-capture 1.1.2

package/src/browser.js

@@ -0,0 +1,207 @@
+// Browser abstraction layer to support both Puppeteer and Playwright
+import puppeteer from 'puppeteer';
+import playwright from 'playwright';
+
+/**
+ * Unified browser interface that works with both Puppeteer and Playwright
+ * @typedef {Object} BrowserAdapter
+ * @property {Function} newPage - Create a new page
+ * @property {Function} close - Close the browser
+ * @property {string} type - Browser type ('puppeteer' or 'playwright')
+ */
+
+/**
+ * Unified page interface
+ * @typedef {Object} PageAdapter
+ * @property {Function} setExtraHTTPHeaders - Set HTTP headers
+ * @property {Function} setUserAgent - Set user agent
+ * @property {Function} setViewport - Set viewport size
+ * @property {Function} goto - Navigate to URL
+ * @property {Function} content - Get page HTML content
+ * @property {Function} screenshot - Take screenshot
+ * @property {Function} close - Close the page
+ * @property {Object} _page - Original page object
+ * @property {string} _type - Browser type
+ */
+
+/**
+ * Create a browser instance using the specified engine
+ * @param {string} engine - 'puppeteer' or 'playwright' (defaults to puppeteer)
+ * @param {Object} options - Browser launch options
+ * @returns {Promise<BrowserAdapter>}
+ */
+export async function createBrowser(engine = 'puppeteer', options = {}) {
+  const normalizedEngine = engine.toLowerCase();
+
+  if (normalizedEngine === 'playwright') {
+    return createPlaywrightBrowser(options);
+  } else {
+    return createPuppeteerBrowser(options);
+  }
+}
+
+/**
+ * Create a Puppeteer browser instance
+ * @param {Object} options - Puppeteer launch options
+ * @returns {Promise<BrowserAdapter>}
+ */
+async function createPuppeteerBrowser(options = {}) {
+  const defaultOptions = {
+    args: [
+      '--no-sandbox',
+      '--disable-setuid-sandbox',
+      '--disable-dev-shm-usage',
+    ],
+  };
+
+  const browser = await puppeteer.launch({ ...defaultOptions, ...options });
+
+  return {
+    async newPage() {
+      const page = await browser.newPage();
+      return createPuppeteerPageAdapter(page);
+    },
+    async close() {
+      await browser.close();
+    },
+    type: 'puppeteer',
+    _browser: browser,
+  };
+}
+
+/**
+ * Create a Playwright browser instance
+ * @param {Object} options - Playwright launch options
+ * @returns {Promise<BrowserAdapter>}
+ */
+async function createPlaywrightBrowser(options = {}) {
+  const defaultOptions = {
+    args: [
+      '--no-sandbox',
+      '--disable-setuid-sandbox',
+      '--disable-dev-shm-usage',
+    ],
+  };
+
+  // Playwright uses chromium by default
+  const browser = await playwright.chromium.launch({
+    ...defaultOptions,
+    ...options,
+  });
+
+  return {
+    async newPage() {
+      const page = await browser.newPage();
+      return createPlaywrightPageAdapter(page);
+    },
+    async close() {
+      await browser.close();
+    },
+    type: 'playwright',
+    _browser: browser,
+  };
+}
+
+/**
+ * Create a page adapter for Puppeteer
+ * @param {Object} page - Puppeteer page object
+ * @returns {PageAdapter}
+ */
+function createPuppeteerPageAdapter(page) {
+  return {
+    async setExtraHTTPHeaders(headers) {
+      await page.setExtraHTTPHeaders(headers);
+    },
+    async setUserAgent(userAgent) {
+      await page.setUserAgent(userAgent);
+    },
+    async setViewport(viewport) {
+      await page.setViewport(viewport);
+    },
+    async goto(url, options = {}) {
+      await page.goto(url, options);
+    },
+    async content() {
+      return await page.content();
+    },
+    async screenshot(options = {}) {
+      return await page.screenshot(options);
+    },
+    async close() {
+      await page.close();
+    },
+    _page: page,
+    _type: 'puppeteer',
+  };
+}
+
+/**
+ * Create a page adapter for Playwright
+ * @param {Object} page - Playwright page object
+ * @returns {PageAdapter}
+ */
+function createPlaywrightPageAdapter(page) {
+  return {
+    async setExtraHTTPHeaders(headers) {
+      await page.setExtraHTTPHeaders(headers);
+    },
+    async setUserAgent(userAgent) {
+      // Playwright doesn't have page.setUserAgent, use extraHTTPHeaders instead
+      await page.setExtraHTTPHeaders({ 'User-Agent': userAgent });
+    },
+    async setViewport(viewport) {
+      // Playwright uses setViewportSize instead of setViewport
+      await page.setViewportSize(viewport);
+    },
+    async goto(url, options = {}) {
+      // Convert Puppeteer waitUntil options to Playwright equivalents
+      const playwrightOptions = { ...options };
+      if (playwrightOptions.waitUntil === 'networkidle0') {
+        playwrightOptions.waitUntil = 'networkidle';
+      }
+      await page.goto(url, playwrightOptions);
+    },
+    async content() {
+      return await page.content();
+    },
+    async screenshot(options = {}) {
+      return await page.screenshot(options);
+    },
+    async close() {
+      await page.close();
+    },
+    _page: page,
+    _type: 'playwright',
+  };
+}
+
+/**
+ * Get the browser engine from query parameters or environment variable
+ * @param {Object} req - Express request object
+ * @returns {string} - 'puppeteer' or 'playwright'
+ */
+export function getBrowserEngine(req) {
+  // Check query parameter first
+  const engineParam = req.query.engine || req.query.browser;
+  if (engineParam) {
+    const normalized = engineParam.toLowerCase();
+    if (normalized === 'playwright' || normalized === 'pw') {
+      return 'playwright';
+    }
+    if (normalized === 'puppeteer' || normalized === 'pptr') {
+      return 'puppeteer';
+    }
+  }
+
+  // Check environment variable
+  const envEngine = process.env.BROWSER_ENGINE;
+  if (envEngine) {
+    const normalized = envEngine.toLowerCase();
+    if (normalized === 'playwright') {
+      return 'playwright';
+    }
+  }
+
+  // Default to puppeteer for backward compatibility
+  return 'puppeteer';
+}

package/src/fetch.js

@@ -0,0 +1,38 @@
+import fetch from 'node-fetch';
+
+export async function fetchHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+  try {
+    const response = await fetch(url);
+    // Copy status and headers
+    res.status(response.status);
+
+    // Set default content type if not present
+    const contentType = response.headers.get('content-type') || 'text/plain';
+    res.setHeader('Content-Type', contentType);
+
+    // Copy other headers
+    for (const [key, value] of response.headers.entries()) {
+      if (
+        key.toLowerCase() !== 'transfer-encoding' &&
+        key.toLowerCase() !== 'content-encoding' &&
+        key.toLowerCase() !== 'content-length'
+      ) {
+        res.setHeader(key, value);
+      }
+    }
+
+    // Get the response body as buffer and send it
+    const buffer = await response.buffer();
+    res.send(buffer);
+  } catch (err) {
+    console.error('Fetch error:', err);
+    if (!res.headersSent) {
+      res.status(500);
+      res.end('Error fetching content');
+    }
+  }
+}

package/src/html.js

@@ -0,0 +1,64 @@
+import { fetchHtml, convertToUtf8, convertRelativeUrls } from './lib.js';
+import { createBrowser, getBrowserEngine } from './browser.js';
+
+export async function htmlHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+  try {
+    // Ensure URL is absolute
+    const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
+
+    // First try to fetch HTML directly
+    const html = await fetchHtml(absoluteUrl);
+
+    // Check if it's valid HTML and contains JavaScript
+    const hasJavaScript =
+      /<script[^>]*>[\s\S]*?<\/script>|<script[^>]*\/>|javascript:/i.test(html);
+    const isHtml = /<html[^>]*>[\s\S]*?<\/html>/i.test(html);
+
+    if (!isHtml || hasJavaScript) {
+      // If not HTML or contains JavaScript, use browser to get rendered HTML
+      const engine = getBrowserEngine(req);
+      const browser = await createBrowser(engine);
+      try {
+        const page = await browser.newPage();
+
+        // Set proper encoding headers
+        await page.setExtraHTTPHeaders({
+          'Accept-Language': 'en-US,en;q=0.9',
+          'Accept-Charset': 'utf-8',
+        });
+
+        await page.setUserAgent(
+          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        );
+        await page.setViewport({ width: 1280, height: 800 });
+        // Navigate to the page
+        await page.goto(absoluteUrl, {
+          waitUntil: 'networkidle0',
+          timeout: 30000,
+        });
+        // Wait for 5 seconds after page load
+        await new Promise((resolve) => setTimeout(resolve, 5000));
+        // Get the rendered HTML, convert to UTF-8, and make URLs absolute (and inject runtime JS hook)
+        const renderedHtml = await page.content();
+        const utf8Html = convertToUtf8(renderedHtml);
+        const absoluteHtml = convertRelativeUrls(utf8Html, absoluteUrl);
+
+        res.type('text/html; charset=utf-8').send(absoluteHtml);
+      } finally {
+        await browser.close();
+      }
+    } else {
+      // If it's plain HTML without JavaScript, convert to UTF-8 and make URLs absolute
+      const utf8Html = convertToUtf8(html);
+      const absoluteHtml = convertRelativeUrls(utf8Html, absoluteUrl);
+      res.type('text/html; charset=utf-8').send(absoluteHtml);
+    }
+  } catch (err) {
+    console.error('HTML fetch error:', err);
+    res.status(500).send('Error fetching HTML');
+  }
+}

package/src/image.js

@@ -0,0 +1,41 @@
+import { createBrowser, getBrowserEngine } from './browser.js';
+
+export async function imageHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+  try {
+    // Ensure URL is absolute
+    const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
+    const engine = getBrowserEngine(req);
+    const browser = await createBrowser(engine);
+    try {
+      const page = await browser.newPage();
+      await page.setExtraHTTPHeaders({
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Accept-Charset': 'utf-8',
+      });
+      await page.setUserAgent(
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+      );
+      await page.setViewport({ width: 1280, height: 800 });
+      await page.goto(absoluteUrl, {
+        waitUntil: 'networkidle0',
+        timeout: 30000,
+      });
+      // Wait for 5 seconds after page load
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+      // Take a screenshot of just the viewport (not the full page)
+      const buffer = await page.screenshot({ type: 'png' });
+      res.set('Content-Type', 'image/png');
+      res.set('Content-Disposition', 'inline; filename="screenshot.png"');
+      res.end(buffer);
+    } finally {
+      await browser.close();
+    }
+  } catch (err) {
+    console.error(err);
+    res.status(500).send('Error capturing screenshot');
+  }
+}

package/src/index.js

@@ -0,0 +1,55 @@
+import express from 'express';
+import { fileURLToPath } from 'url';
+import { htmlHandler } from './html.js';
+import { markdownHandler } from './markdown.js';
+import { imageHandler } from './image.js';
+import { streamHandler } from './stream.js';
+import { fetchHandler } from './fetch.js';
+
+const app = express();
+const port = process.env.PORT || 3000;
+
+app.get('/html', htmlHandler);
+app.get('/markdown', markdownHandler);
+app.get('/image', imageHandler);
+app.get('/stream', streamHandler);
+app.get('/fetch', fetchHandler);
+
+// Start the server if this is the main module
+const isMainModule =
+  process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
+let server;
+if (isMainModule) {
+  console.log('Process PID:', process.pid);
+  server = app.listen(port, () => {
+    console.log(`Renderer service listening on http://localhost:${port}`);
+  });
+
+  // Graceful shutdown for Docker
+  function shutdown(signal) {
+    console.log(`Received shutdown signal (${signal}), closing server...`);
+    server.close(() => {
+      console.log('Server closed. Exiting process.');
+      process.exit(0);
+    });
+    // Force exit if not closed in 2 seconds
+    setTimeout(() => {
+      console.error('Force exiting after 2s');
+      process.exit(1);
+    }, 2000);
+  }
+  process.on('SIGTERM', () => shutdown('SIGTERM'));
+  process.on('SIGINT', () => shutdown('SIGINT'));
+
+  process.on('exit', (code) => {
+    console.log('Process exit event with code:', code);
+  });
+  process.on('uncaughtException', (err) => {
+    console.error('Uncaught Exception:', err);
+  });
+  process.on('unhandledRejection', (reason, promise) => {
+    console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+  });
+}
+
+export { app };