@link-assistant/web-capture 1.1.2 → 1.4.2

package/src/figures.js

@@ -0,0 +1,208 @@
+/**
+ * Figure image extraction and download module (R4).
+ *
+ * Extracts figure images from web pages and downloads them locally.
+ * Supports multi-language figure detection (English/Russian).
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/download.mjs
+ *
+ * @module figures
+ */
+
+import fetch from 'node-fetch';
+import * as cheerio from 'cheerio';
+import { URL } from 'url';
+import { retry } from './retry.js';
+import { createBrowser, getBrowserEngine } from './browser.js';
+import { dismissPopups, scrollToLoadContent } from './popups.js';
+
+/**
+ * Extract figure elements from HTML content.
+ *
+ * @param {string} html - HTML content
+ * @param {string} baseUrl - Base URL for resolving relative URLs
+ * @returns {Object[]} Array of figure objects with src, alt, caption, figureNum
+ */
+export function extractFigures(html, baseUrl) {
+  const $ = cheerio.load(html);
+  const figures = [];
+  let sequentialIndex = 0;
+
+  $('figure').each(function () {
+    const img = $(this).find('img').first();
+    if (!img.length) {
+      return;
+    }
+
+    const src = img.attr('src');
+    if (!src || src.startsWith('data:') || src.includes('.svg')) {
+      return;
+    }
+
+    sequentialIndex++;
+    const captionEl = $(this).find('figcaption');
+    const captionText = captionEl.length ? captionEl.text().trim() : '';
+
+    // Match multi-language figure numbers: "Figure X", "Рис. X", "Рисунок X"
+    const figureMatch = captionText.match(/(?:Figure|Рис\.?|Рисунок)\s*(\d+)/i);
+    const figureNum = figureMatch ? parseInt(figureMatch[1]) : sequentialIndex;
+
+    let resolvedSrc;
+    try {
+      resolvedSrc = new URL(src, baseUrl).href;
+    } catch {
+      resolvedSrc = src;
+    }
+
+    figures.push({
+      figureNum,
+      src: resolvedSrc,
+      alt: img.attr('alt') || '',
+      caption: captionText,
+      sequentialIndex,
+    });
+  });
+
+  return figures;
+}
+
+/**
+ * Download figure images to a local directory.
+ *
+ * @param {Object[]} figures - Array of figure objects from extractFigures
+ * @param {Object} [options] - Download options
+ * @param {Function} [options.onProgress] - Callback(figureNum, status)
+ * @returns {Promise<Object[]>} Array of download results with metadata
+ */
+export async function downloadFigures(figures, options = {}) {
+  const results = [];
+
+  for (const figure of figures) {
+    const ext =
+      figure.src.includes('.jpeg') || figure.src.includes('.jpg')
+        ? 'jpg'
+        : 'png';
+    const filename = `figure-${figure.figureNum}.${ext}`;
+
+    try {
+      const resp = await retry(() => fetch(figure.src), {
+        retries: 3,
+        baseDelay: 1000,
+      });
+
+      if (!resp.ok) {
+        throw new Error(`HTTP ${resp.status}`);
+      }
+
+      const buffer = await resp.buffer();
+
+      if (options.onProgress) {
+        options.onProgress(figure.figureNum, 'downloaded');
+      }
+
+      results.push({
+        figureNum: figure.figureNum,
+        filename,
+        buffer,
+        caption: figure.caption,
+        originalUrl: figure.src,
+      });
+    } catch (err) {
+      if (options.onProgress) {
+        options.onProgress(figure.figureNum, 'failed');
+      }
+      results.push({
+        figureNum: figure.figureNum,
+        filename,
+        buffer: null,
+        caption: figure.caption,
+        originalUrl: figure.src,
+        error: err.message,
+      });
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Extract and download figures from a URL using browser rendering.
+ * This handles JavaScript-rendered pages that require a browser.
+ *
+ * @param {string} url - URL to extract figures from
+ * @param {Object} [options] - Options
+ * @param {string} [options.engine] - Browser engine
+ * @returns {Promise<Object>} Result with figures and metadata
+ */
+export async function extractFiguresFromUrl(url, options = {}) {
+  const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
+  const engine = options.engine || 'puppeteer';
+
+  const browser = await createBrowser(engine);
+  try {
+    const page = await browser.newPage();
+    await page.setUserAgent(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+    );
+    await page.setViewport({ width: 1920, height: 1080 });
+    await page.goto(absoluteUrl, {
+      waitUntil: 'networkidle0',
+      timeout: 30000,
+    });
+
+    await new Promise((resolve) => setTimeout(resolve, 5000));
+    await scrollToLoadContent(page);
+    await dismissPopups(page);
+
+    const html = await page.content();
+    const figures = extractFigures(html, absoluteUrl);
+    const downloaded = await downloadFigures(figures, options);
+
+    return {
+      url: absoluteUrl,
+      figures: downloaded,
+      totalFound: figures.length,
+      totalDownloaded: downloaded.filter((d) => d.buffer).length,
+    };
+  } finally {
+    await browser.close();
+  }
+}
+
+/**
+ * Figures extraction handler for Express API.
+ *
+ * Query parameters:
+ *   url    (required) - URL to extract figures from
+ *   engine - 'puppeteer' or 'playwright'
+ */
+export async function figuresHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+
+  try {
+    const engine = getBrowserEngine(req);
+    const result = await extractFiguresFromUrl(url, { engine });
+
+    // Return metadata (not the actual image buffers via JSON)
+    res.json({
+      url: result.url,
+      totalFound: result.totalFound,
+      totalDownloaded: result.totalDownloaded,
+      figures: result.figures.map((f) => ({
+        figureNum: f.figureNum,
+        filename: f.filename,
+        caption: f.caption,
+        originalUrl: f.originalUrl,
+        downloaded: !!f.buffer,
+        error: f.error || null,
+      })),
+    });
+  } catch (err) {
+    console.error(err);
+    res.status(500).send('Error extracting figures');
+  }
+}

package/src/gdocs.js

@@ -0,0 +1,333 @@
+// Google Docs capture module
+// Supports both API-based and browser-based capture of Google Docs documents.
+// API-based capture uses the Google Docs export URL pattern:
+//   https://docs.google.com/document/d/{DOCUMENT_ID}/export?format={FORMAT}
+// Browser-based capture navigates to the export URL using Puppeteer or Playwright.
+
+import fetch from 'node-fetch';
+import he from 'he';
+import archiver from 'archiver';
+import { convertHtmlToMarkdown } from './lib.js';
+
+const GDOCS_URL_PATTERN = /docs\.google\.com\/document\/d\/([a-zA-Z0-9_-]+)/;
+
+const GDOCS_EXPORT_BASE = 'https://docs.google.com/document/d';
+
+/**
+ * Supported Google Docs export formats.
+ * These map to the `format` query parameter in the export URL.
+ */
+export const GDOCS_EXPORT_FORMATS = {
+  html: 'html',
+  txt: 'txt',
+  md: 'md',
+  pdf: 'pdf',
+  docx: 'docx',
+  epub: 'epub',
+  zip: 'zip',
+};
+
+/**
+ * Check if a URL is a Google Docs document URL.
+ *
+ * @param {string} url - URL to check
+ * @returns {boolean} True if the URL is a Google Docs document URL
+ */
+export function isGoogleDocsUrl(url) {
+  if (!url || typeof url !== 'string') {
+    return false;
+  }
+  return GDOCS_URL_PATTERN.test(url);
+}
+
+/**
+ * Extract the document ID from a Google Docs URL.
+ *
+ * @param {string} url - Google Docs URL
+ * @returns {string|null} Document ID or null if not a valid Google Docs URL
+ */
+export function extractDocumentId(url) {
+  if (!url || typeof url !== 'string') {
+    return null;
+  }
+  const match = url.match(GDOCS_URL_PATTERN);
+  return match ? match[1] : null;
+}
+
+/**
+ * Build a Google Docs export URL.
+ *
+ * @param {string} documentId - Google Docs document ID
+ * @param {string} [format='html'] - Export format (html, txt, md, pdf, docx, epub, zip)
+ * @returns {string} Export URL
+ */
+export function buildExportUrl(documentId, format = 'html') {
+  const exportFormat = GDOCS_EXPORT_FORMATS[format] || 'html';
+  return `${GDOCS_EXPORT_BASE}/${documentId}/export?format=${exportFormat}`;
+}
+
+/**
+ * Fetch a Google Docs document via the export URL (API-based capture).
+ *
+ * For public documents, no authentication is needed.
+ * For private documents, pass an API token via the apiToken option.
+ *
+ * @param {string} url - Google Docs URL (edit URL or any URL containing the document ID)
+ * @param {Object} [options] - Fetch options
+ * @param {string} [options.format='html'] - Export format
+ * @param {string} [options.apiToken] - API token for private documents (Bearer token)
+ * @returns {Promise<{content: string, format: string, documentId: string, exportUrl: string}>}
+ */
+export async function fetchGoogleDoc(url, options = {}) {
+  const { format = 'html', apiToken } = options;
+
+  const documentId = extractDocumentId(url);
+  if (!documentId) {
+    throw new Error(`Not a valid Google Docs URL: ${url}`);
+  }
+
+  const exportUrl = buildExportUrl(documentId, format);
+
+  const headers = {
+    'User-Agent':
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    'Accept-Charset': 'utf-8',
+    'Accept-Language': 'en-US,en;q=0.9',
+  };
+
+  if (apiToken) {
+    headers['Authorization'] = `Bearer ${apiToken}`;
+  }
+
+  const response = await fetch(exportUrl, {
+    headers,
+    redirect: 'follow',
+  });
+
+  if (!response.ok) {
+    const statusText = response.statusText || 'Unknown error';
+    throw new Error(
+      `Failed to fetch Google Doc (${response.status} ${statusText}): ${exportUrl}`
+    );
+  }
+
+  const rawContent = await response.text();
+
+  // Decode HTML entities to unicode for text-based formats
+  const content =
+    format === 'html' || format === 'txt' || format === 'md'
+      ? he.decode(rawContent)
+      : rawContent;
+
+  return {
+    content,
+    format,
+    documentId,
+    exportUrl,
+  };
+}
+
+/**
+ * Fetch a Google Docs document and convert to Markdown.
+ *
+ * Uses the HTML export format internally, then converts to Markdown
+ * using the existing web-capture HTML-to-Markdown pipeline.
+ *
+ * @param {string} url - Google Docs URL
+ * @param {Object} [options] - Options
+ * @param {string} [options.apiToken] - API token for private documents
+ * @returns {Promise<{markdown: string, documentId: string, exportUrl: string}>}
+ */
+export async function fetchGoogleDocAsMarkdown(url, options = {}) {
+  const { apiToken } = options;
+
+  // Always fetch as HTML first, then convert to Markdown
+  const result = await fetchGoogleDoc(url, {
+    format: 'html',
+    apiToken,
+  });
+
+  const markdown = convertHtmlToMarkdown(result.content, result.exportUrl);
+
+  return {
+    markdown,
+    documentId: result.documentId,
+    exportUrl: result.exportUrl,
+  };
+}
+
+/**
+ * Extract base64 data URI images from HTML content.
+ *
+ * Google Docs HTML exports embed images as base64 data URIs.
+ * This function extracts them and replaces with local file paths.
+ *
+ * @param {string} html - HTML content with data URI images
+ * @returns {{html: string, images: Array<{filename: string, data: Buffer, mimeType: string}>}}
+ */
+export function extractBase64Images(html) {
+  const images = [];
+  let idx = 1;
+
+  const updatedHtml = html.replace(
+    /(<img\s[^>]*src=")data:image\/(png|jpeg|jpg|gif|webp|svg\+xml);base64,([^"]+)(")/gi,
+    (match, prefix, mimeExt, base64Data, suffix) => {
+      const ext =
+        mimeExt === 'jpeg' ? 'jpg' : mimeExt === 'svg+xml' ? 'svg' : mimeExt;
+      const filename = `image-${String(idx).padStart(2, '0')}.${ext}`;
+      const mimeType = `image/${mimeExt}`;
+      images.push({
+        filename,
+        data: Buffer.from(base64Data, 'base64'),
+        mimeType,
+      });
+      idx++;
+      return `${prefix}images/${filename}${suffix}`;
+    }
+  );
+
+  return { html: updatedHtml, images };
+}
+
+/**
+ * Fetch a Google Docs document as a ZIP archive with images.
+ *
+ * Fetches the document as HTML, extracts embedded base64 images,
+ * and bundles everything into a ZIP archive containing:
+ * - article.md (Markdown version)
+ * - article.html (HTML version with local image paths)
+ * - images/ (extracted images)
+ *
+ * @param {string} url - Google Docs URL
+ * @param {Object} [options] - Options
+ * @param {string} [options.apiToken] - API token for private documents
+ * @returns {Promise<{documentId: string, exportUrl: string, createArchive: function}>}
+ */
+export async function fetchGoogleDocAsArchive(url, options = {}) {
+  const { apiToken } = options;
+
+  // Fetch HTML with embedded base64 images
+  const result = await fetchGoogleDoc(url, { format: 'html', apiToken });
+
+  // Extract base64 images from HTML
+  const { html: localHtml, images } = extractBase64Images(result.content);
+
+  // Convert the localized HTML to Markdown
+  const markdown = convertHtmlToMarkdown(localHtml, result.exportUrl);
+
+  return {
+    documentId: result.documentId,
+    exportUrl: result.exportUrl,
+    html: localHtml,
+    markdown,
+    images,
+  };
+}
+
+/**
+ * Express handler for the /gdocs API endpoint.
+ *
+ * Query parameters:
+ * - url (required): Google Docs URL
+ * - format (optional): Export format (html, markdown, md, txt, pdf, docx)
+ * - apiToken (optional): API token for private documents
+ *
+ * The API token can also be provided via:
+ * - Authorization header: Bearer <token>
+ * - X-Api-Token header: <token>
+ *
+ * @param {import('express').Request} req
+ * @param {import('express').Response} res
+ */
+export async function gdocsHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+
+  if (!isGoogleDocsUrl(url)) {
+    return res.status(400).send('URL is not a Google Docs document URL');
+  }
+
+  // Resolve API token from query, headers, or environment
+  const apiToken =
+    req.query.apiToken ||
+    extractBearerToken(req.headers.authorization) ||
+    req.headers['x-api-token'] ||
+    undefined;
+
+  const format = (req.query.format || 'markdown').toLowerCase();
+
+  try {
+    if (format === 'archive' || format === 'zip') {
+      return await sendGDocsArchive(res, url, apiToken);
+    }
+
+    if (format === 'markdown' || format === 'md') {
+      const result = await fetchGoogleDocAsMarkdown(url, { apiToken });
+      return res.type('text/markdown').send(result.markdown);
+    }
+
+    return await sendGDocsExport(res, url, format, apiToken);
+  } catch (err) {
+    console.error('Google Docs capture error:', err.message);
+    return res.status(500).send(`Error capturing Google Doc: ${err.message}`);
+  }
+}
+
+async function sendGDocsArchive(res, url, apiToken) {
+  const archiveResult = await fetchGoogleDocAsArchive(url, { apiToken });
+
+  res.set('Content-Type', 'application/zip');
+  res.set(
+    'Content-Disposition',
+    `attachment; filename="gdoc-${archiveResult.documentId}.zip"`
+  );
+
+  const archive = archiver('zip', { zlib: { level: 9 } });
+  archive.pipe(res);
+  archive.append(archiveResult.markdown, { name: 'article.md' });
+  archive.append(archiveResult.html, { name: 'article.html' });
+  for (const img of archiveResult.images) {
+    archive.append(img.data, { name: `images/${img.filename}` });
+  }
+  await archive.finalize();
+}
+
+const FORMAT_CONTENT_TYPES = {
+  html: 'text/html',
+  txt: 'text/plain',
+  pdf: 'application/pdf',
+  docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+};
+
+const FORMAT_DISPOSITIONS = {
+  pdf: 'inline; filename="document.pdf"',
+  docx: 'inline; filename="document.docx"',
+};
+
+async function sendGDocsExport(res, url, format, apiToken) {
+  const result = await fetchGoogleDoc(url, { format, apiToken });
+  const contentType = FORMAT_CONTENT_TYPES[format] || 'text/html';
+  const disposition = FORMAT_DISPOSITIONS[format];
+
+  res.type(contentType);
+  if (disposition) {
+    res.set('Content-Disposition', disposition);
+  }
+  return res.send(result.content);
+}
+
+/**
+ * Extract Bearer token from Authorization header value.
+ *
+ * @param {string|undefined} authHeader - Authorization header value
+ * @returns {string|undefined} Token or undefined
+ */
+function extractBearerToken(authHeader) {
+  if (!authHeader || typeof authHeader !== 'string') {
+    return undefined;
+  }
+  const match = authHeader.match(/^Bearer\s+(.+)$/i);
+  return match ? match[1] : undefined;
+}

package/src/image.js

@@ -1,15 +1,46 @@
 import { createBrowser, getBrowserEngine } from './browser.js';
+import { dismissPopups, scrollToLoadContent } from './popups.js';
 
+/**
+ * Image screenshot handler.
+ *
+ * Query parameters:
+ *   url       (required) - URL to capture
+ *   engine    - 'puppeteer' or 'playwright'
+ *   format    - 'png' (default, lossless) or 'jpeg'
+ *   quality   - JPEG quality 0-100 (default 80, ignored for PNG)
+ *   width     - viewport width in px (default 1280)
+ *   height    - viewport height in px (default 800)
+ *   fullPage  - 'true' to capture full scrollable page (default 'false')
+ *   theme     - 'light', 'dark', or 'no-preference' (default: browser default)
+ *   dismissPopups - 'true' to auto-close cookie/consent popups before capture (default 'true')
+ */
 export async function imageHandler(req, res) {
   const url = req.query.url;
   if (!url) {
     return res.status(400).send('Missing `url` parameter');
   }
+
+  // Parse options from query parameters
+  const format = (req.query.format || 'png').toLowerCase();
+  if (format !== 'png' && format !== 'jpeg') {
+    return res.status(400).send('Invalid `format`: must be "png" or "jpeg"');
+  }
+  const quality =
+    format === 'jpeg'
+      ? Math.min(100, Math.max(0, parseInt(req.query.quality, 10) || 80))
+      : undefined;
+  const width = parseInt(req.query.width, 10) || 1280;
+  const height = parseInt(req.query.height, 10) || 800;
+  const fullPage = req.query.fullPage === 'true';
+  const theme = req.query.theme; // 'light', 'dark', or undefined
+  const shouldDismissPopups = req.query.dismissPopups !== 'false'; // default true
+
   try {
-    // Ensure URL is absolute
     const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
     const engine = getBrowserEngine(req);
-    const browser = await createBrowser(engine);
+    const browserOpts = theme ? { colorScheme: theme } : {};
+    const browser = await createBrowser(engine, browserOpts);
     try {
       const page = await browser.newPage();
       await page.setExtraHTTPHeaders({
@@ -19,17 +50,43 @@ export async function imageHandler(req, res) {
       await page.setUserAgent(
         'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
       );
-      await page.setViewport({ width: 1280, height: 800 });
+      await page.setViewport({ width, height });
       await page.goto(absoluteUrl, {
         waitUntil: 'networkidle0',
         timeout: 30000,
       });
-      // Wait for 5 seconds after page load
+
+      // Wait for dynamic content
       await new Promise((resolve) => setTimeout(resolve, 5000));
-      // Take a screenshot of just the viewport (not the full page)
-      const buffer = await page.screenshot({ type: 'png' });
-      res.set('Content-Type', 'image/png');
-      res.set('Content-Disposition', 'inline; filename="screenshot.png"');
+
+      if (fullPage) {
+        // Scroll to trigger lazy-loaded images
+        await scrollToLoadContent(page);
+      }
+
+      if (shouldDismissPopups) {
+        await dismissPopups(page);
+        // Scroll back to top after dismissing popups using browser-commander's evaluate API
+        // eslint-disable-next-line no-undef
+        await page.evaluate(() => window.scrollTo(0, 0));
+        await new Promise((resolve) => setTimeout(resolve, 500));
+      }
+
+      // Build screenshot options
+      const screenshotOpts = {
+        type: format,
+        fullPage,
+      };
+      if (format === 'jpeg') {
+        screenshotOpts.quality = quality;
+      }
+
+      const buffer = await page.screenshot(screenshotOpts);
+
+      const contentType = format === 'jpeg' ? 'image/jpeg' : 'image/png';
+      const ext = format === 'jpeg' ? 'jpg' : 'png';
+      res.set('Content-Type', contentType);
+      res.set('Content-Disposition', `inline; filename="screenshot.${ext}"`);
       res.end(buffer);
     } finally {
       await browser.close();

package/src/index.js

@@ -5,15 +5,30 @@ import { markdownHandler } from './markdown.js';
 import { imageHandler } from './image.js';
 import { streamHandler } from './stream.js';
 import { fetchHandler } from './fetch.js';
+import { archiveHandler } from './archive.js';
+import { pdfHandler } from './pdf.js';
+import { docxHandler } from './docx.js';
+import { animationHandler } from './animation.js';
+import { figuresHandler } from './figures.js';
+import { themedImageHandler } from './themed-image.js';
+import { gdocsHandler } from './gdocs.js';
 
 const app = express();
 const port = process.env.PORT || 3000;
 
+app.get('/health', (req, res) => res.json({ status: 'ok' }));
 app.get('/html', htmlHandler);
 app.get('/markdown', markdownHandler);
 app.get('/image', imageHandler);
 app.get('/stream', streamHandler);
 app.get('/fetch', fetchHandler);
+app.get('/archive', archiveHandler);
+app.get('/pdf', pdfHandler);
+app.get('/docx', docxHandler);
+app.get('/animation', animationHandler);
+app.get('/figures', figuresHandler);
+app.get('/themed-image', themedImageHandler);
+app.get('/gdocs', gdocsHandler);
 
 // Start the server if this is the main module
 const isMainModule =