@link-assistant/web-capture 1.1.2 → 1.7.1

package/src/batch.js

@@ -0,0 +1,180 @@
+/**
+ * Batch processing and configuration module (R7).
+ *
+ * Supports processing multiple URLs from a configuration file.
+ * Configuration format matches the articles-config pattern from meta-theory.
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/articles-config.mjs
+ *
+ * @module batch
+ */
+
+import fs from 'fs';
+import { URL } from 'url';
+
+/**
+ * @typedef {Object} ArticleConfig
+ * @property {string} url - Article URL
+ * @property {string} [title] - Article title
+ * @property {string} [language] - Article language (e.g., 'en', 'ru')
+ * @property {string} [archivePath] - Local archive directory path
+ * @property {string} [markdownFile] - Output markdown filename
+ * @property {string} [screenshotLightFile] - Light theme screenshot filename
+ * @property {string} [screenshotDarkFile] - Dark theme screenshot filename
+ * @property {string} [imagesDir] - Images directory name
+ * @property {boolean} [hasLocalImages] - Whether to download images locally
+ * @property {number} [expectedFigures] - Expected number of figure images
+ * @property {string} [format] - Output format override
+ */
+
+/**
+ * @typedef {Object} BatchConfig
+ * @property {Object<string, ArticleConfig>} articles - Map of version/id to article config
+ * @property {Object} [defaults] - Default options applied to all articles
+ */
+
+/**
+ * Load batch configuration from a JSON or JavaScript file.
+ *
+ * Supports:
+ * - JSON files (.json)
+ * - JavaScript/ESM files (.mjs, .js) with default or named exports
+ *
+ * @param {string} configPath - Path to configuration file
+ * @returns {Promise<BatchConfig>} Loaded configuration
+ */
+export async function loadConfig(configPath) {
+  const ext = configPath.split('.').pop().toLowerCase();
+
+  if (ext === 'json') {
+    const content = fs.readFileSync(configPath, 'utf-8');
+    return JSON.parse(content);
+  }
+
+  if (ext === 'mjs' || ext === 'js') {
+    // Dynamic import for ESM modules
+    const fileUrl = new URL(`file://${configPath}`);
+    const module = await import(fileUrl.href);
+
+    // Support both default export and named exports
+    if (module.default) {
+      return module.default;
+    }
+
+    // If module exports ARTICLES (matching meta-theory pattern)
+    if (module.ARTICLES) {
+      return { articles: module.ARTICLES };
+    }
+
+    return module;
+  }
+
+  throw new Error(
+    `Unsupported config format: .${ext}. Use .json, .mjs, or .js`
+  );
+}
+
+/**
+ * Get article configuration by version/id.
+ *
+ * @param {BatchConfig} config - Batch configuration
+ * @param {string} version - Article version/id
+ * @returns {ArticleConfig} Article configuration
+ */
+export function getArticle(config, version) {
+  const article = config.articles[version];
+  if (!article) {
+    throw new Error(
+      `Unknown article version: ${version}. Available: ${Object.keys(config.articles).join(', ')}`
+    );
+  }
+  // Merge defaults
+  return { ...config.defaults, ...article };
+}
+
+/**
+ * Get all article versions from configuration.
+ *
+ * @param {BatchConfig} config - Batch configuration
+ * @returns {string[]} Array of version/id strings
+ */
+export function getAllVersions(config) {
+  return Object.keys(config.articles);
+}
+
+/**
+ * Get all article configurations.
+ *
+ * @param {BatchConfig} config - Batch configuration
+ * @returns {ArticleConfig[]} Array of article configs (with defaults merged)
+ */
+export function getAllArticles(config) {
+  return Object.values(config.articles).map((article) => ({
+    ...config.defaults,
+    ...article,
+  }));
+}
+
+/**
+ * Create a default batch configuration for a list of URLs.
+ *
+ * @param {string[]} urls - Array of URLs to process
+ * @param {Object} [defaults] - Default options for all articles
+ * @returns {BatchConfig} Generated configuration
+ */
+export function createConfigFromUrls(urls, defaults = {}) {
+  const articles = {};
+
+  urls.forEach((url, index) => {
+    const id = String(index + 1);
+    let hostname;
+    try {
+      hostname = new URL(url).hostname;
+    } catch {
+      hostname = 'article';
+    }
+
+    articles[id] = {
+      url,
+      title: `Article ${id}`,
+      archivePath: `archive/${hostname.replace(/\./g, '-')}/${id}`,
+      markdownFile: 'document.md',
+      screenshotLightFile: 'document-light.png',
+      screenshotDarkFile: 'document-dark.png',
+      imagesDir: 'images',
+      hasLocalImages: true,
+    };
+  });
+
+  return { articles, defaults };
+}
+
+/**
+ * Validate a batch configuration.
+ *
+ * @param {BatchConfig} config - Configuration to validate
+ * @returns {Object} Validation result with {valid, errors}
+ */
+export function validateConfig(config) {
+  const errors = [];
+
+  if (!config.articles || typeof config.articles !== 'object') {
+    errors.push('Configuration must have an "articles" object');
+    return { valid: false, errors };
+  }
+
+  for (const [id, article] of Object.entries(config.articles)) {
+    if (!article.url) {
+      errors.push(`Article "${id}" missing required "url" field`);
+    } else {
+      try {
+        new URL(article.url);
+      } catch {
+        errors.push(`Article "${id}" has invalid URL: ${article.url}`);
+      }
+    }
+  }
+
+  return { valid: errors.length === 0, errors };
+}

package/src/browser.js

@@ -1,17 +1,37 @@
-// Browser abstraction layer to support both Puppeteer and Playwright
-import puppeteer from 'puppeteer';
-import playwright from 'playwright';
+// Browser abstraction layer using browser-commander for all browser operations
+// See: https://github.com/link-foundation/browser-commander
+
+import {
+  launchBrowser,
+  makeBrowserCommander,
+  emulateMedia,
+} from 'browser-commander';
+import os from 'os';
+import path from 'path';
 
 /**
- * Unified browser interface that works with both Puppeteer and Playwright
- * @typedef {Object} BrowserAdapter
- * @property {Function} newPage - Create a new page
- * @property {Function} close - Close the browser
- * @property {string} type - Browser type ('puppeteer' or 'playwright')
+ * Additional Chrome args needed for headless server environments
+ * These are appended to browser-commander's default CHROME_ARGS
  */
+const SERVER_CHROME_ARGS = [
+  '--no-sandbox',
+  '--disable-setuid-sandbox',
+  '--disable-dev-shm-usage',
+];
 
 /**
- * Unified page interface
+ * Unified page interface wrapping browser-commander's makeBrowserCommander.
+ *
+ * Exposes:
+ * - setExtraHTTPHeaders / setUserAgent / setViewport / goto / content / screenshot / close
+ *   (via the underlying raw page for cross-engine compatibility)
+ * - commander.pdf()           – PDF generation (browser-commander v0.8.0)
+ * - commander.emulateMedia()  – Color scheme emulation (browser-commander v0.7.0)
+ * - commander.keyboard        – Keyboard interaction (browser-commander v0.7.0)
+ * - commander.onDialog()      – Dialog event handling (browser-commander v0.7.0)
+ * - commander.evaluate()      – Page evaluation (browser-commander)
+ * All other browser-commander APIs are available on the returned commander object.
+ *
  * @typedef {Object} PageAdapter
  * @property {Function} setExtraHTTPHeaders - Set HTTP headers
  * @property {Function} setUserAgent - Set user agent
@@ -20,158 +40,177 @@ import playwright from 'playwright';
  * @property {Function} content - Get page HTML content
  * @property {Function} screenshot - Take screenshot
  * @property {Function} close - Close the page
- * @property {Object} _page - Original page object
- * @property {string} _type - Browser type
+ * @property {Function} pdf - Generate PDF (browser-commander v0.8.0)
+ * @property {Function} emulateMedia - Color scheme emulation (browser-commander v0.7.0)
+ * @property {Object} keyboard - Keyboard interaction (browser-commander v0.7.0)
+ * @property {Function} onDialog - Dialog event handling (browser-commander v0.7.0)
+ * @property {Function} evaluate - Page evaluation
+ * @property {string} type - Browser type ('puppeteer' or 'playwright')
  */
 
 /**
- * Create a browser instance using the specified engine
+ * Create a browser instance using the specified engine.
+ * Uses browser-commander's launchBrowser + makeBrowserCommander facade.
+ *
  * @param {string} engine - 'puppeteer' or 'playwright' (defaults to puppeteer)
  * @param {Object} options - Browser launch options
- * @returns {Promise<BrowserAdapter>}
+ * @param {string} [options.colorScheme] - Color scheme: 'light', 'dark', or 'no-preference'
+ * @returns {Promise<Object>} - Browser handle with newPage() and close()
  */
 export async function createBrowser(engine = 'puppeteer', options = {}) {
   const normalizedEngine = engine.toLowerCase();
+  const engineType =
+    normalizedEngine === 'playwright' || normalizedEngine === 'pw'
+      ? 'playwright'
+      : 'puppeteer';
 
-  if (normalizedEngine === 'playwright') {
-    return createPlaywrightBrowser(options);
-  } else {
-    return createPuppeteerBrowser(options);
-  }
-}
+  const { colorScheme, ...launchOpts } = options;
 
-/**
- * Create a Puppeteer browser instance
- * @param {Object} options - Puppeteer launch options
- * @returns {Promise<BrowserAdapter>}
- */
-async function createPuppeteerBrowser(options = {}) {
-  const defaultOptions = {
-    args: [
-      '--no-sandbox',
-      '--disable-setuid-sandbox',
-      '--disable-dev-shm-usage',
-    ],
-  };
+  // Generate unique userDataDir for this session to avoid conflicts
+  const userDataDir = path.join(
+    os.tmpdir(),
+    `web-capture-${engineType}-${Date.now()}`
+  );
+
+  // Use browser-commander's launchBrowser with server-specific args
+  const { browser, page: initialPage } = await launchBrowser({
+    engine: engineType,
+    args: SERVER_CHROME_ARGS,
+    headless: true,
+    userDataDir,
+    slowMo: 0,
+    ...launchOpts,
+  });
 
-  const browser = await puppeteer.launch({ ...defaultOptions, ...options });
+  // Close the initial page since we'll create new ones via newPage()
+  await initialPage.close();
 
   return {
     async newPage() {
-      const page = await browser.newPage();
-      return createPuppeteerPageAdapter(page);
-    },
-    async close() {
-      await browser.close();
-    },
-    type: 'puppeteer',
-    _browser: browser,
-  };
-}
+      const rawPage = await browser.newPage();
 
-/**
- * Create a Playwright browser instance
- * @param {Object} options - Playwright launch options
- * @returns {Promise<BrowserAdapter>}
- */
-async function createPlaywrightBrowser(options = {}) {
-  const defaultOptions = {
-    args: [
-      '--no-sandbox',
-      '--disable-setuid-sandbox',
-      '--disable-dev-shm-usage',
-    ],
-  };
+      // Apply color scheme emulation using browser-commander's unified API (v0.7.0+)
+      if (colorScheme) {
+        try {
+          await emulateMedia({
+            page: rawPage,
+            engine: engineType,
+            colorScheme,
+          });
+        } catch {
+          /* emulateMedia not available in all environments */
+        }
+      }
 
-  // Playwright uses chromium by default
-  const browser = await playwright.chromium.launch({
-    ...defaultOptions,
-    ...options,
-  });
+      // Wrap with the full browser-commander facade for unified API access
+      const commander = makeBrowserCommander({ page: rawPage });
 
-  return {
-    async newPage() {
-      const page = await browser.newPage();
-      return createPlaywrightPageAdapter(page);
+      // Build a unified page adapter that exposes both:
+      // 1. The familiar cross-engine adapter interface (setExtraHTTPHeaders, goto, etc.)
+      // 2. All browser-commander v0.7.0+ APIs (pdf, emulateMedia, keyboard, onDialog, evaluate)
+      return createPageAdapter(rawPage, commander, engineType);
     },
     async close() {
       await browser.close();
     },
-    type: 'playwright',
-    _browser: browser,
+    type: engineType,
+    rawBrowser: browser,
   };
 }
 
 /**
- * Create a page adapter for Puppeteer
- * @param {Object} page - Puppeteer page object
+ * Create a unified page adapter from a raw page and its browser-commander facade.
+ * Provides a stable cross-engine interface plus all new browser-commander APIs.
+ *
+ * @param {Object} rawPage - Raw Playwright or Puppeteer page
+ * @param {Object} commander - browser-commander facade from makeBrowserCommander
+ * @param {string} engineType - 'puppeteer' or 'playwright'
  * @returns {PageAdapter}
  */
-function createPuppeteerPageAdapter(page) {
+function createPageAdapter(rawPage, commander, engineType) {
   return {
+    // --- Cross-engine adapter methods (stable interface) ---
+
     async setExtraHTTPHeaders(headers) {
-      await page.setExtraHTTPHeaders(headers);
+      await rawPage.setExtraHTTPHeaders(headers);
     },
     async setUserAgent(userAgent) {
-      await page.setUserAgent(userAgent);
+      if (engineType === 'playwright') {
+        // Playwright doesn't have page.setUserAgent, use extraHTTPHeaders instead
+        await rawPage.setExtraHTTPHeaders({ 'User-Agent': userAgent });
+      } else {
+        await rawPage.setUserAgent(userAgent);
+      }
     },
     async setViewport(viewport) {
-      await page.setViewport(viewport);
+      if (engineType === 'playwright') {
+        // Playwright uses setViewportSize instead of setViewport
+        await rawPage.setViewportSize(viewport);
+      } else {
+        await rawPage.setViewport(viewport);
+      }
     },
     async goto(url, options = {}) {
-      await page.goto(url, options);
+      if (engineType === 'playwright') {
+        // Convert Puppeteer waitUntil options to Playwright equivalents
+        const playwrightOptions = { ...options };
+        if (playwrightOptions.waitUntil === 'networkidle0') {
+          playwrightOptions.waitUntil = 'networkidle';
+        }
+        await rawPage.goto(url, playwrightOptions);
+      } else {
+        await rawPage.goto(url, options);
+      }
     },
     async content() {
-      return await page.content();
+      return await rawPage.content();
     },
     async screenshot(options = {}) {
-      return await page.screenshot(options);
+      return await rawPage.screenshot(options);
     },
     async close() {
-      await page.close();
+      await commander.destroy();
+      await rawPage.close();
     },
-    _page: page,
-    _type: 'puppeteer',
-  };
-}
 
-/**
- * Create a page adapter for Playwright
- * @param {Object} page - Playwright page object
- * @returns {PageAdapter}
- */
-function createPlaywrightPageAdapter(page) {
-  return {
-    async setExtraHTTPHeaders(headers) {
-      await page.setExtraHTTPHeaders(headers);
+    // --- browser-commander v0.8.0+: PDF generation ---
+    // Usage: await page.pdf({ pdfOptions: { format: 'A4', printBackground: true } })
+    async pdf(opts = {}) {
+      return await commander.pdf(opts);
     },
-    async setUserAgent(userAgent) {
-      // Playwright doesn't have page.setUserAgent, use extraHTTPHeaders instead
-      await page.setExtraHTTPHeaders({ 'User-Agent': userAgent });
+
+    // --- browser-commander v0.7.0+: Color scheme emulation ---
+    // Usage: await page.emulateMedia({ colorScheme: 'dark' })
+    async emulateMedia(opts = {}) {
+      return await commander.emulateMedia(opts);
     },
-    async setViewport(viewport) {
-      // Playwright uses setViewportSize instead of setViewport
-      await page.setViewportSize(viewport);
+
+    // --- browser-commander v0.7.0+: Keyboard interaction ---
+    // Usage: await page.keyboard.press('Escape')
+    get keyboard() {
+      return commander.keyboard;
     },
-    async goto(url, options = {}) {
-      // Convert Puppeteer waitUntil options to Playwright equivalents
-      const playwrightOptions = { ...options };
-      if (playwrightOptions.waitUntil === 'networkidle0') {
-        playwrightOptions.waitUntil = 'networkidle';
-      }
-      await page.goto(url, playwrightOptions);
+
+    // --- browser-commander v0.7.0+: Dialog event handling ---
+    // Usage: page.onDialog(async (dialog) => { await dialog.dismiss(); })
+    onDialog(handler) {
+      return commander.onDialog(handler);
     },
-    async content() {
-      return await page.content();
+    offDialog(handler) {
+      return commander.offDialog(handler);
     },
-    async screenshot(options = {}) {
-      return await page.screenshot(options);
+    clearDialogHandlers() {
+      return commander.clearDialogHandlers();
     },
-    async close() {
-      await page.close();
+
+    // --- browser-commander: Page evaluation ---
+    // Usage: await page.evaluate(() => window.scrollTo(0, 0))
+    async evaluate(fn, ...args) {
+      return await commander.evaluate({ fn, args });
     },
-    _page: page,
-    _type: 'playwright',
+
+    // Engine type identifier
+    type: engineType,
   };
 }
 

package/src/docx.js

@@ -0,0 +1,182 @@
+/**
+ * DOCX export handler.
+ *
+ * Converts a web page to a DOCX document with embedded images.
+ * Uses the page's markdown conversion and the `docx` library to build the document.
+ *
+ * Query parameters:
+ *   url    (required) - URL to convert
+ *   engine - 'puppeteer' or 'playwright' (only used for browser rendering fallback)
+ */
+
+import fetch from 'node-fetch';
+import * as cheerio from 'cheerio';
+import { URL } from 'url';
+import {
+  Document,
+  Packer,
+  Paragraph,
+  TextRun,
+  HeadingLevel,
+  ImageRun,
+  ExternalHyperlink,
+} from 'docx';
+import { fetchHtml, convertRelativeUrls } from './lib.js';
+
+export async function docxHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+
+  try {
+    const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
+    const html = await fetchHtml(absoluteUrl);
+    const absHtml = convertRelativeUrls(html, absoluteUrl);
+    const $ = cheerio.load(absHtml);
+
+    // Remove unwanted elements
+    $('style, script, noscript, nav, footer, header').remove();
+
+    // Build DOCX sections from HTML structure
+    const children = [];
+
+    // Extract title
+    const title = $('h1').first().text().trim() || $('title').text().trim();
+    if (title) {
+      children.push(
+        new Paragraph({ text: title, heading: HeadingLevel.TITLE })
+      );
+    }
+
+    // Process body content
+    const body = $('article').length ? $('article') : $('body');
+    for (const el of body.children().toArray()) {
+      const tagName = (el.tagName || '').toLowerCase();
+      const text = $(el).text().trim();
+
+      if (!text && tagName !== 'img' && tagName !== 'figure') {
+        continue;
+      }
+
+      if (tagName === 'h1') {
+        children.push(new Paragraph({ text, heading: HeadingLevel.HEADING_1 }));
+      } else if (tagName === 'h2') {
+        children.push(new Paragraph({ text, heading: HeadingLevel.HEADING_2 }));
+      } else if (tagName === 'h3') {
+        children.push(new Paragraph({ text, heading: HeadingLevel.HEADING_3 }));
+      } else if (tagName === 'h4' || tagName === 'h5' || tagName === 'h6') {
+        children.push(new Paragraph({ text, heading: HeadingLevel.HEADING_4 }));
+      } else if (tagName === 'figure' || tagName === 'img') {
+        const img = tagName === 'img' ? $(el) : $(el).find('img').first();
+        const src = img.attr('src');
+        if (src && !src.startsWith('data:')) {
+          try {
+            const imgUrl = new URL(src, absoluteUrl).href;
+            const imgResp = await fetch(imgUrl);
+            if (imgResp.ok) {
+              const buffer = await imgResp.buffer();
+              children.push(
+                new Paragraph({
+                  children: [
+                    new ImageRun({
+                      data: buffer,
+                      transformation: { width: 600, height: 400 },
+                      type: guessImageType(imgUrl),
+                    }),
+                  ],
+                })
+              );
+            }
+          } catch {
+            /* skip failed image embeds */
+          }
+        }
+        // Add caption if present
+        const caption = $(el).find('figcaption').text().trim();
+        if (caption) {
+          children.push(
+            new Paragraph({
+              children: [new TextRun({ text: caption, italics: true })],
+            })
+          );
+        }
+      } else if (tagName === 'ul' || tagName === 'ol') {
+        $(el)
+          .find('li')
+          .each(function () {
+            const liText = $(this).text().trim();
+            if (liText) {
+              children.push(
+                new Paragraph({
+                  text: liText,
+                  bullet: { level: 0 },
+                })
+              );
+            }
+          });
+      } else if (tagName === 'pre' || tagName === 'code') {
+        children.push(
+          new Paragraph({
+            children: [new TextRun({ text, font: 'Courier New', size: 20 })],
+          })
+        );
+      } else if (tagName === 'a') {
+        const href = $(el).attr('href');
+        if (href && text) {
+          children.push(
+            new Paragraph({
+              children: [
+                new ExternalHyperlink({
+                  children: [
+                    new TextRun({
+                      text,
+                      style: 'Hyperlink',
+                    }),
+                  ],
+                  link: href,
+                }),
+              ],
+            })
+          );
+        }
+      } else {
+        // Default: treat as paragraph
+        if (text) {
+          children.push(new Paragraph({ text }));
+        }
+      }
+    }
+
+    if (children.length === 0) {
+      children.push(new Paragraph({ text: 'No content extracted.' }));
+    }
+
+    const doc = new Document({
+      sections: [{ children }],
+    });
+
+    const buffer = await Packer.toBuffer(doc);
+
+    res.set(
+      'Content-Type',
+      'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
+    );
+    res.set('Content-Disposition', 'attachment; filename="page.docx"');
+    res.end(buffer);
+  } catch (err) {
+    console.error(err);
+    res.status(500).send('Error generating DOCX');
+  }
+}
+
+function guessImageType(url) {
+  const pathname = new URL(url).pathname.toLowerCase();
+  if (pathname.endsWith('.jpg') || pathname.endsWith('.jpeg')) {
+    return 'jpg';
+  }
+  if (pathname.endsWith('.gif')) {
+    return 'gif';
+  }
+  return 'png';
+}