@link-assistant/web-capture 1.1.2 → 1.4.2

package/src/animation.js

@@ -0,0 +1,283 @@
+/**
+ * Animation capture module (R2).
+ *
+ * Captures web animations as GIF, MP4, or WebM by taking screenshots
+ * at regular intervals and detecting when the animation loops.
+ *
+ * Supports three capture modes:
+ * - screencast: CDP-based push capture (30-60 FPS, Chromium only)
+ * - beginframe: Deterministic frame-perfect capture (Chromium only)
+ * - screenshot: Polling-based capture (3-8 FPS, cross-browser)
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/capture-animation.mjs
+ *
+ * @module animation
+ */
+
+import { createBrowser, getBrowserEngine } from './browser.js';
+// eslint-disable-next-line no-unused-vars
+import { dismissPopups, scrollToLoadContent } from './popups.js';
+
+/**
+ * Default animation capture options.
+ */
+const DEFAULTS = {
+  maxSize: 1024,
+  viewportWidth: 1920,
+  viewportHeight: 1080,
+  interval: 0,
+  fps: null,
+  speed: 1.0,
+  delay: null,
+  minFrames: 120,
+  loopTimeout: 60,
+  staticTimeout: 60,
+  similarity: 0.99,
+  crop: true,
+  cropPadding: null,
+  captureMode: 'screenshot',
+  format: 'gif',
+  extractKeyframes: false,
+  dismissPopups: true,
+};
+
+/**
+ * Compare two frame buffers for pixel similarity.
+ *
+ * @param {Buffer} frame1 - First PNG frame buffer
+ * @param {Buffer} frame2 - Second PNG frame buffer
+ * @returns {number} Similarity score 0-1
+ */
+export function compareFrames(frame1, frame2) {
+  if (!frame1 || !frame2) {
+    return 0;
+  }
+  if (frame1.length !== frame2.length) {
+    return 0;
+  }
+
+  let matchingBytes = 0;
+  const totalBytes = frame1.length;
+
+  for (let i = 0; i < totalBytes; i++) {
+    if (frame1[i] === frame2[i]) {
+      matchingBytes++;
+    }
+  }
+
+  return matchingBytes / totalBytes;
+}
+
+/**
+ * Capture animation frames from a web page.
+ *
+ * @param {string} url - URL of the page with animation
+ * @param {Object} [options] - Capture options
+ * @param {string} [options.engine] - Browser engine ('puppeteer' or 'playwright')
+ * @param {string} [options.captureMode='screenshot'] - Capture mode
+ * @param {number} [options.maxSize=1024] - Max dimension for output
+ * @param {number} [options.viewportWidth=1920] - Viewport width
+ * @param {number} [options.viewportHeight=1080] - Viewport height
+ * @param {number} [options.interval=0] - Capture interval in ms (0 = as fast as possible)
+ * @param {number} [options.minFrames=120] - Minimum frames to capture per cycle
+ * @param {number} [options.loopTimeout=60] - Max seconds to wait for loop
+ * @param {number} [options.staticTimeout=60] - Max seconds with no change
+ * @param {number} [options.similarity=0.99] - Pixel similarity threshold for loop detection
+ * @param {boolean} [options.crop=true] - Auto-crop to content
+ * @param {boolean} [options.dismissPopups=true] - Dismiss popups before capture
+ * @param {Function} [options.onFrame] - Callback(frameIndex, buffer) for each frame
+ * @param {Function} [options.onLoop] - Callback(frameIndex) when loop detected
+ * @returns {Promise<Object>} Capture result with frames array and metadata
+ */
+export async function captureAnimationFrames(url, options = {}) {
+  const opts = { ...DEFAULTS, ...options };
+  const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
+
+  const engine = opts.engine || 'puppeteer';
+  const browser = await createBrowser(engine, {});
+
+  try {
+    const page = await browser.newPage();
+    await page.setViewport({
+      width: opts.viewportWidth,
+      height: opts.viewportHeight,
+    });
+    await page.setUserAgent(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+    );
+
+    await page.goto(absoluteUrl, {
+      waitUntil: 'networkidle0',
+      timeout: 30000,
+    });
+
+    await new Promise((resolve) => setTimeout(resolve, 3000));
+
+    if (opts.dismissPopups) {
+      await dismissPopups(page);
+    }
+
+    // Capture frames using screenshot mode (cross-browser compatible)
+    const frames = [];
+    const timestamps = [];
+    const startTime = Date.now();
+    let loopDetected = false;
+    let loopFrame = -1;
+
+    // Capture first frame as reference
+    const firstFrame = await page.screenshot({ type: 'png' });
+    frames.push(firstFrame);
+    timestamps.push(Date.now() - startTime);
+
+    if (opts.onFrame) {
+      opts.onFrame(0, firstFrame);
+    }
+
+    // Capture subsequent frames
+    let lastChangeTime = Date.now();
+    let previousFrame = firstFrame;
+
+    for (let i = 1; i < opts.minFrames * 3; i++) {
+      // Check timeouts
+      const elapsed = (Date.now() - startTime) / 1000;
+      if (elapsed > opts.loopTimeout) {
+        break;
+      }
+      if ((Date.now() - lastChangeTime) / 1000 > opts.staticTimeout) {
+        break;
+      }
+
+      if (opts.interval > 0) {
+        await new Promise((resolve) => setTimeout(resolve, opts.interval));
+      }
+
+      const frame = await page.screenshot({ type: 'png' });
+      const similarity = compareFrames(frame, previousFrame);
+
+      // Check if frame changed
+      if (similarity < 0.999) {
+        lastChangeTime = Date.now();
+      }
+
+      frames.push(frame);
+      timestamps.push(Date.now() - startTime);
+
+      if (opts.onFrame) {
+        opts.onFrame(i, frame);
+      }
+
+      // Check for loop (compare with first frame)
+      if (i >= opts.minFrames) {
+        const loopSimilarity = compareFrames(frame, firstFrame);
+        if (loopSimilarity >= opts.similarity) {
+          loopDetected = true;
+          loopFrame = i;
+          if (opts.onLoop) {
+            opts.onLoop(i);
+          }
+          break;
+        }
+      }
+
+      previousFrame = frame;
+    }
+
+    // Extract key frames if requested
+    let keyframes = null;
+    if (opts.extractKeyframes && frames.length >= 3) {
+      const midIdx = Math.floor(frames.length / 2);
+      keyframes = [
+        { index: 0, buffer: frames[0] },
+        { index: midIdx, buffer: frames[midIdx] },
+        { index: frames.length - 1, buffer: frames[frames.length - 1] },
+      ];
+    }
+
+    return {
+      frames: loopDetected ? frames.slice(0, loopFrame) : frames,
+      timestamps: loopDetected ? timestamps.slice(0, loopFrame) : timestamps,
+      loopDetected,
+      loopFrame,
+      totalFrames: frames.length,
+      duration: timestamps[timestamps.length - 1],
+      keyframes,
+      width: opts.viewportWidth,
+      height: opts.viewportHeight,
+    };
+  } finally {
+    await browser.close();
+  }
+}
+
+/**
+ * Animation capture handler for Express API.
+ *
+ * Query parameters:
+ *   url            (required) - URL to capture
+ *   engine         - 'puppeteer' or 'playwright'
+ *   format         - 'gif' (default), 'png-sequence'
+ *   maxSize        - Max output dimension (default 1024)
+ *   interval       - Capture interval ms (default 0)
+ *   minFrames      - Min frames per cycle (default 120)
+ *   loopTimeout    - Max seconds for loop (default 60)
+ *   similarity     - Loop detection threshold (default 0.99)
+ *   captureMode    - 'screenshot' (default), 'screencast', 'beginframe'
+ *   extractKeyframes - 'true' to extract key frames
+ */
+export async function animationHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+
+  const format = (req.query.format || 'gif').toLowerCase();
+  if (!['gif', 'png-sequence'].includes(format)) {
+    return res
+      .status(400)
+      .send('Invalid `format`: must be "gif" or "png-sequence"');
+  }
+
+  try {
+    const engine = getBrowserEngine(req);
+    const result = await captureAnimationFrames(url, {
+      engine,
+      format,
+      maxSize: parseInt(req.query.maxSize, 10) || DEFAULTS.maxSize,
+      interval: parseInt(req.query.interval, 10) || DEFAULTS.interval,
+      minFrames: parseInt(req.query.minFrames, 10) || DEFAULTS.minFrames,
+      loopTimeout: parseInt(req.query.loopTimeout, 10) || DEFAULTS.loopTimeout,
+      similarity: parseFloat(req.query.similarity) || DEFAULTS.similarity,
+      captureMode: req.query.captureMode || DEFAULTS.captureMode,
+      extractKeyframes: req.query.extractKeyframes === 'true',
+    });
+
+    if (format === 'png-sequence') {
+      // Return as JSON with base64-encoded frames
+      res.json({
+        frames: result.frames.map((f) => f.toString('base64')),
+        timestamps: result.timestamps,
+        loopDetected: result.loopDetected,
+        loopFrame: result.loopFrame,
+        totalFrames: result.totalFrames,
+        duration: result.duration,
+      });
+    } else {
+      // Return frames info as JSON (GIF encoding requires gif-encoder-2
+      // which is an optional dependency)
+      res.json({
+        frameCount: result.frames.length,
+        loopDetected: result.loopDetected,
+        loopFrame: result.loopFrame,
+        duration: result.duration,
+        message:
+          'GIF encoding requires gif-encoder-2 package. ' +
+          'Use CLI with --format gif for full GIF output, ' +
+          'or use png-sequence format for raw frames.',
+      });
+    }
+  } catch (err) {
+    console.error(err);
+    res.status(500).send('Error capturing animation');
+  }
+}

package/src/archive.js

@@ -0,0 +1,212 @@
+/**
+ * ZIP archive handler.
+ *
+ * Downloads a web page as a self-contained ZIP archive containing:
+ * - article.md or article.html  (document with either remote or local asset links)
+ * - images/     (directory of downloaded images, when localImages=true)
+ * - css/        (directory for stylesheets, when documentFormat=html and localImages=true)
+ * - js/         (directory for scripts, when documentFormat=html and localImages=true)
+ *
+ * Query parameters:
+ *   url            (required) - URL to capture
+ *   engine         - 'puppeteer' or 'playwright'
+ *   localImages    - 'true' (default) to download images locally into the archive,
+ *                    'false' to keep original remote URLs in the document
+ *   documentFormat - 'markdown' (default) or 'html' - format of the main document
+ */
+
+import archiver from 'archiver';
+import fetch from 'node-fetch';
+import * as cheerio from 'cheerio';
+import { URL } from 'url';
+import {
+  fetchHtml,
+  convertHtmlToMarkdown,
+  convertRelativeUrls,
+} from './lib.js';
+import { retry } from './retry.js';
+
+export async function archiveHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+
+  const localImages = req.query.localImages !== 'false'; // default true
+  const documentFormat =
+    req.query.documentFormat === 'html' ? 'html' : 'markdown';
+
+  try {
+    const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
+
+    const html = await retry(() => fetchHtml(absoluteUrl), {
+      retries: 3,
+      baseDelay: 1000,
+      onRetry: (err, attempt, delay) => {
+        console.log(
+          `Retry ${attempt} fetching ${absoluteUrl} after ${delay}ms: ${err.message}`
+        );
+      },
+    });
+
+    // Collect images from the HTML
+    const $ = cheerio.load(html);
+    const images = [];
+    $('img').each(function () {
+      const src = $(this).attr('src');
+      if (src && !src.startsWith('data:') && !src.startsWith('blob:')) {
+        try {
+          const imgUrl = new URL(src, absoluteUrl).href;
+          images.push(imgUrl);
+        } catch {
+          /* skip invalid URLs */
+        }
+      }
+    });
+
+    // Deduplicate
+    const uniqueImages = [...new Set(images)];
+
+    // Set up the ZIP archive stream
+    res.set('Content-Type', 'application/zip');
+    const hostname = new URL(absoluteUrl).hostname.replace(/\./g, '-');
+    res.set(
+      'Content-Disposition',
+      `attachment; filename="${hostname}-archive.zip"`
+    );
+
+    const archive = archiver('zip', { zlib: { level: 9 } });
+    archive.pipe(res);
+
+    // Build the image map for local downloads
+    const imageMap = new Map();
+    if (localImages && uniqueImages.length > 0) {
+      let idx = 1;
+      for (const imgUrl of uniqueImages) {
+        const ext = guessImageExtension(imgUrl);
+        const filename = `image-${idx}.${ext}`;
+        imageMap.set(imgUrl, `images/${filename}`);
+        idx++;
+      }
+    }
+
+    if (documentFormat === 'html') {
+      // HTML format: produce a local HTML document with assets in folders
+      let outputHtml = convertRelativeUrls(html, absoluteUrl);
+
+      // Remove scripts and non-essential elements for a clean local copy
+      const $out = cheerio.load(outputHtml);
+      $out('script, noscript').remove();
+
+      if (localImages && imageMap.size > 0) {
+        // Rewrite image URLs to local paths
+        $out('img').each(function () {
+          const src = $out(this).attr('src');
+          if (src) {
+            try {
+              const resolvedUrl = new URL(src, absoluteUrl).href;
+              if (imageMap.has(resolvedUrl)) {
+                $out(this).attr('src', imageMap.get(resolvedUrl));
+              }
+            } catch {
+              /* skip invalid URLs */
+            }
+          }
+        });
+      }
+
+      // Collect and localize CSS stylesheets
+      const cssFiles = [];
+      if (localImages) {
+        let cssIdx = 1;
+        $out('link[rel="stylesheet"]').each(function () {
+          const href = $out(this).attr('href');
+          if (href && !href.startsWith('data:')) {
+            try {
+              const cssUrl = new URL(href, absoluteUrl).href;
+              const localPath = `css/style-${cssIdx}.css`;
+              cssFiles.push({ url: cssUrl, localPath });
+              $out(this).attr('href', localPath);
+              cssIdx++;
+            } catch {
+              /* skip invalid URLs */
+            }
+          }
+        });
+      }
+
+      outputHtml = $out.html();
+      archive.append(outputHtml, { name: 'article.html' });
+
+      // Download and add CSS files
+      for (const { url: cssUrl, localPath } of cssFiles) {
+        try {
+          const cssResp = await retry(() => fetch(cssUrl), {
+            retries: 2,
+            baseDelay: 500,
+          });
+          if (cssResp.ok) {
+            const cssText = await cssResp.text();
+            archive.append(cssText, { name: localPath });
+          }
+        } catch {
+          /* skip failed CSS downloads */
+        }
+      }
+    } else {
+      // Markdown format (default)
+      let markdown = convertHtmlToMarkdown(html, absoluteUrl);
+
+      if (localImages && imageMap.size > 0) {
+        // Rewrite image URLs in markdown to local paths
+        for (const [remoteUrl, localPath] of imageMap) {
+          markdown = markdown.split(remoteUrl).join(localPath);
+        }
+      }
+
+      archive.append(markdown, { name: 'article.md' });
+    }
+
+    // Download and add images if local mode
+    if (localImages && imageMap.size > 0) {
+      for (const [imgUrl, localPath] of imageMap) {
+        try {
+          const imgResp = await retry(() => fetch(imgUrl), {
+            retries: 2,
+            baseDelay: 500,
+          });
+          if (imgResp.ok) {
+            const buffer = await imgResp.buffer();
+            archive.append(buffer, { name: localPath });
+          }
+        } catch {
+          /* skip failed image downloads */
+        }
+      }
+    }
+
+    await archive.finalize();
+  } catch (err) {
+    console.error(err);
+    if (!res.headersSent) {
+      res.status(500).send('Error creating archive');
+    }
+  }
+}
+
+function guessImageExtension(url) {
+  const pathname = new URL(url).pathname.toLowerCase();
+  if (pathname.endsWith('.jpg') || pathname.endsWith('.jpeg')) {
+    return 'jpg';
+  }
+  if (pathname.endsWith('.gif')) {
+    return 'gif';
+  }
+  if (pathname.endsWith('.webp')) {
+    return 'webp';
+  }
+  if (pathname.endsWith('.svg')) {
+    return 'svg';
+  }
+  return 'png';
+}

package/src/batch.js

@@ -0,0 +1,180 @@
+/**
+ * Batch processing and configuration module (R7).
+ *
+ * Supports processing multiple URLs from a configuration file.
+ * Configuration format matches the articles-config pattern from meta-theory.
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/articles-config.mjs
+ *
+ * @module batch
+ */
+
+import fs from 'fs';
+import { URL } from 'url';
+
+/**
+ * @typedef {Object} ArticleConfig
+ * @property {string} url - Article URL
+ * @property {string} [title] - Article title
+ * @property {string} [language] - Article language (e.g., 'en', 'ru')
+ * @property {string} [archivePath] - Local archive directory path
+ * @property {string} [markdownFile] - Output markdown filename
+ * @property {string} [screenshotLightFile] - Light theme screenshot filename
+ * @property {string} [screenshotDarkFile] - Dark theme screenshot filename
+ * @property {string} [imagesDir] - Images directory name
+ * @property {boolean} [hasLocalImages] - Whether to download images locally
+ * @property {number} [expectedFigures] - Expected number of figure images
+ * @property {string} [format] - Output format override
+ */
+
+/**
+ * @typedef {Object} BatchConfig
+ * @property {Object<string, ArticleConfig>} articles - Map of version/id to article config
+ * @property {Object} [defaults] - Default options applied to all articles
+ */
+
+/**
+ * Load batch configuration from a JSON or JavaScript file.
+ *
+ * Supports:
+ * - JSON files (.json)
+ * - JavaScript/ESM files (.mjs, .js) with default or named exports
+ *
+ * @param {string} configPath - Path to configuration file
+ * @returns {Promise<BatchConfig>} Loaded configuration
+ */
+export async function loadConfig(configPath) {
+  const ext = configPath.split('.').pop().toLowerCase();
+
+  if (ext === 'json') {
+    const content = fs.readFileSync(configPath, 'utf-8');
+    return JSON.parse(content);
+  }
+
+  if (ext === 'mjs' || ext === 'js') {
+    // Dynamic import for ESM modules
+    const fileUrl = new URL(`file://${configPath}`);
+    const module = await import(fileUrl.href);
+
+    // Support both default export and named exports
+    if (module.default) {
+      return module.default;
+    }
+
+    // If module exports ARTICLES (matching meta-theory pattern)
+    if (module.ARTICLES) {
+      return { articles: module.ARTICLES };
+    }
+
+    return module;
+  }
+
+  throw new Error(
+    `Unsupported config format: .${ext}. Use .json, .mjs, or .js`
+  );
+}
+
+/**
+ * Get article configuration by version/id.
+ *
+ * @param {BatchConfig} config - Batch configuration
+ * @param {string} version - Article version/id
+ * @returns {ArticleConfig} Article configuration
+ */
+export function getArticle(config, version) {
+  const article = config.articles[version];
+  if (!article) {
+    throw new Error(
+      `Unknown article version: ${version}. Available: ${Object.keys(config.articles).join(', ')}`
+    );
+  }
+  // Merge defaults
+  return { ...config.defaults, ...article };
+}
+
+/**
+ * Get all article versions from configuration.
+ *
+ * @param {BatchConfig} config - Batch configuration
+ * @returns {string[]} Array of version/id strings
+ */
+export function getAllVersions(config) {
+  return Object.keys(config.articles);
+}
+
+/**
+ * Get all article configurations.
+ *
+ * @param {BatchConfig} config - Batch configuration
+ * @returns {ArticleConfig[]} Array of article configs (with defaults merged)
+ */
+export function getAllArticles(config) {
+  return Object.values(config.articles).map((article) => ({
+    ...config.defaults,
+    ...article,
+  }));
+}
+
+/**
+ * Create a default batch configuration for a list of URLs.
+ *
+ * @param {string[]} urls - Array of URLs to process
+ * @param {Object} [defaults] - Default options for all articles
+ * @returns {BatchConfig} Generated configuration
+ */
+export function createConfigFromUrls(urls, defaults = {}) {
+  const articles = {};
+
+  urls.forEach((url, index) => {
+    const id = String(index + 1);
+    let hostname;
+    try {
+      hostname = new URL(url).hostname;
+    } catch {
+      hostname = 'article';
+    }
+
+    articles[id] = {
+      url,
+      title: `Article ${id}`,
+      archivePath: `archive/${hostname.replace(/\./g, '-')}/${id}`,
+      markdownFile: 'article.md',
+      screenshotLightFile: 'article-light.png',
+      screenshotDarkFile: 'article-dark.png',
+      imagesDir: 'images',
+      hasLocalImages: true,
+    };
+  });
+
+  return { articles, defaults };
+}
+
+/**
+ * Validate a batch configuration.
+ *
+ * @param {BatchConfig} config - Configuration to validate
+ * @returns {Object} Validation result with {valid, errors}
+ */
+export function validateConfig(config) {
+  const errors = [];
+
+  if (!config.articles || typeof config.articles !== 'object') {
+    errors.push('Configuration must have an "articles" object');
+    return { valid: false, errors };
+  }
+
+  for (const [id, article] of Object.entries(config.articles)) {
+    if (!article.url) {
+      errors.push(`Article "${id}" missing required "url" field`);
+    } else {
+      try {
+        new URL(article.url);
+      } catch {
+        errors.push(`Article "${id}" has invalid URL: ${article.url}`);
+      }
+    }
+  }
+
+  return { valid: errors.length === 0, errors };
+}