@link-assistant/web-capture 1.1.2 → 1.7.1

package/src/popups.js

@@ -0,0 +1,171 @@
+/* eslint-disable no-undef */
+/**
+ * Popup/modal/cookie banner detection and auto-close utilities.
+ *
+ * Handles multiple types of overlays including:
+ * - Google Funding Choices (FC) consent dialogs
+ * - Habr cookie banners (tm-cookie-banner)
+ * - Generic modals, overlays, and notification popups
+ *
+ * Based on best practices from https://github.com/link-foundation/meta-theory
+ */
+
+/**
+ * Close any popup overlays/modals on a page adapter (from browser.js).
+ * Should be called after page content has loaded and before taking screenshots.
+ *
+ * Uses browser-commander APIs:
+ * - page.onDialog()   – dismiss browser-level alert/confirm dialogs (v0.7.0+)
+ * - page.keyboard     – press Escape to dismiss remaining popups (v0.7.0+)
+ * - page.evaluate()   – run JS in the browser context to click close buttons
+ *
+ * @param {Object} page - A PageAdapter from browser.js
+ */
+export async function dismissPopups(page) {
+  // Handle browser-level dialog events (alert, confirm, prompt) using
+  // browser-commander's dialog API (v0.7.0+)
+  page.onDialog(async (dialog) => {
+    try {
+      await dialog.dismiss();
+    } catch {
+      /* ignore */
+    }
+  });
+
+  // Try to click the Google Funding Choices "Consent" button first
+  const fcConsentClicked = await page.evaluate(() => {
+    const consentBtn = document.querySelector('.fc-cta-consent');
+    if (consentBtn) {
+      try {
+        consentBtn.click();
+        return true;
+      } catch {
+        return false;
+      }
+    }
+    return false;
+  });
+
+  if (fcConsentClicked) {
+    await sleep(page, 1000);
+  }
+
+  // Click all known close/dismiss buttons and remove overlay elements
+  await page.evaluate(() => {
+    const closeSelectors = [
+      // Google Funding Choices (FC) consent dialog
+      '.fc-cta-consent',
+      '.fc-close',
+      '.fc-dismiss-button',
+      // Cookie consent
+      '.tm-cookie-banner__close',
+      '.cookie-banner__close',
+      '[data-test-id="cookie-banner-close"]',
+      // Consent popup buttons
+      '.consent-popup__close',
+      '.consent__close',
+      'button[data-testid="consent-close"]',
+      'button[data-testid="consent-reject"]',
+      // Generic close buttons
+      '.tm-popup__close',
+      '.popup__close',
+      '.modal__close',
+      '.overlay__close',
+      '[aria-label="Close"]',
+      '[aria-label="Закрыть"]',
+      '.tm-base-modal__close',
+      '.tm-notification__close',
+    ];
+
+    for (const selector of closeSelectors) {
+      const els = document.querySelectorAll(selector);
+      for (const el of els) {
+        try {
+          el.click();
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+
+    // Remove Google FC consent dialog elements from DOM
+    for (const sel of ['.fc-consent-root', '.fc-dialog-overlay']) {
+      const el = document.querySelector(sel);
+      if (el) {
+        try {
+          el.remove();
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+
+    // Hide fixed-position overlays that cover content
+    const allEls = document.querySelectorAll('*');
+    for (const el of allEls) {
+      const style = getComputedStyle(el);
+      if (style.position === 'fixed' && el.offsetParent !== null) {
+        const rect = el.getBoundingClientRect();
+        if (
+          rect.height > 200 ||
+          (el.className &&
+            el.className.match &&
+            el.className.match(
+              /popup|modal|overlay|banner|cookie|notification|consent|fc-/i
+            ))
+        ) {
+          el.style.display = 'none';
+        }
+      }
+    }
+  });
+
+  await sleep(page, 500);
+
+  // Press Escape to dismiss remaining popups using browser-commander's keyboard API (v0.7.0+)
+  try {
+    await page.keyboard.press('Escape');
+    await sleep(page, 300);
+  } catch {
+    /* ignore */
+  }
+}
+
+/**
+ * Scroll through the page to trigger lazy-loaded content.
+ * Makes multiple passes for media-heavy pages.
+ *
+ * @param {Object} page - PageAdapter from browser.js
+ * @param {Object} options
+ * @param {number} options.passes - Number of scroll passes (default: 2)
+ * @param {number} options.stepDelay - Delay between scroll steps in ms (default: 100)
+ * @param {number} options.passDelay - Delay between passes in ms (default: 1000)
+ */
+export async function scrollToLoadContent(page, options = {}) {
+  const { passes = 2, stepDelay = 100, passDelay = 1000 } = options;
+
+  for (let pass = 0; pass < passes; pass++) {
+    await page.evaluate(
+      async ({ delay }) => {
+        const scrollHeight = document.documentElement.scrollHeight;
+        const viewportHeight = window.innerHeight;
+        const scrollSteps = Math.ceil(scrollHeight / viewportHeight);
+        for (let i = 0; i < scrollSteps; i++) {
+          window.scrollTo(0, i * viewportHeight);
+          await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+        window.scrollTo(0, 0);
+      },
+      { delay: stepDelay }
+    );
+    await sleep(page, passDelay);
+  }
+
+  // Extra wait for images to finish loading
+  await sleep(page, 2000);
+}
+
+/** Portable sleep using standard Promise-based timeout */
+async function sleep(_page, ms) {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}

package/src/postprocess.js

@@ -0,0 +1,263 @@
+/**
+ * Markdown post-processing pipeline.
+ *
+ * Applies a series of text transformations to improve markdown quality:
+ * - Unicode normalization (non-breaking spaces, curly quotes, dashes)
+ * - LaTeX formula spacing fixes for GitHub rendering
+ * - Bold formatting cleanup
+ * - Percent sign fix for GitHub KaTeX
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/download-article.mjs
+ *
+ * @module postprocess
+ */
+
+/**
+ * Apply all post-processing transformations to markdown text.
+ *
+ * @param {string} markdown - Raw markdown text
+ * @param {Object} [options] - Processing options
+ * @param {boolean} [options.normalizeUnicode=true] - Normalize unicode characters
+ * @param {boolean} [options.fixLatexSpacing=true] - Fix spacing around LaTeX formulas
+ * @param {boolean} [options.fixBoldFormatting=true] - Clean up bold formatting artifacts
+ * @param {boolean} [options.fixPercentSign=true] - Fix percent signs in formulas
+ * @returns {string} Post-processed markdown
+ */
+export function postProcessMarkdown(markdown, options = {}) {
+  const {
+    normalizeUnicode = true,
+    fixLatexSpacing = true,
+    fixBoldFormatting = true,
+    fixPercentSign = true,
+  } = options;
+
+  let result = markdown;
+
+  if (normalizeUnicode) {
+    result = applyUnicodeNormalization(result);
+  }
+
+  if (fixLatexSpacing) {
+    result = applyLatexSpacingFixes(result);
+  }
+
+  if (fixPercentSign) {
+    result = applyPercentSignFix(result);
+  }
+
+  if (fixBoldFormatting) {
+    result = applyBoldFormattingFixes(result);
+  }
+
+  // Fix double spaces (but not in code blocks)
+  result = result.replace(/([^\n`]) +/g, (match, char) => `${char} `);
+
+  // Clean up extra spaces around em-dashes
+  result = result.replace(/\s+—\s+/g, ' — ');
+
+  // Fix stray standalone $ signs on their own line
+  result = result.replace(/^\$\s*$/gm, '');
+
+  return result;
+}
+
+/**
+ * Normalize unicode characters for consistent rendering.
+ *
+ * @param {string} text - Input text
+ * @returns {string} Normalized text
+ */
+export function applyUnicodeNormalization(text) {
+  let result = text;
+
+  // Preserve non-breaking spaces as   entities for clear marking
+  result = result.replace(/\u00A0/g, ' ');
+
+  // Normalize curly quotes to straight quotes
+  result = result.replace(/[\u2018\u2019]/g, "'");
+  result = result.replace(/[\u201C\u201D]/g, '"');
+
+  // Normalize em-dash and en-dash
+  result = result.replace(/\u2014/g, ' \u2014 '); // em-dash with spaces
+  result = result.replace(/\u2013/g, '-'); // en-dash to hyphen
+
+  // Normalize ellipsis
+  result = result.replace(/\u2026/g, '...');
+
+  return result;
+}
+
+/**
+ * Fix spacing around inline LaTeX formulas for GitHub rendering.
+ * Uses a line-by-line token-based approach to correctly identify
+ * opening/closing $ delimiters.
+ *
+ * @param {string} text - Input text
+ * @returns {string} Text with fixed formula spacing
+ */
+export function applyLatexSpacingFixes(text) {
+  return text
+    .split('\n')
+    .map((line) => {
+      // Skip block formula lines ($$...$$) and blockquote block formulas
+      const trimmedLine = line.replace(/^>\s*/, '');
+      if (trimmedLine.startsWith('$$') && trimmedLine.endsWith('$$')) {
+        return line;
+      }
+
+      // Find all inline formula spans by tracking $ delimiters
+      const formulas = [];
+      let i = 0;
+      while (i < line.length) {
+        if (line[i] === '$' && (i === 0 || line[i - 1] !== '\\')) {
+          // Skip $$ block delimiters
+          if (line[i + 1] === '$') {
+            i += 2;
+            continue;
+          }
+          // Found opening $, find closing $
+          const start = i;
+          i++;
+          while (i < line.length && (line[i] !== '$' || line[i - 1] === '\\')) {
+            i++;
+          }
+          if (i < line.length) {
+            formulas.push({ start, end: i });
+            i++;
+          }
+        } else {
+          i++;
+        }
+      }
+
+      if (formulas.length === 0) {
+        return line;
+      }
+
+      // Build the line with fixes applied
+      let fixed = '';
+      let pos = 0;
+      for (const f of formulas) {
+        fixed += line.substring(pos, f.start);
+
+        const rawInner = line.substring(f.start + 1, f.end);
+        const inner = rawInner.trim();
+
+        // Add space before formula if preceded by word char, comma, colon, etc.
+        if (
+          fixed.length > 0 &&
+          /[a-zA-Z\u0430-\u044F\u0410-\u042F\u0451\u0401,:;\u00BB)\]]$/.test(
+            fixed
+          )
+        ) {
+          fixed += ' ';
+        }
+
+        fixed += `$${inner}$`;
+
+        // Add space after formula if followed by word character
+        const afterPos = f.end + 1;
+        if (
+          afterPos < line.length &&
+          /^[a-zA-Z\u0430-\u044F\u0410-\u042F\u0451\u0401]/.test(line[afterPos])
+        ) {
+          fixed += ' ';
+        }
+
+        pos = f.end + 1;
+      }
+      fixed += line.substring(pos);
+
+      return fixed;
+    })
+    .join('\n');
+}
+
+/**
+ * Fix percent sign in inline formulas for GitHub KaTeX rendering.
+ * GitHub's KaTeX treats % as a LaTeX comment character.
+ * Workaround: use \\% which GitHub's preprocessor converts to \%.
+ *
+ * @param {string} text - Input text
+ * @returns {string} Text with fixed percent signs
+ */
+export function applyPercentSignFix(text) {
+  let result = text;
+  result = result.replace(/\$(\d+)\\+%\$/g, '$$$1\\\\%$$');
+  result = result.replace(/\$(\d+)\\text\{%\}\$/g, '$$$1\\\\%$$');
+  return result;
+}
+
+/**
+ * Clean up bold formatting artifacts from HTML-to-markdown conversion.
+ *
+ * @param {string} text - Input text
+ * @returns {string} Text with cleaned bold formatting
+ */
+export function applyBoldFormattingFixes(text) {
+  let result = text;
+
+  // Remove empty bold markers
+  result = result.replace(/(\S)\*\*[^\S\n]*\*\*(\S)/g, '$1 $2');
+  result = result.replace(/\*\*[^\S\n]*\*\*/g, '');
+
+  // Fix bold marker spacing: trim content inside **...**
+  result = result
+    .split('\n')
+    .map((line) => {
+      const parts = [];
+      let lastIndex = 0;
+      const boldRegex = /\*\*(.+?)\*\*/g;
+      let m;
+      while ((m = boldRegex.exec(line)) !== null) {
+        parts.push({
+          type: 'text',
+          content: line.substring(lastIndex, m.index),
+        });
+        parts.push({ type: 'bold', content: m[1].trim() });
+        lastIndex = m.index + m[0].length;
+      }
+      parts.push({
+        type: 'text',
+        content: line.substring(lastIndex),
+      });
+
+      if (parts.filter((p) => p.type === 'bold').length === 0) {
+        return line;
+      }
+
+      let rebuilt = '';
+      for (let i = 0; i < parts.length; i++) {
+        const part = parts[i];
+        if (part.type === 'bold') {
+          if (!part.content) {
+            continue;
+          }
+          if (
+            rebuilt.length > 0 &&
+            /[a-zA-Z\u0430-\u044F\u0410-\u042F\u0451\u04010-9).]$/.test(rebuilt)
+          ) {
+            rebuilt += ' ';
+          }
+          rebuilt += `**${part.content}**`;
+          const nextPart = parts[i + 1];
+          if (
+            nextPart &&
+            nextPart.content &&
+            /^[a-zA-Z\u0430-\u044F\u0410-\u042F\u0451\u0401[(]/.test(
+              nextPart.content
+            )
+          ) {
+            rebuilt += ' ';
+          }
+        } else {
+          rebuilt += part.content;
+        }
+      }
+      return rebuilt;
+    })
+    .join('\n');
+
+  return result;
+}

package/src/retry.js

@@ -0,0 +1,48 @@
+/**
+ * Retry utility with exponential backoff.
+ *
+ * Used for network-dependent operations (fetching HTML, downloading images, etc.)
+ * to handle transient failures gracefully.
+ *
+ * @param {Function} fn - Async function to retry
+ * @param {Object} options
+ * @param {number} options.retries - Maximum number of retries (default: 3)
+ * @param {number} options.baseDelay - Initial delay in ms (default: 1000)
+ * @param {number} options.maxDelay - Maximum delay in ms (default: 30000)
+ * @param {number} options.factor - Exponential backoff factor (default: 2)
+ * @param {Function} options.onRetry - Optional callback(error, attempt) called before each retry
+ * @returns {Promise} Result of the function call
+ */
+export async function retry(fn, options = {}) {
+  const {
+    retries = 3,
+    baseDelay = 1000,
+    maxDelay = 30000,
+    factor = 2,
+    onRetry,
+  } = options;
+
+  let lastError;
+
+  for (let attempt = 0; attempt <= retries; attempt++) {
+    try {
+      return await fn(attempt);
+    } catch (error) {
+      lastError = error;
+
+      if (attempt >= retries) {
+        break;
+      }
+
+      const delay = Math.min(baseDelay * Math.pow(factor, attempt), maxDelay);
+
+      if (onRetry) {
+        onRetry(error, attempt + 1, delay);
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+
+  throw lastError;
+}

package/src/themed-image.js

@@ -0,0 +1,167 @@
+/**
+ * Dual-themed screenshot capture module (R3).
+ *
+ * Captures screenshots in both light and dark themes in a single operation.
+ * Uses separate browser contexts for reliable colorScheme application.
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/download.mjs
+ *
+ * @module themed-image
+ */
+
+import { createBrowser, getBrowserEngine } from './browser.js';
+import { dismissPopups, scrollToLoadContent } from './popups.js';
+
+/**
+ * Capture a single themed screenshot.
+ *
+ * @param {Object} browser - Browser instance from createBrowser
+ * @param {string} url - URL to capture
+ * @param {string} theme - 'light' or 'dark'
+ * @param {Object} [options] - Screenshot options
+ * @param {number} [options.width=1920] - Viewport width
+ * @param {number} [options.height=1080] - Viewport height
+ * @param {boolean} [options.fullPage=true] - Full page capture
+ * @param {boolean} [options.dismissPopups=true] - Dismiss popups
+ * @returns {Promise<Buffer>} Screenshot buffer
+ */
+async function captureThemedScreenshot(browser, url, theme, options = {}) {
+  const {
+    width = 1920,
+    height = 1080,
+    fullPage = true,
+    dismissPopups: shouldDismissPopups = true,
+  } = options;
+
+  // Create browser with theme-specific color scheme
+  const themeOpts = { colorScheme: theme };
+  const themedBrowser = await createBrowser(
+    browser.type || 'puppeteer',
+    themeOpts
+  );
+
+  try {
+    const page = await themedBrowser.newPage();
+    await page.setViewport({ width, height });
+    await page.setUserAgent(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+    );
+
+    await page.goto(url, {
+      waitUntil: 'networkidle0',
+      timeout: 30000,
+    });
+
+    await new Promise((resolve) => setTimeout(resolve, 5000));
+
+    if (fullPage) {
+      await scrollToLoadContent(page);
+    }
+
+    if (shouldDismissPopups) {
+      await dismissPopups(page);
+      // eslint-disable-next-line no-undef
+      await page.evaluate(() => window.scrollTo(0, 0));
+      await new Promise((resolve) => setTimeout(resolve, 500));
+    }
+
+    const buffer = await page.screenshot({
+      type: 'png',
+      fullPage,
+    });
+
+    return buffer;
+  } finally {
+    await themedBrowser.close();
+  }
+}
+
+/**
+ * Capture screenshots in both light and dark themes.
+ *
+ * @param {string} url - URL to capture
+ * @param {Object} [options] - Options
+ * @param {string} [options.engine='puppeteer'] - Browser engine
+ * @param {number} [options.width=1920] - Viewport width
+ * @param {number} [options.height=1080] - Viewport height
+ * @param {boolean} [options.fullPage=true] - Full page capture
+ * @param {boolean} [options.dismissPopups=true] - Dismiss popups
+ * @returns {Promise<Object>} Result with light and dark screenshot buffers
+ */
+export async function captureDualThemeScreenshots(url, options = {}) {
+  const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
+  const engine = options.engine || 'puppeteer';
+
+  // Capture light theme
+  const lightBuffer = await captureThemedScreenshot(
+    { type: engine },
+    absoluteUrl,
+    'light',
+    options
+  );
+
+  // Capture dark theme
+  const darkBuffer = await captureThemedScreenshot(
+    { type: engine },
+    absoluteUrl,
+    'dark',
+    options
+  );
+
+  return {
+    light: lightBuffer,
+    dark: darkBuffer,
+    url: absoluteUrl,
+    width: options.width || 1920,
+    height: options.height || 1080,
+  };
+}
+
+/**
+ * Dual-themed screenshot handler for Express API.
+ *
+ * Query parameters:
+ *   url        (required) - URL to capture
+ *   engine     - 'puppeteer' or 'playwright'
+ *   width      - viewport width (default 1920)
+ *   height     - viewport height (default 1080)
+ *   fullPage   - 'true' (default) to capture full page
+ *   dismissPopups - 'true' (default) to dismiss popups
+ */
+export async function themedImageHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+
+  try {
+    const engine = getBrowserEngine(req);
+    const width = parseInt(req.query.width, 10) || 1920;
+    const height = parseInt(req.query.height, 10) || 1080;
+    const fullPage = req.query.fullPage !== 'false';
+    const shouldDismissPopups = req.query.dismissPopups !== 'false';
+
+    const result = await captureDualThemeScreenshots(url, {
+      engine,
+      width,
+      height,
+      fullPage,
+      dismissPopups: shouldDismissPopups,
+    });
+
+    // Return as JSON with base64-encoded images
+    res.json({
+      url: result.url,
+      width: result.width,
+      height: result.height,
+      light: result.light.toString('base64'),
+      dark: result.dark.toString('base64'),
+      lightSize: result.light.length,
+      darkSize: result.dark.length,
+    });
+  } catch (err) {
+    console.error(err);
+    res.status(500).send('Error capturing themed screenshots');
+  }
+}