@link-assistant/web-capture 1.1.2 → 1.7.1

package/src/image.js

@@ -1,15 +1,46 @@
 import { createBrowser, getBrowserEngine } from './browser.js';
+import { dismissPopups, scrollToLoadContent } from './popups.js';
 
+/**
+ * Image screenshot handler.
+ *
+ * Query parameters:
+ *   url       (required) - URL to capture
+ *   engine    - 'puppeteer' or 'playwright'
+ *   format    - 'png' (default, lossless) or 'jpeg'
+ *   quality   - JPEG quality 0-100 (default 80, ignored for PNG)
+ *   width     - viewport width in px (default 1280)
+ *   height    - viewport height in px (default 800)
+ *   fullPage  - 'true' to capture full scrollable page (default 'false')
+ *   theme     - 'light', 'dark', or 'no-preference' (default: browser default)
+ *   dismissPopups - 'true' to auto-close cookie/consent popups before capture (default 'true')
+ */
 export async function imageHandler(req, res) {
   const url = req.query.url;
   if (!url) {
     return res.status(400).send('Missing `url` parameter');
   }
+
+  // Parse options from query parameters
+  const format = (req.query.format || 'png').toLowerCase();
+  if (format !== 'png' && format !== 'jpeg') {
+    return res.status(400).send('Invalid `format`: must be "png" or "jpeg"');
+  }
+  const quality =
+    format === 'jpeg'
+      ? Math.min(100, Math.max(0, parseInt(req.query.quality, 10) || 80))
+      : undefined;
+  const width = parseInt(req.query.width, 10) || 1280;
+  const height = parseInt(req.query.height, 10) || 800;
+  const fullPage = req.query.fullPage === 'true';
+  const theme = req.query.theme; // 'light', 'dark', or undefined
+  const shouldDismissPopups = req.query.dismissPopups !== 'false'; // default true
+
   try {
-    // Ensure URL is absolute
     const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
     const engine = getBrowserEngine(req);
-    const browser = await createBrowser(engine);
+    const browserOpts = theme ? { colorScheme: theme } : {};
+    const browser = await createBrowser(engine, browserOpts);
     try {
       const page = await browser.newPage();
       await page.setExtraHTTPHeaders({
@@ -19,17 +50,43 @@ export async function imageHandler(req, res) {
       await page.setUserAgent(
         'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
       );
-      await page.setViewport({ width: 1280, height: 800 });
+      await page.setViewport({ width, height });
       await page.goto(absoluteUrl, {
         waitUntil: 'networkidle0',
         timeout: 30000,
       });
-      // Wait for 5 seconds after page load
+
+      // Wait for dynamic content
       await new Promise((resolve) => setTimeout(resolve, 5000));
-      // Take a screenshot of just the viewport (not the full page)
-      const buffer = await page.screenshot({ type: 'png' });
-      res.set('Content-Type', 'image/png');
-      res.set('Content-Disposition', 'inline; filename="screenshot.png"');
+
+      if (fullPage) {
+        // Scroll to trigger lazy-loaded images
+        await scrollToLoadContent(page);
+      }
+
+      if (shouldDismissPopups) {
+        await dismissPopups(page);
+        // Scroll back to top after dismissing popups using browser-commander's evaluate API
+        // eslint-disable-next-line no-undef
+        await page.evaluate(() => window.scrollTo(0, 0));
+        await new Promise((resolve) => setTimeout(resolve, 500));
+      }
+
+      // Build screenshot options
+      const screenshotOpts = {
+        type: format,
+        fullPage,
+      };
+      if (format === 'jpeg') {
+        screenshotOpts.quality = quality;
+      }
+
+      const buffer = await page.screenshot(screenshotOpts);
+
+      const contentType = format === 'jpeg' ? 'image/jpeg' : 'image/png';
+      const ext = format === 'jpeg' ? 'jpg' : 'png';
+      res.set('Content-Type', contentType);
+      res.set('Content-Disposition', `inline; filename="screenshot.${ext}"`);
       res.end(buffer);
     } finally {
       await browser.close();

package/src/index.js

@@ -5,15 +5,30 @@ import { markdownHandler } from './markdown.js';
 import { imageHandler } from './image.js';
 import { streamHandler } from './stream.js';
 import { fetchHandler } from './fetch.js';
+import { archiveHandler } from './archive.js';
+import { pdfHandler } from './pdf.js';
+import { docxHandler } from './docx.js';
+import { animationHandler } from './animation.js';
+import { figuresHandler } from './figures.js';
+import { themedImageHandler } from './themed-image.js';
+import { gdocsHandler } from './gdocs.js';
 
 const app = express();
 const port = process.env.PORT || 3000;
 
+app.get('/health', (req, res) => res.json({ status: 'ok' }));
 app.get('/html', htmlHandler);
 app.get('/markdown', markdownHandler);
 app.get('/image', imageHandler);
 app.get('/stream', streamHandler);
 app.get('/fetch', fetchHandler);
+app.get('/archive', archiveHandler);
+app.get('/pdf', pdfHandler);
+app.get('/docx', docxHandler);
+app.get('/animation', animationHandler);
+app.get('/figures', figuresHandler);
+app.get('/themed-image', themedImageHandler);
+app.get('/gdocs', gdocsHandler);
 
 // Start the server if this is the main module
 const isMainModule =

package/src/latex.js

@@ -0,0 +1,127 @@
+/**
+ * LaTeX formula extraction module.
+ *
+ * Extracts LaTeX formulas from HTML content, handling multiple sources:
+ * - Habr: img.formula elements with `source` attribute
+ * - KaTeX: .katex elements with annotation[encoding="application/x-tex"]
+ * - MathJax: mjx-container elements with data-tex/data-latex attributes
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/download-article.mjs
+ */
+
+/**
+ * Extract LaTeX source from a formula image element (Habr-specific).
+ * Habr renders formulas as SVG/PNG images with class "formula".
+ * The original LaTeX source is in the `source` attribute.
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element (img.formula)
+ * @returns {string|null} LaTeX source or null
+ */
+export function extractHabrFormula($, el) {
+  const source = $(el).attr('source');
+  if (source) {
+    return source.trim();
+  }
+  const alt = $(el).attr('alt');
+  if (alt) {
+    return alt.trim();
+  }
+  return null;
+}
+
+/**
+ * Extract LaTeX from KaTeX elements.
+ * KaTeX stores the TeX source in annotation[encoding="application/x-tex"].
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element (.katex or .math)
+ * @returns {string|null} LaTeX source or null
+ */
+export function extractKatexFormula($, el) {
+  const annotation = $(el).find('annotation[encoding="application/x-tex"]');
+  if (annotation.length > 0) {
+    return annotation.text().trim();
+  }
+  const dataTex = $(el).attr('data-tex') || $(el).attr('data-latex');
+  if (dataTex) {
+    return dataTex.trim();
+  }
+  return null;
+}
+
+/**
+ * Extract LaTeX from MathJax elements.
+ * MathJax stores TeX in data-tex attribute or annotation elements.
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element (mjx-container)
+ * @returns {string|null} LaTeX source or null
+ */
+export function extractMathJaxFormula($, el) {
+  const dataTex = $(el).attr('data-tex') || $(el).attr('data-latex');
+  if (dataTex) {
+    return dataTex.trim();
+  }
+  const annotation = $(el).find('annotation[encoding="application/x-tex"]');
+  if (annotation.length > 0) {
+    return annotation.text().trim();
+  }
+  return null;
+}
+
+/**
+ * Check if an element is a formula image (Habr-specific).
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element
+ * @returns {boolean}
+ */
+export function isFormulaImage($, el) {
+  return (
+    $(el).is('img') &&
+    ($(el).hasClass('formula') ||
+      $(el).attr('source') !== undefined ||
+      ($(el).attr('class') || '').includes('formula'))
+  );
+}
+
+/**
+ * Check if an element is a math element (KaTeX, MathJax, or generic math class).
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element
+ * @returns {boolean}
+ */
+export function isMathElement($, el) {
+  const tag = (el.tagName || el.name || '').toLowerCase();
+  const classes = $(el).attr('class') || '';
+  return (
+    classes.includes('katex') ||
+    classes.includes('math') ||
+    classes.includes('MathJax') ||
+    tag === 'mjx-container'
+  );
+}
+
+/**
+ * Extract formula from any supported element type.
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element
+ * @returns {string|null} LaTeX source or null
+ */
+export function extractFormula($, el) {
+  if (isFormulaImage($, el)) {
+    return extractHabrFormula($, el);
+  }
+  const tag = (el.tagName || el.name || '').toLowerCase();
+  if (tag === 'mjx-container') {
+    return extractMathJaxFormula($, el);
+  }
+  if (isMathElement($, el)) {
+    return extractKatexFormula($, el);
+  }
+  return null;
+}

package/src/lib.js

@@ -6,6 +6,10 @@ import TurndownService from 'turndown';
 import iconv from 'iconv-lite';
 import { URL } from 'url';
 import turndownPluginGfm from 'turndown-plugin-gfm';
+import he from 'he';
+import { isFormulaImage, isMathElement, extractFormula } from './latex.js';
+import { extractMetadata } from './metadata.js';
+import { postProcessMarkdown } from './postprocess.js';
 
 export async function fetchHtml(url) {
   if (!url) {
@@ -162,7 +166,9 @@ export function convertHtmlToMarkdown(html, baseUrl) {
     style: false,
   });
   turndown.use(turndownPluginGfm.gfm);
-  return turndown.turndown($.html());
+  // Decode HTML entities to unicode after markdown conversion
+  // Preserve non-breaking spaces as   entities for clear marking
+  return he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ');
 }
 
 // Convert relative URLs to absolute URLs in HTML content
@@ -287,6 +293,227 @@ export function convertRelativeUrls(html, baseUrl) {
   return result;
 }
 
+/**
+ * Enhanced HTML to Markdown conversion with LaTeX formula extraction,
+ * metadata extraction, code language detection, and post-processing.
+ *
+ * This is the enhanced version of convertHtmlToMarkdown that handles:
+ * - LaTeX formulas from img.formula (Habr), KaTeX, MathJax
+ * - Article metadata (author, date, views, hubs, tags)
+ * - Code language detection with content-based correction
+ * - Blockquote math grouping with \displaystyle
+ * - Unicode normalization, quote straightening, dash normalization
+ * - LaTeX spacing fixes for GitHub rendering
+ *
+ * @param {string} html - HTML content
+ * @param {string} [baseUrl] - Base URL for resolving relative URLs
+ * @param {Object} [options] - Enhanced options
+ * @param {boolean} [options.extractLatex=true] - Extract LaTeX formulas
+ * @param {boolean} [options.extractMetadata=true] - Extract article metadata
+ * @param {boolean} [options.postProcess=true] - Apply post-processing pipeline
+ * @param {boolean} [options.detectCodeLanguage=true] - Detect/correct code languages
+ * @returns {Object} Result with { markdown, metadata }
+ */
+export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
+  const {
+    extractLatex = true,
+    extractMetadata: shouldExtractMetadata = true,
+    postProcess = true,
+    detectCodeLanguage = true,
+  } = options;
+
+  // Ensure all URLs are absolute before Markdown conversion
+  if (baseUrl) {
+    html = convertRelativeUrls(html, baseUrl);
+  }
+
+  const $ = cheerio.load(html);
+
+  // Extract metadata before cleaning
+  let metadata = null;
+  if (shouldExtractMetadata) {
+    metadata = extractMetadata($);
+  }
+
+  // Remove unwanted elements
+  $('style, script, noscript').remove();
+  $('*').each(function () {
+    const attribs = this.attribs || {};
+    Object.keys(attribs).forEach((attr) => {
+      if (attr.toLowerCase().startsWith('on')) {
+        $(this).removeAttr(attr);
+      }
+    });
+  });
+  $('a[href^="javascript:"]').remove();
+  $('[style]').removeAttr('style');
+
+  // Remove empty headings
+  $('h1, h2, h3, h4, h5, h6').each(function () {
+    if (!$(this).text().trim()) {
+      $(this).remove();
+    }
+  });
+
+  // Remove empty links (same logic as convertHtmlToMarkdown)
+  $('a').each(function () {
+    const directText = $(this)
+      .contents()
+      .filter(function () {
+        return this.type === 'text';
+      })
+      .text();
+    if (!directText.trim()) {
+      let allChildrenEmpty = true;
+      $(this)
+        .children()
+        .each(function () {
+          if (
+            $(this).text().trim() ||
+            (this.tagName === 'img' && $(this).attr('alt'))
+          ) {
+            allChildrenEmpty = false;
+          }
+        });
+      if (allChildrenEmpty) {
+        $(this).remove();
+      }
+    }
+  });
+  $('a').each(function () {
+    const children = $(this).children();
+    if (
+      children.length === 1 &&
+      children[0].tagName === 'img' &&
+      !$(children[0]).attr('alt')
+    ) {
+      $(this).remove();
+    }
+  });
+  $('a').each(function () {
+    if (!$(this).text().trim()) {
+      $(this).remove();
+    }
+  });
+
+  // Handle LaTeX formulas before Turndown conversion
+  if (extractLatex) {
+    // Replace Habr formula images with LaTeX text
+    $('img.formula, img[source]').each(function () {
+      if (isFormulaImage($, this)) {
+        const latex = extractFormula($, this);
+        if (latex) {
+          $(this).replaceWith(`$${latex}$`);
+        }
+      }
+    });
+
+    // Replace KaTeX/MathJax elements
+    $('.katex, .math, mjx-container, .MathJax').each(function () {
+      if (isMathElement($, this)) {
+        const latex = extractFormula($, this);
+        if (latex) {
+          $(this).replaceWith(`$${latex}$`);
+        }
+      }
+    });
+  }
+
+  // Handle code language detection/correction
+  if (detectCodeLanguage) {
+    $('pre code').each(function () {
+      const codeText = $(this).text().trim();
+      const language =
+        $(this)
+          .attr('class')
+          ?.match(/language-(\w+)/)?.[1] ||
+        $(this)
+          .attr('class')
+          ?.match(/^(\w+)$/)?.[1] ||
+        '';
+
+      // Content-based language correction
+      if (
+        language === 'matlab' &&
+        /\b(Require\s+Import|Definition|Fixpoint|Lemma|Theorem|Proof|Qed|Notation|Inductive)\b/.test(
+          codeText
+        )
+      ) {
+        $(this).removeClass(`language-${language}`).addClass('language-coq');
+      }
+    });
+  }
+
+  // Preprocess ARIA tables (same as original)
+  $('[role="table"]').each(function () {
+    const $table = $(this);
+    const $newTable = $('<table></table>');
+    const label = $table.attr('aria-label');
+    const descId = $table.attr('aria-describedby');
+    if (label) {
+      $newTable.append(`<caption>${label}</caption>`);
+    } else if (descId && $(`#${descId}`).length) {
+      $newTable.append(`<caption>${$(`#${descId}`).text()}</caption>`);
+    }
+    const rowgroups = $table.find('> [role="rowgroup"]');
+    if (rowgroups.length > 0) {
+      const $thead = $('<thead></thead>');
+      const $tbody = $('<tbody></tbody>');
+      rowgroups.each(function (i) {
+        $(this)
+          .find('> [role="row"]')
+          .each(function () {
+            const $row = $(this);
+            const $tr = $('<tr></tr>');
+            $row.children('[role="columnheader"]').each(function () {
+              $tr.append($('<th></th>').text($(this).text()));
+            });
+            $row.children('[role="cell"]').each(function () {
+              $tr.append($('<td></td>').text($(this).text()));
+            });
+            if (i === 0 && $tr.children('th').length) {
+              $thead.append($tr);
+            } else {
+              $tbody.append($tr);
+            }
+          });
+      });
+      if ($thead.children().length) {
+        $newTable.append($thead);
+      }
+      if ($tbody.children().length) {
+        $newTable.append($tbody);
+      }
+    }
+    $table.replaceWith($newTable);
+  });
+
+  // Convert to Markdown using Turndown
+  const turndown = new TurndownService({
+    headingStyle: 'atx',
+    codeBlockStyle: 'fenced',
+    emDelimiter: '*',
+    bulletListMarker: '-',
+    strongDelimiter: '**',
+    linkStyle: 'inlined',
+    linkReferenceStyle: 'full',
+    hr: '---',
+    style: false,
+  });
+  turndown.use(turndownPluginGfm.gfm);
+
+  // Decode HTML entities to unicode after markdown conversion
+  // Normalize non-breaking spaces to regular spaces in Markdown text
+  let markdown = he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ');
+
+  // Apply post-processing
+  if (postProcess) {
+    markdown = postProcessMarkdown(markdown);
+  }
+
+  return { markdown, metadata };
+}
+
 // Convert HTML content to UTF-8 if it's not already
 export function convertToUtf8(html) {
   // First, try to detect the current encoding from meta tag
@@ -328,3 +555,56 @@ export function ensureUtf8(html) {
   }
   return html;
 }
+
+const VOID_TAGS = new Set([
+  'area',
+  'base',
+  'br',
+  'col',
+  'embed',
+  'hr',
+  'img',
+  'input',
+  'link',
+  'meta',
+  'param',
+  'source',
+  'track',
+  'wbr',
+]);
+
+export function prettyPrintHtml(html) {
+  const tagRe = /(<\/?[a-zA-Z][^>]*?>)/g;
+  const parts = html.split(tagRe).filter(Boolean);
+  let indent = 0;
+  const indentStr = '  ';
+  const lines = [];
+
+  for (const part of parts) {
+    const isTag = part.startsWith('<');
+    if (!isTag) {
+      const text = part.trim();
+      if (text) {
+        lines.push(indentStr.repeat(indent) + text);
+      }
+      continue;
+    }
+    const isClosing = part.startsWith('</');
+    const tagMatch = part.match(/^<\/?([a-zA-Z][a-zA-Z0-9]*)/);
+    const tagName = tagMatch ? tagMatch[1].toLowerCase() : '';
+    const isVoid = VOID_TAGS.has(tagName);
+    const isSelfClosing = part.endsWith('/>');
+
+    if (isClosing) {
+      indent = Math.max(0, indent - 1);
+      lines.push(indentStr.repeat(indent) + part);
+    } else if (isVoid || isSelfClosing) {
+      lines.push(indentStr.repeat(indent) + part);
+    } else {
+      lines.push(indentStr.repeat(indent) + part);
+      indent++;
+    }
+  }
+
+  return `${lines.join('\n')}\n`;
+}