@link-assistant/web-capture 1.1.2 → 1.4.2

package/src/latex.js

@@ -0,0 +1,127 @@
+/**
+ * LaTeX formula extraction module.
+ *
+ * Extracts LaTeX formulas from HTML content, handling multiple sources:
+ * - Habr: img.formula elements with `source` attribute
+ * - KaTeX: .katex elements with annotation[encoding="application/x-tex"]
+ * - MathJax: mjx-container elements with data-tex/data-latex attributes
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/download-article.mjs
+ */
+
+/**
+ * Extract LaTeX source from a formula image element (Habr-specific).
+ * Habr renders formulas as SVG/PNG images with class "formula".
+ * The original LaTeX source is in the `source` attribute.
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element (img.formula)
+ * @returns {string|null} LaTeX source or null
+ */
+export function extractHabrFormula($, el) {
+  const source = $(el).attr('source');
+  if (source) {
+    return source.trim();
+  }
+  const alt = $(el).attr('alt');
+  if (alt) {
+    return alt.trim();
+  }
+  return null;
+}
+
+/**
+ * Extract LaTeX from KaTeX elements.
+ * KaTeX stores the TeX source in annotation[encoding="application/x-tex"].
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element (.katex or .math)
+ * @returns {string|null} LaTeX source or null
+ */
+export function extractKatexFormula($, el) {
+  const annotation = $(el).find('annotation[encoding="application/x-tex"]');
+  if (annotation.length > 0) {
+    return annotation.text().trim();
+  }
+  const dataTex = $(el).attr('data-tex') || $(el).attr('data-latex');
+  if (dataTex) {
+    return dataTex.trim();
+  }
+  return null;
+}
+
+/**
+ * Extract LaTeX from MathJax elements.
+ * MathJax stores TeX in data-tex attribute or annotation elements.
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element (mjx-container)
+ * @returns {string|null} LaTeX source or null
+ */
+export function extractMathJaxFormula($, el) {
+  const dataTex = $(el).attr('data-tex') || $(el).attr('data-latex');
+  if (dataTex) {
+    return dataTex.trim();
+  }
+  const annotation = $(el).find('annotation[encoding="application/x-tex"]');
+  if (annotation.length > 0) {
+    return annotation.text().trim();
+  }
+  return null;
+}
+
+/**
+ * Check if an element is a formula image (Habr-specific).
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element
+ * @returns {boolean}
+ */
+export function isFormulaImage($, el) {
+  return (
+    $(el).is('img') &&
+    ($(el).hasClass('formula') ||
+      $(el).attr('source') !== undefined ||
+      ($(el).attr('class') || '').includes('formula'))
+  );
+}
+
+/**
+ * Check if an element is a math element (KaTeX, MathJax, or generic math class).
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element
+ * @returns {boolean}
+ */
+export function isMathElement($, el) {
+  const tag = (el.tagName || el.name || '').toLowerCase();
+  const classes = $(el).attr('class') || '';
+  return (
+    classes.includes('katex') ||
+    classes.includes('math') ||
+    classes.includes('MathJax') ||
+    tag === 'mjx-container'
+  );
+}
+
+/**
+ * Extract formula from any supported element type.
+ *
+ * @param {Object} $ - Cheerio instance
+ * @param {Object} el - Cheerio element
+ * @returns {string|null} LaTeX source or null
+ */
+export function extractFormula($, el) {
+  if (isFormulaImage($, el)) {
+    return extractHabrFormula($, el);
+  }
+  const tag = (el.tagName || el.name || '').toLowerCase();
+  if (tag === 'mjx-container') {
+    return extractMathJaxFormula($, el);
+  }
+  if (isMathElement($, el)) {
+    return extractKatexFormula($, el);
+  }
+  return null;
+}

package/src/lib.js

@@ -6,6 +6,10 @@ import TurndownService from 'turndown';
 import iconv from 'iconv-lite';
 import { URL } from 'url';
 import turndownPluginGfm from 'turndown-plugin-gfm';
+import he from 'he';
+import { isFormulaImage, isMathElement, extractFormula } from './latex.js';
+import { extractMetadata } from './metadata.js';
+import { postProcessMarkdown } from './postprocess.js';
 
 export async function fetchHtml(url) {
   if (!url) {
@@ -162,7 +166,8 @@ export function convertHtmlToMarkdown(html, baseUrl) {
     style: false,
   });
   turndown.use(turndownPluginGfm.gfm);
-  return turndown.turndown($.html());
+  // Decode HTML entities to unicode after markdown conversion
+  return he.decode(turndown.turndown($.html()));
 }
 
 // Convert relative URLs to absolute URLs in HTML content
@@ -287,6 +292,226 @@ export function convertRelativeUrls(html, baseUrl) {
   return result;
 }
 
+/**
+ * Enhanced HTML to Markdown conversion with LaTeX formula extraction,
+ * metadata extraction, code language detection, and post-processing.
+ *
+ * This is the enhanced version of convertHtmlToMarkdown that handles:
+ * - LaTeX formulas from img.formula (Habr), KaTeX, MathJax
+ * - Article metadata (author, date, views, hubs, tags)
+ * - Code language detection with content-based correction
+ * - Blockquote math grouping with \displaystyle
+ * - Unicode normalization, quote straightening, dash normalization
+ * - LaTeX spacing fixes for GitHub rendering
+ *
+ * @param {string} html - HTML content
+ * @param {string} [baseUrl] - Base URL for resolving relative URLs
+ * @param {Object} [options] - Enhanced options
+ * @param {boolean} [options.extractLatex=true] - Extract LaTeX formulas
+ * @param {boolean} [options.extractMetadata=true] - Extract article metadata
+ * @param {boolean} [options.postProcess=true] - Apply post-processing pipeline
+ * @param {boolean} [options.detectCodeLanguage=true] - Detect/correct code languages
+ * @returns {Object} Result with { markdown, metadata }
+ */
+export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
+  const {
+    extractLatex = true,
+    extractMetadata: shouldExtractMetadata = true,
+    postProcess = true,
+    detectCodeLanguage = true,
+  } = options;
+
+  // Ensure all URLs are absolute before Markdown conversion
+  if (baseUrl) {
+    html = convertRelativeUrls(html, baseUrl);
+  }
+
+  const $ = cheerio.load(html);
+
+  // Extract metadata before cleaning
+  let metadata = null;
+  if (shouldExtractMetadata) {
+    metadata = extractMetadata($);
+  }
+
+  // Remove unwanted elements
+  $('style, script, noscript').remove();
+  $('*').each(function () {
+    const attribs = this.attribs || {};
+    Object.keys(attribs).forEach((attr) => {
+      if (attr.toLowerCase().startsWith('on')) {
+        $(this).removeAttr(attr);
+      }
+    });
+  });
+  $('a[href^="javascript:"]').remove();
+  $('[style]').removeAttr('style');
+
+  // Remove empty headings
+  $('h1, h2, h3, h4, h5, h6').each(function () {
+    if (!$(this).text().trim()) {
+      $(this).remove();
+    }
+  });
+
+  // Remove empty links (same logic as convertHtmlToMarkdown)
+  $('a').each(function () {
+    const directText = $(this)
+      .contents()
+      .filter(function () {
+        return this.type === 'text';
+      })
+      .text();
+    if (!directText.trim()) {
+      let allChildrenEmpty = true;
+      $(this)
+        .children()
+        .each(function () {
+          if (
+            $(this).text().trim() ||
+            (this.tagName === 'img' && $(this).attr('alt'))
+          ) {
+            allChildrenEmpty = false;
+          }
+        });
+      if (allChildrenEmpty) {
+        $(this).remove();
+      }
+    }
+  });
+  $('a').each(function () {
+    const children = $(this).children();
+    if (
+      children.length === 1 &&
+      children[0].tagName === 'img' &&
+      !$(children[0]).attr('alt')
+    ) {
+      $(this).remove();
+    }
+  });
+  $('a').each(function () {
+    if (!$(this).text().trim()) {
+      $(this).remove();
+    }
+  });
+
+  // Handle LaTeX formulas before Turndown conversion
+  if (extractLatex) {
+    // Replace Habr formula images with LaTeX text
+    $('img.formula, img[source]').each(function () {
+      if (isFormulaImage($, this)) {
+        const latex = extractFormula($, this);
+        if (latex) {
+          $(this).replaceWith(`$${latex}$`);
+        }
+      }
+    });
+
+    // Replace KaTeX/MathJax elements
+    $('.katex, .math, mjx-container, .MathJax').each(function () {
+      if (isMathElement($, this)) {
+        const latex = extractFormula($, this);
+        if (latex) {
+          $(this).replaceWith(`$${latex}$`);
+        }
+      }
+    });
+  }
+
+  // Handle code language detection/correction
+  if (detectCodeLanguage) {
+    $('pre code').each(function () {
+      const codeText = $(this).text().trim();
+      const language =
+        $(this)
+          .attr('class')
+          ?.match(/language-(\w+)/)?.[1] ||
+        $(this)
+          .attr('class')
+          ?.match(/^(\w+)$/)?.[1] ||
+        '';
+
+      // Content-based language correction
+      if (
+        language === 'matlab' &&
+        /\b(Require\s+Import|Definition|Fixpoint|Lemma|Theorem|Proof|Qed|Notation|Inductive)\b/.test(
+          codeText
+        )
+      ) {
+        $(this).removeClass(`language-${language}`).addClass('language-coq');
+      }
+    });
+  }
+
+  // Preprocess ARIA tables (same as original)
+  $('[role="table"]').each(function () {
+    const $table = $(this);
+    const $newTable = $('<table></table>');
+    const label = $table.attr('aria-label');
+    const descId = $table.attr('aria-describedby');
+    if (label) {
+      $newTable.append(`<caption>${label}</caption>`);
+    } else if (descId && $(`#${descId}`).length) {
+      $newTable.append(`<caption>${$(`#${descId}`).text()}</caption>`);
+    }
+    const rowgroups = $table.find('> [role="rowgroup"]');
+    if (rowgroups.length > 0) {
+      const $thead = $('<thead></thead>');
+      const $tbody = $('<tbody></tbody>');
+      rowgroups.each(function (i) {
+        $(this)
+          .find('> [role="row"]')
+          .each(function () {
+            const $row = $(this);
+            const $tr = $('<tr></tr>');
+            $row.children('[role="columnheader"]').each(function () {
+              $tr.append($('<th></th>').text($(this).text()));
+            });
+            $row.children('[role="cell"]').each(function () {
+              $tr.append($('<td></td>').text($(this).text()));
+            });
+            if (i === 0 && $tr.children('th').length) {
+              $thead.append($tr);
+            } else {
+              $tbody.append($tr);
+            }
+          });
+      });
+      if ($thead.children().length) {
+        $newTable.append($thead);
+      }
+      if ($tbody.children().length) {
+        $newTable.append($tbody);
+      }
+    }
+    $table.replaceWith($newTable);
+  });
+
+  // Convert to Markdown using Turndown
+  const turndown = new TurndownService({
+    headingStyle: 'atx',
+    codeBlockStyle: 'fenced',
+    emDelimiter: '*',
+    bulletListMarker: '-',
+    strongDelimiter: '**',
+    linkStyle: 'inlined',
+    linkReferenceStyle: 'full',
+    hr: '---',
+    style: false,
+  });
+  turndown.use(turndownPluginGfm.gfm);
+
+  // Decode HTML entities to unicode after markdown conversion
+  let markdown = he.decode(turndown.turndown($.html()));
+
+  // Apply post-processing
+  if (postProcess) {
+    markdown = postProcessMarkdown(markdown);
+  }
+
+  return { markdown, metadata };
+}
+
 // Convert HTML content to UTF-8 if it's not already
 export function convertToUtf8(html) {
   // First, try to detect the current encoding from meta tag

package/src/localize-images.js

@@ -0,0 +1,202 @@
+/**
+ * Markdown image localization module (R5).
+ *
+ * Post-processing tool that:
+ * 1. Reads markdown files
+ * 2. Extracts all external image URLs
+ * 3. Downloads images to local directory
+ * 4. Updates markdown to reference local paths
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/download-markdown-images.mjs
+ *
+ * @module localize-images
+ */
+
+import fetch from 'node-fetch';
+import { URL } from 'url';
+import { retry } from './retry.js';
+
+/**
+ * Extract image references from markdown text.
+ *
+ * @param {string} markdownText - Markdown content
+ * @returns {Object[]} Array of {fullMatch, altText, url}
+ */
+export function extractImageReferences(markdownText) {
+  const imageRegex = /!\[([^\]]*)\]\((https?:\/\/[^)]+)\)/g;
+  const images = [];
+  let match;
+
+  while ((match = imageRegex.exec(markdownText)) !== null) {
+    images.push({
+      fullMatch: match[0],
+      altText: match[1],
+      url: match[2],
+    });
+  }
+
+  return images;
+}
+
+/**
+ * Get file extension from URL.
+ *
+ * @param {string} url - Image URL
+ * @returns {string} File extension with dot (e.g., '.png')
+ */
+export function getExtensionFromUrl(url) {
+  try {
+    const pathname = new URL(url).pathname.split('?')[0];
+    const ext = pathname.match(/\.(\w+)$/);
+    if (ext) {
+      const lower = ext[1].toLowerCase();
+      if (['png', 'jpg', 'jpeg', 'gif', 'webp', 'svg'].includes(lower)) {
+        return `.${lower}`;
+      }
+    }
+  } catch {
+    // ignore
+  }
+  return '.png';
+}
+
+/**
+ * Generate local filename for a downloaded image.
+ *
+ * @param {string} url - Image URL
+ * @param {number} index - Zero-based index
+ * @returns {string} Local filename (e.g., 'image-01.png')
+ */
+export function generateLocalFilename(url, index) {
+  const ext = getExtensionFromUrl(url);
+  return `image-${String(index + 1).padStart(2, '0')}${ext}`;
+}
+
+/**
+ * Localize images in markdown text by downloading external images
+ * and replacing URLs with local paths.
+ *
+ * @param {string} markdownText - Input markdown
+ * @param {Object} [options] - Options
+ * @param {string} [options.imagesDir='images'] - Local images directory name
+ * @param {boolean} [options.dryRun=false] - Only report what would be done
+ * @param {Function} [options.onProgress] - Callback(index, total, status, url)
+ * @param {string[]} [options.excludeDomains] - Domains to skip (already local)
+ * @returns {Promise<Object>} Result with updated markdown and metadata
+ */
+export async function localizeImages(markdownText, options = {}) {
+  const {
+    imagesDir = 'images',
+    dryRun = false,
+    onProgress,
+    excludeDomains = [],
+  } = options;
+
+  const allImages = extractImageReferences(markdownText);
+
+  // Filter to only external images not already localized
+  const externalImages = allImages.filter((img) => {
+    if (!img.url.startsWith('http')) {
+      return false;
+    }
+    if (img.url.includes(`${imagesDir}/`)) {
+      return false;
+    }
+    for (const domain of excludeDomains) {
+      if (img.url.includes(domain)) {
+        return false;
+      }
+    }
+    return true;
+  });
+
+  if (externalImages.length === 0) {
+    return {
+      markdown: markdownText,
+      downloaded: 0,
+      total: 0,
+      replacements: [],
+      metadata: [],
+    };
+  }
+
+  const replacements = [];
+  const metadata = [];
+  let downloadedCount = 0;
+  let updatedMarkdown = markdownText;
+
+  for (let i = 0; i < externalImages.length; i++) {
+    const image = externalImages[i];
+    const localFilename = generateLocalFilename(image.url, i);
+    const relativePath = `${imagesDir}/${localFilename}`;
+
+    if (onProgress) {
+      onProgress(i + 1, externalImages.length, 'downloading', image.url);
+    }
+
+    if (dryRun) {
+      replacements.push({
+        from: image.fullMatch,
+        to: `![${image.altText}](${relativePath})`,
+      });
+      metadata.push({
+        index: i + 1,
+        originalUrl: image.url,
+        altText: image.altText,
+        localPath: relativePath,
+      });
+      continue;
+    }
+
+    try {
+      const resp = await retry(() => fetch(image.url), {
+        retries: 3,
+        baseDelay: 1000,
+      });
+
+      if (!resp.ok) {
+        throw new Error(`HTTP ${resp.status}`);
+      }
+
+      const buffer = await resp.buffer();
+      downloadedCount++;
+
+      replacements.push({
+        from: image.fullMatch,
+        to: `![${image.altText}](${relativePath})`,
+        buffer,
+        filename: localFilename,
+      });
+
+      metadata.push({
+        index: i + 1,
+        originalUrl: image.url,
+        altText: image.altText,
+        localPath: relativePath,
+      });
+
+      if (onProgress) {
+        onProgress(i + 1, externalImages.length, 'downloaded', image.url);
+      }
+    } catch {
+      if (onProgress) {
+        onProgress(i + 1, externalImages.length, 'failed', image.url);
+      }
+      // Keep original URL if download fails
+    }
+  }
+
+  // Apply replacements to markdown
+  for (const replacement of replacements) {
+    updatedMarkdown = updatedMarkdown.replace(replacement.from, replacement.to);
+  }
+
+  return {
+    markdown: updatedMarkdown,
+    downloaded: downloadedCount,
+    total: externalImages.length,
+    replacements,
+    metadata,
+  };
+}