@link-assistant/web-capture 1.1.2 → 1.7.1

package/src/localize-images.js

@@ -0,0 +1,202 @@
+/**
+ * Markdown image localization module (R5).
+ *
+ * Post-processing tool that:
+ * 1. Reads markdown files
+ * 2. Extracts all external image URLs
+ * 3. Downloads images to local directory
+ * 4. Updates markdown to reference local paths
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/download-markdown-images.mjs
+ *
+ * @module localize-images
+ */
+
+import fetch from 'node-fetch';
+import { URL } from 'url';
+import { retry } from './retry.js';
+
+/**
+ * Extract image references from markdown text.
+ *
+ * @param {string} markdownText - Markdown content
+ * @returns {Object[]} Array of {fullMatch, altText, url}
+ */
+export function extractImageReferences(markdownText) {
+  const imageRegex = /!\[([^\]]*)\]\((https?:\/\/[^)]+)\)/g;
+  const images = [];
+  let match;
+
+  while ((match = imageRegex.exec(markdownText)) !== null) {
+    images.push({
+      fullMatch: match[0],
+      altText: match[1],
+      url: match[2],
+    });
+  }
+
+  return images;
+}
+
+/**
+ * Get file extension from URL.
+ *
+ * @param {string} url - Image URL
+ * @returns {string} File extension with dot (e.g., '.png')
+ */
+export function getExtensionFromUrl(url) {
+  try {
+    const pathname = new URL(url).pathname.split('?')[0];
+    const ext = pathname.match(/\.(\w+)$/);
+    if (ext) {
+      const lower = ext[1].toLowerCase();
+      if (['png', 'jpg', 'jpeg', 'gif', 'webp', 'svg'].includes(lower)) {
+        return `.${lower}`;
+      }
+    }
+  } catch {
+    // ignore
+  }
+  return '.png';
+}
+
+/**
+ * Generate local filename for a downloaded image.
+ *
+ * @param {string} url - Image URL
+ * @param {number} index - Zero-based index
+ * @returns {string} Local filename (e.g., 'image-01.png')
+ */
+export function generateLocalFilename(url, index) {
+  const ext = getExtensionFromUrl(url);
+  return `image-${String(index + 1).padStart(2, '0')}${ext}`;
+}
+
+/**
+ * Localize images in markdown text by downloading external images
+ * and replacing URLs with local paths.
+ *
+ * @param {string} markdownText - Input markdown
+ * @param {Object} [options] - Options
+ * @param {string} [options.imagesDir='images'] - Local images directory name
+ * @param {boolean} [options.dryRun=false] - Only report what would be done
+ * @param {Function} [options.onProgress] - Callback(index, total, status, url)
+ * @param {string[]} [options.excludeDomains] - Domains to skip (already local)
+ * @returns {Promise<Object>} Result with updated markdown and metadata
+ */
+export async function localizeImages(markdownText, options = {}) {
+  const {
+    imagesDir = 'images',
+    dryRun = false,
+    onProgress,
+    excludeDomains = [],
+  } = options;
+
+  const allImages = extractImageReferences(markdownText);
+
+  // Filter to only external images not already localized
+  const externalImages = allImages.filter((img) => {
+    if (!img.url.startsWith('http')) {
+      return false;
+    }
+    if (img.url.includes(`${imagesDir}/`)) {
+      return false;
+    }
+    for (const domain of excludeDomains) {
+      if (img.url.includes(domain)) {
+        return false;
+      }
+    }
+    return true;
+  });
+
+  if (externalImages.length === 0) {
+    return {
+      markdown: markdownText,
+      downloaded: 0,
+      total: 0,
+      replacements: [],
+      metadata: [],
+    };
+  }
+
+  const replacements = [];
+  const metadata = [];
+  let downloadedCount = 0;
+  let updatedMarkdown = markdownText;
+
+  for (let i = 0; i < externalImages.length; i++) {
+    const image = externalImages[i];
+    const localFilename = generateLocalFilename(image.url, i);
+    const relativePath = `${imagesDir}/${localFilename}`;
+
+    if (onProgress) {
+      onProgress(i + 1, externalImages.length, 'downloading', image.url);
+    }
+
+    if (dryRun) {
+      replacements.push({
+        from: image.fullMatch,
+        to: `![${image.altText}](${relativePath})`,
+      });
+      metadata.push({
+        index: i + 1,
+        originalUrl: image.url,
+        altText: image.altText,
+        localPath: relativePath,
+      });
+      continue;
+    }
+
+    try {
+      const resp = await retry(() => fetch(image.url), {
+        retries: 3,
+        baseDelay: 1000,
+      });
+
+      if (!resp.ok) {
+        throw new Error(`HTTP ${resp.status}`);
+      }
+
+      const buffer = await resp.buffer();
+      downloadedCount++;
+
+      replacements.push({
+        from: image.fullMatch,
+        to: `![${image.altText}](${relativePath})`,
+        buffer,
+        filename: localFilename,
+      });
+
+      metadata.push({
+        index: i + 1,
+        originalUrl: image.url,
+        altText: image.altText,
+        localPath: relativePath,
+      });
+
+      if (onProgress) {
+        onProgress(i + 1, externalImages.length, 'downloaded', image.url);
+      }
+    } catch {
+      if (onProgress) {
+        onProgress(i + 1, externalImages.length, 'failed', image.url);
+      }
+      // Keep original URL if download fails
+    }
+  }
+
+  // Apply replacements to markdown
+  for (const replacement of replacements) {
+    updatedMarkdown = updatedMarkdown.replace(replacement.from, replacement.to);
+  }
+
+  return {
+    markdown: updatedMarkdown,
+    downloaded: downloadedCount,
+    total: externalImages.length,
+    replacements,
+    metadata,
+  };
+}

package/src/markdown.js

@@ -1,14 +1,19 @@
 import { fetchHtml, convertHtmlToMarkdown } from './lib.js';
+import { stripBase64Images } from './extract-images.js';
 
 export async function markdownHandler(req, res) {
   const url = req.query.url;
   if (!url) {
     return res.status(400).send('Missing `url` parameter');
   }
+  const embedImages = req.query.embedImages === 'true';
   try {
     const html = await fetchHtml(url);
-    // Pass baseUrl to convertHtmlToMarkdown so all URLs are absolute
-    const markdown = convertHtmlToMarkdown(html, url);
+    let markdown = convertHtmlToMarkdown(html, url);
+    if (!embedImages) {
+      const strip = stripBase64Images(markdown);
+      markdown = strip.markdown;
+    }
     res.type('text/markdown').send(markdown);
   } catch (err) {
     console.error(err);

package/src/metadata.js

@@ -0,0 +1,333 @@
+/**
+ * Article metadata extraction module.
+ *
+ * Extracts metadata from web pages including:
+ * - Author information (name, URL, karma)
+ * - Publication date and modification date
+ * - Reading time and difficulty
+ * - Views, votes, bookmarks, comments
+ * - Hubs and tags (with URLs)
+ * - Translation information
+ * - LD+JSON structured data
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/download-article.mjs
+ */
+
+/**
+ * Extract article metadata from HTML using Cheerio.
+ * Works without a browser by parsing the HTML directly.
+ *
+ * @param {Object} $ - Cheerio instance loaded with the page HTML
+ * @returns {Object} Extracted metadata
+ */
+export function extractMetadata($) {
+  const meta = {};
+
+  // Author
+  const authorEl = $('.tm-user-info__username');
+  if (authorEl.length > 0) {
+    meta.author = authorEl.text().trim();
+    meta.authorUrl = authorEl.attr('href') || null;
+  }
+
+  // Publication date
+  const timeEl = $('time[datetime]').first();
+  if (timeEl.length > 0) {
+    meta.publishDate = timeEl.attr('datetime');
+    meta.publishDateText = timeEl.text().trim();
+  }
+
+  // Reading time
+  const readTimeEl = $('.tm-article-reading-time__label');
+  if (readTimeEl.length > 0) {
+    meta.readingTime = readTimeEl.text().trim();
+  }
+
+  // Difficulty level
+  const diffEl = $('.tm-article-complexity__label');
+  if (diffEl.length > 0) {
+    meta.difficulty = diffEl.text().trim();
+  }
+
+  // Views
+  const viewsEl = $('.tm-icon-counter__value');
+  if (viewsEl.length > 0) {
+    meta.views = viewsEl.attr('title') || viewsEl.text().trim();
+  }
+
+  // Hubs (use specific hub link selector to avoid duplicates)
+  const hubEls = $('.tm-publication-hub__link');
+  if (hubEls.length > 0) {
+    meta.hubs = [];
+    meta.hubUrls = [];
+    hubEls.each(function () {
+      const nameSpan = $(this).find('span:first-child');
+      const name = nameSpan.length
+        ? nameSpan.text().trim()
+        : $(this)
+            .text()
+            .trim()
+            .replace(/\s*\*\s*$/, '');
+      meta.hubs.push(name);
+      meta.hubUrls.push({
+        name,
+        url: $(this).attr('href') || null,
+      });
+    });
+  }
+
+  // Tags from meta keywords
+  const keywordsMeta = $('meta[name="keywords"]');
+  if (keywordsMeta.length > 0) {
+    const content = keywordsMeta.attr('content');
+    if (content) {
+      meta.tags = content
+        .split(',')
+        .map((t) => t.trim())
+        .filter(Boolean);
+    }
+  }
+
+  // Tags with URLs (from article footer)
+  const tagEls = $('.tm-article-body__tags-item a, .tm-tags-list__link');
+  if (tagEls.length > 0) {
+    meta.tagLinks = [];
+    tagEls.each(function () {
+      meta.tagLinks.push({
+        name: $(this).text().trim(),
+        url: $(this).attr('href') || null,
+      });
+    });
+  }
+
+  // Translation badge
+  const translationLabelEl = $('.tm-publication-label_variant-translation');
+  if (translationLabelEl.length > 0) {
+    meta.isTranslation = true;
+    meta.translationLabel = translationLabelEl.text().trim();
+  }
+
+  // Original article link
+  const originLinkEl = $('.tm-article-presenter__origin-link');
+  if (originLinkEl.length > 0) {
+    meta.originalArticleUrl = originLinkEl.attr('href') || null;
+    const authorSpan = originLinkEl.find('span');
+    if (authorSpan.length > 0) {
+      meta.originalAuthors = authorSpan.text().trim();
+    }
+    meta.originalAuthorText = originLinkEl.text().trim();
+  }
+
+  // LD+JSON structured data
+  const ldJsonScript = $('script[type="application/ld+json"]').first();
+  if (ldJsonScript.length > 0) {
+    try {
+      const ldData = JSON.parse(ldJsonScript.html());
+      if (ldData.dateModified) {
+        meta.dateModified = ldData.dateModified;
+      }
+      if (ldData.author?.name) {
+        meta.authorFullName = ldData.author.name;
+      }
+    } catch {
+      // ignore parse errors
+    }
+  }
+
+  // Votes
+  const votesEl = $('.tm-votes-meter__value');
+  if (votesEl.length > 0) {
+    meta.votes = votesEl.text().trim();
+  }
+
+  // Comments count
+  const commentsEl = $('.tm-article-comments-counter-link__value');
+  if (commentsEl.length > 0) {
+    meta.comments = commentsEl.text().trim();
+  }
+
+  // Bookmarks count
+  const bookmarksEl = $('.bookmarks-button__counter');
+  if (bookmarksEl.length > 0) {
+    meta.bookmarks = bookmarksEl.text().trim();
+  }
+
+  // Author karma
+  const karmaEl = $('.tm-karma__votes');
+  if (karmaEl.length > 0) {
+    meta.authorKarma = karmaEl.text().trim();
+  }
+
+  return meta;
+}
+
+/**
+ * Format metadata as a markdown header block.
+ * Placed after the title in the output markdown.
+ *
+ * @param {Object} metadata - Extracted metadata object
+ * @returns {string[]} Array of markdown lines
+ */
+export function formatMetadataBlock(metadata) {
+  if (!metadata) {
+    return [];
+  }
+  const lines = [];
+
+  // Author line
+  if (metadata.author) {
+    const authorName = metadata.authorFullName
+      ? `${metadata.authorFullName} (${metadata.author})`
+      : metadata.author;
+    const authorLink = metadata.authorUrl
+      ? `[${authorName}](${metadata.authorUrl})`
+      : authorName;
+    lines.push(`**Author:** ${authorLink}`);
+  }
+
+  // Article type (translation)
+  if (metadata.isTranslation) {
+    lines.push(`**Type:** ${metadata.translationLabel || 'Translation'}`);
+  }
+
+  // Original article link
+  if (metadata.originalAuthors) {
+    const authorsText = metadata.originalAuthors;
+    if (metadata.originalArticleUrl) {
+      lines.push(
+        `**Original article:** [${authorsText}](${metadata.originalArticleUrl})`
+      );
+    } else {
+      lines.push(`**Original authors:** ${authorsText}`);
+    }
+  }
+
+  // Publication date
+  if (metadata.publishDate) {
+    const date = new Date(metadata.publishDate);
+    const formatted = date.toLocaleDateString('en-US', {
+      year: 'numeric',
+      month: 'long',
+      day: 'numeric',
+    });
+    let dateLine = `**Published:** ${formatted}`;
+    if (metadata.dateModified) {
+      const modDate = new Date(metadata.dateModified);
+      const modFormatted = modDate.toLocaleDateString('en-US', {
+        year: 'numeric',
+        month: 'long',
+        day: 'numeric',
+      });
+      if (modFormatted !== formatted) {
+        dateLine += ` (updated ${modFormatted})`;
+      }
+    }
+    lines.push(dateLine);
+  }
+
+  // Reading time and difficulty
+  const infoItems = [];
+  if (metadata.readingTime) {
+    infoItems.push(`Reading time: ${metadata.readingTime}`);
+  }
+  if (metadata.difficulty) {
+    infoItems.push(`Difficulty: ${metadata.difficulty}`);
+  }
+  if (metadata.views) {
+    infoItems.push(`Views: ${metadata.views}`);
+  }
+  if (infoItems.length > 0) {
+    lines.push(`**${infoItems.join(' | ')}**`);
+  }
+
+  // Hubs
+  if (metadata.hubs && metadata.hubs.length > 0) {
+    lines.push(`**Hubs:** ${metadata.hubs.join(', ')}`);
+  }
+
+  // Tags
+  if (metadata.tags && metadata.tags.length > 0) {
+    lines.push(`**Tags:** ${metadata.tags.join(', ')}`);
+  }
+
+  return lines;
+}
+
+/**
+ * Format footer metadata block.
+ * Placed at the end of the article, matching Habr article footer.
+ *
+ * @param {Object} metadata - Extracted metadata object
+ * @returns {string[]} Array of markdown lines
+ */
+export function formatFooterBlock(metadata) {
+  if (!metadata) {
+    return [];
+  }
+  const lines = [];
+
+  lines.push('---');
+  lines.push('');
+
+  // Tags with links
+  if (metadata.tagLinks && metadata.tagLinks.length > 0) {
+    const tagStrings = metadata.tagLinks.map((t) =>
+      t.url ? `[${t.name}](${t.url})` : t.name
+    );
+    lines.push(`**Tags:** ${tagStrings.join(', ')}`);
+    lines.push('');
+  } else if (metadata.tags && metadata.tags.length > 0) {
+    lines.push(`**Tags:** ${metadata.tags.join(', ')}`);
+    lines.push('');
+  }
+
+  // Hubs with links
+  if (metadata.hubUrls && metadata.hubUrls.length > 0) {
+    const hubStrings = metadata.hubUrls.map((h) =>
+      h.url ? `[${h.name}](${h.url})` : h.name
+    );
+    lines.push(`**Hubs:** ${hubStrings.join(', ')}`);
+    lines.push('');
+  } else if (metadata.hubs && metadata.hubs.length > 0) {
+    lines.push(`**Hubs:** ${metadata.hubs.join(', ')}`);
+    lines.push('');
+  }
+
+  // Article stats
+  const stats = [];
+  if (metadata.votes) {
+    stats.push(`Votes: ${metadata.votes}`);
+  }
+  if (metadata.views) {
+    stats.push(`Views: ${metadata.views}`);
+  }
+  if (metadata.bookmarks) {
+    stats.push(`Bookmarks: ${metadata.bookmarks}`);
+  }
+  if (metadata.comments) {
+    stats.push(`Comments: ${metadata.comments}`);
+  }
+  if (stats.length > 0) {
+    lines.push(`**${stats.join(' | ')}**`);
+    lines.push('');
+  }
+
+  // Author info
+  if (metadata.author) {
+    const authorName = metadata.authorFullName
+      ? `${metadata.authorFullName} (${metadata.author})`
+      : metadata.author;
+    const authorLink = metadata.authorUrl
+      ? `[${authorName}](${metadata.authorUrl})`
+      : authorName;
+    let authorLine = `**Author:** ${authorLink}`;
+    if (metadata.authorKarma) {
+      authorLine += ` | Karma: ${metadata.authorKarma}`;
+    }
+    lines.push(authorLine);
+    lines.push('');
+  }
+
+  return lines;
+}

package/src/pdf.js

@@ -0,0 +1,76 @@
+/**
+ * PDF export handler.
+ *
+ * Renders a web page to PDF using the browser's native print-to-PDF.
+ * All images are embedded automatically by the browser engine.
+ *
+ * Query parameters:
+ *   url       (required) - URL to capture
+ *   engine    - 'puppeteer' or 'playwright'
+ *   width     - viewport width in px (default 1280)
+ *   height    - viewport height in px (default 800)
+ *   theme     - 'light' or 'dark'
+ *   dismissPopups - 'true' (default) to auto-close popups before export
+ */
+
+import { createBrowser, getBrowserEngine } from './browser.js';
+import { dismissPopups, scrollToLoadContent } from './popups.js';
+
+export async function pdfHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+
+  const width = parseInt(req.query.width, 10) || 1280;
+  const height = parseInt(req.query.height, 10) || 800;
+  const theme = req.query.theme;
+  const shouldDismissPopups = req.query.dismissPopups !== 'false';
+
+  try {
+    const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
+    const engine = getBrowserEngine(req);
+    const browserOpts = theme ? { colorScheme: theme } : {};
+    const browser = await createBrowser(engine, browserOpts);
+    try {
+      const page = await browser.newPage();
+      await page.setExtraHTTPHeaders({
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Accept-Charset': 'utf-8',
+      });
+      await page.setUserAgent(
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+      );
+      await page.setViewport({ width, height });
+      await page.goto(absoluteUrl, {
+        waitUntil: 'networkidle0',
+        timeout: 30000,
+      });
+
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+      await scrollToLoadContent(page);
+
+      if (shouldDismissPopups) {
+        await dismissPopups(page);
+      }
+
+      // Generate PDF using browser-commander's unified pdf() API (v0.8.0+)
+      const pdfBuffer = await page.pdf({
+        pdfOptions: {
+          format: 'A4',
+          printBackground: true,
+          margin: { top: '1cm', right: '1cm', bottom: '1cm', left: '1cm' },
+        },
+      });
+
+      res.set('Content-Type', 'application/pdf');
+      res.set('Content-Disposition', 'inline; filename="page.pdf"');
+      res.end(pdfBuffer);
+    } finally {
+      await browser.close();
+    }
+  } catch (err) {
+    console.error(err);
+    res.status(500).send('Error generating PDF');
+  }
+}