@link-assistant/web-capture 1.1.2 → 1.4.2

package/src/metadata.js

@@ -0,0 +1,333 @@
+/**
+ * Article metadata extraction module.
+ *
+ * Extracts metadata from web pages including:
+ * - Author information (name, URL, karma)
+ * - Publication date and modification date
+ * - Reading time and difficulty
+ * - Views, votes, bookmarks, comments
+ * - Hubs and tags (with URLs)
+ * - Translation information
+ * - LD+JSON structured data
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/download-article.mjs
+ */
+
+/**
+ * Extract article metadata from HTML using Cheerio.
+ * Works without a browser by parsing the HTML directly.
+ *
+ * @param {Object} $ - Cheerio instance loaded with the page HTML
+ * @returns {Object} Extracted metadata
+ */
+export function extractMetadata($) {
+  const meta = {};
+
+  // Author
+  const authorEl = $('.tm-user-info__username');
+  if (authorEl.length > 0) {
+    meta.author = authorEl.text().trim();
+    meta.authorUrl = authorEl.attr('href') || null;
+  }
+
+  // Publication date
+  const timeEl = $('time[datetime]').first();
+  if (timeEl.length > 0) {
+    meta.publishDate = timeEl.attr('datetime');
+    meta.publishDateText = timeEl.text().trim();
+  }
+
+  // Reading time
+  const readTimeEl = $('.tm-article-reading-time__label');
+  if (readTimeEl.length > 0) {
+    meta.readingTime = readTimeEl.text().trim();
+  }
+
+  // Difficulty level
+  const diffEl = $('.tm-article-complexity__label');
+  if (diffEl.length > 0) {
+    meta.difficulty = diffEl.text().trim();
+  }
+
+  // Views
+  const viewsEl = $('.tm-icon-counter__value');
+  if (viewsEl.length > 0) {
+    meta.views = viewsEl.attr('title') || viewsEl.text().trim();
+  }
+
+  // Hubs (use specific hub link selector to avoid duplicates)
+  const hubEls = $('.tm-publication-hub__link');
+  if (hubEls.length > 0) {
+    meta.hubs = [];
+    meta.hubUrls = [];
+    hubEls.each(function () {
+      const nameSpan = $(this).find('span:first-child');
+      const name = nameSpan.length
+        ? nameSpan.text().trim()
+        : $(this)
+            .text()
+            .trim()
+            .replace(/\s*\*\s*$/, '');
+      meta.hubs.push(name);
+      meta.hubUrls.push({
+        name,
+        url: $(this).attr('href') || null,
+      });
+    });
+  }
+
+  // Tags from meta keywords
+  const keywordsMeta = $('meta[name="keywords"]');
+  if (keywordsMeta.length > 0) {
+    const content = keywordsMeta.attr('content');
+    if (content) {
+      meta.tags = content
+        .split(',')
+        .map((t) => t.trim())
+        .filter(Boolean);
+    }
+  }
+
+  // Tags with URLs (from article footer)
+  const tagEls = $('.tm-article-body__tags-item a, .tm-tags-list__link');
+  if (tagEls.length > 0) {
+    meta.tagLinks = [];
+    tagEls.each(function () {
+      meta.tagLinks.push({
+        name: $(this).text().trim(),
+        url: $(this).attr('href') || null,
+      });
+    });
+  }
+
+  // Translation badge
+  const translationLabelEl = $('.tm-publication-label_variant-translation');
+  if (translationLabelEl.length > 0) {
+    meta.isTranslation = true;
+    meta.translationLabel = translationLabelEl.text().trim();
+  }
+
+  // Original article link
+  const originLinkEl = $('.tm-article-presenter__origin-link');
+  if (originLinkEl.length > 0) {
+    meta.originalArticleUrl = originLinkEl.attr('href') || null;
+    const authorSpan = originLinkEl.find('span');
+    if (authorSpan.length > 0) {
+      meta.originalAuthors = authorSpan.text().trim();
+    }
+    meta.originalAuthorText = originLinkEl.text().trim();
+  }
+
+  // LD+JSON structured data
+  const ldJsonScript = $('script[type="application/ld+json"]').first();
+  if (ldJsonScript.length > 0) {
+    try {
+      const ldData = JSON.parse(ldJsonScript.html());
+      if (ldData.dateModified) {
+        meta.dateModified = ldData.dateModified;
+      }
+      if (ldData.author?.name) {
+        meta.authorFullName = ldData.author.name;
+      }
+    } catch {
+      // ignore parse errors
+    }
+  }
+
+  // Votes
+  const votesEl = $('.tm-votes-meter__value');
+  if (votesEl.length > 0) {
+    meta.votes = votesEl.text().trim();
+  }
+
+  // Comments count
+  const commentsEl = $('.tm-article-comments-counter-link__value');
+  if (commentsEl.length > 0) {
+    meta.comments = commentsEl.text().trim();
+  }
+
+  // Bookmarks count
+  const bookmarksEl = $('.bookmarks-button__counter');
+  if (bookmarksEl.length > 0) {
+    meta.bookmarks = bookmarksEl.text().trim();
+  }
+
+  // Author karma
+  const karmaEl = $('.tm-karma__votes');
+  if (karmaEl.length > 0) {
+    meta.authorKarma = karmaEl.text().trim();
+  }
+
+  return meta;
+}
+
+/**
+ * Format metadata as a markdown header block.
+ * Placed after the title in the output markdown.
+ *
+ * @param {Object} metadata - Extracted metadata object
+ * @returns {string[]} Array of markdown lines
+ */
+export function formatMetadataBlock(metadata) {
+  if (!metadata) {
+    return [];
+  }
+  const lines = [];
+
+  // Author line
+  if (metadata.author) {
+    const authorName = metadata.authorFullName
+      ? `${metadata.authorFullName} (${metadata.author})`
+      : metadata.author;
+    const authorLink = metadata.authorUrl
+      ? `[${authorName}](${metadata.authorUrl})`
+      : authorName;
+    lines.push(`**Author:** ${authorLink}`);
+  }
+
+  // Article type (translation)
+  if (metadata.isTranslation) {
+    lines.push(`**Type:** ${metadata.translationLabel || 'Translation'}`);
+  }
+
+  // Original article link
+  if (metadata.originalAuthors) {
+    const authorsText = metadata.originalAuthors;
+    if (metadata.originalArticleUrl) {
+      lines.push(
+        `**Original article:** [${authorsText}](${metadata.originalArticleUrl})`
+      );
+    } else {
+      lines.push(`**Original authors:** ${authorsText}`);
+    }
+  }
+
+  // Publication date
+  if (metadata.publishDate) {
+    const date = new Date(metadata.publishDate);
+    const formatted = date.toLocaleDateString('en-US', {
+      year: 'numeric',
+      month: 'long',
+      day: 'numeric',
+    });
+    let dateLine = `**Published:** ${formatted}`;
+    if (metadata.dateModified) {
+      const modDate = new Date(metadata.dateModified);
+      const modFormatted = modDate.toLocaleDateString('en-US', {
+        year: 'numeric',
+        month: 'long',
+        day: 'numeric',
+      });
+      if (modFormatted !== formatted) {
+        dateLine += ` (updated ${modFormatted})`;
+      }
+    }
+    lines.push(dateLine);
+  }
+
+  // Reading time and difficulty
+  const infoItems = [];
+  if (metadata.readingTime) {
+    infoItems.push(`Reading time: ${metadata.readingTime}`);
+  }
+  if (metadata.difficulty) {
+    infoItems.push(`Difficulty: ${metadata.difficulty}`);
+  }
+  if (metadata.views) {
+    infoItems.push(`Views: ${metadata.views}`);
+  }
+  if (infoItems.length > 0) {
+    lines.push(`**${infoItems.join(' | ')}**`);
+  }
+
+  // Hubs
+  if (metadata.hubs && metadata.hubs.length > 0) {
+    lines.push(`**Hubs:** ${metadata.hubs.join(', ')}`);
+  }
+
+  // Tags
+  if (metadata.tags && metadata.tags.length > 0) {
+    lines.push(`**Tags:** ${metadata.tags.join(', ')}`);
+  }
+
+  return lines;
+}
+
+/**
+ * Format footer metadata block.
+ * Placed at the end of the article, matching Habr article footer.
+ *
+ * @param {Object} metadata - Extracted metadata object
+ * @returns {string[]} Array of markdown lines
+ */
+export function formatFooterBlock(metadata) {
+  if (!metadata) {
+    return [];
+  }
+  const lines = [];
+
+  lines.push('---');
+  lines.push('');
+
+  // Tags with links
+  if (metadata.tagLinks && metadata.tagLinks.length > 0) {
+    const tagStrings = metadata.tagLinks.map((t) =>
+      t.url ? `[${t.name}](${t.url})` : t.name
+    );
+    lines.push(`**Tags:** ${tagStrings.join(', ')}`);
+    lines.push('');
+  } else if (metadata.tags && metadata.tags.length > 0) {
+    lines.push(`**Tags:** ${metadata.tags.join(', ')}`);
+    lines.push('');
+  }
+
+  // Hubs with links
+  if (metadata.hubUrls && metadata.hubUrls.length > 0) {
+    const hubStrings = metadata.hubUrls.map((h) =>
+      h.url ? `[${h.name}](${h.url})` : h.name
+    );
+    lines.push(`**Hubs:** ${hubStrings.join(', ')}`);
+    lines.push('');
+  } else if (metadata.hubs && metadata.hubs.length > 0) {
+    lines.push(`**Hubs:** ${metadata.hubs.join(', ')}`);
+    lines.push('');
+  }
+
+  // Article stats
+  const stats = [];
+  if (metadata.votes) {
+    stats.push(`Votes: ${metadata.votes}`);
+  }
+  if (metadata.views) {
+    stats.push(`Views: ${metadata.views}`);
+  }
+  if (metadata.bookmarks) {
+    stats.push(`Bookmarks: ${metadata.bookmarks}`);
+  }
+  if (metadata.comments) {
+    stats.push(`Comments: ${metadata.comments}`);
+  }
+  if (stats.length > 0) {
+    lines.push(`**${stats.join(' | ')}**`);
+    lines.push('');
+  }
+
+  // Author info
+  if (metadata.author) {
+    const authorName = metadata.authorFullName
+      ? `${metadata.authorFullName} (${metadata.author})`
+      : metadata.author;
+    const authorLink = metadata.authorUrl
+      ? `[${authorName}](${metadata.authorUrl})`
+      : authorName;
+    let authorLine = `**Author:** ${authorLink}`;
+    if (metadata.authorKarma) {
+      authorLine += ` | Karma: ${metadata.authorKarma}`;
+    }
+    lines.push(authorLine);
+    lines.push('');
+  }
+
+  return lines;
+}

package/src/pdf.js

@@ -0,0 +1,76 @@
+/**
+ * PDF export handler.
+ *
+ * Renders a web page to PDF using the browser's native print-to-PDF.
+ * All images are embedded automatically by the browser engine.
+ *
+ * Query parameters:
+ *   url       (required) - URL to capture
+ *   engine    - 'puppeteer' or 'playwright'
+ *   width     - viewport width in px (default 1280)
+ *   height    - viewport height in px (default 800)
+ *   theme     - 'light' or 'dark'
+ *   dismissPopups - 'true' (default) to auto-close popups before export
+ */
+
+import { createBrowser, getBrowserEngine } from './browser.js';
+import { dismissPopups, scrollToLoadContent } from './popups.js';
+
+export async function pdfHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+
+  const width = parseInt(req.query.width, 10) || 1280;
+  const height = parseInt(req.query.height, 10) || 800;
+  const theme = req.query.theme;
+  const shouldDismissPopups = req.query.dismissPopups !== 'false';
+
+  try {
+    const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
+    const engine = getBrowserEngine(req);
+    const browserOpts = theme ? { colorScheme: theme } : {};
+    const browser = await createBrowser(engine, browserOpts);
+    try {
+      const page = await browser.newPage();
+      await page.setExtraHTTPHeaders({
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Accept-Charset': 'utf-8',
+      });
+      await page.setUserAgent(
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+      );
+      await page.setViewport({ width, height });
+      await page.goto(absoluteUrl, {
+        waitUntil: 'networkidle0',
+        timeout: 30000,
+      });
+
+      await new Promise((resolve) => setTimeout(resolve, 5000));
+      await scrollToLoadContent(page);
+
+      if (shouldDismissPopups) {
+        await dismissPopups(page);
+      }
+
+      // Generate PDF using browser-commander's unified pdf() API (v0.8.0+)
+      const pdfBuffer = await page.pdf({
+        pdfOptions: {
+          format: 'A4',
+          printBackground: true,
+          margin: { top: '1cm', right: '1cm', bottom: '1cm', left: '1cm' },
+        },
+      });
+
+      res.set('Content-Type', 'application/pdf');
+      res.set('Content-Disposition', 'inline; filename="page.pdf"');
+      res.end(pdfBuffer);
+    } finally {
+      await browser.close();
+    }
+  } catch (err) {
+    console.error(err);
+    res.status(500).send('Error generating PDF');
+  }
+}

package/src/popups.js

@@ -0,0 +1,171 @@
+/* eslint-disable no-undef */
+/**
+ * Popup/modal/cookie banner detection and auto-close utilities.
+ *
+ * Handles multiple types of overlays including:
+ * - Google Funding Choices (FC) consent dialogs
+ * - Habr cookie banners (tm-cookie-banner)
+ * - Generic modals, overlays, and notification popups
+ *
+ * Based on best practices from https://github.com/link-foundation/meta-theory
+ */
+
+/**
+ * Close any popup overlays/modals on a page adapter (from browser.js).
+ * Should be called after page content has loaded and before taking screenshots.
+ *
+ * Uses browser-commander APIs:
+ * - page.onDialog()   – dismiss browser-level alert/confirm dialogs (v0.7.0+)
+ * - page.keyboard     – press Escape to dismiss remaining popups (v0.7.0+)
+ * - page.evaluate()   – run JS in the browser context to click close buttons
+ *
+ * @param {Object} page - A PageAdapter from browser.js
+ */
+export async function dismissPopups(page) {
+  // Handle browser-level dialog events (alert, confirm, prompt) using
+  // browser-commander's dialog API (v0.7.0+)
+  page.onDialog(async (dialog) => {
+    try {
+      await dialog.dismiss();
+    } catch {
+      /* ignore */
+    }
+  });
+
+  // Try to click the Google Funding Choices "Consent" button first
+  const fcConsentClicked = await page.evaluate(() => {
+    const consentBtn = document.querySelector('.fc-cta-consent');
+    if (consentBtn) {
+      try {
+        consentBtn.click();
+        return true;
+      } catch {
+        return false;
+      }
+    }
+    return false;
+  });
+
+  if (fcConsentClicked) {
+    await sleep(page, 1000);
+  }
+
+  // Click all known close/dismiss buttons and remove overlay elements
+  await page.evaluate(() => {
+    const closeSelectors = [
+      // Google Funding Choices (FC) consent dialog
+      '.fc-cta-consent',
+      '.fc-close',
+      '.fc-dismiss-button',
+      // Cookie consent
+      '.tm-cookie-banner__close',
+      '.cookie-banner__close',
+      '[data-test-id="cookie-banner-close"]',
+      // Consent popup buttons
+      '.consent-popup__close',
+      '.consent__close',
+      'button[data-testid="consent-close"]',
+      'button[data-testid="consent-reject"]',
+      // Generic close buttons
+      '.tm-popup__close',
+      '.popup__close',
+      '.modal__close',
+      '.overlay__close',
+      '[aria-label="Close"]',
+      '[aria-label="Закрыть"]',
+      '.tm-base-modal__close',
+      '.tm-notification__close',
+    ];
+
+    for (const selector of closeSelectors) {
+      const els = document.querySelectorAll(selector);
+      for (const el of els) {
+        try {
+          el.click();
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+
+    // Remove Google FC consent dialog elements from DOM
+    for (const sel of ['.fc-consent-root', '.fc-dialog-overlay']) {
+      const el = document.querySelector(sel);
+      if (el) {
+        try {
+          el.remove();
+        } catch {
+          /* ignore */
+        }
+      }
+    }
+
+    // Hide fixed-position overlays that cover content
+    const allEls = document.querySelectorAll('*');
+    for (const el of allEls) {
+      const style = getComputedStyle(el);
+      if (style.position === 'fixed' && el.offsetParent !== null) {
+        const rect = el.getBoundingClientRect();
+        if (
+          rect.height > 200 ||
+          (el.className &&
+            el.className.match &&
+            el.className.match(
+              /popup|modal|overlay|banner|cookie|notification|consent|fc-/i
+            ))
+        ) {
+          el.style.display = 'none';
+        }
+      }
+    }
+  });
+
+  await sleep(page, 500);
+
+  // Press Escape to dismiss remaining popups using browser-commander's keyboard API (v0.7.0+)
+  try {
+    await page.keyboard.press('Escape');
+    await sleep(page, 300);
+  } catch {
+    /* ignore */
+  }
+}
+
+/**
+ * Scroll through the page to trigger lazy-loaded content.
+ * Makes multiple passes for media-heavy pages.
+ *
+ * @param {Object} page - PageAdapter from browser.js
+ * @param {Object} options
+ * @param {number} options.passes - Number of scroll passes (default: 2)
+ * @param {number} options.stepDelay - Delay between scroll steps in ms (default: 100)
+ * @param {number} options.passDelay - Delay between passes in ms (default: 1000)
+ */
+export async function scrollToLoadContent(page, options = {}) {
+  const { passes = 2, stepDelay = 100, passDelay = 1000 } = options;
+
+  for (let pass = 0; pass < passes; pass++) {
+    await page.evaluate(
+      async ({ delay }) => {
+        const scrollHeight = document.documentElement.scrollHeight;
+        const viewportHeight = window.innerHeight;
+        const scrollSteps = Math.ceil(scrollHeight / viewportHeight);
+        for (let i = 0; i < scrollSteps; i++) {
+          window.scrollTo(0, i * viewportHeight);
+          await new Promise((resolve) => setTimeout(resolve, delay));
+        }
+        window.scrollTo(0, 0);
+      },
+      { delay: stepDelay }
+    );
+    await sleep(page, passDelay);
+  }
+
+  // Extra wait for images to finish loading
+  await sleep(page, 2000);
+}
+
+/** Portable sleep using standard Promise-based timeout */
+async function sleep(_page, ms) {
+  await new Promise((resolve) => setTimeout(resolve, ms));
+}