@link-assistant/web-capture 1.1.2 → 1.7.1

package/src/extract-images.js

@@ -0,0 +1,120 @@
+import { createHash } from 'crypto';
+import fs from 'fs';
+import path from 'path';
+
+const BASE64_IMAGE_REGEX =
+  /!\[([^\]]*)\]\(data:image\/(png|jpeg|jpg|gif|webp|svg\+xml);base64,([^)]+)\)/gi;
+
+/**
+ * @param {string} markdown - Markdown content with data URI images
+ * @param {string} outputDir - Directory where the markdown file is being written
+ * @param {Object} [options]
+ * @param {string} [options.imagesDir='images'] - Subdirectory name for images
+ * @returns {{markdown: string, extracted: number}}
+ */
+export function extractAndSaveImages(markdown, outputDir, options = {}) {
+  const { imagesDir = 'images' } = options;
+  const imagesPath = path.resolve(outputDir, imagesDir);
+
+  const images = [];
+
+  const updatedMarkdown = markdown.replace(
+    BASE64_IMAGE_REGEX,
+    (match, altText, mimeExt, base64Data) => {
+      const ext =
+        mimeExt === 'jpeg' ? 'jpg' : mimeExt === 'svg+xml' ? 'svg' : mimeExt;
+
+      let buffer;
+      try {
+        buffer = Buffer.from(base64Data, 'base64');
+      } catch {
+        return match;
+      }
+
+      const hash = createHash('sha256')
+        .update(buffer)
+        .digest('hex')
+        .slice(0, 8);
+      const filename = `image-${hash}.${ext}`;
+      const relativePath = `${imagesDir}/${filename}`;
+
+      images.push({ filename, buffer });
+
+      return `![${altText}](${relativePath})`;
+    }
+  );
+
+  if (images.length > 0) {
+    fs.mkdirSync(imagesPath, { recursive: true });
+    for (const img of images) {
+      fs.writeFileSync(path.join(imagesPath, img.filename), img.buffer);
+    }
+  }
+
+  return { markdown: updatedMarkdown, extracted: images.length };
+}
+
+/**
+ * Extract base64 images from markdown into memory buffers and rewrite paths.
+ * Does NOT write to disk — intended for streaming into archives.
+ *
+ * @param {string} markdown - Markdown content with data URI images
+ * @param {string} [imagesDir='images'] - Directory prefix for image paths
+ * @returns {{markdown: string, images: Array<{filename: string, buffer: Buffer}>}}
+ */
+export function extractBase64ToBuffers(markdown, imagesDir = 'images') {
+  const images = [];
+  const updatedMarkdown = markdown.replace(
+    BASE64_IMAGE_REGEX,
+    (match, altText, mimeExt, base64Data) => {
+      const ext =
+        mimeExt === 'jpeg' ? 'jpg' : mimeExt === 'svg+xml' ? 'svg' : mimeExt;
+
+      let buffer;
+      try {
+        buffer = Buffer.from(base64Data, 'base64');
+      } catch {
+        return match;
+      }
+
+      const hash = createHash('sha256')
+        .update(buffer)
+        .digest('hex')
+        .slice(0, 8);
+      const filename = `image-${hash}.${ext}`;
+      const relativePath = `${imagesDir}/${filename}`;
+
+      images.push({ filename, buffer });
+      return `![${altText}](${relativePath})`;
+    }
+  );
+  return { markdown: updatedMarkdown, images };
+}
+
+/**
+ * Strip base64 data URI images from markdown, leaving only the alt text
+ * as a placeholder. Used when keepOriginalLinks is enabled — base64 images
+ * have no original URL to restore, so we remove the heavy data URI.
+ *
+ * @param {string} markdown - Markdown content with data URI images
+ * @returns {{markdown: string, stripped: number}}
+ */
+export function stripBase64Images(markdown) {
+  let stripped = 0;
+  const updatedMarkdown = markdown.replace(
+    BASE64_IMAGE_REGEX,
+    (_match, altText) => {
+      stripped++;
+      return altText ? `*[image: ${altText}]*` : '';
+    }
+  );
+  return { markdown: updatedMarkdown, stripped };
+}
+
+/**
+ * @param {string} markdown - Markdown content
+ * @returns {boolean}
+ */
+export function hasBase64Images(markdown) {
+  return BASE64_IMAGE_REGEX.test(markdown);
+}

package/src/figures.js

@@ -0,0 +1,208 @@
+/**
+ * Figure image extraction and download module (R4).
+ *
+ * Extracts figure images from web pages and downloads them locally.
+ * Supports multi-language figure detection (English/Russian).
+ *
+ * Based on reference implementation from:
+ * https://github.com/link-foundation/meta-theory/blob/main/scripts/download.mjs
+ *
+ * @module figures
+ */
+
+import fetch from 'node-fetch';
+import * as cheerio from 'cheerio';
+import { URL } from 'url';
+import { retry } from './retry.js';
+import { createBrowser, getBrowserEngine } from './browser.js';
+import { dismissPopups, scrollToLoadContent } from './popups.js';
+
+/**
+ * Extract figure elements from HTML content.
+ *
+ * @param {string} html - HTML content
+ * @param {string} baseUrl - Base URL for resolving relative URLs
+ * @returns {Object[]} Array of figure objects with src, alt, caption, figureNum
+ */
+export function extractFigures(html, baseUrl) {
+  const $ = cheerio.load(html);
+  const figures = [];
+  let sequentialIndex = 0;
+
+  $('figure').each(function () {
+    const img = $(this).find('img').first();
+    if (!img.length) {
+      return;
+    }
+
+    const src = img.attr('src');
+    if (!src || src.startsWith('data:') || src.includes('.svg')) {
+      return;
+    }
+
+    sequentialIndex++;
+    const captionEl = $(this).find('figcaption');
+    const captionText = captionEl.length ? captionEl.text().trim() : '';
+
+    // Match multi-language figure numbers: "Figure X", "Рис. X", "Рисунок X"
+    const figureMatch = captionText.match(/(?:Figure|Рис\.?|Рисунок)\s*(\d+)/i);
+    const figureNum = figureMatch ? parseInt(figureMatch[1]) : sequentialIndex;
+
+    let resolvedSrc;
+    try {
+      resolvedSrc = new URL(src, baseUrl).href;
+    } catch {
+      resolvedSrc = src;
+    }
+
+    figures.push({
+      figureNum,
+      src: resolvedSrc,
+      alt: img.attr('alt') || '',
+      caption: captionText,
+      sequentialIndex,
+    });
+  });
+
+  return figures;
+}
+
+/**
+ * Download figure images to a local directory.
+ *
+ * @param {Object[]} figures - Array of figure objects from extractFigures
+ * @param {Object} [options] - Download options
+ * @param {Function} [options.onProgress] - Callback(figureNum, status)
+ * @returns {Promise<Object[]>} Array of download results with metadata
+ */
+export async function downloadFigures(figures, options = {}) {
+  const results = [];
+
+  for (const figure of figures) {
+    const ext =
+      figure.src.includes('.jpeg') || figure.src.includes('.jpg')
+        ? 'jpg'
+        : 'png';
+    const filename = `figure-${figure.figureNum}.${ext}`;
+
+    try {
+      const resp = await retry(() => fetch(figure.src), {
+        retries: 3,
+        baseDelay: 1000,
+      });
+
+      if (!resp.ok) {
+        throw new Error(`HTTP ${resp.status}`);
+      }
+
+      const buffer = await resp.buffer();
+
+      if (options.onProgress) {
+        options.onProgress(figure.figureNum, 'downloaded');
+      }
+
+      results.push({
+        figureNum: figure.figureNum,
+        filename,
+        buffer,
+        caption: figure.caption,
+        originalUrl: figure.src,
+      });
+    } catch (err) {
+      if (options.onProgress) {
+        options.onProgress(figure.figureNum, 'failed');
+      }
+      results.push({
+        figureNum: figure.figureNum,
+        filename,
+        buffer: null,
+        caption: figure.caption,
+        originalUrl: figure.src,
+        error: err.message,
+      });
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Extract and download figures from a URL using browser rendering.
+ * This handles JavaScript-rendered pages that require a browser.
+ *
+ * @param {string} url - URL to extract figures from
+ * @param {Object} [options] - Options
+ * @param {string} [options.engine] - Browser engine
+ * @returns {Promise<Object>} Result with figures and metadata
+ */
+export async function extractFiguresFromUrl(url, options = {}) {
+  const absoluteUrl = url.startsWith('http') ? url : `https://${url}`;
+  const engine = options.engine || 'puppeteer';
+
+  const browser = await createBrowser(engine);
+  try {
+    const page = await browser.newPage();
+    await page.setUserAgent(
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+    );
+    await page.setViewport({ width: 1920, height: 1080 });
+    await page.goto(absoluteUrl, {
+      waitUntil: 'networkidle0',
+      timeout: 30000,
+    });
+
+    await new Promise((resolve) => setTimeout(resolve, 5000));
+    await scrollToLoadContent(page);
+    await dismissPopups(page);
+
+    const html = await page.content();
+    const figures = extractFigures(html, absoluteUrl);
+    const downloaded = await downloadFigures(figures, options);
+
+    return {
+      url: absoluteUrl,
+      figures: downloaded,
+      totalFound: figures.length,
+      totalDownloaded: downloaded.filter((d) => d.buffer).length,
+    };
+  } finally {
+    await browser.close();
+  }
+}
+
+/**
+ * Figures extraction handler for Express API.
+ *
+ * Query parameters:
+ *   url    (required) - URL to extract figures from
+ *   engine - 'puppeteer' or 'playwright'
+ */
+export async function figuresHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+
+  try {
+    const engine = getBrowserEngine(req);
+    const result = await extractFiguresFromUrl(url, { engine });
+
+    // Return metadata (not the actual image buffers via JSON)
+    res.json({
+      url: result.url,
+      totalFound: result.totalFound,
+      totalDownloaded: result.totalDownloaded,
+      figures: result.figures.map((f) => ({
+        figureNum: f.figureNum,
+        filename: f.filename,
+        caption: f.caption,
+        originalUrl: f.originalUrl,
+        downloaded: !!f.buffer,
+        error: f.error || null,
+      })),
+    });
+  } catch (err) {
+    console.error(err);
+    res.status(500).send('Error extracting figures');
+  }
+}

package/src/gdocs.js

@@ -0,0 +1,335 @@
+// Google Docs capture module
+// Supports both API-based and browser-based capture of Google Docs documents.
+// API-based capture uses the Google Docs export URL pattern:
+//   https://docs.google.com/document/d/{DOCUMENT_ID}/export?format={FORMAT}
+// Browser-based capture navigates to the export URL using Puppeteer or Playwright.
+
+import fetch from 'node-fetch';
+import he from 'he';
+import archiver from 'archiver';
+import { convertHtmlToMarkdown, prettyPrintHtml } from './lib.js';
+
+const GDOCS_URL_PATTERN = /docs\.google\.com\/document\/d\/([a-zA-Z0-9_-]+)/;
+
+const GDOCS_EXPORT_BASE = 'https://docs.google.com/document/d';
+
+/**
+ * Supported Google Docs export formats.
+ * These map to the `format` query parameter in the export URL.
+ */
+export const GDOCS_EXPORT_FORMATS = {
+  html: 'html',
+  txt: 'txt',
+  md: 'md',
+  pdf: 'pdf',
+  docx: 'docx',
+  epub: 'epub',
+  zip: 'zip',
+};
+
+/**
+ * Check if a URL is a Google Docs document URL.
+ *
+ * @param {string} url - URL to check
+ * @returns {boolean} True if the URL is a Google Docs document URL
+ */
+export function isGoogleDocsUrl(url) {
+  if (!url || typeof url !== 'string') {
+    return false;
+  }
+  return GDOCS_URL_PATTERN.test(url);
+}
+
+/**
+ * Extract the document ID from a Google Docs URL.
+ *
+ * @param {string} url - Google Docs URL
+ * @returns {string|null} Document ID or null if not a valid Google Docs URL
+ */
+export function extractDocumentId(url) {
+  if (!url || typeof url !== 'string') {
+    return null;
+  }
+  const match = url.match(GDOCS_URL_PATTERN);
+  return match ? match[1] : null;
+}
+
+/**
+ * Build a Google Docs export URL.
+ *
+ * @param {string} documentId - Google Docs document ID
+ * @param {string} [format='html'] - Export format (html, txt, md, pdf, docx, epub, zip)
+ * @returns {string} Export URL
+ */
+export function buildExportUrl(documentId, format = 'html') {
+  const exportFormat = GDOCS_EXPORT_FORMATS[format] || 'html';
+  return `${GDOCS_EXPORT_BASE}/${documentId}/export?format=${exportFormat}`;
+}
+
+/**
+ * Fetch a Google Docs document via the export URL (API-based capture).
+ *
+ * For public documents, no authentication is needed.
+ * For private documents, pass an API token via the apiToken option.
+ *
+ * @param {string} url - Google Docs URL (edit URL or any URL containing the document ID)
+ * @param {Object} [options] - Fetch options
+ * @param {string} [options.format='html'] - Export format
+ * @param {string} [options.apiToken] - API token for private documents (Bearer token)
+ * @returns {Promise<{content: string, format: string, documentId: string, exportUrl: string}>}
+ */
+export async function fetchGoogleDoc(url, options = {}) {
+  const { format = 'html', apiToken } = options;
+
+  const documentId = extractDocumentId(url);
+  if (!documentId) {
+    throw new Error(`Not a valid Google Docs URL: ${url}`);
+  }
+
+  const exportUrl = buildExportUrl(documentId, format);
+
+  const headers = {
+    'User-Agent':
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    'Accept-Charset': 'utf-8',
+    'Accept-Language': 'en-US,en;q=0.9',
+  };
+
+  if (apiToken) {
+    headers['Authorization'] = `Bearer ${apiToken}`;
+  }
+
+  const response = await fetch(exportUrl, {
+    headers,
+    redirect: 'follow',
+  });
+
+  if (!response.ok) {
+    const statusText = response.statusText || 'Unknown error';
+    throw new Error(
+      `Failed to fetch Google Doc (${response.status} ${statusText}): ${exportUrl}`
+    );
+  }
+
+  const rawContent = await response.text();
+
+  // Decode HTML entities to unicode for text-based formats
+  const content =
+    format === 'html' || format === 'txt' || format === 'md'
+      ? he.decode(rawContent)
+      : rawContent;
+
+  return {
+    content,
+    format,
+    documentId,
+    exportUrl,
+  };
+}
+
+/**
+ * Fetch a Google Docs document and convert to Markdown.
+ *
+ * Uses the HTML export format internally, then converts to Markdown
+ * using the existing web-capture HTML-to-Markdown pipeline.
+ *
+ * @param {string} url - Google Docs URL
+ * @param {Object} [options] - Options
+ * @param {string} [options.apiToken] - API token for private documents
+ * @returns {Promise<{markdown: string, documentId: string, exportUrl: string}>}
+ */
+export async function fetchGoogleDocAsMarkdown(url, options = {}) {
+  const { apiToken } = options;
+
+  // Always fetch as HTML first, then convert to Markdown
+  const result = await fetchGoogleDoc(url, {
+    format: 'html',
+    apiToken,
+  });
+
+  const markdown = convertHtmlToMarkdown(result.content, result.exportUrl);
+
+  return {
+    markdown,
+    documentId: result.documentId,
+    exportUrl: result.exportUrl,
+  };
+}
+
+/**
+ * Extract base64 data URI images from HTML content.
+ *
+ * Google Docs HTML exports embed images as base64 data URIs.
+ * This function extracts them and replaces with local file paths.
+ *
+ * @param {string} html - HTML content with data URI images
+ * @returns {{html: string, images: Array<{filename: string, data: Buffer, mimeType: string}>}}
+ */
+export function extractBase64Images(html) {
+  const images = [];
+  let idx = 1;
+
+  const updatedHtml = html.replace(
+    /(<img\s[^>]*src=")data:image\/(png|jpeg|jpg|gif|webp|svg\+xml);base64,([^"]+)(")/gi,
+    (match, prefix, mimeExt, base64Data, suffix) => {
+      const ext =
+        mimeExt === 'jpeg' ? 'jpg' : mimeExt === 'svg+xml' ? 'svg' : mimeExt;
+      const filename = `image-${String(idx).padStart(2, '0')}.${ext}`;
+      const mimeType = `image/${mimeExt}`;
+      images.push({
+        filename,
+        data: Buffer.from(base64Data, 'base64'),
+        mimeType,
+      });
+      idx++;
+      return `${prefix}images/${filename}${suffix}`;
+    }
+  );
+
+  return { html: updatedHtml, images };
+}
+
+/**
+ * Fetch a Google Docs document as a ZIP archive with images.
+ *
+ * Fetches the document as HTML, extracts embedded base64 images,
+ * and bundles everything into a ZIP archive containing:
+ * - document.md (Markdown version)
+ * - document.html (HTML version with local image paths)
+ * - images/ (extracted images)
+ *
+ * @param {string} url - Google Docs URL
+ * @param {Object} [options] - Options
+ * @param {string} [options.apiToken] - API token for private documents
+ * @returns {Promise<{documentId: string, exportUrl: string, createArchive: function}>}
+ */
+export async function fetchGoogleDocAsArchive(url, options = {}) {
+  const { apiToken } = options;
+
+  // Fetch HTML with embedded base64 images
+  const result = await fetchGoogleDoc(url, { format: 'html', apiToken });
+
+  // Extract base64 images from HTML
+  const { html: localHtml, images } = extractBase64Images(result.content);
+
+  // Convert the localized HTML to Markdown
+  const markdown = convertHtmlToMarkdown(localHtml);
+
+  return {
+    documentId: result.documentId,
+    exportUrl: result.exportUrl,
+    html: localHtml,
+    markdown,
+    images,
+  };
+}
+
+/**
+ * Express handler for the /gdocs API endpoint.
+ *
+ * Query parameters:
+ * - url (required): Google Docs URL
+ * - format (optional): Export format (html, markdown, md, txt, pdf, docx)
+ * - apiToken (optional): API token for private documents
+ *
+ * The API token can also be provided via:
+ * - Authorization header: Bearer <token>
+ * - X-Api-Token header: <token>
+ *
+ * @param {import('express').Request} req
+ * @param {import('express').Response} res
+ */
+export async function gdocsHandler(req, res) {
+  const url = req.query.url;
+  if (!url) {
+    return res.status(400).send('Missing `url` parameter');
+  }
+
+  if (!isGoogleDocsUrl(url)) {
+    return res.status(400).send('URL is not a Google Docs document URL');
+  }
+
+  // Resolve API token from query, headers, or environment
+  const apiToken =
+    req.query.apiToken ||
+    extractBearerToken(req.headers.authorization) ||
+    req.headers['x-api-token'] ||
+    undefined;
+
+  const format = (req.query.format || 'markdown').toLowerCase();
+
+  try {
+    if (format === 'archive' || format === 'zip') {
+      return await sendGDocsArchive(res, url, apiToken);
+    }
+
+    if (format === 'markdown' || format === 'md') {
+      const result = await fetchGoogleDocAsMarkdown(url, { apiToken });
+      return res.type('text/markdown').send(result.markdown);
+    }
+
+    return await sendGDocsExport(res, url, format, apiToken);
+  } catch (err) {
+    console.error('Google Docs capture error:', err.message);
+    return res.status(500).send(`Error capturing Google Doc: ${err.message}`);
+  }
+}
+
+async function sendGDocsArchive(res, url, apiToken) {
+  const archiveResult = await fetchGoogleDocAsArchive(url, { apiToken });
+
+  res.set('Content-Type', 'application/zip');
+  res.set(
+    'Content-Disposition',
+    `attachment; filename="gdoc-${archiveResult.documentId}.zip"`
+  );
+
+  const archive = archiver('zip', { zlib: { level: 9 } });
+  archive.pipe(res);
+  archive.append(archiveResult.markdown, { name: 'document.md' });
+  archive.append(prettyPrintHtml(archiveResult.html), {
+    name: 'document.html',
+  });
+  for (const img of archiveResult.images) {
+    archive.append(img.data, { name: `images/${img.filename}` });
+  }
+  await archive.finalize();
+}
+
+const FORMAT_CONTENT_TYPES = {
+  html: 'text/html',
+  txt: 'text/plain',
+  pdf: 'application/pdf',
+  docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+};
+
+const FORMAT_DISPOSITIONS = {
+  pdf: 'inline; filename="document.pdf"',
+  docx: 'inline; filename="document.docx"',
+};
+
+async function sendGDocsExport(res, url, format, apiToken) {
+  const result = await fetchGoogleDoc(url, { format, apiToken });
+  const contentType = FORMAT_CONTENT_TYPES[format] || 'text/html';
+  const disposition = FORMAT_DISPOSITIONS[format];
+
+  res.type(contentType);
+  if (disposition) {
+    res.set('Content-Disposition', disposition);
+  }
+  return res.send(result.content);
+}
+
+/**
+ * Extract Bearer token from Authorization header value.
+ *
+ * @param {string|undefined} authHeader - Authorization header value
+ * @returns {string|undefined} Token or undefined
+ */
+function extractBearerToken(authHeader) {
+  if (!authHeader || typeof authHeader !== 'string') {
+    return undefined;
+  }
+  const match = authHeader.match(/^Bearer\s+(.+)$/i);
+  return match ? match[1] : undefined;
+}