@link-assistant/web-capture 1.7.8 → 1.7.10

package/CHANGELOG.md

@@ -1,5 +1,36 @@
 # @link-assistant/web-capture
 
+## 1.7.10
+
+### Patch Changes
+
+- fcc031f: Fix remaining Google Docs capture gaps from issue #92 in both the JS and
+  Rust CLIs:
+  - Browser capture now keeps multi-column table rows intact, renders
+    ordered lists with sequential `1. 2. 3.` numbering, and joins same-list
+    items with a single newline so tight-list markdown matches the source
+    document.
+  - Archive mode downloads `docs-images-rt/...` image URLs into the
+    archive's `images/` directory and rewrites markdown/html references so
+    exports are self-contained.
+  - API mode (`--capture api`) runs the export HTML through a shared
+    preprocessor that hoists inline bold / italic / strikethrough spans to
+    semantic tags, strips Google Docs' heading-numbering spans and empty
+    anchor wrappers, unwraps `google.com/url?q=` redirects, and normalizes
+    non-breaking spaces.
+
+## 1.7.9
+
+### Patch Changes
+
+- cee7590: Add a live Google Docs integration test that captures the public markdown
+  round-trip reference document
+  (https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit)
+  and verifies every documented URL variation, capture-method selection path,
+  and feature-section heading. The test is gated behind `GDOCS_INTEGRATION=true`
+  and is wired into CI so regressions in the `--capture api` HTML-to-Markdown
+  pipeline surface against a real Google Doc. Addresses issue #90.
+
 ## 1.7.8
 
 ### Patch Changes

package/bin/web-capture.js

@@ -527,8 +527,24 @@ async function captureUrl(url, options) {
           textBytes: Buffer.byteLength(result.text || ''),
         }));
         if (normalizedFormat === 'archive' || normalizedFormat === 'zip') {
+          const { localizeGoogleDocsModelImages } =
+            await import('../src/gdocs.js');
+          const localized = await localizeGoogleDocsModelImages(result, {
+            log,
+          });
+          log.debug(() => ({
+            event: 'gdocs.capture.browser-model.archive.localized',
+            images: localized.images.length,
+            markdownBytes: Buffer.byteLength(localized.markdown),
+            htmlBytes: Buffer.byteLength(localized.html),
+          }));
           await writeGoogleDocsArchive({
-            archiveResult: { ...result, images: [] },
+            archiveResult: {
+              ...result,
+              markdown: localized.markdown,
+              html: localized.html,
+              images: localized.images,
+            },
             absoluteUrl,
             explicitOutput,
             dataDir,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.8",
+  "version": "1.7.10",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/gdocs-images.js

@@ -0,0 +1,130 @@
+// Google Docs editor-model image localization (issue #92 R5).
+//
+// The browser-model capture returns `docs-images-rt/…` URLs inside the
+// markdown/html. Archive mode needs those images copied into `images/` so
+// the archive is self-contained. This module downloads the images and
+// rewrites the captured output accordingly.
+
+import fetch from 'node-fetch';
+import { URL } from 'url';
+
+const DEFAULT_USER_AGENT =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
+function getExtensionFromImageUrl(url) {
+  try {
+    const u = new URL(url);
+    const match = u.pathname.match(/\.(png|jpe?g|gif|webp|svg)$/iu);
+    if (match) {
+      const ext = match[1].toLowerCase();
+      return ext === 'jpeg' ? '.jpg' : `.${ext}`;
+    }
+  } catch {
+    // fall through to default
+  }
+  return '.png';
+}
+
+function mimeTypeForExtension(filename) {
+  const dot = filename.lastIndexOf('.');
+  const ext = dot >= 0 ? filename.slice(dot + 1).toLowerCase() : 'png';
+  switch (ext) {
+    case 'jpg':
+    case 'jpeg':
+      return 'image/jpeg';
+    case 'gif':
+      return 'image/gif';
+    case 'webp':
+      return 'image/webp';
+    case 'svg':
+      return 'image/svg+xml';
+    default:
+      return 'image/png';
+  }
+}
+
+/**
+ * Download the `docs-images-rt` URLs captured from an editor-model capture.
+ *
+ * Produces `{ filename, data, mimeType }` entries compatible with
+ * `writeGoogleDocsArchive` and rewrites the markdown/html to reference the
+ * local `images/<filename>` paths.
+ *
+ * @param {{markdown: string, html: string, capture: Object}} modelResult
+ * @param {Object} [options] - Options
+ * @param {typeof fetch} [options.fetchImpl=fetch] - Injected fetch for tests
+ * @param {Object} [options.log] - Verbose logger
+ * @returns {Promise<{markdown: string, html: string, images: Array}>}
+ */
+export async function localizeGoogleDocsModelImages(modelResult, options = {}) {
+  const { fetchImpl = fetch, log } = options;
+  const images = [];
+  const seen = new Map();
+  let nextIndex = 1;
+
+  const resolveFilename = (url, existingAlt) => {
+    if (seen.has(url)) {
+      return seen.get(url);
+    }
+    const ext = getExtensionFromImageUrl(url);
+    const filename = `image-${String(nextIndex).padStart(2, '0')}${ext}`;
+    nextIndex += 1;
+    seen.set(url, filename);
+    log?.debug?.(() => ({
+      event: 'gdocs.archive.image.scheduled',
+      url,
+      filename,
+      alt: existingAlt || null,
+    }));
+    return filename;
+  };
+
+  const candidates = (modelResult?.capture?.images || []).filter(
+    (img) => img?.url
+  );
+  for (const img of candidates) {
+    const filename = resolveFilename(img.url, img.alt);
+    try {
+      const response = await fetchImpl(img.url, {
+        headers: {
+          'User-Agent': DEFAULT_USER_AGENT,
+          Accept: 'image/*,*/*;q=0.8',
+        },
+        redirect: 'follow',
+      });
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}`);
+      }
+      const arrayBuffer = await response.arrayBuffer();
+      const data = Buffer.from(arrayBuffer);
+      const mimeType =
+        response.headers?.get?.('content-type') ||
+        mimeTypeForExtension(filename);
+      images.push({ filename, data, mimeType });
+      log?.debug?.(() => ({
+        event: 'gdocs.archive.image.downloaded',
+        url: img.url,
+        filename,
+        bytes: data.length,
+        mimeType,
+      }));
+    } catch (err) {
+      log?.debug?.(() => ({
+        event: 'gdocs.archive.image.failed',
+        url: img.url,
+        filename,
+        error: err?.message || String(err),
+      }));
+    }
+  }
+
+  let markdown = modelResult?.markdown || '';
+  let html = modelResult?.html || '';
+  for (const [url, filename] of seen.entries()) {
+    const localPath = `images/${filename}`;
+    markdown = markdown.split(url).join(localPath);
+    html = html.split(url).join(localPath);
+  }
+
+  return { markdown, html, images };
+}

package/src/gdocs-preprocess.js

@@ -0,0 +1,105 @@
+// Google Docs export HTML preprocessor (issue #92 R6).
+//
+// Google Drive's `/export?format=html` payload serves bold/italic/
+// strikethrough as inline `style` spans, numbers headings with
+// `<a id><span>1. </span>`, and routes every link through a
+// `google.com/url?q=` redirect. The generic Cheerio + Turndown pipeline
+// discards those signals, so the browser-less API path loses the
+// formatting and gains noisy redirect URLs.
+//
+// This module rewrites the export HTML into semantic markup before the
+// generic converter runs.
+
+import * as cheerio from 'cheerio';
+
+/**
+ * Pre-process Google Docs export HTML so the generic Cheerio+Turndown
+ * pipeline preserves inline formatting, heading numbers, and link targets.
+ *
+ * @param {string} html - Raw export HTML returned by Google Docs.
+ * @returns {{html: string, hoisted: number, unwrappedLinks: number}}
+ */
+export function preprocessGoogleDocsExportHtml(html) {
+  if (!html || typeof html !== 'string') {
+    return { html: html || '', hoisted: 0, unwrappedLinks: 0 };
+  }
+
+  const $ = cheerio.load(html, { decodeEntities: false });
+  let hoisted = 0;
+  let unwrappedLinks = 0;
+
+  $('span[style], p[style]').each(function () {
+    const style = String($(this).attr('style') || '');
+    const hasBold = /font-weight\s*:\s*(?:bold|[6-9]\d{2})/i.test(style);
+    const hasItalic = /font-style\s*:\s*italic/i.test(style);
+    const hasStrike = /text-decoration[^;]*\bline-through\b/i.test(style);
+    if (!hasBold && !hasItalic && !hasStrike) {
+      return;
+    }
+    const inner = $(this).html() || '';
+    let wrapped = inner;
+    if (hasStrike) {
+      wrapped = `<del>${wrapped}</del>`;
+    }
+    if (hasItalic) {
+      wrapped = `<em>${wrapped}</em>`;
+    }
+    if (hasBold) {
+      wrapped = `<strong>${wrapped}</strong>`;
+    }
+    $(this).replaceWith(wrapped);
+    hoisted += 1;
+  });
+
+  $('h1, h2, h3, h4, h5, h6').each(function () {
+    const $h = $(this);
+    $h.find('a').each(function () {
+      const $a = $(this);
+      if (!$a.attr('href') && !$a.text().trim() && !$a.find('img').length) {
+        $a.remove();
+      }
+    });
+    $h.find('> span, > strong > span').each(function () {
+      const $span = $(this);
+      const text = $span.text();
+      if (/^\s*\d+(?:\.\d+)*\.?\s*$/.test(text)) {
+        $span.remove();
+      }
+    });
+  });
+
+  $('a[href*="google.com/url?q="]').each(function () {
+    const href = String($(this).attr('href') || '');
+    const match = href.match(/[?&]q=([^&]+)/);
+    if (match) {
+      try {
+        const decoded = decodeURIComponent(match[1]);
+        $(this).attr('href', decoded);
+        unwrappedLinks += 1;
+      } catch {
+        // Leave the original href in place if it can't be decoded.
+      }
+    }
+  });
+
+  $('*').each(function () {
+    if (this.type !== 'tag') {
+      return;
+    }
+    $(this)
+      .contents()
+      .each(function () {
+        if (this.type === 'text' && this.data) {
+          const replaced = this.data.replace(/\u00A0/g, ' ');
+          if (replaced !== this.data) {
+            this.data = replaced;
+          }
+        }
+      });
+  });
+
+  let rewritten = $.html();
+  rewritten = rewritten.replace(/&nbsp;/g, ' ');
+
+  return { html: rewritten, hoisted, unwrappedLinks };
+}