@link-assistant/web-capture 1.7.14 → 1.7.15

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @link-assistant/web-capture
 
+## 1.7.15
+
+### Patch Changes
+
+- bf7eaec: Improve Google Docs public-export Markdown fidelity for headings, strikethrough, blockquotes, nested lists, and tables.
+
 ## 1.7.14
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.14",
+  "version": "1.7.15",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/gdocs-preprocess.js

@@ -30,6 +30,12 @@ export function preprocessGoogleDocsExportHtml(html) {
   let unwrappedLinks = 0;
 
   $('span, p').each(function () {
+    if (
+      this.tagName === 'span' &&
+      $(this).closest('h1, h2, h3, h4, h5, h6').length
+    ) {
+      return;
+    }
     const style = combinedElementStyle($, this, classStyles);
     const hasBold = hasBoldStyle(style);
     const hasItalic = hasItalicStyle(style);
@@ -56,15 +62,9 @@ export function preprocessGoogleDocsExportHtml(html) {
     hoisted += 1;
   });
 
-  $('p').each(function () {
-    const style = combinedElementStyle($, this, classStyles);
-    if (!isBlockquoteStyle(style)) {
-      return;
-    }
-    $(this).replaceWith(
-      `<blockquote><p>${$(this).html() || ''}</p></blockquote>`
-    );
-  });
+  groupGoogleDocsBlockquotes($, classStyles);
+  nestGoogleDocsLists($, classStyles);
+  normalizeGoogleDocsTables($);
 
   $('a').each(function () {
     const $a = $(this);
@@ -126,6 +126,24 @@ export function preprocessGoogleDocsExportHtml(html) {
   return { html: rewritten, hoisted, unwrappedLinks };
 }
 
+/**
+ * Normalize Markdown emitted from Google Docs public-export HTML converters.
+ *
+ * @param {string} markdown - Markdown rendered from preprocessed export HTML.
+ * @returns {string}
+ */
+export function normalizeGoogleDocsExportMarkdown(markdown) {
+  if (!markdown || typeof markdown !== 'string') {
+    return markdown || '';
+  }
+
+  return markdown
+    .replace(/\\([.!()[\]])/g, '$1')
+    .replace(/(^|[^\w~])~([^~\n]+)~(?=$|[^\w~])/g, '$1~~$2~~')
+    .replace(/\n{3,}(?=> )/g, '\n\n')
+    .replace(/\n{3,}/g, '\n\n');
+}
+
 function parseCssClassStyles($) {
   const classStyles = new Map();
   $('style').each(function () {
@@ -157,6 +175,213 @@ function combinedElementStyle($, element, classStyles) {
   return styles.join(';');
 }
 
+function groupGoogleDocsBlockquotes($, classStyles) {
+  const processed = new Set();
+
+  $('p').each(function () {
+    if (
+      processed.has(this) ||
+      !isBlockquoteStyle(combinedElementStyle($, this, classStyles))
+    ) {
+      return;
+    }
+
+    const paragraphs = [];
+    let current = $(this);
+    while (
+      current.length &&
+      current[0].tagName === 'p' &&
+      isBlockquoteStyle(combinedElementStyle($, current[0], classStyles))
+    ) {
+      paragraphs.push(current[0]);
+      processed.add(current[0]);
+      current = current.next();
+    }
+
+    const quoteHtml = paragraphs
+      .map((paragraph) => `<p>${$(paragraph).html() || ''}</p>`)
+      .join('');
+    $(paragraphs[0]).before(`<blockquote>${quoteHtml}</blockquote>`);
+    for (const paragraph of paragraphs) {
+      $(paragraph).remove();
+    }
+  });
+}
+
+function nestGoogleDocsLists($, classStyles) {
+  const processed = new Set();
+
+  $('body')
+    .children('ul, ol')
+    .each(function () {
+      if (processed.has(this)) {
+        return;
+      }
+
+      const group = [this];
+      processed.add(this);
+      let next = $(this).next();
+      while (next.length && ['ul', 'ol'].includes(next[0].tagName)) {
+        group.push(next[0]);
+        processed.add(next[0]);
+        next = next.next();
+      }
+
+      if (group.length < 2) {
+        return;
+      }
+
+      const nested = renderNestedListGroup($, group, classStyles);
+      $(group[0]).before(nested);
+      for (const list of group) {
+        $(list).remove();
+      }
+    });
+}
+
+function renderNestedListGroup($, group, classStyles) {
+  let html = '';
+  let currentLevel = -1;
+  const openTags = [];
+  const itemOpen = [];
+
+  const closeItem = (level) => {
+    if (itemOpen[level]) {
+      html += '</li>';
+      itemOpen[level] = false;
+    }
+  };
+
+  const closeList = (level) => {
+    closeItem(level);
+    html += `</${openTags[level]}>`;
+    openTags[level] = undefined;
+  };
+
+  const openList = (level, tagName) => {
+    openTags[level] = tagName;
+    itemOpen[level] = false;
+    html += `<${tagName}>`;
+  };
+
+  for (const list of group) {
+    const tagName = list.tagName.toLowerCase();
+    $(list)
+      .children('li')
+      .each(function () {
+        const level = googleDocsListItemLevel($, this, classStyles);
+        while (currentLevel > level) {
+          closeList(currentLevel);
+          currentLevel -= 1;
+        }
+
+        while (currentLevel < level) {
+          currentLevel += 1;
+          openList(currentLevel, tagName);
+        }
+
+        if (openTags[level] && openTags[level] !== tagName) {
+          closeList(level);
+          openList(level, tagName);
+        } else if (!openTags[level]) {
+          openList(level, tagName);
+        }
+
+        closeItem(level);
+        html += `<li>${$(this).html() || ''}`;
+        itemOpen[level] = true;
+
+        for (let deeper = level + 1; deeper < itemOpen.length; deeper += 1) {
+          itemOpen[deeper] = false;
+          openTags[deeper] = undefined;
+        }
+      });
+  }
+
+  while (currentLevel >= 0) {
+    closeList(currentLevel);
+    currentLevel -= 1;
+  }
+
+  return html;
+}
+
+function googleDocsListItemLevel($, li, classStyles) {
+  const style = combinedElementStyle($, li, classStyles);
+  const marginLeft = numericCssPointValue(style, 'margin-left');
+  if (marginLeft <= 0) {
+    return 0;
+  }
+  return Math.max(0, Math.round((marginLeft - 36) / 36));
+}
+
+function normalizeGoogleDocsTables($) {
+  $('table').each(function () {
+    const $table = $(this);
+    const rows = $table.find('tr').toArray();
+    if (!rows.length) {
+      return;
+    }
+
+    const $cleanTable = $('<table></table>');
+    const $thead = $('<thead></thead>');
+    const $tbody = $('<tbody></tbody>');
+
+    rows.forEach((row, rowIndex) => {
+      const $tr = $('<tr></tr>');
+      $(row)
+        .children('td, th')
+        .each(function () {
+          const tagName = rowIndex === 0 ? 'th' : 'td';
+          const $cell = $(`<${tagName}></${tagName}>`);
+          $cell.html(inlineTableCellHtml($, this));
+          $tr.append($cell);
+        });
+
+      if (!$tr.children().length) {
+        return;
+      }
+      if (rowIndex === 0) {
+        $thead.append($tr);
+      } else {
+        $tbody.append($tr);
+      }
+    });
+
+    if ($thead.children().length) {
+      $cleanTable.append($thead);
+    }
+    if ($tbody.children().length) {
+      $cleanTable.append($tbody);
+    }
+
+    $table.replaceWith($cleanTable);
+  });
+}
+
+function inlineTableCellHtml($, cell) {
+  const parts = [];
+  $(cell)
+    .contents()
+    .each(function () {
+      if (this.type === 'tag' && ['p', 'div'].includes(this.tagName)) {
+        const inner = $(this).html() || '';
+        if (inner.trim()) {
+          parts.push(inner);
+        }
+        return;
+      }
+      if (this.type === 'tag') {
+        parts.push($.html(this));
+        return;
+      }
+      if (this.type === 'text' && this.data.trim()) {
+        parts.push(this.data);
+      }
+    });
+  return parts.join('<br>');
+}
+
 function hasBoldStyle(style) {
   return /font-weight\s*:\s*(?:bold|[6-9]\d{2})/i.test(style);
 }

package/src/gdocs.js

@@ -11,7 +11,10 @@ import fetch from 'node-fetch';
 import he from 'he';
 import { convertHtmlToMarkdown } from './lib.js';
 import { createBrowser as defaultCreateBrowser } from './browser.js';
-import { preprocessGoogleDocsExportHtml } from './gdocs-preprocess.js';
+import {
+  normalizeGoogleDocsExportMarkdown,
+  preprocessGoogleDocsExportHtml,
+} from './gdocs-preprocess.js';
 import { localizeGoogleDocsModelImages } from './gdocs-images.js';
 import { renderBlocksMarkdown } from './gdocs-render-markdown.js';
 import {
@@ -22,6 +25,7 @@ import {
 } from './gdocs-fallback.js';
 
 export {
+  normalizeGoogleDocsExportMarkdown,
   preprocessGoogleDocsExportHtml,
   localizeGoogleDocsModelImages,
   isGoogleDocsBrowserModelUnavailableError,
@@ -305,7 +309,9 @@ export async function fetchGoogleDocAsMarkdown(url, options = {}) {
     hoisted: preprocessed.hoisted,
     unwrappedLinks: preprocessed.unwrappedLinks,
   }));
-  const markdown = convertHtmlToMarkdown(preprocessed.html, result.exportUrl);
+  const markdown = normalizeGoogleDocsExportMarkdown(
+    convertHtmlToMarkdown(preprocessed.html, result.exportUrl)
+  );
   log?.debug?.(() => ({
     event: 'gdocs.public-export.markdown',
     documentId: result.documentId,
@@ -1354,7 +1360,9 @@ export async function fetchGoogleDocAsArchive(url, options = {}) {
   const { html: localHtml, images } = extractBase64Images(preprocessed.html);
 
   // Convert the localized HTML to Markdown
-  const markdown = convertHtmlToMarkdown(localHtml);
+  const markdown = normalizeGoogleDocsExportMarkdown(
+    convertHtmlToMarkdown(localHtml)
+  );
   log?.debug?.(() => ({
     event: 'gdocs.public-export.archive',
     documentId: result.documentId,

package/tests/integration/gdocs-public-doc.test.js

@@ -172,17 +172,6 @@ describe('Google Docs public test document (issue #90)', () => {
     // the captured markdown. This is our regression net for any future
     // changes to the public-export or HTML-to-markdown pipeline.
     //
-    // The captured markdown from `--capture api` still escapes leading
-    // numeric prefixes (e.g. `1\.` instead of `1.`) and strips inline
-    // formatting — both are documented as R2 in
-    // docs/case-studies/issue-90/README.md and are not yet fixed. For the
-    // purposes of this regression test we normalise Turndown's backslash
-    // escapes before checking that every section heading is present so the
-    // test locks down content preservation rather than format fidelity. The
-    // next wave of work (R2) will tighten this matcher once the HTML-to-
-    // Markdown pipeline stops escaping the dots.
-    const normalizeEscapes = (text) => text.replace(/\\([.!()[\]])/g, '$1');
-
     // Google Docs occasionally returns transient 500s on the public-export
     // endpoint. The Habr integration suite has the same flake and solves it
     // with retry + exponential backoff, so we reuse the shared helper here.
@@ -207,10 +196,21 @@ describe('Google Docs public test document (issue #90)', () => {
       expect(exportUrl).toBe(buildExportUrl(PUBLIC_TEST_DOCUMENT.id, 'html'));
       expect(markdown.length).toBeGreaterThan(1000);
 
-      const normalized = normalizeEscapes(markdown);
       for (const section of PUBLIC_TEST_DOCUMENT.sections) {
-        expect(normalized).toContain(section);
+        expect(markdown).toContain(section);
       }
+      expect(markdown).toContain('## 1. Headings');
+      expect(markdown).toContain('**This text is bold**');
+      expect(markdown).toContain('*This text is italic*');
+      expect(markdown).toContain('~~This text has strikethrough~~');
+      expect(markdown).toContain('> This is a single-level blockquote');
+      expect(markdown).toContain('| Feature | Supported | Notes |');
+      expect(markdown).not.toContain('| Feature |  | Supported |');
+      expect(markdown).toContain('|  | x |  |');
+      expect(markdown).toContain('1.  Parent item 1');
+      expect(markdown).toContain('    1.  Child item 1.1');
+      expect(markdown).toContain('        1.  Grandchild item 1.2.1');
+      expect(markdown).not.toContain('-   Child item 1.1');
     });
 
     it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(

package/tests/unit/gdocs.test.js

@@ -13,9 +13,11 @@ import {
   renderDocsApiDocument,
   selectGoogleDocsCaptureMethod,
   localizeGoogleDocsModelImages,
+  normalizeGoogleDocsExportMarkdown,
   preprocessGoogleDocsExportHtml,
   GDOCS_EXPORT_FORMATS,
 } from '../../src/gdocs.js';
+import { convertHtmlToMarkdown } from '../../src/lib.js';
 import {
   extractAndSaveImages,
   extractBase64ToBuffers,
@@ -1042,6 +1044,46 @@ describe('gdocs', () => {
       expect(unwrappedLinks).toBe(0);
       expect(out).toContain('<p>Plain text with <strong>bold</strong>.</p>');
     });
+
+    it('recovers public-export markdown structure for issue #102', () => {
+      const html = `
+        <style>
+          .c5{margin-left:36pt}
+          .c8{margin-left:72pt}
+          .c19{margin-left:108pt}
+          .q{margin-left:24pt;margin-right:24pt}
+          .i{font-style:italic}
+          .s{text-decoration:line-through}
+        </style>
+        <h2><span class="i">1. Headings</span></h2>
+        <p class="q">Quote one.</p>
+        <p class="q">Quote two.</p>
+        <ol><li class="c5">Parent</li></ol>
+        <ol><li class="c8">Child</li></ol>
+        <ol><li class="c19">Grandchild</li></ol>
+        <ol><li class="c5">Parent 2</li></ol>
+        <table>
+          <thead>
+            <tr><td><p>Feature</p></td><td><p>Supported</p></td><tbody></tbody></tr>
+            <tr><td><p><span class="s">Strike</span></p></td><td><p>Yes</p></td></tr>
+          </thead>
+        </table>
+      `;
+
+      const preprocessed = preprocessGoogleDocsExportHtml(html);
+      const markdown = normalizeGoogleDocsExportMarkdown(
+        convertHtmlToMarkdown(preprocessed.html)
+      );
+
+      expect(markdown).toContain('## 1. Headings');
+      expect(markdown).not.toContain('*1. Headings*');
+      expect(markdown).toContain('> Quote one.\n> \n> Quote two.');
+      expect(markdown).toMatch(
+        /1\.\s+Parent\n\s+1\.\s+Child\n\s+1\.\s+Grandchild\n2\.\s+Parent 2/
+      );
+      expect(markdown).toContain('| Feature | Supported |');
+      expect(markdown).toContain('| ~~Strike~~ | Yes |');
+    });
   });
 
   describe('extractBase64Images', () => {