@link-assistant/web-capture 1.7.14 → 1.7.16

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/web-capture
 
+## 1.7.16
+
+### Patch Changes
+
+- bf7eaec: Keep Google Docs browser-model soft line breaks outside inline mark tags and preserve image dimensions in HTML output.
+
+## 1.7.15
+
+### Patch Changes
+
+- bf7eaec: Improve Google Docs public-export Markdown fidelity for headings, strikethrough, blockquotes, nested lists, and tables.
+
 ## 1.7.14
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.14",
+  "version": "1.7.16",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {

package/src/gdocs-preprocess.js

@@ -30,6 +30,12 @@ export function preprocessGoogleDocsExportHtml(html) {
   let unwrappedLinks = 0;
 
   $('span, p').each(function () {
+    if (
+      this.tagName === 'span' &&
+      $(this).closest('h1, h2, h3, h4, h5, h6').length
+    ) {
+      return;
+    }
     const style = combinedElementStyle($, this, classStyles);
     const hasBold = hasBoldStyle(style);
     const hasItalic = hasItalicStyle(style);
@@ -56,15 +62,9 @@ export function preprocessGoogleDocsExportHtml(html) {
     hoisted += 1;
   });
 
-  $('p').each(function () {
-    const style = combinedElementStyle($, this, classStyles);
-    if (!isBlockquoteStyle(style)) {
-      return;
-    }
-    $(this).replaceWith(
-      `<blockquote><p>${$(this).html() || ''}</p></blockquote>`
-    );
-  });
+  groupGoogleDocsBlockquotes($, classStyles);
+  nestGoogleDocsLists($, classStyles);
+  normalizeGoogleDocsTables($);
 
   $('a').each(function () {
     const $a = $(this);
@@ -126,6 +126,24 @@ export function preprocessGoogleDocsExportHtml(html) {
   return { html: rewritten, hoisted, unwrappedLinks };
 }
 
+/**
+ * Normalize Markdown emitted from Google Docs public-export HTML converters.
+ *
+ * @param {string} markdown - Markdown rendered from preprocessed export HTML.
+ * @returns {string}
+ */
+export function normalizeGoogleDocsExportMarkdown(markdown) {
+  if (!markdown || typeof markdown !== 'string') {
+    return markdown || '';
+  }
+
+  return markdown
+    .replace(/\\([.!()[\]])/g, '$1')
+    .replace(/(^|[^\w~])~([^~\n]+)~(?=$|[^\w~])/g, '$1~~$2~~')
+    .replace(/\n{3,}(?=> )/g, '\n\n')
+    .replace(/\n{3,}/g, '\n\n');
+}
+
 function parseCssClassStyles($) {
   const classStyles = new Map();
   $('style').each(function () {
@@ -157,6 +175,213 @@ function combinedElementStyle($, element, classStyles) {
   return styles.join(';');
 }
 
+function groupGoogleDocsBlockquotes($, classStyles) {
+  const processed = new Set();
+
+  $('p').each(function () {
+    if (
+      processed.has(this) ||
+      !isBlockquoteStyle(combinedElementStyle($, this, classStyles))
+    ) {
+      return;
+    }
+
+    const paragraphs = [];
+    let current = $(this);
+    while (
+      current.length &&
+      current[0].tagName === 'p' &&
+      isBlockquoteStyle(combinedElementStyle($, current[0], classStyles))
+    ) {
+      paragraphs.push(current[0]);
+      processed.add(current[0]);
+      current = current.next();
+    }
+
+    const quoteHtml = paragraphs
+      .map((paragraph) => `<p>${$(paragraph).html() || ''}</p>`)
+      .join('');
+    $(paragraphs[0]).before(`<blockquote>${quoteHtml}</blockquote>`);
+    for (const paragraph of paragraphs) {
+      $(paragraph).remove();
+    }
+  });
+}
+
+function nestGoogleDocsLists($, classStyles) {
+  const processed = new Set();
+
+  $('body')
+    .children('ul, ol')
+    .each(function () {
+      if (processed.has(this)) {
+        return;
+      }
+
+      const group = [this];
+      processed.add(this);
+      let next = $(this).next();
+      while (next.length && ['ul', 'ol'].includes(next[0].tagName)) {
+        group.push(next[0]);
+        processed.add(next[0]);
+        next = next.next();
+      }
+
+      if (group.length < 2) {
+        return;
+      }
+
+      const nested = renderNestedListGroup($, group, classStyles);
+      $(group[0]).before(nested);
+      for (const list of group) {
+        $(list).remove();
+      }
+    });
+}
+
+function renderNestedListGroup($, group, classStyles) {
+  let html = '';
+  let currentLevel = -1;
+  const openTags = [];
+  const itemOpen = [];
+
+  const closeItem = (level) => {
+    if (itemOpen[level]) {
+      html += '</li>';
+      itemOpen[level] = false;
+    }
+  };
+
+  const closeList = (level) => {
+    closeItem(level);
+    html += `</${openTags[level]}>`;
+    openTags[level] = undefined;
+  };
+
+  const openList = (level, tagName) => {
+    openTags[level] = tagName;
+    itemOpen[level] = false;
+    html += `<${tagName}>`;
+  };
+
+  for (const list of group) {
+    const tagName = list.tagName.toLowerCase();
+    $(list)
+      .children('li')
+      .each(function () {
+        const level = googleDocsListItemLevel($, this, classStyles);
+        while (currentLevel > level) {
+          closeList(currentLevel);
+          currentLevel -= 1;
+        }
+
+        while (currentLevel < level) {
+          currentLevel += 1;
+          openList(currentLevel, tagName);
+        }
+
+        if (openTags[level] && openTags[level] !== tagName) {
+          closeList(level);
+          openList(level, tagName);
+        } else if (!openTags[level]) {
+          openList(level, tagName);
+        }
+
+        closeItem(level);
+        html += `<li>${$(this).html() || ''}`;
+        itemOpen[level] = true;
+
+        for (let deeper = level + 1; deeper < itemOpen.length; deeper += 1) {
+          itemOpen[deeper] = false;
+          openTags[deeper] = undefined;
+        }
+      });
+  }
+
+  while (currentLevel >= 0) {
+    closeList(currentLevel);
+    currentLevel -= 1;
+  }
+
+  return html;
+}
+
+function googleDocsListItemLevel($, li, classStyles) {
+  const style = combinedElementStyle($, li, classStyles);
+  const marginLeft = numericCssPointValue(style, 'margin-left');
+  if (marginLeft <= 0) {
+    return 0;
+  }
+  return Math.max(0, Math.round((marginLeft - 36) / 36));
+}
+
+function normalizeGoogleDocsTables($) {
+  $('table').each(function () {
+    const $table = $(this);
+    const rows = $table.find('tr').toArray();
+    if (!rows.length) {
+      return;
+    }
+
+    const $cleanTable = $('<table></table>');
+    const $thead = $('<thead></thead>');
+    const $tbody = $('<tbody></tbody>');
+
+    rows.forEach((row, rowIndex) => {
+      const $tr = $('<tr></tr>');
+      $(row)
+        .children('td, th')
+        .each(function () {
+          const tagName = rowIndex === 0 ? 'th' : 'td';
+          const $cell = $(`<${tagName}></${tagName}>`);
+          $cell.html(inlineTableCellHtml($, this));
+          $tr.append($cell);
+        });
+
+      if (!$tr.children().length) {
+        return;
+      }
+      if (rowIndex === 0) {
+        $thead.append($tr);
+      } else {
+        $tbody.append($tr);
+      }
+    });
+
+    if ($thead.children().length) {
+      $cleanTable.append($thead);
+    }
+    if ($tbody.children().length) {
+      $cleanTable.append($tbody);
+    }
+
+    $table.replaceWith($cleanTable);
+  });
+}
+
+function inlineTableCellHtml($, cell) {
+  const parts = [];
+  $(cell)
+    .contents()
+    .each(function () {
+      if (this.type === 'tag' && ['p', 'div'].includes(this.tagName)) {
+        const inner = $(this).html() || '';
+        if (inner.trim()) {
+          parts.push(inner);
+        }
+        return;
+      }
+      if (this.type === 'tag') {
+        parts.push($.html(this));
+        return;
+      }
+      if (this.type === 'text' && this.data.trim()) {
+        parts.push(this.data);
+      }
+    });
+  return parts.join('<br>');
+}
+
 function hasBoldStyle(style) {
   return /font-weight\s*:\s*(?:bold|[6-9]\d{2})/i.test(style);
 }

package/src/gdocs-render-markdown.js

@@ -138,9 +138,27 @@ function renderTextRunsMarkdown(nodes = []) {
       italic: Boolean(node.italic),
       strike: Boolean(node.strike),
     };
-    output += markdownStyleTransition(active, next);
-    output += node.text || '';
-    active = next;
+    const text = node.text || '';
+    let start = 0;
+    for (let idx = 0; idx < text.length; idx++) {
+      if (text[idx] !== '\n') {
+        continue;
+      }
+      if (idx > start) {
+        output += markdownStyleTransition(active, next);
+        output += text.slice(start, idx);
+        active = next;
+      }
+      output += markdownStyleTransition(active, inactive);
+      output += '\n';
+      active = inactive;
+      start = idx + 1;
+    }
+    if (start < text.length) {
+      output += markdownStyleTransition(active, next);
+      output += text.slice(start);
+      active = next;
+    }
   }
   output += markdownStyleTransition(active, inactive);
   return output;

package/src/gdocs.js

@@ -11,7 +11,10 @@ import fetch from 'node-fetch';
 import he from 'he';
 import { convertHtmlToMarkdown } from './lib.js';
 import { createBrowser as defaultCreateBrowser } from './browser.js';
-import { preprocessGoogleDocsExportHtml } from './gdocs-preprocess.js';
+import {
+  normalizeGoogleDocsExportMarkdown,
+  preprocessGoogleDocsExportHtml,
+} from './gdocs-preprocess.js';
 import { localizeGoogleDocsModelImages } from './gdocs-images.js';
 import { renderBlocksMarkdown } from './gdocs-render-markdown.js';
 import {
@@ -22,6 +25,7 @@ import {
 } from './gdocs-fallback.js';
 
 export {
+  normalizeGoogleDocsExportMarkdown,
   preprocessGoogleDocsExportHtml,
   localizeGoogleDocsModelImages,
   isGoogleDocsBrowserModelUnavailableError,
@@ -305,7 +309,9 @@ export async function fetchGoogleDocAsMarkdown(url, options = {}) {
     hoisted: preprocessed.hoisted,
     unwrappedLinks: preprocessed.unwrappedLinks,
   }));
-  const markdown = convertHtmlToMarkdown(preprocessed.html, result.exportUrl);
+  const markdown = normalizeGoogleDocsExportMarkdown(
+    convertHtmlToMarkdown(preprocessed.html, result.exportUrl)
+  );
   log?.debug?.(() => ({
     event: 'gdocs.public-export.markdown',
     documentId: result.documentId,
@@ -1213,7 +1219,17 @@ function renderContentHtml(content = []) {
 }
 
 function renderMarkedHtml(node) {
-  let output = escapeHtml(node.text || '').replace(/\n/g, '<br>');
+  return String(node.text || '')
+    .split('\n')
+    .map((segment) => renderMarkedHtmlSegment(node, segment))
+    .join('<br>');
+}
+
+function renderMarkedHtmlSegment(node, text) {
+  if (!text) {
+    return '';
+  }
+  let output = escapeHtml(text);
   if (node.bold) {
     output = `<strong>${output}</strong>`;
   }
@@ -1354,7 +1370,9 @@ export async function fetchGoogleDocAsArchive(url, options = {}) {
   const { html: localHtml, images } = extractBase64Images(preprocessed.html);
 
   // Convert the localized HTML to Markdown
-  const markdown = convertHtmlToMarkdown(localHtml);
+  const markdown = normalizeGoogleDocsExportMarkdown(
+    convertHtmlToMarkdown(localHtml)
+  );
   log?.debug?.(() => ({
     event: 'gdocs.public-export.archive',
     documentId: result.documentId,

package/tests/integration/gdocs-public-doc.test.js

@@ -172,17 +172,6 @@ describe('Google Docs public test document (issue #90)', () => {
     // the captured markdown. This is our regression net for any future
     // changes to the public-export or HTML-to-markdown pipeline.
     //
-    // The captured markdown from `--capture api` still escapes leading
-    // numeric prefixes (e.g. `1\.` instead of `1.`) and strips inline
-    // formatting — both are documented as R2 in
-    // docs/case-studies/issue-90/README.md and are not yet fixed. For the
-    // purposes of this regression test we normalise Turndown's backslash
-    // escapes before checking that every section heading is present so the
-    // test locks down content preservation rather than format fidelity. The
-    // next wave of work (R2) will tighten this matcher once the HTML-to-
-    // Markdown pipeline stops escaping the dots.
-    const normalizeEscapes = (text) => text.replace(/\\([.!()[\]])/g, '$1');
-
     // Google Docs occasionally returns transient 500s on the public-export
     // endpoint. The Habr integration suite has the same flake and solves it
     // with retry + exponential backoff, so we reuse the shared helper here.
@@ -207,10 +196,21 @@ describe('Google Docs public test document (issue #90)', () => {
       expect(exportUrl).toBe(buildExportUrl(PUBLIC_TEST_DOCUMENT.id, 'html'));
       expect(markdown.length).toBeGreaterThan(1000);
 
-      const normalized = normalizeEscapes(markdown);
       for (const section of PUBLIC_TEST_DOCUMENT.sections) {
-        expect(normalized).toContain(section);
+        expect(markdown).toContain(section);
       }
+      expect(markdown).toContain('## 1. Headings');
+      expect(markdown).toContain('**This text is bold**');
+      expect(markdown).toContain('*This text is italic*');
+      expect(markdown).toContain('~~This text has strikethrough~~');
+      expect(markdown).toContain('> This is a single-level blockquote');
+      expect(markdown).toContain('| Feature | Supported | Notes |');
+      expect(markdown).not.toContain('| Feature |  | Supported |');
+      expect(markdown).toContain('|  | x |  |');
+      expect(markdown).toContain('1.  Parent item 1');
+      expect(markdown).toContain('    1.  Child item 1.1');
+      expect(markdown).toContain('        1.  Grandchild item 1.2.1');
+      expect(markdown).not.toContain('-   Child item 1.1');
     });
 
     it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(

package/tests/unit/gdocs.test.js

@@ -13,9 +13,11 @@ import {
   renderDocsApiDocument,
   selectGoogleDocsCaptureMethod,
   localizeGoogleDocsModelImages,
+  normalizeGoogleDocsExportMarkdown,
   preprocessGoogleDocsExportHtml,
   GDOCS_EXPORT_FORMATS,
 } from '../../src/gdocs.js';
+import { convertHtmlToMarkdown } from '../../src/lib.js';
 import {
   extractAndSaveImages,
   extractBase64ToBuffers,
@@ -24,10 +26,26 @@ import {
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
+import { fileURLToPath } from 'url';
 import nock from 'nock';
 
 jest.setTimeout(30000);
 
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const REPO_ROOT = path.resolve(__dirname, '..', '..', '..');
+const ISSUE_104_FIXTURE_DIR = path.join(
+  REPO_ROOT,
+  'docs',
+  'case-studies',
+  'issue-104',
+  'fixtures'
+);
+
+function readIssue104Fixture(filename) {
+  return fs.readFileSync(path.join(ISSUE_104_FIXTURE_DIR, filename), 'utf-8');
+}
+
 describe('gdocs', () => {
   describe('isGoogleDocsUrl', () => {
     it('returns true for standard Google Docs edit URL', () => {
@@ -646,6 +664,74 @@ describe('gdocs', () => {
       );
     });
 
+    it('renders soft breaks outside marks and preserves image dimensions (issue #104)', () => {
+      const fixture = JSON.parse(
+        readIssue104Fixture('multiline-marked-inline-image-model.json')
+      );
+      const capture = parseGoogleDocsModelChunks(
+        fixture.chunks,
+        fixture.cidUrlMap
+      );
+
+      expect(capture.blocks[0].content).toEqual([
+        expect.objectContaining({
+          type: 'text',
+          text: 'Line one of bold text.',
+          bold: true,
+        }),
+        expect.objectContaining({ type: 'text', text: '\n', bold: false }),
+        expect.objectContaining({
+          type: 'text',
+          text: 'Line two of bold text.',
+          bold: true,
+        }),
+        expect.objectContaining({
+          type: 'image',
+          alt: 'Inline diagram',
+          width: 320,
+          height: 180,
+        }),
+        expect.objectContaining({ type: 'text', text: '\n\n', bold: false }),
+        expect.objectContaining({
+          type: 'text',
+          text: 'Line three of bold text.',
+          bold: true,
+        }),
+      ]);
+      expect(renderGoogleDocsCapture(capture, 'html')).toBe(
+        readIssue104Fixture(
+          'multiline-marked-inline-image.expected.html'
+        ).trimEnd()
+      );
+      expect(renderGoogleDocsCapture(capture, 'markdown')).toBe(
+        readIssue104Fixture('multiline-marked-inline-image.expected.md')
+      );
+    });
+
+    it('does not keep inline marks open across embedded newlines (issue #104)', () => {
+      const capture = {
+        blocks: [
+          {
+            type: 'paragraph',
+            content: [
+              {
+                type: 'text',
+                text: 'Alpha\nBeta',
+                bold: true,
+              },
+            ],
+          },
+        ],
+      };
+
+      expect(renderGoogleDocsCapture(capture, 'html')).toBe(
+        '<!doctype html><html><body><p><strong>Alpha</strong><br><strong>Beta</strong></p></body></html>'
+      );
+      expect(renderGoogleDocsCapture(capture, 'markdown')).toBe(
+        '**Alpha**\n**Beta**\n'
+      );
+    });
+
     it('keeps nested bold and italic markers balanced (issue #96)', () => {
       const text = 'Bold text with italic inside and back to bold\n';
       const chunks = [
@@ -1042,6 +1128,46 @@ describe('gdocs', () => {
       expect(unwrappedLinks).toBe(0);
       expect(out).toContain('<p>Plain text with <strong>bold</strong>.</p>');
     });
+
+    it('recovers public-export markdown structure for issue #102', () => {
+      const html = `
+        <style>
+          .c5{margin-left:36pt}
+          .c8{margin-left:72pt}
+          .c19{margin-left:108pt}
+          .q{margin-left:24pt;margin-right:24pt}
+          .i{font-style:italic}
+          .s{text-decoration:line-through}
+        </style>
+        <h2><span class="i">1. Headings</span></h2>
+        <p class="q">Quote one.</p>
+        <p class="q">Quote two.</p>
+        <ol><li class="c5">Parent</li></ol>
+        <ol><li class="c8">Child</li></ol>
+        <ol><li class="c19">Grandchild</li></ol>
+        <ol><li class="c5">Parent 2</li></ol>
+        <table>
+          <thead>
+            <tr><td><p>Feature</p></td><td><p>Supported</p></td><tbody></tbody></tr>
+            <tr><td><p><span class="s">Strike</span></p></td><td><p>Yes</p></td></tr>
+          </thead>
+        </table>
+      `;
+
+      const preprocessed = preprocessGoogleDocsExportHtml(html);
+      const markdown = normalizeGoogleDocsExportMarkdown(
+        convertHtmlToMarkdown(preprocessed.html)
+      );
+
+      expect(markdown).toContain('## 1. Headings');
+      expect(markdown).not.toContain('*1. Headings*');
+      expect(markdown).toContain('> Quote one.\n> \n> Quote two.');
+      expect(markdown).toMatch(
+        /1\.\s+Parent\n\s+1\.\s+Child\n\s+1\.\s+Grandchild\n2\.\s+Parent 2/
+      );
+      expect(markdown).toContain('| Feature | Supported |');
+      expect(markdown).toContain('| ~~Strike~~ | Yes |');
+    });
   });
 
   describe('extractBase64Images', () => {