@link-assistant/web-capture 1.7.22 → 1.7.24

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/web-capture
 
+## 1.7.24
+
+### Patch Changes
+
+- 52ccb91: Fix structurally broken bold markup in `--capture api` output. Adjacent `<span style="font-weight:700">` siblings that the Google Docs export emits for headings, list items, and notes used to be wrapped in independent `<strong>...</strong>` pairs, producing split runs (`**13.1** **First subsection**`), stray `****` between adjacent bold blocks, and bold ranges that leaked across `<br>`/`<img>` boundaries (`**Caption A:![](images/x.png)Caption B:**`). The Google Docs preprocessor now closes `<strong>` runs at `<br>`/`<img>` boundaries, splits the wrapping `<p>` into separate paragraphs at those boundaries, drops empty `<strong></strong>` pairs, and coalesces adjacent bold siblings into a single run. Renderers can now interpret the output as valid CommonMark again.
+
+## 1.7.23
+
+### Patch Changes
+
+- 6b36b5b: Fix `<br><br>` collapsing into two CommonMark hard breaks (`  \n  \n`) instead of a paragraph break (`\n\n`). Google Docs export-html marks paragraph boundaries with `<br><br>`, which Turndown faithfully emitted as two trailing-two-space-newline pairs. Renderers (GitHub, MkDocs, Pandoc) then joined the surrounding lines into a single `<p>` with a `<br>`, cramming captions against images with no vertical spacing, and the "blank" separator line in the markdown source actually carried trailing whitespace that polluted diffs. Two or more adjacent hard breaks now coalesce to `\n\n` after Turndown runs, restoring true paragraph breaks. Applied in both `convertHtmlToMarkdown` (used by `--capture api`) and `convertHtmlToMarkdownEnhanced`.
+
 ## 1.7.22
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.22",
+  "version": "1.7.24",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {
@@ -62,6 +62,7 @@
     "jest": "^29.7.0",
     "jscpd": "^4.0.5",
     "lint-staged": "^16.2.6",
+    "markdown-it": "^14.1.1",
     "nock": "^13.5.4",
     "prettier": "^3.6.2",
     "supertest": "^6.3.4"

package/src/gdocs-preprocess.js

@@ -62,6 +62,11 @@ export function preprocessGoogleDocsExportHtml(html) {
     hoisted += 1;
   });
 
+  splitStrongAtBlockBoundaries($);
+  splitParagraphsAtBoldBoundaries($);
+  removeEmptyStrong($);
+  coalesceAdjacentStrong($);
+
   groupGoogleDocsBlockquotes($, classStyles);
   nestGoogleDocsLists($, classStyles);
   normalizeGoogleDocsTables($);
@@ -144,6 +149,172 @@ export function normalizeGoogleDocsExportMarkdown(markdown) {
     .replace(/\n{3,}/g, '\n\n');
 }
 
+// Issue #120: a `<strong>` whose closing tag would land in a different block
+// boundary leaks bolding across paragraphs and images. Split such a strong
+// into multiple bold runs that each stop at the boundary.
+function splitStrongAtBlockBoundaries($) {
+  let changed = true;
+  while (changed) {
+    changed = false;
+    const offenders = $('strong').filter(function () {
+      return $(this).find('br, img, p, div, li').length > 0;
+    });
+    if (!offenders.length) {
+      break;
+    }
+    offenders.each(function () {
+      const $strong = $(this);
+      const segments = [];
+      let current = [];
+      const flush = () => {
+        if (current.length) {
+          segments.push({ type: 'inline', nodes: current });
+          current = [];
+        }
+      };
+      $strong.contents().each(function () {
+        if (this.type === 'tag' && this.tagName === 'br') {
+          flush();
+          segments.push({ type: 'br' });
+        } else if (this.type === 'tag' && this.tagName === 'img') {
+          flush();
+          segments.push({ type: 'img', node: this });
+        } else if (
+          this.type === 'tag' &&
+          ['p', 'div', 'li'].includes(this.tagName)
+        ) {
+          flush();
+          segments.push({ type: 'block', node: this });
+        } else {
+          current.push(this);
+        }
+      });
+      flush();
+
+      const pieces = segments.map((segment) => {
+        if (segment.type === 'br') {
+          return '<br>';
+        }
+        if (segment.type === 'img') {
+          return $.html(segment.node);
+        }
+        if (segment.type === 'block') {
+          const inner = $(segment.node).html() || '';
+          const tag = segment.node.tagName;
+          const innerWrapped = inner.trim() ? `<strong>${inner}</strong>` : '';
+          return `<${tag}>${innerWrapped}</${tag}>`;
+        }
+        const inner = segment.nodes.map((node) => $.html(node)).join('');
+        if (!inner.trim()) {
+          return inner;
+        }
+        return `<strong>${inner}</strong>`;
+      });
+      $strong.replaceWith(pieces.join(''));
+      changed = true;
+    });
+  }
+}
+
+// Issue #120: when a `<p>` contains a `<br>` adjacent to a `<strong>` or an
+// `<img>`, split it into multiple paragraphs so each bold run and image
+// becomes its own block. Without this, the markdown converter renders
+// `**Caption**  \n![](x)` (inline) instead of separate blocks.
+function splitParagraphsAtBoldBoundaries($) {
+  $('p').each(function () {
+    const $p = $(this);
+    const contents = $p.contents().toArray();
+    if (!contents.some((n) => n.type === 'tag' && n.tagName === 'br')) {
+      return;
+    }
+    const hasStrongOrImg = contents.some(
+      (n) => n.type === 'tag' && (n.tagName === 'strong' || n.tagName === 'img')
+    );
+    if (!hasStrongOrImg) {
+      return;
+    }
+    const segments = [];
+    let current = [];
+    const flush = () => {
+      const html = current
+        .map((node) => $.html(node))
+        .join('')
+        .trim();
+      if (html) {
+        segments.push(html);
+      }
+      current = [];
+    };
+    for (const node of contents) {
+      if (node.type === 'tag' && node.tagName === 'br') {
+        flush();
+      } else if (node.type === 'tag' && node.tagName === 'img') {
+        flush();
+        segments.push($.html(node));
+      } else {
+        current.push(node);
+      }
+    }
+    flush();
+    if (segments.length <= 1) {
+      return;
+    }
+    const replacement = segments.map((html) => `<p>${html}</p>`).join('');
+    $p.replaceWith(replacement);
+  });
+}
+
+// Issue #120: drop empty `<strong></strong>` so they cannot pair with a
+// neighbour and emit `****`.
+function removeEmptyStrong($) {
+  $('strong').each(function () {
+    const $strong = $(this);
+    if (!$strong.html() || !$strong.text().trim()) {
+      if (!$strong.find('img').length) {
+        $strong.remove();
+      }
+    }
+  });
+}
+
+// Issue #120: merge adjacent `<strong>` siblings (optionally separated by
+// whitespace text nodes) into a single `<strong>` so the converter emits
+// `**a b**` instead of `**a** **b**`.
+function coalesceAdjacentStrong($) {
+  let changed = true;
+  while (changed) {
+    changed = false;
+    $('strong').each(function () {
+      const $strong = $(this);
+      if (!$strong.parent().length) {
+        return;
+      }
+      let sibling = this.next;
+      const between = [];
+      while (sibling) {
+        if (sibling.type === 'text' && /^\s*$/.test(sibling.data || '')) {
+          between.push(sibling);
+          sibling = sibling.next;
+          continue;
+        }
+        if (sibling.type === 'tag' && sibling.tagName === 'strong') {
+          const innerSelf = $strong.html() || '';
+          const innerNext = $(sibling).html() || '';
+          const joiner = between.map((node) => node.data || '').join('');
+          $strong.html(`${innerSelf}${joiner}${innerNext}`);
+          for (const node of between) {
+            $(node).remove();
+          }
+          $(sibling).remove();
+          changed = true;
+          return;
+        }
+        break;
+      }
+    });
+  }
+}
+
 function parseCssClassStyles($) {
   const classStyles = new Map();
   $('style').each(function () {

package/src/lib.js

@@ -184,7 +184,19 @@ export function convertHtmlToMarkdown(html, baseUrl) {
   preserveTableCellLineBreaks(turndown);
   // Decode HTML entities to unicode after markdown conversion
   // Preserve non-breaking spaces as   entities for clear marking
-  return he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ');
+  return coalesceBrRunsToParagraphBreak(
+    he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ')
+  );
+}
+
+// Coalesce runs of CommonMark hard breaks (`  \n`) into a single paragraph
+// break. Google Docs export wraps every visual line break in `<br>`, including
+// `<br><br>` at paragraph boundaries; Turndown faithfully emits two trailing-
+// two-space-newline pairs for that, which renders as `<br><br>` joined inside
+// one `<p>` instead of two paragraphs. Two or more adjacent hard breaks always
+// mean "paragraph break" in the source HTML, so collapse them to `\n\n`.
+function coalesceBrRunsToParagraphBreak(markdown) {
+  return markdown.replace(/(?: {2,}\n){2,}/g, '\n\n');
 }
 
 // Lift <br> nodes out of inline parents when they sit at the leading or
@@ -764,7 +776,9 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
 
   // Decode HTML entities to unicode after markdown conversion
   // Normalize non-breaking spaces to regular spaces in Markdown text
-  let markdown = he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ');
+  let markdown = coalesceBrRunsToParagraphBreak(
+    he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ')
+  );
 
   // Apply post-processing
   if (postProcess) {

package/tests/fixtures/bold-coalesce-and-leakage.html

@@ -0,0 +1,27 @@
+<!doctype html>
+<html>
+  <body>
+    <!-- Symptom A: two adjacent bold spans -->
+    <p>
+      <span style="font-weight: 700">13.1 </span
+      ><span style="font-weight: 700">First subsection</span>
+    </p>
+
+    <!-- Symptom B: outer bold ends, inner bold begins, no separator -->
+    <ol>
+      <li><span style="font-weight: 700">Item header.</span></li>
+    </ol>
+    <p><span style="font-weight: 700">Note:</span> body text.</p>
+
+    <!-- Symptom C: bold range spans across <br> and <img> -->
+    <p>
+      <span style="font-weight: 700"
+        >Caption A:<img
+          alt=""
+          src="data:image/png;base64,iVBORw0KGgo=" /><br />Caption B:<img
+          alt=""
+          src="data:image/png;base64,iVBORw0KGgo="
+      /></span>
+    </p>
+  </body>
+</html>

package/tests/fixtures/paragraph-vs-line-break.html

@@ -0,0 +1,11 @@
+<!doctype html>
+<html>
+  <body>
+    <p>
+      <span>Caption A:</span><br />
+      <img alt="" src="data:image/png;base64,iVBORw0KGgo=" /><br /><br />
+      <span>Caption B:</span><br />
+      <img alt="" src="data:image/png;base64,iVBORw0KGgo=" />
+    </p>
+  </body>
+</html>

package/tests/unit/bold-coalesce-and-leakage.test.js

@@ -0,0 +1,54 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import {
+  normalizeGoogleDocsExportMarkdown,
+  preprocessGoogleDocsExportHtml,
+} from '../../src/gdocs.js';
+import { convertHtmlToMarkdown } from '../../src/lib.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const HTML = fs.readFileSync(
+  path.join(__dirname, '..', 'fixtures', 'bold-coalesce-and-leakage.html'),
+  'utf8'
+);
+
+describe('bold-run coalescing and block boundary handling (issue #120)', () => {
+  let md;
+  beforeAll(() => {
+    const preprocessed = preprocessGoogleDocsExportHtml(HTML);
+    md = normalizeGoogleDocsExportMarkdown(
+      convertHtmlToMarkdown(preprocessed.html)
+    );
+  });
+
+  it('A: coalesces adjacent bold spans into a single bold run', () => {
+    expect(md).toMatch(/\*\*13\.1 First subsection\*\*/);
+    expect(md).not.toMatch(/\*\*13\.1\*\*\s+\*\*First subsection\*\*/);
+  });
+
+  it('B: never emits an empty "****" pair between adjacent bold runs', () => {
+    expect(md).not.toMatch(/\*{4}/);
+  });
+
+  it('C: closes bold at block-level boundaries (<br>, <img>)', () => {
+    // No single bold run may contain an image. CommonMark bold cannot span
+    // a blank line (`\n\n`), so the inner content is restricted accordingly.
+    const strongRe = /\*\*((?:(?!\*\*|\n\n)[\s\S])+?)\*\*/g;
+    const imageRe = /!\[[^\]]*\]\([^)]+\)/;
+    for (const match of md.matchAll(strongRe)) {
+      expect(imageRe.test(match[1])).toBe(false);
+    }
+    expect(md).toMatch(/\*\*Caption A:\*\*/);
+    expect(md).toMatch(/\*\*Caption B:\*\*/);
+  });
+
+  it('D: every "**" opener has a matching closer (balanced asterisks)', () => {
+    const stripped = md
+      .replace(/`[^`]*`/g, '')
+      .replace(/!\[[^\]]*\]\([^)]*\)/g, '');
+    const count = (stripped.match(/\*\*/g) || []).length;
+    expect(count % 2).toBe(0);
+  });
+});

package/tests/unit/paragraph-vs-line-break.test.js

@@ -0,0 +1,56 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import MarkdownIt from 'markdown-it';
+import {
+  convertHtmlToMarkdown,
+  convertHtmlToMarkdownEnhanced,
+} from '../../src/lib.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const HTML = fs.readFileSync(
+  path.join(__dirname, '..', 'fixtures', 'paragraph-vs-line-break.html'),
+  'utf8'
+);
+
+describe('<br><br> coalesces to a paragraph break (not two hard breaks)', () => {
+  it('renders a paragraph break between Caption A and Caption B (convertHtmlToMarkdown)', () => {
+    const md = convertHtmlToMarkdown(HTML);
+    const lines = md.split('\n');
+    const idxA = lines.findIndex((l) => l.includes('Caption A:'));
+    const idxB = lines.findIndex((l) => l.includes('Caption B:'));
+    expect(idxA).toBeGreaterThanOrEqual(0);
+    expect(idxB).toBeGreaterThan(idxA);
+
+    const between = lines.slice(idxA + 1, idxB);
+    expect(between.some((l) => l === '')).toBe(true);
+
+    expect(md).not.toMatch(/^[ \t]+ {2}$/m);
+  });
+
+  it('renders a paragraph break between Caption A and Caption B (convertHtmlToMarkdownEnhanced)', () => {
+    const { markdown } = convertHtmlToMarkdownEnhanced(HTML);
+    const lines = markdown.split('\n');
+    const idxA = lines.findIndex((l) => l.includes('Caption A:'));
+    const idxB = lines.findIndex((l) => l.includes('Caption B:'));
+    expect(idxA).toBeGreaterThanOrEqual(0);
+    expect(idxB).toBeGreaterThan(idxA);
+
+    const between = lines.slice(idxA + 1, idxB);
+    expect(between.some((l) => l === '')).toBe(true);
+
+    expect(markdown).not.toMatch(/^[ \t]+ {2}$/m);
+  });
+
+  it('CommonMark renders Caption A and Caption B as separate paragraphs', () => {
+    const md = convertHtmlToMarkdown(HTML);
+    const html = new MarkdownIt('commonmark').render(md);
+    const aIdx = html.indexOf('Caption A:');
+    const bIdx = html.indexOf('Caption B:');
+    expect(aIdx).toBeGreaterThan(-1);
+    expect(bIdx).toBeGreaterThan(aIdx);
+    const between = html.slice(aIdx, bIdx);
+    expect(between).toMatch(/<\/p>\s*<p>/);
+  });
+});