@link-assistant/web-capture 1.7.21 → 1.7.23

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @link-assistant/web-capture
 
+## 1.7.23
+
+### Patch Changes
+
+- 6b36b5b: Fix `<br><br>` collapsing into two CommonMark hard breaks (`  \n  \n`) instead of a paragraph break (`\n\n`). Google Docs export-html marks paragraph boundaries with `<br><br>`, which Turndown faithfully emitted as two trailing-two-space-newline pairs. Renderers (GitHub, MkDocs, Pandoc) then joined the surrounding lines into a single `<p>` with a `<br>`, cramming captions against images with no vertical spacing, and the "blank" separator line in the markdown source actually carried trailing whitespace that polluted diffs. Two or more adjacent hard breaks now coalesce to `\n\n` after Turndown runs, restoring true paragraph breaks. Applied in both `convertHtmlToMarkdown` (used by `--capture api`) and `convertHtmlToMarkdownEnhanced`.
+
+## 1.7.22
+
+### Patch Changes
+
+- a339ca2: Fix `--capture api` collapsing `<br>`-separated lines inside list items into one run. The Google Docs export-html path lost line breaks when a `<br>` was the leading or trailing child of an inline element (e.g. a `<span>` between bold runs), because Turndown trims inner content of inline elements with edge whitespace. The HTML pre-processing now hoists those edge `<br>`s out of their inline parents before Turndown sees them, restoring CommonMark hard breaks. Additionally, the post-processor's double-space collapse no longer eats the two trailing spaces that mark a hard break.
+
 ## 1.7.21
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.21",
+  "version": "1.7.23",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {
@@ -62,6 +62,7 @@
     "jest": "^29.7.0",
     "jscpd": "^4.0.5",
     "lint-staged": "^16.2.6",
+    "markdown-it": "^14.1.1",
     "nock": "^13.5.4",
     "prettier": "^3.6.2",
     "supertest": "^6.3.4"

package/src/lib.js

@@ -164,6 +164,10 @@ export function convertHtmlToMarkdown(html, baseUrl) {
     $table.replaceWith($newTable);
   });
 
+  // Hoist <br>s out of inline edges so Turndown's flanking-whitespace trim
+  // does not eat the hard break. See liftBrFromInlineEdges for details.
+  liftBrFromInlineEdges($);
+
   // Convert cleaned HTML to Markdown
   const turndown = new TurndownService({
     headingStyle: 'atx',
@@ -180,7 +184,85 @@ export function convertHtmlToMarkdown(html, baseUrl) {
   preserveTableCellLineBreaks(turndown);
   // Decode HTML entities to unicode after markdown conversion
   // Preserve non-breaking spaces as &nbsp; entities for clear marking
-  return he.decode(turndown.turndown($.html())).replace(/\u00A0/g, '&nbsp;');
+  return coalesceBrRunsToParagraphBreak(
+    he.decode(turndown.turndown($.html())).replace(/\u00A0/g, '&nbsp;')
+  );
+}
+
+// Coalesce runs of CommonMark hard breaks (`  \n`) into a single paragraph
+// break. Google Docs export wraps every visual line break in `<br>`, including
+// `<br><br>` at paragraph boundaries; Turndown faithfully emits two trailing-
+// two-space-newline pairs for that, which renders as `<br><br>` joined inside
+// one `<p>` instead of two paragraphs. Two or more adjacent hard breaks always
+// mean "paragraph break" in the source HTML, so collapse them to `\n\n`.
+function coalesceBrRunsToParagraphBreak(markdown) {
+  return markdown.replace(/(?: {2,}\n){2,}/g, '\n\n');
+}
+
+// Lift <br> nodes out of inline parents when they sit at the leading or
+// trailing edge.
+//
+// Turndown calls `content.trim()` on the inner markdown of any inline element
+// whose `flankingWhitespace` is non-empty (turndown.cjs.js:902), which strips
+// the `  \n` hard break that the <br> rule emits. Google Docs export-html
+// commonly produces `<strong>X</strong><span> Y.<br></span><strong>Z</strong>`
+// — the trailing <br> inside a leading-space <span> gets `.trim()`-ed away,
+// so X/Y/Z collapse onto one line.
+//
+// Hoisting trailing/leading <br>s out of their inline wrapper sidesteps the
+// trim entirely. The hard break then attaches at the parent level, which
+// Turndown handles correctly.
+function liftBrFromInlineEdges($) {
+  const inlineParents = new Set([
+    'span',
+    'a',
+    'strong',
+    'b',
+    'em',
+    'i',
+    'u',
+    's',
+    'del',
+    'ins',
+    'sub',
+    'sup',
+    'mark',
+    'small',
+    'q',
+    'cite',
+    'code',
+    'abbr',
+    'time',
+    'kbd',
+    'samp',
+    'var',
+    'font',
+  ]);
+
+  // Hoist <br>s repeatedly until none sit at an inline edge — this handles
+  // nested wrappers like `<span><em>foo<br></em></span>`.
+  let changed = true;
+  while (changed) {
+    changed = false;
+    $('br').each(function () {
+      const el = this;
+      const parent = el.parent;
+      if (!parent || !inlineParents.has(parent.tagName)) {
+        return;
+      }
+      const isFirst = parent.children[0] === el;
+      const isLast = parent.children[parent.children.length - 1] === el;
+      if (!isFirst && !isLast) {
+        return;
+      }
+      if (isLast) {
+        $(parent).after(el);
+      } else {
+        $(parent).before(el);
+      }
+      changed = true;
+    });
+  }
 }
 
 function preserveLeadingHeadingNumbering($) {
@@ -674,6 +756,9 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
     $table.replaceWith($newTable);
   });
 
+  // Hoist <br>s out of inline edges (see liftBrFromInlineEdges).
+  liftBrFromInlineEdges($);
+
   // Convert to Markdown using Turndown
   const turndown = new TurndownService({
     headingStyle: 'atx',
@@ -691,7 +776,9 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
 
   // Decode HTML entities to unicode after markdown conversion
   // Normalize non-breaking spaces to regular spaces in Markdown text
-  let markdown = he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ');
+  let markdown = coalesceBrRunsToParagraphBreak(
+    he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ')
+  );
 
   // Apply post-processing
   if (postProcess) {

package/src/postprocess.js

@@ -50,8 +50,9 @@ export function postProcessMarkdown(markdown, options = {}) {
     result = applyBoldFormattingFixes(result);
   }
 
-  // Fix double spaces (but not in code blocks)
-  result = result.replace(/([^\n`]) +/g, (match, char) => `${char} `);
+  // Fix double spaces (but not in code blocks, and preserve CommonMark hard
+  // breaks which require two trailing spaces before a newline).
+  result = result.replace(/([^\n`]) +(?!\n)/g, (match, char) => `${char} `);
 
   // Clean up extra spaces around em-dashes
   result = result.replace(/\s+—\s+/g, ' — ');

package/tests/fixtures/list-item-with-br.html

@@ -0,0 +1,8 @@
+<ol>
+  <li>
+    <strong>Glossary entry.</strong><span> Some lead text.<br /></span
+    ><strong>TAG1</strong><span> – Definition one.<br /></span
+    ><strong>TAG2</strong><span> – Definition two.<br /></span
+    ><strong>TAG3</strong><span> – Definition three.</span>
+  </li>
+</ol>

package/tests/fixtures/paragraph-vs-line-break.html

@@ -0,0 +1,11 @@
+<!doctype html>
+<html>
+  <body>
+    <p>
+      <span>Caption A:</span><br />
+      <img alt="" src="data:image/png;base64,iVBORw0KGgo=" /><br /><br />
+      <span>Caption B:</span><br />
+      <img alt="" src="data:image/png;base64,iVBORw0KGgo=" />
+    </p>
+  </body>
+</html>

package/tests/unit/html2md-br-in-list-item.test.js

@@ -0,0 +1,23 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import { convertHtmlToMarkdown } from '../../src/lib.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const HTML = fs.readFileSync(
+  path.join(__dirname, '..', 'fixtures', 'list-item-with-br.html'),
+  'utf8'
+);
+
+it('preserves <br> as a line separator inside list items', () => {
+  const md = convertHtmlToMarkdown(HTML);
+  for (const tag of ['TAG1', 'TAG2', 'TAG3']) {
+    expect(md).toMatch(
+      new RegExp(`(^|\\n)\\s*\\*{0,2}${tag}\\*{0,2}\\s*–`, 'm')
+    );
+  }
+  // Definitions must not be glued on the same line (no <br> dropped).
+  expect(md).not.toMatch(/Definition one\.[ \t]*\*{0,2}TAG2/);
+  expect(md).not.toMatch(/Definition two\.[ \t]*\*{0,2}TAG3/);
+});

package/tests/unit/paragraph-vs-line-break.test.js

@@ -0,0 +1,56 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import MarkdownIt from 'markdown-it';
+import {
+  convertHtmlToMarkdown,
+  convertHtmlToMarkdownEnhanced,
+} from '../../src/lib.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const HTML = fs.readFileSync(
+  path.join(__dirname, '..', 'fixtures', 'paragraph-vs-line-break.html'),
+  'utf8'
+);
+
+describe('<br><br> coalesces to a paragraph break (not two hard breaks)', () => {
+  it('renders a paragraph break between Caption A and Caption B (convertHtmlToMarkdown)', () => {
+    const md = convertHtmlToMarkdown(HTML);
+    const lines = md.split('\n');
+    const idxA = lines.findIndex((l) => l.includes('Caption A:'));
+    const idxB = lines.findIndex((l) => l.includes('Caption B:'));
+    expect(idxA).toBeGreaterThanOrEqual(0);
+    expect(idxB).toBeGreaterThan(idxA);
+
+    const between = lines.slice(idxA + 1, idxB);
+    expect(between.some((l) => l === '')).toBe(true);
+
+    expect(md).not.toMatch(/^[ \t]+ {2}$/m);
+  });
+
+  it('renders a paragraph break between Caption A and Caption B (convertHtmlToMarkdownEnhanced)', () => {
+    const { markdown } = convertHtmlToMarkdownEnhanced(HTML);
+    const lines = markdown.split('\n');
+    const idxA = lines.findIndex((l) => l.includes('Caption A:'));
+    const idxB = lines.findIndex((l) => l.includes('Caption B:'));
+    expect(idxA).toBeGreaterThanOrEqual(0);
+    expect(idxB).toBeGreaterThan(idxA);
+
+    const between = lines.slice(idxA + 1, idxB);
+    expect(between.some((l) => l === '')).toBe(true);
+
+    expect(markdown).not.toMatch(/^[ \t]+ {2}$/m);
+  });
+
+  it('CommonMark renders Caption A and Caption B as separate paragraphs', () => {
+    const md = convertHtmlToMarkdown(HTML);
+    const html = new MarkdownIt('commonmark').render(md);
+    const aIdx = html.indexOf('Caption A:');
+    const bIdx = html.indexOf('Caption B:');
+    expect(aIdx).toBeGreaterThan(-1);
+    expect(bIdx).toBeGreaterThan(aIdx);
+    const between = html.slice(aIdx, bIdx);
+    expect(between).toMatch(/<\/p>\s*<p>/);
+  });
+});