@link-assistant/web-capture 1.7.22 → 1.7.23

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @link-assistant/web-capture
 
+## 1.7.23
+
+### Patch Changes
+
+- 6b36b5b: Fix `<br><br>` collapsing into two CommonMark hard breaks (`  \n  \n`) instead of a paragraph break (`\n\n`). Google Docs export-html marks paragraph boundaries with `<br><br>`, which Turndown faithfully emitted as two trailing-two-space-newline pairs. Renderers (GitHub, MkDocs, Pandoc) then joined the surrounding lines into a single `<p>` with a `<br>`, cramming captions against images with no vertical spacing, and the "blank" separator line in the markdown source actually carried trailing whitespace that polluted diffs. Two or more adjacent hard breaks now coalesce to `\n\n` after Turndown runs, restoring true paragraph breaks. Applied in both `convertHtmlToMarkdown` (used by `--capture api`) and `convertHtmlToMarkdownEnhanced`.
+
 ## 1.7.22
 
 ### Patch Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-assistant/web-capture",
-  "version": "1.7.22",
+  "version": "1.7.23",
   "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
   "main": "src/index.js",
   "bin": {
@@ -62,6 +62,7 @@
     "jest": "^29.7.0",
     "jscpd": "^4.0.5",
     "lint-staged": "^16.2.6",
+    "markdown-it": "^14.1.1",
     "nock": "^13.5.4",
     "prettier": "^3.6.2",
     "supertest": "^6.3.4"

package/src/lib.js

@@ -184,7 +184,19 @@ export function convertHtmlToMarkdown(html, baseUrl) {
   preserveTableCellLineBreaks(turndown);
   // Decode HTML entities to unicode after markdown conversion
   // Preserve non-breaking spaces as   entities for clear marking
-  return he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ');
+  return coalesceBrRunsToParagraphBreak(
+    he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ')
+  );
+}
+
+// Coalesce runs of CommonMark hard breaks (`  \n`) into a single paragraph
+// break. Google Docs export wraps every visual line break in `<br>`, including
+// `<br><br>` at paragraph boundaries; Turndown faithfully emits two trailing-
+// two-space-newline pairs for that, which renders as `<br><br>` joined inside
+// one `<p>` instead of two paragraphs. Two or more adjacent hard breaks always
+// mean "paragraph break" in the source HTML, so collapse them to `\n\n`.
+function coalesceBrRunsToParagraphBreak(markdown) {
+  return markdown.replace(/(?: {2,}\n){2,}/g, '\n\n');
 }
 
 // Lift <br> nodes out of inline parents when they sit at the leading or
@@ -764,7 +776,9 @@ export function convertHtmlToMarkdownEnhanced(html, baseUrl, options = {}) {
 
   // Decode HTML entities to unicode after markdown conversion
   // Normalize non-breaking spaces to regular spaces in Markdown text
-  let markdown = he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ');
+  let markdown = coalesceBrRunsToParagraphBreak(
+    he.decode(turndown.turndown($.html())).replace(/\u00A0/g, ' ')
+  );
 
   // Apply post-processing
   if (postProcess) {

package/tests/fixtures/paragraph-vs-line-break.html

@@ -0,0 +1,11 @@
+<!doctype html>
+<html>
+  <body>
+    <p>
+      <span>Caption A:</span><br />
+      <img alt="" src="data:image/png;base64,iVBORw0KGgo=" /><br /><br />
+      <span>Caption B:</span><br />
+      <img alt="" src="data:image/png;base64,iVBORw0KGgo=" />
+    </p>
+  </body>
+</html>

package/tests/unit/paragraph-vs-line-break.test.js

@@ -0,0 +1,56 @@
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+import MarkdownIt from 'markdown-it';
+import {
+  convertHtmlToMarkdown,
+  convertHtmlToMarkdownEnhanced,
+} from '../../src/lib.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const HTML = fs.readFileSync(
+  path.join(__dirname, '..', 'fixtures', 'paragraph-vs-line-break.html'),
+  'utf8'
+);
+
+describe('<br><br> coalesces to a paragraph break (not two hard breaks)', () => {
+  it('renders a paragraph break between Caption A and Caption B (convertHtmlToMarkdown)', () => {
+    const md = convertHtmlToMarkdown(HTML);
+    const lines = md.split('\n');
+    const idxA = lines.findIndex((l) => l.includes('Caption A:'));
+    const idxB = lines.findIndex((l) => l.includes('Caption B:'));
+    expect(idxA).toBeGreaterThanOrEqual(0);
+    expect(idxB).toBeGreaterThan(idxA);
+
+    const between = lines.slice(idxA + 1, idxB);
+    expect(between.some((l) => l === '')).toBe(true);
+
+    expect(md).not.toMatch(/^[ \t]+ {2}$/m);
+  });
+
+  it('renders a paragraph break between Caption A and Caption B (convertHtmlToMarkdownEnhanced)', () => {
+    const { markdown } = convertHtmlToMarkdownEnhanced(HTML);
+    const lines = markdown.split('\n');
+    const idxA = lines.findIndex((l) => l.includes('Caption A:'));
+    const idxB = lines.findIndex((l) => l.includes('Caption B:'));
+    expect(idxA).toBeGreaterThanOrEqual(0);
+    expect(idxB).toBeGreaterThan(idxA);
+
+    const between = lines.slice(idxA + 1, idxB);
+    expect(between.some((l) => l === '')).toBe(true);
+
+    expect(markdown).not.toMatch(/^[ \t]+ {2}$/m);
+  });
+
+  it('CommonMark renders Caption A and Caption B as separate paragraphs', () => {
+    const md = convertHtmlToMarkdown(HTML);
+    const html = new MarkdownIt('commonmark').render(md);
+    const aIdx = html.indexOf('Caption A:');
+    const bIdx = html.indexOf('Caption B:');
+    expect(aIdx).toBeGreaterThan(-1);
+    expect(bIdx).toBeGreaterThan(aIdx);
+    const between = html.slice(aIdx, bIdx);
+    expect(between).toMatch(/<\/p>\s*<p>/);
+  });
+});