@link-assistant/web-capture 1.7.9 → 1.7.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,24 @@
1
1
  # @link-assistant/web-capture
2
2
 
3
+ ## 1.7.10
4
+
5
+ ### Patch Changes
6
+
7
+ - fcc031f: Fix remaining Google Docs capture gaps from issue #92 in both the JS and
8
+ Rust CLIs:
9
+ - Browser capture now keeps multi-column table rows intact, renders
10
+ ordered lists with sequential `1. 2. 3.` numbering, and joins same-list
11
+ items with a single newline so tight-list markdown matches the source
12
+ document.
13
+ - Archive mode downloads `docs-images-rt/...` image URLs into the
14
+ archive's `images/` directory and rewrites markdown/html references so
15
+ exports are self-contained.
16
+ - API mode (`--capture api`) runs the export HTML through a shared
17
+ preprocessor that hoists inline bold / italic / strikethrough spans to
18
+ semantic tags, strips Google Docs' heading-numbering spans and empty
19
+ anchor wrappers, unwraps `google.com/url?q=` redirects, and normalizes
20
+ non-breaking spaces.
21
+
3
22
  ## 1.7.9
4
23
 
5
24
  ### Patch Changes
@@ -527,8 +527,24 @@ async function captureUrl(url, options) {
527
527
  textBytes: Buffer.byteLength(result.text || ''),
528
528
  }));
529
529
  if (normalizedFormat === 'archive' || normalizedFormat === 'zip') {
530
+ const { localizeGoogleDocsModelImages } =
531
+ await import('../src/gdocs.js');
532
+ const localized = await localizeGoogleDocsModelImages(result, {
533
+ log,
534
+ });
535
+ log.debug(() => ({
536
+ event: 'gdocs.capture.browser-model.archive.localized',
537
+ images: localized.images.length,
538
+ markdownBytes: Buffer.byteLength(localized.markdown),
539
+ htmlBytes: Buffer.byteLength(localized.html),
540
+ }));
530
541
  await writeGoogleDocsArchive({
531
- archiveResult: { ...result, images: [] },
542
+ archiveResult: {
543
+ ...result,
544
+ markdown: localized.markdown,
545
+ html: localized.html,
546
+ images: localized.images,
547
+ },
532
548
  absoluteUrl,
533
549
  explicitOutput,
534
550
  dataDir,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@link-assistant/web-capture",
3
- "version": "1.7.9",
3
+ "version": "1.7.10",
4
4
  "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
5
5
  "main": "src/index.js",
6
6
  "bin": {
@@ -0,0 +1,130 @@
1
+ // Google Docs editor-model image localization (issue #92 R5).
2
+ //
3
+ // The browser-model capture returns `docs-images-rt/…` URLs inside the
4
+ // markdown/html. Archive mode needs those images copied into `images/` so
5
+ // the archive is self-contained. This module downloads the images and
6
+ // rewrites the captured output accordingly.
7
+
8
+ import fetch from 'node-fetch';
9
+ import { URL } from 'url';
10
+
11
+ const DEFAULT_USER_AGENT =
12
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
13
+
14
+ function getExtensionFromImageUrl(url) {
15
+ try {
16
+ const u = new URL(url);
17
+ const match = u.pathname.match(/\.(png|jpe?g|gif|webp|svg)$/iu);
18
+ if (match) {
19
+ const ext = match[1].toLowerCase();
20
+ return ext === 'jpeg' ? '.jpg' : `.${ext}`;
21
+ }
22
+ } catch {
23
+ // fall through to default
24
+ }
25
+ return '.png';
26
+ }
27
+
28
+ function mimeTypeForExtension(filename) {
29
+ const dot = filename.lastIndexOf('.');
30
+ const ext = dot >= 0 ? filename.slice(dot + 1).toLowerCase() : 'png';
31
+ switch (ext) {
32
+ case 'jpg':
33
+ case 'jpeg':
34
+ return 'image/jpeg';
35
+ case 'gif':
36
+ return 'image/gif';
37
+ case 'webp':
38
+ return 'image/webp';
39
+ case 'svg':
40
+ return 'image/svg+xml';
41
+ default:
42
+ return 'image/png';
43
+ }
44
+ }
45
+
46
+ /**
47
+ * Download the `docs-images-rt` URLs captured from an editor-model capture.
48
+ *
49
+ * Produces `{ filename, data, mimeType }` entries compatible with
50
+ * `writeGoogleDocsArchive` and rewrites the markdown/html to reference the
51
+ * local `images/<filename>` paths.
52
+ *
53
+ * @param {{markdown: string, html: string, capture: Object}} modelResult
54
+ * @param {Object} [options] - Options
55
+ * @param {typeof fetch} [options.fetchImpl=fetch] - Injected fetch for tests
56
+ * @param {Object} [options.log] - Verbose logger
57
+ * @returns {Promise<{markdown: string, html: string, images: Array}>}
58
+ */
59
+ export async function localizeGoogleDocsModelImages(modelResult, options = {}) {
60
+ const { fetchImpl = fetch, log } = options;
61
+ const images = [];
62
+ const seen = new Map();
63
+ let nextIndex = 1;
64
+
65
+ const resolveFilename = (url, existingAlt) => {
66
+ if (seen.has(url)) {
67
+ return seen.get(url);
68
+ }
69
+ const ext = getExtensionFromImageUrl(url);
70
+ const filename = `image-${String(nextIndex).padStart(2, '0')}${ext}`;
71
+ nextIndex += 1;
72
+ seen.set(url, filename);
73
+ log?.debug?.(() => ({
74
+ event: 'gdocs.archive.image.scheduled',
75
+ url,
76
+ filename,
77
+ alt: existingAlt || null,
78
+ }));
79
+ return filename;
80
+ };
81
+
82
+ const candidates = (modelResult?.capture?.images || []).filter(
83
+ (img) => img?.url
84
+ );
85
+ for (const img of candidates) {
86
+ const filename = resolveFilename(img.url, img.alt);
87
+ try {
88
+ const response = await fetchImpl(img.url, {
89
+ headers: {
90
+ 'User-Agent': DEFAULT_USER_AGENT,
91
+ Accept: 'image/*,*/*;q=0.8',
92
+ },
93
+ redirect: 'follow',
94
+ });
95
+ if (!response.ok) {
96
+ throw new Error(`HTTP ${response.status}`);
97
+ }
98
+ const arrayBuffer = await response.arrayBuffer();
99
+ const data = Buffer.from(arrayBuffer);
100
+ const mimeType =
101
+ response.headers?.get?.('content-type') ||
102
+ mimeTypeForExtension(filename);
103
+ images.push({ filename, data, mimeType });
104
+ log?.debug?.(() => ({
105
+ event: 'gdocs.archive.image.downloaded',
106
+ url: img.url,
107
+ filename,
108
+ bytes: data.length,
109
+ mimeType,
110
+ }));
111
+ } catch (err) {
112
+ log?.debug?.(() => ({
113
+ event: 'gdocs.archive.image.failed',
114
+ url: img.url,
115
+ filename,
116
+ error: err?.message || String(err),
117
+ }));
118
+ }
119
+ }
120
+
121
+ let markdown = modelResult?.markdown || '';
122
+ let html = modelResult?.html || '';
123
+ for (const [url, filename] of seen.entries()) {
124
+ const localPath = `images/${filename}`;
125
+ markdown = markdown.split(url).join(localPath);
126
+ html = html.split(url).join(localPath);
127
+ }
128
+
129
+ return { markdown, html, images };
130
+ }
@@ -0,0 +1,105 @@
1
+ // Google Docs export HTML preprocessor (issue #92 R6).
2
+ //
3
+ // Google Drive's `/export?format=html` payload serves bold/italic/
4
+ // strikethrough as inline `style` spans, numbers headings with
5
+ // `<a id><span>1. </span>`, and routes every link through a
6
+ // `google.com/url?q=` redirect. The generic Cheerio + Turndown pipeline
7
+ // discards those signals, so the browser-less API path loses the
8
+ // formatting and gains noisy redirect URLs.
9
+ //
10
+ // This module rewrites the export HTML into semantic markup before the
11
+ // generic converter runs.
12
+
13
+ import * as cheerio from 'cheerio';
14
+
15
+ /**
16
+ * Pre-process Google Docs export HTML so the generic Cheerio+Turndown
17
+ * pipeline preserves inline formatting, heading numbers, and link targets.
18
+ *
19
+ * @param {string} html - Raw export HTML returned by Google Docs.
20
+ * @returns {{html: string, hoisted: number, unwrappedLinks: number}}
21
+ */
22
+ export function preprocessGoogleDocsExportHtml(html) {
23
+ if (!html || typeof html !== 'string') {
24
+ return { html: html || '', hoisted: 0, unwrappedLinks: 0 };
25
+ }
26
+
27
+ const $ = cheerio.load(html, { decodeEntities: false });
28
+ let hoisted = 0;
29
+ let unwrappedLinks = 0;
30
+
31
+ $('span[style], p[style]').each(function () {
32
+ const style = String($(this).attr('style') || '');
33
+ const hasBold = /font-weight\s*:\s*(?:bold|[6-9]\d{2})/i.test(style);
34
+ const hasItalic = /font-style\s*:\s*italic/i.test(style);
35
+ const hasStrike = /text-decoration[^;]*\bline-through\b/i.test(style);
36
+ if (!hasBold && !hasItalic && !hasStrike) {
37
+ return;
38
+ }
39
+ const inner = $(this).html() || '';
40
+ let wrapped = inner;
41
+ if (hasStrike) {
42
+ wrapped = `<del>${wrapped}</del>`;
43
+ }
44
+ if (hasItalic) {
45
+ wrapped = `<em>${wrapped}</em>`;
46
+ }
47
+ if (hasBold) {
48
+ wrapped = `<strong>${wrapped}</strong>`;
49
+ }
50
+ $(this).replaceWith(wrapped);
51
+ hoisted += 1;
52
+ });
53
+
54
+ $('h1, h2, h3, h4, h5, h6').each(function () {
55
+ const $h = $(this);
56
+ $h.find('a').each(function () {
57
+ const $a = $(this);
58
+ if (!$a.attr('href') && !$a.text().trim() && !$a.find('img').length) {
59
+ $a.remove();
60
+ }
61
+ });
62
+ $h.find('> span, > strong > span').each(function () {
63
+ const $span = $(this);
64
+ const text = $span.text();
65
+ if (/^\s*\d+(?:\.\d+)*\.?\s*$/.test(text)) {
66
+ $span.remove();
67
+ }
68
+ });
69
+ });
70
+
71
+ $('a[href*="google.com/url?q="]').each(function () {
72
+ const href = String($(this).attr('href') || '');
73
+ const match = href.match(/[?&]q=([^&]+)/);
74
+ if (match) {
75
+ try {
76
+ const decoded = decodeURIComponent(match[1]);
77
+ $(this).attr('href', decoded);
78
+ unwrappedLinks += 1;
79
+ } catch {
80
+ // Leave the original href in place if it can't be decoded.
81
+ }
82
+ }
83
+ });
84
+
85
+ $('*').each(function () {
86
+ if (this.type !== 'tag') {
87
+ return;
88
+ }
89
+ $(this)
90
+ .contents()
91
+ .each(function () {
92
+ if (this.type === 'text' && this.data) {
93
+ const replaced = this.data.replace(/\u00A0/g, ' ');
94
+ if (replaced !== this.data) {
95
+ this.data = replaced;
96
+ }
97
+ }
98
+ });
99
+ });
100
+
101
+ let rewritten = $.html();
102
+ rewritten = rewritten.replace(/&nbsp;/g, ' ');
103
+
104
+ return { html: rewritten, hoisted, unwrappedLinks };
105
+ }
package/src/gdocs.js CHANGED
@@ -11,6 +11,10 @@ import fetch from 'node-fetch';
11
11
  import he from 'he';
12
12
  import { convertHtmlToMarkdown } from './lib.js';
13
13
  import { createBrowser as defaultCreateBrowser } from './browser.js';
14
+ import { preprocessGoogleDocsExportHtml } from './gdocs-preprocess.js';
15
+ import { localizeGoogleDocsModelImages } from './gdocs-images.js';
16
+
17
+ export { preprocessGoogleDocsExportHtml, localizeGoogleDocsModelImages };
14
18
 
15
19
  const GDOCS_URL_PATTERN = /docs\.google\.com\/document\/d\/([a-zA-Z0-9_-]+)/;
16
20
 
@@ -283,7 +287,14 @@ export async function fetchGoogleDocAsMarkdown(url, options = {}) {
283
287
  log,
284
288
  });
285
289
 
286
- const markdown = convertHtmlToMarkdown(result.content, result.exportUrl);
290
+ const preprocessed = preprocessGoogleDocsExportHtml(result.content);
291
+ log?.debug?.(() => ({
292
+ event: 'gdocs.export.style-hoist',
293
+ documentId: result.documentId,
294
+ hoisted: preprocessed.hoisted,
295
+ unwrappedLinks: preprocessed.unwrappedLinks,
296
+ }));
297
+ const markdown = convertHtmlToMarkdown(preprocessed.html, result.exportUrl);
287
298
  log?.debug?.(() => ({
288
299
  event: 'gdocs.public-export.markdown',
289
300
  documentId: result.documentId,
@@ -384,7 +395,8 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
384
395
 
385
396
  const capture = parseGoogleDocsModelChunks(
386
397
  modelData.chunks,
387
- modelData.cidUrlMap
398
+ modelData.cidUrlMap,
399
+ { log }
388
400
  );
389
401
  log?.debug?.(() => ({
390
402
  event: 'gdocs.browser-model.parsed',
@@ -531,9 +543,16 @@ export function extractCidUrlMapFromHtml(html) {
531
543
  *
532
544
  * @param {Array} chunks - Captured DOCS_modelChunk values
533
545
  * @param {Object<string,string>} [cidUrlMap={}] - CID-to-docs-images-rt URL map
546
+ * @param {Object} [options] - Parse options
547
+ * @param {Object} [options.log] - Optional verbose logger
534
548
  * @returns {{blocks: Array, tables: Array, images: Array, text: string}}
535
549
  */
536
- export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
550
+ export function parseGoogleDocsModelChunks(
551
+ chunks = [],
552
+ cidUrlMap = {},
553
+ options = {}
554
+ ) {
555
+ const { log } = options;
537
556
  const items = collectModelItems(chunks);
538
557
  const fullText = items
539
558
  .filter((item) => item.ty === 'is' || item.ty === 'iss')
@@ -581,6 +600,16 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
581
600
  let table = null;
582
601
  let row = null;
583
602
  let cell = null;
603
+ const tableHistograms = [];
604
+ let currentHistogram = null;
605
+
606
+ const bumpHistogram = (code) => {
607
+ if (!currentHistogram) {
608
+ return;
609
+ }
610
+ const key = `0x${code.toString(16).padStart(2, '0')}`;
611
+ currentHistogram[key] = (currentHistogram[key] || 0) + 1;
612
+ };
584
613
 
585
614
  const flushParagraph = (endPos = null) => {
586
615
  const text = contentToText(paragraph).trim();
@@ -594,15 +623,18 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
594
623
  paragraph = [];
595
624
  };
596
625
 
597
- const flushCell = () => {
626
+ const flushCell = ({ dropEmpty = false } = {}) => {
598
627
  if (cell && row) {
599
- row.cells.push(cell);
628
+ const isEmpty = cell.content.length === 0;
629
+ if (!dropEmpty || !isEmpty) {
630
+ row.cells.push(cell);
631
+ }
600
632
  }
601
633
  cell = null;
602
634
  };
603
635
 
604
- const flushRow = () => {
605
- flushCell();
636
+ const flushRow = ({ dropEmptyTrailingCell = false } = {}) => {
637
+ flushCell({ dropEmpty: dropEmptyTrailingCell });
606
638
  if (row && table) {
607
639
  table.rows.push(row);
608
640
  }
@@ -610,11 +642,23 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
610
642
  };
611
643
 
612
644
  const flushTable = () => {
613
- flushRow();
645
+ flushRow({ dropEmptyTrailingCell: true });
614
646
  if (table) {
647
+ // Drop trailing empty rows that Google Docs sometimes emits after the
648
+ // last cell content (from `\n` just before the `0x11` close marker).
649
+ while (
650
+ table.rows.length > 1 &&
651
+ table.rows[table.rows.length - 1].cells.length === 0
652
+ ) {
653
+ table.rows.pop();
654
+ }
615
655
  tables.push(table);
616
656
  blocks.push(table);
617
657
  }
658
+ if (currentHistogram) {
659
+ tableHistograms.push(currentHistogram);
660
+ }
661
+ currentHistogram = null;
618
662
  table = null;
619
663
  };
620
664
 
@@ -636,18 +680,29 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
636
680
  if (code === 0x10) {
637
681
  flushParagraph(i + 1);
638
682
  table = { type: 'table', rows: [] };
683
+ currentHistogram = {
684
+ '0x0a': 0,
685
+ '0x0b': 0,
686
+ '0x10': 1,
687
+ '0x11': 0,
688
+ '0x12': 0,
689
+ '0x1c': 0,
690
+ };
639
691
  continue;
640
692
  }
641
693
  if (code === 0x11) {
694
+ bumpHistogram(code);
642
695
  flushTable();
643
696
  continue;
644
697
  }
645
698
  if (code === 0x12) {
646
- flushRow();
699
+ bumpHistogram(code);
700
+ flushRow({ dropEmptyTrailingCell: true });
647
701
  row = { cells: [] };
648
702
  continue;
649
703
  }
650
704
  if (code === 0x1c) {
705
+ bumpHistogram(code);
651
706
  flushCell();
652
707
  if (!row) {
653
708
  row = { cells: [] };
@@ -656,14 +711,24 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
656
711
  continue;
657
712
  }
658
713
  if (code === 0x0a) {
714
+ bumpHistogram(code);
659
715
  if (table) {
660
- flushRow();
716
+ // Inside a Google Docs table, `\n` (0x0a) separates CELLS, not rows.
717
+ // Row boundaries are communicated via the explicit `0x12` marker and
718
+ // the closing `0x11` marker. Earlier versions collapsed multi-column
719
+ // tables to one column because they flushed the row on `\n`.
720
+ flushCell();
721
+ if (!row) {
722
+ row = { cells: [] };
723
+ }
724
+ cell = { content: [] };
661
725
  } else {
662
726
  flushParagraph(i + 1);
663
727
  }
664
728
  continue;
665
729
  }
666
730
  if (code === 0x0b) {
731
+ bumpHistogram(code);
667
732
  appendText(targetContent(), '\n');
668
733
  continue;
669
734
  }
@@ -684,6 +749,19 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
684
749
  }
685
750
  flushParagraph(fullText.length);
686
751
 
752
+ if (log?.debug && tableHistograms.length > 0) {
753
+ for (let t = 0; t < tableHistograms.length; t++) {
754
+ const tbl = tables[t];
755
+ log.debug(() => ({
756
+ event: 'gdocs.table.histogram',
757
+ tableIndex: t,
758
+ rows: tbl?.rows.length || 0,
759
+ columns: Math.max(...(tbl?.rows || []).map((r) => r.cells.length), 0),
760
+ controls: tableHistograms[t],
761
+ }));
762
+ }
763
+ }
764
+
687
765
  return {
688
766
  blocks,
689
767
  tables,
@@ -981,19 +1059,56 @@ function sameTextStyle(a, b) {
981
1059
  }
982
1060
 
983
1061
  function renderBlocksMarkdown(blocks) {
984
- return blocks
985
- .map((block) => {
986
- if (block.type === 'table') {
987
- return renderTableMarkdown(block);
1062
+ const counters = new Map();
1063
+ const rendered = blocks.map((block) => {
1064
+ if (block.type === 'table') {
1065
+ counters.clear();
1066
+ return { text: renderTableMarkdown(block), list: null };
1067
+ }
1068
+ const counterKey = block.list
1069
+ ? `${block.list.id || ''}\u0000${block.list.level || 0}`
1070
+ : null;
1071
+ if (block.list) {
1072
+ // Reset deeper siblings when we re-enter a shallower level so
1073
+ // nested lists restart from 1 on each new parent.
1074
+ for (const key of [...counters.keys()]) {
1075
+ const level = Number(key.split('\u0000')[1]);
1076
+ if (level > (block.list.level || 0)) {
1077
+ counters.delete(key);
1078
+ }
988
1079
  }
989
- return renderParagraphMarkdown(block);
990
- })
991
- .filter(Boolean)
992
- .join('\n\n')
993
- .trimEnd();
1080
+ const next = (counters.get(counterKey) || 0) + 1;
1081
+ counters.set(counterKey, next);
1082
+ return {
1083
+ text: renderParagraphMarkdown(block, { orderedIndex: next }),
1084
+ list: block.list,
1085
+ };
1086
+ }
1087
+ counters.clear();
1088
+ return { text: renderParagraphMarkdown(block), list: null };
1089
+ });
1090
+
1091
+ const joined = [];
1092
+ for (let i = 0; i < rendered.length; i++) {
1093
+ const cur = rendered[i];
1094
+ if (!cur.text) {
1095
+ continue;
1096
+ }
1097
+ if (joined.length > 0) {
1098
+ const prev = rendered
1099
+ .slice(0, i)
1100
+ .reverse()
1101
+ .find((entry) => entry.text);
1102
+ const sameList =
1103
+ cur.list && prev?.list && (cur.list.id || '') === (prev.list.id || '');
1104
+ joined.push(sameList ? '\n' : '\n\n');
1105
+ }
1106
+ joined.push(cur.text);
1107
+ }
1108
+ return joined.join('').replace(/\s+$/u, '');
994
1109
  }
995
1110
 
996
- function renderParagraphMarkdown(block) {
1111
+ function renderParagraphMarkdown(block, context = {}) {
997
1112
  const text = renderContentMarkdown(block.content).trim();
998
1113
  if (block.horizontalRule) {
999
1114
  return '---';
@@ -1011,7 +1126,8 @@ function renderParagraphMarkdown(block) {
1011
1126
  }
1012
1127
  if (block.list) {
1013
1128
  const indent = ' '.repeat(Math.max(0, block.list.level || 0));
1014
- const marker = block.list.ordered ? '1.' : '-';
1129
+ const orderedIndex = Math.max(1, Number(context.orderedIndex) || 1);
1130
+ const marker = block.list.ordered ? `${orderedIndex}.` : '-';
1015
1131
  return `${indent}${marker} ${text}`;
1016
1132
  }
1017
1133
  if (block.quote) {
@@ -1277,8 +1393,16 @@ export async function fetchGoogleDocAsArchive(url, options = {}) {
1277
1393
  // Fetch HTML with embedded base64 images
1278
1394
  const result = await fetchGoogleDoc(url, { format: 'html', apiToken, log });
1279
1395
 
1396
+ const preprocessed = preprocessGoogleDocsExportHtml(result.content);
1397
+ log?.debug?.(() => ({
1398
+ event: 'gdocs.export.style-hoist',
1399
+ documentId: result.documentId,
1400
+ hoisted: preprocessed.hoisted,
1401
+ unwrappedLinks: preprocessed.unwrappedLinks,
1402
+ }));
1403
+
1280
1404
  // Extract base64 images from HTML
1281
- const { html: localHtml, images } = extractBase64Images(result.content);
1405
+ const { html: localHtml, images } = extractBase64Images(preprocessed.html);
1282
1406
 
1283
1407
  // Convert the localized HTML to Markdown
1284
1408
  const markdown = convertHtmlToMarkdown(localHtml);
@@ -10,6 +10,8 @@ import {
10
10
  renderGoogleDocsCapture,
11
11
  renderDocsApiDocument,
12
12
  selectGoogleDocsCaptureMethod,
13
+ localizeGoogleDocsModelImages,
14
+ preprocessGoogleDocsExportHtml,
13
15
  GDOCS_EXPORT_FORMATS,
14
16
  } from '../../src/gdocs.js';
15
17
  import {
@@ -250,6 +252,115 @@ describe('gdocs', () => {
250
252
  );
251
253
  });
252
254
 
255
+ it('parses multi-column tables when rows are separated by 0x0a (issue #92 R2)', () => {
256
+ // Google Docs sometimes emits cell boundaries as `\n` (0x0a) alongside
257
+ // the 0x12 new-row marker. The parser must keep all cells on the same
258
+ // row rather than collapsing the table to a single column.
259
+ const chunks = [
260
+ {
261
+ chunk: [
262
+ {
263
+ ty: 'is',
264
+ s: `${
265
+ String.fromCharCode(0x10) + String.fromCharCode(0x12)
266
+ }A\nB\nC\n${String.fromCharCode(
267
+ 0x12
268
+ )}D\nE\nF\n${String.fromCharCode(0x11)}`,
269
+ },
270
+ ],
271
+ },
272
+ ];
273
+ const capture = parseGoogleDocsModelChunks(chunks);
274
+
275
+ expect(capture.tables).toHaveLength(1);
276
+ expect(capture.tables[0].rows).toHaveLength(2);
277
+ expect(capture.tables[0].rows[0].cells).toHaveLength(3);
278
+ expect(capture.tables[0].rows[1].cells).toHaveLength(3);
279
+
280
+ const markdown = renderGoogleDocsCapture(capture, 'markdown');
281
+ expect(markdown).toContain('| A | B | C |');
282
+ expect(markdown).toContain('| D | E | F |');
283
+ });
284
+
285
+ it('numbers ordered list items sequentially (issue #92 R3)', () => {
286
+ // Three list items sharing the same list id should render as 1. 2. 3.
287
+ const text = 'First item\nSecond item\nThird item\n';
288
+ const endOfLine = (n) => {
289
+ let idx = -1;
290
+ for (let k = 0; k < n; k++) {
291
+ idx = text.indexOf('\n', idx + 1);
292
+ }
293
+ return idx + 1;
294
+ };
295
+ const chunks = [
296
+ {
297
+ chunk: [
298
+ { ty: 'is', s: text },
299
+ {
300
+ ty: 'as',
301
+ st: 'list',
302
+ si: endOfLine(1),
303
+ ei: endOfLine(1),
304
+ sm: { ls_id: 'kix.list.7' },
305
+ },
306
+ {
307
+ ty: 'as',
308
+ st: 'list',
309
+ si: endOfLine(2),
310
+ ei: endOfLine(2),
311
+ sm: { ls_id: 'kix.list.7' },
312
+ },
313
+ {
314
+ ty: 'as',
315
+ st: 'list',
316
+ si: endOfLine(3),
317
+ ei: endOfLine(3),
318
+ sm: { ls_id: 'kix.list.7' },
319
+ },
320
+ ],
321
+ },
322
+ ];
323
+ const capture = parseGoogleDocsModelChunks(chunks);
324
+ const markdown = renderGoogleDocsCapture(capture, 'markdown');
325
+ const lines = markdown.split('\n');
326
+
327
+ expect(lines[0]).toBe('1. First item');
328
+ expect(lines[1]).toBe('2. Second item');
329
+ expect(lines[2]).toBe('3. Third item');
330
+ });
331
+
332
+ it('joins consecutive list items with a single newline (issue #92 R4)', () => {
333
+ const text = 'First item\nSecond item\n';
334
+ const endFirst = text.indexOf('\n') + 1;
335
+ const endSecond = text.length;
336
+ const chunks = [
337
+ {
338
+ chunk: [
339
+ { ty: 'is', s: text },
340
+ {
341
+ ty: 'as',
342
+ st: 'list',
343
+ si: endFirst,
344
+ ei: endFirst,
345
+ sm: { ls_id: 'kix.list.7' },
346
+ },
347
+ {
348
+ ty: 'as',
349
+ st: 'list',
350
+ si: endSecond,
351
+ ei: endSecond,
352
+ sm: { ls_id: 'kix.list.7' },
353
+ },
354
+ ],
355
+ },
356
+ ];
357
+ const capture = parseGoogleDocsModelChunks(chunks);
358
+ const markdown = renderGoogleDocsCapture(capture, 'markdown');
359
+
360
+ expect(markdown).not.toMatch(/First item\n\n2\. Second item/u);
361
+ expect(markdown).toMatch(/First item\n2\. Second item/u);
362
+ });
363
+
253
364
  it('renders model style records for headings, inline formatting, links, lists, blockquotes, rules, and images', () => {
254
365
  const text = [
255
366
  'Title',
@@ -442,6 +553,146 @@ describe('gdocs', () => {
442
553
  });
443
554
  });
444
555
 
556
+ describe('localizeGoogleDocsModelImages (issue #92 R5)', () => {
557
+ it('downloads docs-images-rt URLs and rewrites markdown/html', async () => {
558
+ const url =
559
+ 'https://docs.google.com/docs-images-rt/doc-id/cid/example.png';
560
+ const modelResult = {
561
+ markdown: `![pic](${url})`,
562
+ html: `<img src="${url}" alt="pic">`,
563
+ capture: {
564
+ images: [{ type: 'image', url, alt: 'pic', cid: 'cid-abc' }],
565
+ },
566
+ };
567
+ const fakeFetch = jest.fn().mockResolvedValue({
568
+ ok: true,
569
+ status: 200,
570
+ headers: new Map([['content-type', 'image/png']]),
571
+ arrayBuffer: async () => new Uint8Array([1, 2, 3, 4]).buffer,
572
+ });
573
+ // The map-based headers need a `get` method in the correct shape:
574
+ fakeFetch.mockResolvedValue({
575
+ ok: true,
576
+ status: 200,
577
+ headers: { get: () => 'image/png' },
578
+ arrayBuffer: async () =>
579
+ new Uint8Array([0x89, 0x50, 0x4e, 0x47]).buffer,
580
+ });
581
+
582
+ const localized = await localizeGoogleDocsModelImages(modelResult, {
583
+ fetchImpl: fakeFetch,
584
+ });
585
+
586
+ expect(fakeFetch).toHaveBeenCalledTimes(1);
587
+ expect(localized.images).toHaveLength(1);
588
+ expect(localized.images[0].filename).toBe('image-01.png');
589
+ expect(localized.images[0].mimeType).toBe('image/png');
590
+ expect(localized.images[0].data).toBeInstanceOf(Buffer);
591
+ expect(localized.markdown).toBe('![pic](images/image-01.png)');
592
+ expect(localized.html).toContain('images/image-01.png');
593
+ expect(localized.html).not.toContain('docs-images-rt');
594
+ });
595
+
596
+ it('keeps original URL when download fails', async () => {
597
+ const url =
598
+ 'https://docs.google.com/docs-images-rt/doc-id/cid/missing.png';
599
+ const modelResult = {
600
+ markdown: `![pic](${url})`,
601
+ html: `<img src="${url}" alt="pic">`,
602
+ capture: { images: [{ url, alt: 'pic' }] },
603
+ };
604
+ const fakeFetch = jest.fn().mockResolvedValue({
605
+ ok: false,
606
+ status: 404,
607
+ headers: { get: () => null },
608
+ });
609
+
610
+ const localized = await localizeGoogleDocsModelImages(modelResult, {
611
+ fetchImpl: fakeFetch,
612
+ });
613
+
614
+ expect(localized.images).toHaveLength(0);
615
+ // Markdown still gets rewritten (local path) even if download fails —
616
+ // callers may choose to ignore this; the key guarantee is that no
617
+ // exception leaks out.
618
+ expect(localized.markdown).toBe('![pic](images/image-01.png)');
619
+ });
620
+ });
621
+
622
+ describe('preprocessGoogleDocsExportHtml (issue #92 R6)', () => {
623
+ it('hoists font-weight:700 spans into <strong>', () => {
624
+ const html = '<p><span style="font-weight:700">Bold text</span></p>';
625
+ const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
626
+ expect(hoisted).toBe(1);
627
+ expect(out).toContain('<strong>');
628
+ expect(out).toContain('Bold text');
629
+ });
630
+
631
+ it('hoists font-style:italic spans into <em>', () => {
632
+ const html = '<p><span style="font-style:italic">Italic text</span></p>';
633
+ const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
634
+ expect(hoisted).toBe(1);
635
+ expect(out).toContain('<em>');
636
+ });
637
+
638
+ it('hoists text-decoration:line-through spans into <del>', () => {
639
+ const html =
640
+ '<p><span style="text-decoration:line-through">Strike</span></p>';
641
+ const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
642
+ expect(hoisted).toBe(1);
643
+ expect(out).toContain('<del>');
644
+ });
645
+
646
+ it('combines multiple styles on a single span', () => {
647
+ const html =
648
+ '<p><span style="font-weight:700;font-style:italic;text-decoration:line-through">Mixed</span></p>';
649
+ const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
650
+ expect(hoisted).toBe(1);
651
+ // Wrappers appear in <strong><em><del>... order (outer-to-inner).
652
+ expect(out).toContain('<strong>');
653
+ expect(out).toContain('<em>');
654
+ expect(out).toContain('<del>');
655
+ });
656
+
657
+ it('unwraps google.com/url?q= redirect links', () => {
658
+ const html =
659
+ '<p><a href="https://www.google.com/url?q=https://example.com&sa=D&source=editors&usg=ABC">Link</a></p>';
660
+ const { html: out, unwrappedLinks } =
661
+ preprocessGoogleDocsExportHtml(html);
662
+ expect(unwrappedLinks).toBe(1);
663
+ expect(out).toContain('href="https://example.com"');
664
+ expect(out).not.toContain('google.com/url?q=');
665
+ });
666
+
667
+ it('strips leading empty anchors and numbering span inside headings', () => {
668
+ const html = '<h1><a id="anchor-1"></a><span>1. </span>Headings</h1>';
669
+ const { html: out } = preprocessGoogleDocsExportHtml(html);
670
+ expect(out).toContain('<h1>Headings</h1>');
671
+ expect(out).not.toContain('1. ');
672
+ expect(out).not.toContain('<a id=');
673
+ });
674
+
675
+ it('replaces &nbsp; entity and U+00A0 with regular spaces', () => {
676
+ const html = '<p>A\u00A0B&nbsp;C</p>';
677
+ const { html: out } = preprocessGoogleDocsExportHtml(html);
678
+ expect(out).toContain('A B C');
679
+ expect(out).not.toMatch(/\u00A0/);
680
+ expect(out).not.toContain('&nbsp;');
681
+ });
682
+
683
+ it('is a no-op for HTML without Google Docs markers', () => {
684
+ const html = '<p>Plain text with <strong>bold</strong>.</p>';
685
+ const {
686
+ html: out,
687
+ hoisted,
688
+ unwrappedLinks,
689
+ } = preprocessGoogleDocsExportHtml(html);
690
+ expect(hoisted).toBe(0);
691
+ expect(unwrappedLinks).toBe(0);
692
+ expect(out).toContain('<p>Plain text with <strong>bold</strong>.</p>');
693
+ });
694
+ });
695
+
445
696
  describe('extractBase64Images', () => {
446
697
  it('extracts a single PNG data URI image', () => {
447
698
  const html = '<img src="data:image/png;base64,iVBORw0KGgo=" alt="test">';