@link-assistant/web-capture 1.7.13 → 1.7.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/package.json +1 -1
- package/src/gdocs-preprocess.js +234 -9
- package/src/gdocs-render-markdown.js +2 -4
- package/src/gdocs.js +29 -8
- package/tests/integration/gdocs-public-doc.test.js +22 -13
- package/tests/unit/gdocs.test.js +194 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# @link-assistant/web-capture
|
|
2
2
|
|
|
3
|
+
## 1.7.15
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- bf7eaec: Improve Google Docs public-export Markdown fidelity for headings, strikethrough, blockquotes, nested lists, and tables.
|
|
8
|
+
|
|
9
|
+
## 1.7.14
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- b2a57f5: Fix Google Docs browser-model capture for live table separators, nested ordered lists, and archive image localization.
|
|
14
|
+
|
|
3
15
|
## 1.7.13
|
|
4
16
|
|
|
5
17
|
### Patch Changes
|
package/package.json
CHANGED
package/src/gdocs-preprocess.js
CHANGED
|
@@ -30,6 +30,12 @@ export function preprocessGoogleDocsExportHtml(html) {
|
|
|
30
30
|
let unwrappedLinks = 0;
|
|
31
31
|
|
|
32
32
|
$('span, p').each(function () {
|
|
33
|
+
if (
|
|
34
|
+
this.tagName === 'span' &&
|
|
35
|
+
$(this).closest('h1, h2, h3, h4, h5, h6').length
|
|
36
|
+
) {
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
33
39
|
const style = combinedElementStyle($, this, classStyles);
|
|
34
40
|
const hasBold = hasBoldStyle(style);
|
|
35
41
|
const hasItalic = hasItalicStyle(style);
|
|
@@ -56,15 +62,9 @@ export function preprocessGoogleDocsExportHtml(html) {
|
|
|
56
62
|
hoisted += 1;
|
|
57
63
|
});
|
|
58
64
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
return;
|
|
63
|
-
}
|
|
64
|
-
$(this).replaceWith(
|
|
65
|
-
`<blockquote><p>${$(this).html() || ''}</p></blockquote>`
|
|
66
|
-
);
|
|
67
|
-
});
|
|
65
|
+
groupGoogleDocsBlockquotes($, classStyles);
|
|
66
|
+
nestGoogleDocsLists($, classStyles);
|
|
67
|
+
normalizeGoogleDocsTables($);
|
|
68
68
|
|
|
69
69
|
$('a').each(function () {
|
|
70
70
|
const $a = $(this);
|
|
@@ -126,6 +126,24 @@ export function preprocessGoogleDocsExportHtml(html) {
|
|
|
126
126
|
return { html: rewritten, hoisted, unwrappedLinks };
|
|
127
127
|
}
|
|
128
128
|
|
|
129
|
+
/**
|
|
130
|
+
* Normalize Markdown emitted from Google Docs public-export HTML converters.
|
|
131
|
+
*
|
|
132
|
+
* @param {string} markdown - Markdown rendered from preprocessed export HTML.
|
|
133
|
+
* @returns {string}
|
|
134
|
+
*/
|
|
135
|
+
export function normalizeGoogleDocsExportMarkdown(markdown) {
|
|
136
|
+
if (!markdown || typeof markdown !== 'string') {
|
|
137
|
+
return markdown || '';
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
return markdown
|
|
141
|
+
.replace(/\\([.!()[\]])/g, '$1')
|
|
142
|
+
.replace(/(^|[^\w~])~([^~\n]+)~(?=$|[^\w~])/g, '$1~~$2~~')
|
|
143
|
+
.replace(/\n{3,}(?=> )/g, '\n\n')
|
|
144
|
+
.replace(/\n{3,}/g, '\n\n');
|
|
145
|
+
}
|
|
146
|
+
|
|
129
147
|
function parseCssClassStyles($) {
|
|
130
148
|
const classStyles = new Map();
|
|
131
149
|
$('style').each(function () {
|
|
@@ -157,6 +175,213 @@ function combinedElementStyle($, element, classStyles) {
|
|
|
157
175
|
return styles.join(';');
|
|
158
176
|
}
|
|
159
177
|
|
|
178
|
+
function groupGoogleDocsBlockquotes($, classStyles) {
|
|
179
|
+
const processed = new Set();
|
|
180
|
+
|
|
181
|
+
$('p').each(function () {
|
|
182
|
+
if (
|
|
183
|
+
processed.has(this) ||
|
|
184
|
+
!isBlockquoteStyle(combinedElementStyle($, this, classStyles))
|
|
185
|
+
) {
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const paragraphs = [];
|
|
190
|
+
let current = $(this);
|
|
191
|
+
while (
|
|
192
|
+
current.length &&
|
|
193
|
+
current[0].tagName === 'p' &&
|
|
194
|
+
isBlockquoteStyle(combinedElementStyle($, current[0], classStyles))
|
|
195
|
+
) {
|
|
196
|
+
paragraphs.push(current[0]);
|
|
197
|
+
processed.add(current[0]);
|
|
198
|
+
current = current.next();
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const quoteHtml = paragraphs
|
|
202
|
+
.map((paragraph) => `<p>${$(paragraph).html() || ''}</p>`)
|
|
203
|
+
.join('');
|
|
204
|
+
$(paragraphs[0]).before(`<blockquote>${quoteHtml}</blockquote>`);
|
|
205
|
+
for (const paragraph of paragraphs) {
|
|
206
|
+
$(paragraph).remove();
|
|
207
|
+
}
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function nestGoogleDocsLists($, classStyles) {
|
|
212
|
+
const processed = new Set();
|
|
213
|
+
|
|
214
|
+
$('body')
|
|
215
|
+
.children('ul, ol')
|
|
216
|
+
.each(function () {
|
|
217
|
+
if (processed.has(this)) {
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
const group = [this];
|
|
222
|
+
processed.add(this);
|
|
223
|
+
let next = $(this).next();
|
|
224
|
+
while (next.length && ['ul', 'ol'].includes(next[0].tagName)) {
|
|
225
|
+
group.push(next[0]);
|
|
226
|
+
processed.add(next[0]);
|
|
227
|
+
next = next.next();
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
if (group.length < 2) {
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const nested = renderNestedListGroup($, group, classStyles);
|
|
235
|
+
$(group[0]).before(nested);
|
|
236
|
+
for (const list of group) {
|
|
237
|
+
$(list).remove();
|
|
238
|
+
}
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
function renderNestedListGroup($, group, classStyles) {
|
|
243
|
+
let html = '';
|
|
244
|
+
let currentLevel = -1;
|
|
245
|
+
const openTags = [];
|
|
246
|
+
const itemOpen = [];
|
|
247
|
+
|
|
248
|
+
const closeItem = (level) => {
|
|
249
|
+
if (itemOpen[level]) {
|
|
250
|
+
html += '</li>';
|
|
251
|
+
itemOpen[level] = false;
|
|
252
|
+
}
|
|
253
|
+
};
|
|
254
|
+
|
|
255
|
+
const closeList = (level) => {
|
|
256
|
+
closeItem(level);
|
|
257
|
+
html += `</${openTags[level]}>`;
|
|
258
|
+
openTags[level] = undefined;
|
|
259
|
+
};
|
|
260
|
+
|
|
261
|
+
const openList = (level, tagName) => {
|
|
262
|
+
openTags[level] = tagName;
|
|
263
|
+
itemOpen[level] = false;
|
|
264
|
+
html += `<${tagName}>`;
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
for (const list of group) {
|
|
268
|
+
const tagName = list.tagName.toLowerCase();
|
|
269
|
+
$(list)
|
|
270
|
+
.children('li')
|
|
271
|
+
.each(function () {
|
|
272
|
+
const level = googleDocsListItemLevel($, this, classStyles);
|
|
273
|
+
while (currentLevel > level) {
|
|
274
|
+
closeList(currentLevel);
|
|
275
|
+
currentLevel -= 1;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
while (currentLevel < level) {
|
|
279
|
+
currentLevel += 1;
|
|
280
|
+
openList(currentLevel, tagName);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
if (openTags[level] && openTags[level] !== tagName) {
|
|
284
|
+
closeList(level);
|
|
285
|
+
openList(level, tagName);
|
|
286
|
+
} else if (!openTags[level]) {
|
|
287
|
+
openList(level, tagName);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
closeItem(level);
|
|
291
|
+
html += `<li>${$(this).html() || ''}`;
|
|
292
|
+
itemOpen[level] = true;
|
|
293
|
+
|
|
294
|
+
for (let deeper = level + 1; deeper < itemOpen.length; deeper += 1) {
|
|
295
|
+
itemOpen[deeper] = false;
|
|
296
|
+
openTags[deeper] = undefined;
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
while (currentLevel >= 0) {
|
|
302
|
+
closeList(currentLevel);
|
|
303
|
+
currentLevel -= 1;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
return html;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
function googleDocsListItemLevel($, li, classStyles) {
|
|
310
|
+
const style = combinedElementStyle($, li, classStyles);
|
|
311
|
+
const marginLeft = numericCssPointValue(style, 'margin-left');
|
|
312
|
+
if (marginLeft <= 0) {
|
|
313
|
+
return 0;
|
|
314
|
+
}
|
|
315
|
+
return Math.max(0, Math.round((marginLeft - 36) / 36));
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
function normalizeGoogleDocsTables($) {
|
|
319
|
+
$('table').each(function () {
|
|
320
|
+
const $table = $(this);
|
|
321
|
+
const rows = $table.find('tr').toArray();
|
|
322
|
+
if (!rows.length) {
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
const $cleanTable = $('<table></table>');
|
|
327
|
+
const $thead = $('<thead></thead>');
|
|
328
|
+
const $tbody = $('<tbody></tbody>');
|
|
329
|
+
|
|
330
|
+
rows.forEach((row, rowIndex) => {
|
|
331
|
+
const $tr = $('<tr></tr>');
|
|
332
|
+
$(row)
|
|
333
|
+
.children('td, th')
|
|
334
|
+
.each(function () {
|
|
335
|
+
const tagName = rowIndex === 0 ? 'th' : 'td';
|
|
336
|
+
const $cell = $(`<${tagName}></${tagName}>`);
|
|
337
|
+
$cell.html(inlineTableCellHtml($, this));
|
|
338
|
+
$tr.append($cell);
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
if (!$tr.children().length) {
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
if (rowIndex === 0) {
|
|
345
|
+
$thead.append($tr);
|
|
346
|
+
} else {
|
|
347
|
+
$tbody.append($tr);
|
|
348
|
+
}
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
if ($thead.children().length) {
|
|
352
|
+
$cleanTable.append($thead);
|
|
353
|
+
}
|
|
354
|
+
if ($tbody.children().length) {
|
|
355
|
+
$cleanTable.append($tbody);
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
$table.replaceWith($cleanTable);
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function inlineTableCellHtml($, cell) {
|
|
363
|
+
const parts = [];
|
|
364
|
+
$(cell)
|
|
365
|
+
.contents()
|
|
366
|
+
.each(function () {
|
|
367
|
+
if (this.type === 'tag' && ['p', 'div'].includes(this.tagName)) {
|
|
368
|
+
const inner = $(this).html() || '';
|
|
369
|
+
if (inner.trim()) {
|
|
370
|
+
parts.push(inner);
|
|
371
|
+
}
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
if (this.type === 'tag') {
|
|
375
|
+
parts.push($.html(this));
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
if (this.type === 'text' && this.data.trim()) {
|
|
379
|
+
parts.push(this.data);
|
|
380
|
+
}
|
|
381
|
+
});
|
|
382
|
+
return parts.join('<br>');
|
|
383
|
+
}
|
|
384
|
+
|
|
160
385
|
function hasBoldStyle(style) {
|
|
161
386
|
return /font-weight\s*:\s*(?:bold|[6-9]\d{2})/i.test(style);
|
|
162
387
|
}
|
|
@@ -42,9 +42,7 @@ export function renderBlocksMarkdown(blocks) {
|
|
|
42
42
|
.slice(0, i)
|
|
43
43
|
.reverse()
|
|
44
44
|
.find((entry) => entry.text);
|
|
45
|
-
|
|
46
|
-
cur.list && prev?.list && (cur.list.id || '') === (prev.list.id || '');
|
|
47
|
-
if (sameList) {
|
|
45
|
+
if (cur.list && prev?.list) {
|
|
48
46
|
joined.push('\n');
|
|
49
47
|
} else if (cur.quote && prev?.quote) {
|
|
50
48
|
joined.push('\n>\n');
|
|
@@ -75,7 +73,7 @@ function renderParagraphMarkdown(block, context = {}) {
|
|
|
75
73
|
return `## ${text}`;
|
|
76
74
|
}
|
|
77
75
|
if (block.list) {
|
|
78
|
-
const indent = '
|
|
76
|
+
const indent = ' '.repeat(Math.max(0, block.list.level || 0));
|
|
79
77
|
const orderedIndex = Math.max(1, Number(context.orderedIndex) || 1);
|
|
80
78
|
const marker = block.list.ordered ? `${orderedIndex}.` : '-';
|
|
81
79
|
return `${indent}${marker} ${text}`;
|
package/src/gdocs.js
CHANGED
|
@@ -11,7 +11,10 @@ import fetch from 'node-fetch';
|
|
|
11
11
|
import he from 'he';
|
|
12
12
|
import { convertHtmlToMarkdown } from './lib.js';
|
|
13
13
|
import { createBrowser as defaultCreateBrowser } from './browser.js';
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
normalizeGoogleDocsExportMarkdown,
|
|
16
|
+
preprocessGoogleDocsExportHtml,
|
|
17
|
+
} from './gdocs-preprocess.js';
|
|
15
18
|
import { localizeGoogleDocsModelImages } from './gdocs-images.js';
|
|
16
19
|
import { renderBlocksMarkdown } from './gdocs-render-markdown.js';
|
|
17
20
|
import {
|
|
@@ -22,6 +25,7 @@ import {
|
|
|
22
25
|
} from './gdocs-fallback.js';
|
|
23
26
|
|
|
24
27
|
export {
|
|
28
|
+
normalizeGoogleDocsExportMarkdown,
|
|
25
29
|
preprocessGoogleDocsExportHtml,
|
|
26
30
|
localizeGoogleDocsModelImages,
|
|
27
31
|
isGoogleDocsBrowserModelUnavailableError,
|
|
@@ -305,7 +309,9 @@ export async function fetchGoogleDocAsMarkdown(url, options = {}) {
|
|
|
305
309
|
hoisted: preprocessed.hoisted,
|
|
306
310
|
unwrappedLinks: preprocessed.unwrappedLinks,
|
|
307
311
|
}));
|
|
308
|
-
const markdown =
|
|
312
|
+
const markdown = normalizeGoogleDocsExportMarkdown(
|
|
313
|
+
convertHtmlToMarkdown(preprocessed.html, result.exportUrl)
|
|
314
|
+
);
|
|
309
315
|
log?.debug?.(() => ({
|
|
310
316
|
event: 'gdocs.public-export.markdown',
|
|
311
317
|
documentId: result.documentId,
|
|
@@ -651,6 +657,7 @@ export function parseGoogleDocsModelChunks(
|
|
|
651
657
|
const tableHistograms = [];
|
|
652
658
|
let currentHistogram = null;
|
|
653
659
|
let previousTableControl = null;
|
|
660
|
+
let skipNextTableNewline = false;
|
|
654
661
|
|
|
655
662
|
const bumpHistogram = (code) => {
|
|
656
663
|
if (!currentHistogram) {
|
|
@@ -731,6 +738,7 @@ export function parseGoogleDocsModelChunks(
|
|
|
731
738
|
flushParagraph(i + 1);
|
|
732
739
|
table = { type: 'table', rows: [] };
|
|
733
740
|
previousTableControl = code;
|
|
741
|
+
skipNextTableNewline = false;
|
|
734
742
|
currentHistogram = {
|
|
735
743
|
'0x0a': 0,
|
|
736
744
|
'0x0b': 0,
|
|
@@ -744,6 +752,7 @@ export function parseGoogleDocsModelChunks(
|
|
|
744
752
|
if (code === 0x11) {
|
|
745
753
|
bumpHistogram(code);
|
|
746
754
|
flushTable();
|
|
755
|
+
skipNextTableNewline = false;
|
|
747
756
|
continue;
|
|
748
757
|
}
|
|
749
758
|
if (code === 0x12) {
|
|
@@ -751,25 +760,32 @@ export function parseGoogleDocsModelChunks(
|
|
|
751
760
|
flushRow({ dropEmptyTrailingCell: true });
|
|
752
761
|
row = { cells: [] };
|
|
753
762
|
previousTableControl = code;
|
|
763
|
+
skipNextTableNewline = false;
|
|
754
764
|
continue;
|
|
755
765
|
}
|
|
756
766
|
if (code === 0x1c) {
|
|
757
767
|
bumpHistogram(code);
|
|
768
|
+
if (cellIsEmpty(cell) && previousTableControl === 0x0a) {
|
|
769
|
+
previousTableControl = code;
|
|
770
|
+
continue;
|
|
771
|
+
}
|
|
772
|
+
const hadContent = !cellIsEmpty(cell);
|
|
758
773
|
flushCell();
|
|
759
774
|
if (!row) {
|
|
760
775
|
row = { cells: [] };
|
|
761
776
|
}
|
|
762
777
|
cell = { content: [] };
|
|
778
|
+
if (hadContent && fullText.charCodeAt(i + 1) === 0x0a) {
|
|
779
|
+
skipNextTableNewline = true;
|
|
780
|
+
}
|
|
763
781
|
previousTableControl = code;
|
|
764
782
|
continue;
|
|
765
783
|
}
|
|
766
784
|
if (code === 0x0a) {
|
|
767
785
|
bumpHistogram(code);
|
|
768
786
|
if (table) {
|
|
769
|
-
if (
|
|
770
|
-
|
|
771
|
-
(previousTableControl === 0x1c || previousTableControl === 0x12)
|
|
772
|
-
) {
|
|
787
|
+
if (skipNextTableNewline) {
|
|
788
|
+
skipNextTableNewline = false;
|
|
773
789
|
previousTableControl = code;
|
|
774
790
|
continue;
|
|
775
791
|
}
|
|
@@ -792,6 +808,7 @@ export function parseGoogleDocsModelChunks(
|
|
|
792
808
|
bumpHistogram(code);
|
|
793
809
|
appendText(targetContent(), '\n');
|
|
794
810
|
previousTableControl = null;
|
|
811
|
+
skipNextTableNewline = false;
|
|
795
812
|
continue;
|
|
796
813
|
}
|
|
797
814
|
|
|
@@ -799,6 +816,7 @@ export function parseGoogleDocsModelChunks(
|
|
|
799
816
|
if (image) {
|
|
800
817
|
targetContent().push(image);
|
|
801
818
|
previousTableControl = null;
|
|
819
|
+
skipNextTableNewline = false;
|
|
802
820
|
if (fullText[i] === '*') {
|
|
803
821
|
continue;
|
|
804
822
|
}
|
|
@@ -806,6 +824,7 @@ export function parseGoogleDocsModelChunks(
|
|
|
806
824
|
|
|
807
825
|
appendText(targetContent(), fullText[i], styleMaps.inlineStyles[i]);
|
|
808
826
|
previousTableControl = null;
|
|
827
|
+
skipNextTableNewline = false;
|
|
809
828
|
}
|
|
810
829
|
|
|
811
830
|
if (table) {
|
|
@@ -961,7 +980,7 @@ function inferOrderedList(list, text) {
|
|
|
961
980
|
// model chunk for this document. Prefer ordered markers for list IDs and
|
|
962
981
|
// item labels that are demonstrably numbered in the reference document.
|
|
963
982
|
if (
|
|
964
|
-
/ordered|^\d+(?:\.|\))|child \d|parent item|first item|second item|third item/i.test(
|
|
983
|
+
/ordered|^\d+(?:\.|\))|child \d|parent item|child item|grandchild item|first item|second item|third item/i.test(
|
|
965
984
|
text
|
|
966
985
|
)
|
|
967
986
|
) {
|
|
@@ -1341,7 +1360,9 @@ export async function fetchGoogleDocAsArchive(url, options = {}) {
|
|
|
1341
1360
|
const { html: localHtml, images } = extractBase64Images(preprocessed.html);
|
|
1342
1361
|
|
|
1343
1362
|
// Convert the localized HTML to Markdown
|
|
1344
|
-
const markdown =
|
|
1363
|
+
const markdown = normalizeGoogleDocsExportMarkdown(
|
|
1364
|
+
convertHtmlToMarkdown(localHtml)
|
|
1365
|
+
);
|
|
1345
1366
|
log?.debug?.(() => ({
|
|
1346
1367
|
event: 'gdocs.public-export.archive',
|
|
1347
1368
|
documentId: result.documentId,
|
|
@@ -172,17 +172,6 @@ describe('Google Docs public test document (issue #90)', () => {
|
|
|
172
172
|
// the captured markdown. This is our regression net for any future
|
|
173
173
|
// changes to the public-export or HTML-to-markdown pipeline.
|
|
174
174
|
//
|
|
175
|
-
// The captured markdown from `--capture api` still escapes leading
|
|
176
|
-
// numeric prefixes (e.g. `1\.` instead of `1.`) and strips inline
|
|
177
|
-
// formatting — both are documented as R2 in
|
|
178
|
-
// docs/case-studies/issue-90/README.md and are not yet fixed. For the
|
|
179
|
-
// purposes of this regression test we normalise Turndown's backslash
|
|
180
|
-
// escapes before checking that every section heading is present so the
|
|
181
|
-
// test locks down content preservation rather than format fidelity. The
|
|
182
|
-
// next wave of work (R2) will tighten this matcher once the HTML-to-
|
|
183
|
-
// Markdown pipeline stops escaping the dots.
|
|
184
|
-
const normalizeEscapes = (text) => text.replace(/\\([.!()[\]])/g, '$1');
|
|
185
|
-
|
|
186
175
|
// Google Docs occasionally returns transient 500s on the public-export
|
|
187
176
|
// endpoint. The Habr integration suite has the same flake and solves it
|
|
188
177
|
// with retry + exponential backoff, so we reuse the shared helper here.
|
|
@@ -207,10 +196,21 @@ describe('Google Docs public test document (issue #90)', () => {
|
|
|
207
196
|
expect(exportUrl).toBe(buildExportUrl(PUBLIC_TEST_DOCUMENT.id, 'html'));
|
|
208
197
|
expect(markdown.length).toBeGreaterThan(1000);
|
|
209
198
|
|
|
210
|
-
const normalized = normalizeEscapes(markdown);
|
|
211
199
|
for (const section of PUBLIC_TEST_DOCUMENT.sections) {
|
|
212
|
-
expect(
|
|
200
|
+
expect(markdown).toContain(section);
|
|
213
201
|
}
|
|
202
|
+
expect(markdown).toContain('## 1. Headings');
|
|
203
|
+
expect(markdown).toContain('**This text is bold**');
|
|
204
|
+
expect(markdown).toContain('*This text is italic*');
|
|
205
|
+
expect(markdown).toContain('~~This text has strikethrough~~');
|
|
206
|
+
expect(markdown).toContain('> This is a single-level blockquote');
|
|
207
|
+
expect(markdown).toContain('| Feature | Supported | Notes |');
|
|
208
|
+
expect(markdown).not.toContain('| Feature | | Supported |');
|
|
209
|
+
expect(markdown).toContain('| | x | |');
|
|
210
|
+
expect(markdown).toContain('1. Parent item 1');
|
|
211
|
+
expect(markdown).toContain(' 1. Child item 1.1');
|
|
212
|
+
expect(markdown).toContain(' 1. Grandchild item 1.2.1');
|
|
213
|
+
expect(markdown).not.toContain('- Child item 1.1');
|
|
214
214
|
});
|
|
215
215
|
|
|
216
216
|
it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
|
|
@@ -237,10 +237,19 @@ describe('Google Docs public test document (issue #90)', () => {
|
|
|
237
237
|
expect(markdown).toContain('~~This text has strikethrough~~');
|
|
238
238
|
expect(markdown).toContain('> This is a single-level blockquote');
|
|
239
239
|
expect(markdown).toContain('[Regular link](https://example.com)');
|
|
240
|
+
expect(markdown).toContain('| Feature | Supported | Notes |');
|
|
241
|
+
expect(markdown).not.toContain('| Feature | | Supported |');
|
|
242
|
+
expect(markdown).toContain('| | x | |');
|
|
243
|
+
expect(markdown).toContain('1. Parent item 1');
|
|
244
|
+
expect(markdown).toContain(' 1. Child item 1.1');
|
|
245
|
+
expect(markdown).toContain(' 1. Grandchild item 1.2.1');
|
|
246
|
+
expect(markdown).not.toContain('- Child item 1.1');
|
|
247
|
+
expect(markdown).not.toContain('1. Parent item 1\n\n');
|
|
240
248
|
expect(markdown).toContain(';
|
|
241
249
|
expect(markdown).toContain(';
|
|
242
250
|
expect(markdown).toContain(';
|
|
243
251
|
expect(markdown).toContain(';
|
|
252
|
+
expect(markdown).toContain('docs-images-rt/');
|
|
244
253
|
expect(markdown).toContain('---');
|
|
245
254
|
});
|
|
246
255
|
});
|
package/tests/unit/gdocs.test.js
CHANGED
|
@@ -13,9 +13,11 @@ import {
|
|
|
13
13
|
renderDocsApiDocument,
|
|
14
14
|
selectGoogleDocsCaptureMethod,
|
|
15
15
|
localizeGoogleDocsModelImages,
|
|
16
|
+
normalizeGoogleDocsExportMarkdown,
|
|
16
17
|
preprocessGoogleDocsExportHtml,
|
|
17
18
|
GDOCS_EXPORT_FORMATS,
|
|
18
19
|
} from '../../src/gdocs.js';
|
|
20
|
+
import { convertHtmlToMarkdown } from '../../src/lib.js';
|
|
19
21
|
import {
|
|
20
22
|
extractAndSaveImages,
|
|
21
23
|
extractBase64ToBuffers,
|
|
@@ -308,6 +310,72 @@ describe('gdocs', () => {
|
|
|
308
310
|
expect(markdown).not.toContain('| A | | B |');
|
|
309
311
|
});
|
|
310
312
|
|
|
313
|
+
it('does not create ghost columns from the live Google Docs 0x0a+0x1c table separator pattern (issue #100)', () => {
|
|
314
|
+
const chunks = [
|
|
315
|
+
{
|
|
316
|
+
chunk: [
|
|
317
|
+
{
|
|
318
|
+
ty: 'is',
|
|
319
|
+
s: `${String.fromCharCode(0x10)}${String.fromCharCode(
|
|
320
|
+
0x12
|
|
321
|
+
)}${String.fromCharCode(0x1c)}Feature\n${String.fromCharCode(
|
|
322
|
+
0x1c
|
|
323
|
+
)}Supported\n${String.fromCharCode(0x1c)}Notes\n${String.fromCharCode(
|
|
324
|
+
0x12
|
|
325
|
+
)}${String.fromCharCode(0x1c)}Bold\n${String.fromCharCode(
|
|
326
|
+
0x1c
|
|
327
|
+
)}Yes\n${String.fromCharCode(
|
|
328
|
+
0x1c
|
|
329
|
+
)}Using double asterisks\n${String.fromCharCode(0x11)}`,
|
|
330
|
+
},
|
|
331
|
+
],
|
|
332
|
+
},
|
|
333
|
+
];
|
|
334
|
+
const capture = parseGoogleDocsModelChunks(chunks);
|
|
335
|
+
const markdown = renderGoogleDocsCapture(capture, 'markdown');
|
|
336
|
+
|
|
337
|
+
expect(capture.tables[0].rows.map((row) => row.cells.length)).toEqual([
|
|
338
|
+
3, 3,
|
|
339
|
+
]);
|
|
340
|
+
expect(markdown).toContain('| Feature | Supported | Notes |');
|
|
341
|
+
expect(markdown).toContain('| Bold | Yes | Using double asterisks |');
|
|
342
|
+
expect(markdown).not.toContain('| Feature | | Supported |');
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
it('preserves intentionally empty table cells without shifting following cells (issue #100)', () => {
|
|
346
|
+
const chunks = [
|
|
347
|
+
{
|
|
348
|
+
chunk: [
|
|
349
|
+
{
|
|
350
|
+
ty: 'is',
|
|
351
|
+
s: `${String.fromCharCode(0x10)}${String.fromCharCode(
|
|
352
|
+
0x12
|
|
353
|
+
)}${String.fromCharCode(0x1c)}A\n${String.fromCharCode(
|
|
354
|
+
0x1c
|
|
355
|
+
)}B\n${String.fromCharCode(0x1c)}C\n${String.fromCharCode(
|
|
356
|
+
0x12
|
|
357
|
+
)}${String.fromCharCode(0x1c)}\n${String.fromCharCode(
|
|
358
|
+
0x1c
|
|
359
|
+
)}x\n${String.fromCharCode(0x1c)}\n${String.fromCharCode(
|
|
360
|
+
0x12
|
|
361
|
+
)}${String.fromCharCode(0x1c)}y\n${String.fromCharCode(
|
|
362
|
+
0x1c
|
|
363
|
+
)}\n${String.fromCharCode(0x1c)}z\n${String.fromCharCode(0x11)}`,
|
|
364
|
+
},
|
|
365
|
+
],
|
|
366
|
+
},
|
|
367
|
+
];
|
|
368
|
+
const capture = parseGoogleDocsModelChunks(chunks);
|
|
369
|
+
const markdown = renderGoogleDocsCapture(capture, 'markdown');
|
|
370
|
+
|
|
371
|
+
expect(capture.tables[0].rows.map((row) => row.cells.length)).toEqual([
|
|
372
|
+
3, 3, 3,
|
|
373
|
+
]);
|
|
374
|
+
expect(markdown).toContain('| A | B | C |');
|
|
375
|
+
expect(markdown).toContain('| | x | |');
|
|
376
|
+
expect(markdown).toContain('| y | | z |');
|
|
377
|
+
});
|
|
378
|
+
|
|
311
379
|
it('numbers ordered list items sequentially (issue #92 R3)', () => {
|
|
312
380
|
// Three list items sharing the same list id should render as 1. 2. 3.
|
|
313
381
|
const text = 'First item\nSecond item\nThird item\n';
|
|
@@ -387,6 +455,92 @@ describe('gdocs', () => {
|
|
|
387
455
|
expect(markdown).toMatch(/First item\n2\. Second item/u);
|
|
388
456
|
});
|
|
389
457
|
|
|
458
|
+
it('keeps nested ordered lists ordered and tight when Google Docs uses separate list ids per level (issue #100)', () => {
|
|
459
|
+
const text = [
|
|
460
|
+
'Parent item 1',
|
|
461
|
+
'Child item 1.1',
|
|
462
|
+
'Child item 1.2',
|
|
463
|
+
'Grandchild item 1.2.1',
|
|
464
|
+
'Grandchild item 1.2.2',
|
|
465
|
+
'Child item 1.3',
|
|
466
|
+
'Parent item 2',
|
|
467
|
+
'',
|
|
468
|
+
].join('\n');
|
|
469
|
+
const lineEnd = (needle) => text.indexOf('\n', text.indexOf(needle)) + 1;
|
|
470
|
+
const chunks = [
|
|
471
|
+
{
|
|
472
|
+
chunk: [
|
|
473
|
+
{ ty: 'is', s: text },
|
|
474
|
+
{
|
|
475
|
+
ty: 'as',
|
|
476
|
+
st: 'list',
|
|
477
|
+
si: lineEnd('Parent item 1'),
|
|
478
|
+
ei: lineEnd('Parent item 1'),
|
|
479
|
+
sm: { ls_id: 'kix.list.8' },
|
|
480
|
+
},
|
|
481
|
+
{
|
|
482
|
+
ty: 'as',
|
|
483
|
+
st: 'list',
|
|
484
|
+
si: lineEnd('Child item 1.1'),
|
|
485
|
+
ei: lineEnd('Child item 1.1'),
|
|
486
|
+
sm: { ls_id: 'kix.list.9', ls_nest: 1 },
|
|
487
|
+
},
|
|
488
|
+
{
|
|
489
|
+
ty: 'as',
|
|
490
|
+
st: 'list',
|
|
491
|
+
si: lineEnd('Child item 1.2'),
|
|
492
|
+
ei: lineEnd('Child item 1.2'),
|
|
493
|
+
sm: { ls_id: 'kix.list.9', ls_nest: 1 },
|
|
494
|
+
},
|
|
495
|
+
{
|
|
496
|
+
ty: 'as',
|
|
497
|
+
st: 'list',
|
|
498
|
+
si: lineEnd('Grandchild item 1.2.1'),
|
|
499
|
+
ei: lineEnd('Grandchild item 1.2.1'),
|
|
500
|
+
sm: { ls_id: 'kix.list.10', ls_nest: 2 },
|
|
501
|
+
},
|
|
502
|
+
{
|
|
503
|
+
ty: 'as',
|
|
504
|
+
st: 'list',
|
|
505
|
+
si: lineEnd('Grandchild item 1.2.2'),
|
|
506
|
+
ei: lineEnd('Grandchild item 1.2.2'),
|
|
507
|
+
sm: { ls_id: 'kix.list.10', ls_nest: 2 },
|
|
508
|
+
},
|
|
509
|
+
{
|
|
510
|
+
ty: 'as',
|
|
511
|
+
st: 'list',
|
|
512
|
+
si: lineEnd('Child item 1.3'),
|
|
513
|
+
ei: lineEnd('Child item 1.3'),
|
|
514
|
+
sm: { ls_id: 'kix.list.9', ls_nest: 1 },
|
|
515
|
+
},
|
|
516
|
+
{
|
|
517
|
+
ty: 'as',
|
|
518
|
+
st: 'list',
|
|
519
|
+
si: lineEnd('Parent item 2'),
|
|
520
|
+
ei: lineEnd('Parent item 2'),
|
|
521
|
+
sm: { ls_id: 'kix.list.8' },
|
|
522
|
+
},
|
|
523
|
+
],
|
|
524
|
+
},
|
|
525
|
+
];
|
|
526
|
+
const capture = parseGoogleDocsModelChunks(chunks);
|
|
527
|
+
const markdown = renderGoogleDocsCapture(capture, 'markdown');
|
|
528
|
+
|
|
529
|
+
expect(markdown).toContain(
|
|
530
|
+
[
|
|
531
|
+
'1. Parent item 1',
|
|
532
|
+
' 1. Child item 1.1',
|
|
533
|
+
' 2. Child item 1.2',
|
|
534
|
+
' 1. Grandchild item 1.2.1',
|
|
535
|
+
' 2. Grandchild item 1.2.2',
|
|
536
|
+
' 3. Child item 1.3',
|
|
537
|
+
'2. Parent item 2',
|
|
538
|
+
].join('\n')
|
|
539
|
+
);
|
|
540
|
+
expect(markdown).not.toContain('Parent item 1\n\n');
|
|
541
|
+
expect(markdown).not.toContain('- Child item 1.1');
|
|
542
|
+
});
|
|
543
|
+
|
|
390
544
|
it('renders model style records for headings, inline formatting, links, lists, blockquotes, rules, and images', () => {
|
|
391
545
|
const text = [
|
|
392
546
|
'Title',
|
|
@@ -890,6 +1044,46 @@ describe('gdocs', () => {
|
|
|
890
1044
|
expect(unwrappedLinks).toBe(0);
|
|
891
1045
|
expect(out).toContain('<p>Plain text with <strong>bold</strong>.</p>');
|
|
892
1046
|
});
|
|
1047
|
+
|
|
1048
|
+
it('recovers public-export markdown structure for issue #102', () => {
|
|
1049
|
+
const html = `
|
|
1050
|
+
<style>
|
|
1051
|
+
.c5{margin-left:36pt}
|
|
1052
|
+
.c8{margin-left:72pt}
|
|
1053
|
+
.c19{margin-left:108pt}
|
|
1054
|
+
.q{margin-left:24pt;margin-right:24pt}
|
|
1055
|
+
.i{font-style:italic}
|
|
1056
|
+
.s{text-decoration:line-through}
|
|
1057
|
+
</style>
|
|
1058
|
+
<h2><span class="i">1. Headings</span></h2>
|
|
1059
|
+
<p class="q">Quote one.</p>
|
|
1060
|
+
<p class="q">Quote two.</p>
|
|
1061
|
+
<ol><li class="c5">Parent</li></ol>
|
|
1062
|
+
<ol><li class="c8">Child</li></ol>
|
|
1063
|
+
<ol><li class="c19">Grandchild</li></ol>
|
|
1064
|
+
<ol><li class="c5">Parent 2</li></ol>
|
|
1065
|
+
<table>
|
|
1066
|
+
<thead>
|
|
1067
|
+
<tr><td><p>Feature</p></td><td><p>Supported</p></td><tbody></tbody></tr>
|
|
1068
|
+
<tr><td><p><span class="s">Strike</span></p></td><td><p>Yes</p></td></tr>
|
|
1069
|
+
</thead>
|
|
1070
|
+
</table>
|
|
1071
|
+
`;
|
|
1072
|
+
|
|
1073
|
+
const preprocessed = preprocessGoogleDocsExportHtml(html);
|
|
1074
|
+
const markdown = normalizeGoogleDocsExportMarkdown(
|
|
1075
|
+
convertHtmlToMarkdown(preprocessed.html)
|
|
1076
|
+
);
|
|
1077
|
+
|
|
1078
|
+
expect(markdown).toContain('## 1. Headings');
|
|
1079
|
+
expect(markdown).not.toContain('*1. Headings*');
|
|
1080
|
+
expect(markdown).toContain('> Quote one.\n> \n> Quote two.');
|
|
1081
|
+
expect(markdown).toMatch(
|
|
1082
|
+
/1\.\s+Parent\n\s+1\.\s+Child\n\s+1\.\s+Grandchild\n2\.\s+Parent 2/
|
|
1083
|
+
);
|
|
1084
|
+
expect(markdown).toContain('| Feature | Supported |');
|
|
1085
|
+
expect(markdown).toContain('| ~~Strike~~ | Yes |');
|
|
1086
|
+
});
|
|
893
1087
|
});
|
|
894
1088
|
|
|
895
1089
|
describe('extractBase64Images', () => {
|