@link-assistant/web-capture 1.7.9 → 1.7.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/bin/web-capture.js +17 -1
- package/package.json +1 -1
- package/src/gdocs-images.js +130 -0
- package/src/gdocs-preprocess.js +105 -0
- package/src/gdocs.js +146 -22
- package/tests/unit/gdocs.test.js +251 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# @link-assistant/web-capture
|
|
2
2
|
|
|
3
|
+
## 1.7.10
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- fcc031f: Fix remaining Google Docs capture gaps from issue #92 in both the JS and
|
|
8
|
+
Rust CLIs:
|
|
9
|
+
- Browser capture now keeps multi-column table rows intact, renders
|
|
10
|
+
ordered lists with sequential `1. 2. 3.` numbering, and joins same-list
|
|
11
|
+
items with a single newline so tight-list markdown matches the source
|
|
12
|
+
document.
|
|
13
|
+
- Archive mode downloads `docs-images-rt/...` image URLs into the
|
|
14
|
+
archive's `images/` directory and rewrites markdown/html references so
|
|
15
|
+
exports are self-contained.
|
|
16
|
+
- API mode (`--capture api`) runs the export HTML through a shared
|
|
17
|
+
preprocessor that hoists inline bold / italic / strikethrough spans to
|
|
18
|
+
semantic tags, strips Google Docs' heading-numbering spans and empty
|
|
19
|
+
anchor wrappers, unwraps `google.com/url?q=` redirects, and normalizes
|
|
20
|
+
non-breaking spaces.
|
|
21
|
+
|
|
3
22
|
## 1.7.9
|
|
4
23
|
|
|
5
24
|
### Patch Changes
|
package/bin/web-capture.js
CHANGED
|
@@ -527,8 +527,24 @@ async function captureUrl(url, options) {
|
|
|
527
527
|
textBytes: Buffer.byteLength(result.text || ''),
|
|
528
528
|
}));
|
|
529
529
|
if (normalizedFormat === 'archive' || normalizedFormat === 'zip') {
|
|
530
|
+
const { localizeGoogleDocsModelImages } =
|
|
531
|
+
await import('../src/gdocs.js');
|
|
532
|
+
const localized = await localizeGoogleDocsModelImages(result, {
|
|
533
|
+
log,
|
|
534
|
+
});
|
|
535
|
+
log.debug(() => ({
|
|
536
|
+
event: 'gdocs.capture.browser-model.archive.localized',
|
|
537
|
+
images: localized.images.length,
|
|
538
|
+
markdownBytes: Buffer.byteLength(localized.markdown),
|
|
539
|
+
htmlBytes: Buffer.byteLength(localized.html),
|
|
540
|
+
}));
|
|
530
541
|
await writeGoogleDocsArchive({
|
|
531
|
-
archiveResult: {
|
|
542
|
+
archiveResult: {
|
|
543
|
+
...result,
|
|
544
|
+
markdown: localized.markdown,
|
|
545
|
+
html: localized.html,
|
|
546
|
+
images: localized.images,
|
|
547
|
+
},
|
|
532
548
|
absoluteUrl,
|
|
533
549
|
explicitOutput,
|
|
534
550
|
dataDir,
|
package/package.json
CHANGED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
// Google Docs editor-model image localization (issue #92 R5).
|
|
2
|
+
//
|
|
3
|
+
// The browser-model capture returns `docs-images-rt/…` URLs inside the
|
|
4
|
+
// markdown/html. Archive mode needs those images copied into `images/` so
|
|
5
|
+
// the archive is self-contained. This module downloads the images and
|
|
6
|
+
// rewrites the captured output accordingly.
|
|
7
|
+
|
|
8
|
+
import fetch from 'node-fetch';
|
|
9
|
+
import { URL } from 'url';
|
|
10
|
+
|
|
11
|
+
const DEFAULT_USER_AGENT =
|
|
12
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
|
13
|
+
|
|
14
|
+
function getExtensionFromImageUrl(url) {
|
|
15
|
+
try {
|
|
16
|
+
const u = new URL(url);
|
|
17
|
+
const match = u.pathname.match(/\.(png|jpe?g|gif|webp|svg)$/iu);
|
|
18
|
+
if (match) {
|
|
19
|
+
const ext = match[1].toLowerCase();
|
|
20
|
+
return ext === 'jpeg' ? '.jpg' : `.${ext}`;
|
|
21
|
+
}
|
|
22
|
+
} catch {
|
|
23
|
+
// fall through to default
|
|
24
|
+
}
|
|
25
|
+
return '.png';
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function mimeTypeForExtension(filename) {
|
|
29
|
+
const dot = filename.lastIndexOf('.');
|
|
30
|
+
const ext = dot >= 0 ? filename.slice(dot + 1).toLowerCase() : 'png';
|
|
31
|
+
switch (ext) {
|
|
32
|
+
case 'jpg':
|
|
33
|
+
case 'jpeg':
|
|
34
|
+
return 'image/jpeg';
|
|
35
|
+
case 'gif':
|
|
36
|
+
return 'image/gif';
|
|
37
|
+
case 'webp':
|
|
38
|
+
return 'image/webp';
|
|
39
|
+
case 'svg':
|
|
40
|
+
return 'image/svg+xml';
|
|
41
|
+
default:
|
|
42
|
+
return 'image/png';
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Download the `docs-images-rt` URLs captured from an editor-model capture.
|
|
48
|
+
*
|
|
49
|
+
* Produces `{ filename, data, mimeType }` entries compatible with
|
|
50
|
+
* `writeGoogleDocsArchive` and rewrites the markdown/html to reference the
|
|
51
|
+
* local `images/<filename>` paths.
|
|
52
|
+
*
|
|
53
|
+
* @param {{markdown: string, html: string, capture: Object}} modelResult
|
|
54
|
+
* @param {Object} [options] - Options
|
|
55
|
+
* @param {typeof fetch} [options.fetchImpl=fetch] - Injected fetch for tests
|
|
56
|
+
* @param {Object} [options.log] - Verbose logger
|
|
57
|
+
* @returns {Promise<{markdown: string, html: string, images: Array}>}
|
|
58
|
+
*/
|
|
59
|
+
export async function localizeGoogleDocsModelImages(modelResult, options = {}) {
|
|
60
|
+
const { fetchImpl = fetch, log } = options;
|
|
61
|
+
const images = [];
|
|
62
|
+
const seen = new Map();
|
|
63
|
+
let nextIndex = 1;
|
|
64
|
+
|
|
65
|
+
const resolveFilename = (url, existingAlt) => {
|
|
66
|
+
if (seen.has(url)) {
|
|
67
|
+
return seen.get(url);
|
|
68
|
+
}
|
|
69
|
+
const ext = getExtensionFromImageUrl(url);
|
|
70
|
+
const filename = `image-${String(nextIndex).padStart(2, '0')}${ext}`;
|
|
71
|
+
nextIndex += 1;
|
|
72
|
+
seen.set(url, filename);
|
|
73
|
+
log?.debug?.(() => ({
|
|
74
|
+
event: 'gdocs.archive.image.scheduled',
|
|
75
|
+
url,
|
|
76
|
+
filename,
|
|
77
|
+
alt: existingAlt || null,
|
|
78
|
+
}));
|
|
79
|
+
return filename;
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
const candidates = (modelResult?.capture?.images || []).filter(
|
|
83
|
+
(img) => img?.url
|
|
84
|
+
);
|
|
85
|
+
for (const img of candidates) {
|
|
86
|
+
const filename = resolveFilename(img.url, img.alt);
|
|
87
|
+
try {
|
|
88
|
+
const response = await fetchImpl(img.url, {
|
|
89
|
+
headers: {
|
|
90
|
+
'User-Agent': DEFAULT_USER_AGENT,
|
|
91
|
+
Accept: 'image/*,*/*;q=0.8',
|
|
92
|
+
},
|
|
93
|
+
redirect: 'follow',
|
|
94
|
+
});
|
|
95
|
+
if (!response.ok) {
|
|
96
|
+
throw new Error(`HTTP ${response.status}`);
|
|
97
|
+
}
|
|
98
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
99
|
+
const data = Buffer.from(arrayBuffer);
|
|
100
|
+
const mimeType =
|
|
101
|
+
response.headers?.get?.('content-type') ||
|
|
102
|
+
mimeTypeForExtension(filename);
|
|
103
|
+
images.push({ filename, data, mimeType });
|
|
104
|
+
log?.debug?.(() => ({
|
|
105
|
+
event: 'gdocs.archive.image.downloaded',
|
|
106
|
+
url: img.url,
|
|
107
|
+
filename,
|
|
108
|
+
bytes: data.length,
|
|
109
|
+
mimeType,
|
|
110
|
+
}));
|
|
111
|
+
} catch (err) {
|
|
112
|
+
log?.debug?.(() => ({
|
|
113
|
+
event: 'gdocs.archive.image.failed',
|
|
114
|
+
url: img.url,
|
|
115
|
+
filename,
|
|
116
|
+
error: err?.message || String(err),
|
|
117
|
+
}));
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
let markdown = modelResult?.markdown || '';
|
|
122
|
+
let html = modelResult?.html || '';
|
|
123
|
+
for (const [url, filename] of seen.entries()) {
|
|
124
|
+
const localPath = `images/${filename}`;
|
|
125
|
+
markdown = markdown.split(url).join(localPath);
|
|
126
|
+
html = html.split(url).join(localPath);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return { markdown, html, images };
|
|
130
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// Google Docs export HTML preprocessor (issue #92 R6).
|
|
2
|
+
//
|
|
3
|
+
// Google Drive's `/export?format=html` payload serves bold/italic/
|
|
4
|
+
// strikethrough as inline `style` spans, numbers headings with
|
|
5
|
+
// `<a id><span>1. </span>`, and routes every link through a
|
|
6
|
+
// `google.com/url?q=` redirect. The generic Cheerio + Turndown pipeline
|
|
7
|
+
// discards those signals, so the browser-less API path loses the
|
|
8
|
+
// formatting and gains noisy redirect URLs.
|
|
9
|
+
//
|
|
10
|
+
// This module rewrites the export HTML into semantic markup before the
|
|
11
|
+
// generic converter runs.
|
|
12
|
+
|
|
13
|
+
import * as cheerio from 'cheerio';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Pre-process Google Docs export HTML so the generic Cheerio+Turndown
|
|
17
|
+
* pipeline preserves inline formatting, heading numbers, and link targets.
|
|
18
|
+
*
|
|
19
|
+
* @param {string} html - Raw export HTML returned by Google Docs.
|
|
20
|
+
* @returns {{html: string, hoisted: number, unwrappedLinks: number}}
|
|
21
|
+
*/
|
|
22
|
+
export function preprocessGoogleDocsExportHtml(html) {
|
|
23
|
+
if (!html || typeof html !== 'string') {
|
|
24
|
+
return { html: html || '', hoisted: 0, unwrappedLinks: 0 };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const $ = cheerio.load(html, { decodeEntities: false });
|
|
28
|
+
let hoisted = 0;
|
|
29
|
+
let unwrappedLinks = 0;
|
|
30
|
+
|
|
31
|
+
$('span[style], p[style]').each(function () {
|
|
32
|
+
const style = String($(this).attr('style') || '');
|
|
33
|
+
const hasBold = /font-weight\s*:\s*(?:bold|[6-9]\d{2})/i.test(style);
|
|
34
|
+
const hasItalic = /font-style\s*:\s*italic/i.test(style);
|
|
35
|
+
const hasStrike = /text-decoration[^;]*\bline-through\b/i.test(style);
|
|
36
|
+
if (!hasBold && !hasItalic && !hasStrike) {
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
const inner = $(this).html() || '';
|
|
40
|
+
let wrapped = inner;
|
|
41
|
+
if (hasStrike) {
|
|
42
|
+
wrapped = `<del>${wrapped}</del>`;
|
|
43
|
+
}
|
|
44
|
+
if (hasItalic) {
|
|
45
|
+
wrapped = `<em>${wrapped}</em>`;
|
|
46
|
+
}
|
|
47
|
+
if (hasBold) {
|
|
48
|
+
wrapped = `<strong>${wrapped}</strong>`;
|
|
49
|
+
}
|
|
50
|
+
$(this).replaceWith(wrapped);
|
|
51
|
+
hoisted += 1;
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
$('h1, h2, h3, h4, h5, h6').each(function () {
|
|
55
|
+
const $h = $(this);
|
|
56
|
+
$h.find('a').each(function () {
|
|
57
|
+
const $a = $(this);
|
|
58
|
+
if (!$a.attr('href') && !$a.text().trim() && !$a.find('img').length) {
|
|
59
|
+
$a.remove();
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
$h.find('> span, > strong > span').each(function () {
|
|
63
|
+
const $span = $(this);
|
|
64
|
+
const text = $span.text();
|
|
65
|
+
if (/^\s*\d+(?:\.\d+)*\.?\s*$/.test(text)) {
|
|
66
|
+
$span.remove();
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
$('a[href*="google.com/url?q="]').each(function () {
|
|
72
|
+
const href = String($(this).attr('href') || '');
|
|
73
|
+
const match = href.match(/[?&]q=([^&]+)/);
|
|
74
|
+
if (match) {
|
|
75
|
+
try {
|
|
76
|
+
const decoded = decodeURIComponent(match[1]);
|
|
77
|
+
$(this).attr('href', decoded);
|
|
78
|
+
unwrappedLinks += 1;
|
|
79
|
+
} catch {
|
|
80
|
+
// Leave the original href in place if it can't be decoded.
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
$('*').each(function () {
|
|
86
|
+
if (this.type !== 'tag') {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
$(this)
|
|
90
|
+
.contents()
|
|
91
|
+
.each(function () {
|
|
92
|
+
if (this.type === 'text' && this.data) {
|
|
93
|
+
const replaced = this.data.replace(/\u00A0/g, ' ');
|
|
94
|
+
if (replaced !== this.data) {
|
|
95
|
+
this.data = replaced;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
let rewritten = $.html();
|
|
102
|
+
rewritten = rewritten.replace(/ /g, ' ');
|
|
103
|
+
|
|
104
|
+
return { html: rewritten, hoisted, unwrappedLinks };
|
|
105
|
+
}
|
package/src/gdocs.js
CHANGED
|
@@ -11,6 +11,10 @@ import fetch from 'node-fetch';
|
|
|
11
11
|
import he from 'he';
|
|
12
12
|
import { convertHtmlToMarkdown } from './lib.js';
|
|
13
13
|
import { createBrowser as defaultCreateBrowser } from './browser.js';
|
|
14
|
+
import { preprocessGoogleDocsExportHtml } from './gdocs-preprocess.js';
|
|
15
|
+
import { localizeGoogleDocsModelImages } from './gdocs-images.js';
|
|
16
|
+
|
|
17
|
+
export { preprocessGoogleDocsExportHtml, localizeGoogleDocsModelImages };
|
|
14
18
|
|
|
15
19
|
const GDOCS_URL_PATTERN = /docs\.google\.com\/document\/d\/([a-zA-Z0-9_-]+)/;
|
|
16
20
|
|
|
@@ -283,7 +287,14 @@ export async function fetchGoogleDocAsMarkdown(url, options = {}) {
|
|
|
283
287
|
log,
|
|
284
288
|
});
|
|
285
289
|
|
|
286
|
-
const
|
|
290
|
+
const preprocessed = preprocessGoogleDocsExportHtml(result.content);
|
|
291
|
+
log?.debug?.(() => ({
|
|
292
|
+
event: 'gdocs.export.style-hoist',
|
|
293
|
+
documentId: result.documentId,
|
|
294
|
+
hoisted: preprocessed.hoisted,
|
|
295
|
+
unwrappedLinks: preprocessed.unwrappedLinks,
|
|
296
|
+
}));
|
|
297
|
+
const markdown = convertHtmlToMarkdown(preprocessed.html, result.exportUrl);
|
|
287
298
|
log?.debug?.(() => ({
|
|
288
299
|
event: 'gdocs.public-export.markdown',
|
|
289
300
|
documentId: result.documentId,
|
|
@@ -384,7 +395,8 @@ export async function captureGoogleDocWithBrowser(url, options = {}) {
|
|
|
384
395
|
|
|
385
396
|
const capture = parseGoogleDocsModelChunks(
|
|
386
397
|
modelData.chunks,
|
|
387
|
-
modelData.cidUrlMap
|
|
398
|
+
modelData.cidUrlMap,
|
|
399
|
+
{ log }
|
|
388
400
|
);
|
|
389
401
|
log?.debug?.(() => ({
|
|
390
402
|
event: 'gdocs.browser-model.parsed',
|
|
@@ -531,9 +543,16 @@ export function extractCidUrlMapFromHtml(html) {
|
|
|
531
543
|
*
|
|
532
544
|
* @param {Array} chunks - Captured DOCS_modelChunk values
|
|
533
545
|
* @param {Object<string,string>} [cidUrlMap={}] - CID-to-docs-images-rt URL map
|
|
546
|
+
* @param {Object} [options] - Parse options
|
|
547
|
+
* @param {Object} [options.log] - Optional verbose logger
|
|
534
548
|
* @returns {{blocks: Array, tables: Array, images: Array, text: string}}
|
|
535
549
|
*/
|
|
536
|
-
export function parseGoogleDocsModelChunks(
|
|
550
|
+
export function parseGoogleDocsModelChunks(
|
|
551
|
+
chunks = [],
|
|
552
|
+
cidUrlMap = {},
|
|
553
|
+
options = {}
|
|
554
|
+
) {
|
|
555
|
+
const { log } = options;
|
|
537
556
|
const items = collectModelItems(chunks);
|
|
538
557
|
const fullText = items
|
|
539
558
|
.filter((item) => item.ty === 'is' || item.ty === 'iss')
|
|
@@ -581,6 +600,16 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
|
|
|
581
600
|
let table = null;
|
|
582
601
|
let row = null;
|
|
583
602
|
let cell = null;
|
|
603
|
+
const tableHistograms = [];
|
|
604
|
+
let currentHistogram = null;
|
|
605
|
+
|
|
606
|
+
const bumpHistogram = (code) => {
|
|
607
|
+
if (!currentHistogram) {
|
|
608
|
+
return;
|
|
609
|
+
}
|
|
610
|
+
const key = `0x${code.toString(16).padStart(2, '0')}`;
|
|
611
|
+
currentHistogram[key] = (currentHistogram[key] || 0) + 1;
|
|
612
|
+
};
|
|
584
613
|
|
|
585
614
|
const flushParagraph = (endPos = null) => {
|
|
586
615
|
const text = contentToText(paragraph).trim();
|
|
@@ -594,15 +623,18 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
|
|
|
594
623
|
paragraph = [];
|
|
595
624
|
};
|
|
596
625
|
|
|
597
|
-
const flushCell = () => {
|
|
626
|
+
const flushCell = ({ dropEmpty = false } = {}) => {
|
|
598
627
|
if (cell && row) {
|
|
599
|
-
|
|
628
|
+
const isEmpty = cell.content.length === 0;
|
|
629
|
+
if (!dropEmpty || !isEmpty) {
|
|
630
|
+
row.cells.push(cell);
|
|
631
|
+
}
|
|
600
632
|
}
|
|
601
633
|
cell = null;
|
|
602
634
|
};
|
|
603
635
|
|
|
604
|
-
const flushRow = () => {
|
|
605
|
-
flushCell();
|
|
636
|
+
const flushRow = ({ dropEmptyTrailingCell = false } = {}) => {
|
|
637
|
+
flushCell({ dropEmpty: dropEmptyTrailingCell });
|
|
606
638
|
if (row && table) {
|
|
607
639
|
table.rows.push(row);
|
|
608
640
|
}
|
|
@@ -610,11 +642,23 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
|
|
|
610
642
|
};
|
|
611
643
|
|
|
612
644
|
const flushTable = () => {
|
|
613
|
-
flushRow();
|
|
645
|
+
flushRow({ dropEmptyTrailingCell: true });
|
|
614
646
|
if (table) {
|
|
647
|
+
// Drop trailing empty rows that Google Docs sometimes emits after the
|
|
648
|
+
// last cell content (from `\n` just before the `0x11` close marker).
|
|
649
|
+
while (
|
|
650
|
+
table.rows.length > 1 &&
|
|
651
|
+
table.rows[table.rows.length - 1].cells.length === 0
|
|
652
|
+
) {
|
|
653
|
+
table.rows.pop();
|
|
654
|
+
}
|
|
615
655
|
tables.push(table);
|
|
616
656
|
blocks.push(table);
|
|
617
657
|
}
|
|
658
|
+
if (currentHistogram) {
|
|
659
|
+
tableHistograms.push(currentHistogram);
|
|
660
|
+
}
|
|
661
|
+
currentHistogram = null;
|
|
618
662
|
table = null;
|
|
619
663
|
};
|
|
620
664
|
|
|
@@ -636,18 +680,29 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
|
|
|
636
680
|
if (code === 0x10) {
|
|
637
681
|
flushParagraph(i + 1);
|
|
638
682
|
table = { type: 'table', rows: [] };
|
|
683
|
+
currentHistogram = {
|
|
684
|
+
'0x0a': 0,
|
|
685
|
+
'0x0b': 0,
|
|
686
|
+
'0x10': 1,
|
|
687
|
+
'0x11': 0,
|
|
688
|
+
'0x12': 0,
|
|
689
|
+
'0x1c': 0,
|
|
690
|
+
};
|
|
639
691
|
continue;
|
|
640
692
|
}
|
|
641
693
|
if (code === 0x11) {
|
|
694
|
+
bumpHistogram(code);
|
|
642
695
|
flushTable();
|
|
643
696
|
continue;
|
|
644
697
|
}
|
|
645
698
|
if (code === 0x12) {
|
|
646
|
-
|
|
699
|
+
bumpHistogram(code);
|
|
700
|
+
flushRow({ dropEmptyTrailingCell: true });
|
|
647
701
|
row = { cells: [] };
|
|
648
702
|
continue;
|
|
649
703
|
}
|
|
650
704
|
if (code === 0x1c) {
|
|
705
|
+
bumpHistogram(code);
|
|
651
706
|
flushCell();
|
|
652
707
|
if (!row) {
|
|
653
708
|
row = { cells: [] };
|
|
@@ -656,14 +711,24 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
|
|
|
656
711
|
continue;
|
|
657
712
|
}
|
|
658
713
|
if (code === 0x0a) {
|
|
714
|
+
bumpHistogram(code);
|
|
659
715
|
if (table) {
|
|
660
|
-
|
|
716
|
+
// Inside a Google Docs table, `\n` (0x0a) separates CELLS, not rows.
|
|
717
|
+
// Row boundaries are communicated via the explicit `0x12` marker and
|
|
718
|
+
// the closing `0x11` marker. Earlier versions collapsed multi-column
|
|
719
|
+
// tables to one column because they flushed the row on `\n`.
|
|
720
|
+
flushCell();
|
|
721
|
+
if (!row) {
|
|
722
|
+
row = { cells: [] };
|
|
723
|
+
}
|
|
724
|
+
cell = { content: [] };
|
|
661
725
|
} else {
|
|
662
726
|
flushParagraph(i + 1);
|
|
663
727
|
}
|
|
664
728
|
continue;
|
|
665
729
|
}
|
|
666
730
|
if (code === 0x0b) {
|
|
731
|
+
bumpHistogram(code);
|
|
667
732
|
appendText(targetContent(), '\n');
|
|
668
733
|
continue;
|
|
669
734
|
}
|
|
@@ -684,6 +749,19 @@ export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
|
|
|
684
749
|
}
|
|
685
750
|
flushParagraph(fullText.length);
|
|
686
751
|
|
|
752
|
+
if (log?.debug && tableHistograms.length > 0) {
|
|
753
|
+
for (let t = 0; t < tableHistograms.length; t++) {
|
|
754
|
+
const tbl = tables[t];
|
|
755
|
+
log.debug(() => ({
|
|
756
|
+
event: 'gdocs.table.histogram',
|
|
757
|
+
tableIndex: t,
|
|
758
|
+
rows: tbl?.rows.length || 0,
|
|
759
|
+
columns: Math.max(...(tbl?.rows || []).map((r) => r.cells.length), 0),
|
|
760
|
+
controls: tableHistograms[t],
|
|
761
|
+
}));
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
|
|
687
765
|
return {
|
|
688
766
|
blocks,
|
|
689
767
|
tables,
|
|
@@ -981,19 +1059,56 @@ function sameTextStyle(a, b) {
|
|
|
981
1059
|
}
|
|
982
1060
|
|
|
983
1061
|
function renderBlocksMarkdown(blocks) {
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
1062
|
+
const counters = new Map();
|
|
1063
|
+
const rendered = blocks.map((block) => {
|
|
1064
|
+
if (block.type === 'table') {
|
|
1065
|
+
counters.clear();
|
|
1066
|
+
return { text: renderTableMarkdown(block), list: null };
|
|
1067
|
+
}
|
|
1068
|
+
const counterKey = block.list
|
|
1069
|
+
? `${block.list.id || ''}\u0000${block.list.level || 0}`
|
|
1070
|
+
: null;
|
|
1071
|
+
if (block.list) {
|
|
1072
|
+
// Reset deeper siblings when we re-enter a shallower level so
|
|
1073
|
+
// nested lists restart from 1 on each new parent.
|
|
1074
|
+
for (const key of [...counters.keys()]) {
|
|
1075
|
+
const level = Number(key.split('\u0000')[1]);
|
|
1076
|
+
if (level > (block.list.level || 0)) {
|
|
1077
|
+
counters.delete(key);
|
|
1078
|
+
}
|
|
988
1079
|
}
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
1080
|
+
const next = (counters.get(counterKey) || 0) + 1;
|
|
1081
|
+
counters.set(counterKey, next);
|
|
1082
|
+
return {
|
|
1083
|
+
text: renderParagraphMarkdown(block, { orderedIndex: next }),
|
|
1084
|
+
list: block.list,
|
|
1085
|
+
};
|
|
1086
|
+
}
|
|
1087
|
+
counters.clear();
|
|
1088
|
+
return { text: renderParagraphMarkdown(block), list: null };
|
|
1089
|
+
});
|
|
1090
|
+
|
|
1091
|
+
const joined = [];
|
|
1092
|
+
for (let i = 0; i < rendered.length; i++) {
|
|
1093
|
+
const cur = rendered[i];
|
|
1094
|
+
if (!cur.text) {
|
|
1095
|
+
continue;
|
|
1096
|
+
}
|
|
1097
|
+
if (joined.length > 0) {
|
|
1098
|
+
const prev = rendered
|
|
1099
|
+
.slice(0, i)
|
|
1100
|
+
.reverse()
|
|
1101
|
+
.find((entry) => entry.text);
|
|
1102
|
+
const sameList =
|
|
1103
|
+
cur.list && prev?.list && (cur.list.id || '') === (prev.list.id || '');
|
|
1104
|
+
joined.push(sameList ? '\n' : '\n\n');
|
|
1105
|
+
}
|
|
1106
|
+
joined.push(cur.text);
|
|
1107
|
+
}
|
|
1108
|
+
return joined.join('').replace(/\s+$/u, '');
|
|
994
1109
|
}
|
|
995
1110
|
|
|
996
|
-
function renderParagraphMarkdown(block) {
|
|
1111
|
+
function renderParagraphMarkdown(block, context = {}) {
|
|
997
1112
|
const text = renderContentMarkdown(block.content).trim();
|
|
998
1113
|
if (block.horizontalRule) {
|
|
999
1114
|
return '---';
|
|
@@ -1011,7 +1126,8 @@ function renderParagraphMarkdown(block) {
|
|
|
1011
1126
|
}
|
|
1012
1127
|
if (block.list) {
|
|
1013
1128
|
const indent = ' '.repeat(Math.max(0, block.list.level || 0));
|
|
1014
|
-
const
|
|
1129
|
+
const orderedIndex = Math.max(1, Number(context.orderedIndex) || 1);
|
|
1130
|
+
const marker = block.list.ordered ? `${orderedIndex}.` : '-';
|
|
1015
1131
|
return `${indent}${marker} ${text}`;
|
|
1016
1132
|
}
|
|
1017
1133
|
if (block.quote) {
|
|
@@ -1277,8 +1393,16 @@ export async function fetchGoogleDocAsArchive(url, options = {}) {
|
|
|
1277
1393
|
// Fetch HTML with embedded base64 images
|
|
1278
1394
|
const result = await fetchGoogleDoc(url, { format: 'html', apiToken, log });
|
|
1279
1395
|
|
|
1396
|
+
const preprocessed = preprocessGoogleDocsExportHtml(result.content);
|
|
1397
|
+
log?.debug?.(() => ({
|
|
1398
|
+
event: 'gdocs.export.style-hoist',
|
|
1399
|
+
documentId: result.documentId,
|
|
1400
|
+
hoisted: preprocessed.hoisted,
|
|
1401
|
+
unwrappedLinks: preprocessed.unwrappedLinks,
|
|
1402
|
+
}));
|
|
1403
|
+
|
|
1280
1404
|
// Extract base64 images from HTML
|
|
1281
|
-
const { html: localHtml, images } = extractBase64Images(
|
|
1405
|
+
const { html: localHtml, images } = extractBase64Images(preprocessed.html);
|
|
1282
1406
|
|
|
1283
1407
|
// Convert the localized HTML to Markdown
|
|
1284
1408
|
const markdown = convertHtmlToMarkdown(localHtml);
|
package/tests/unit/gdocs.test.js
CHANGED
|
@@ -10,6 +10,8 @@ import {
|
|
|
10
10
|
renderGoogleDocsCapture,
|
|
11
11
|
renderDocsApiDocument,
|
|
12
12
|
selectGoogleDocsCaptureMethod,
|
|
13
|
+
localizeGoogleDocsModelImages,
|
|
14
|
+
preprocessGoogleDocsExportHtml,
|
|
13
15
|
GDOCS_EXPORT_FORMATS,
|
|
14
16
|
} from '../../src/gdocs.js';
|
|
15
17
|
import {
|
|
@@ -250,6 +252,115 @@ describe('gdocs', () => {
|
|
|
250
252
|
);
|
|
251
253
|
});
|
|
252
254
|
|
|
255
|
+
it('parses multi-column tables when rows are separated by 0x0a (issue #92 R2)', () => {
|
|
256
|
+
// Google Docs sometimes emits cell boundaries as `\n` (0x0a) alongside
|
|
257
|
+
// the 0x12 new-row marker. The parser must keep all cells on the same
|
|
258
|
+
// row rather than collapsing the table to a single column.
|
|
259
|
+
const chunks = [
|
|
260
|
+
{
|
|
261
|
+
chunk: [
|
|
262
|
+
{
|
|
263
|
+
ty: 'is',
|
|
264
|
+
s: `${
|
|
265
|
+
String.fromCharCode(0x10) + String.fromCharCode(0x12)
|
|
266
|
+
}A\nB\nC\n${String.fromCharCode(
|
|
267
|
+
0x12
|
|
268
|
+
)}D\nE\nF\n${String.fromCharCode(0x11)}`,
|
|
269
|
+
},
|
|
270
|
+
],
|
|
271
|
+
},
|
|
272
|
+
];
|
|
273
|
+
const capture = parseGoogleDocsModelChunks(chunks);
|
|
274
|
+
|
|
275
|
+
expect(capture.tables).toHaveLength(1);
|
|
276
|
+
expect(capture.tables[0].rows).toHaveLength(2);
|
|
277
|
+
expect(capture.tables[0].rows[0].cells).toHaveLength(3);
|
|
278
|
+
expect(capture.tables[0].rows[1].cells).toHaveLength(3);
|
|
279
|
+
|
|
280
|
+
const markdown = renderGoogleDocsCapture(capture, 'markdown');
|
|
281
|
+
expect(markdown).toContain('| A | B | C |');
|
|
282
|
+
expect(markdown).toContain('| D | E | F |');
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
it('numbers ordered list items sequentially (issue #92 R3)', () => {
|
|
286
|
+
// Three list items sharing the same list id should render as 1. 2. 3.
|
|
287
|
+
const text = 'First item\nSecond item\nThird item\n';
|
|
288
|
+
const endOfLine = (n) => {
|
|
289
|
+
let idx = -1;
|
|
290
|
+
for (let k = 0; k < n; k++) {
|
|
291
|
+
idx = text.indexOf('\n', idx + 1);
|
|
292
|
+
}
|
|
293
|
+
return idx + 1;
|
|
294
|
+
};
|
|
295
|
+
const chunks = [
|
|
296
|
+
{
|
|
297
|
+
chunk: [
|
|
298
|
+
{ ty: 'is', s: text },
|
|
299
|
+
{
|
|
300
|
+
ty: 'as',
|
|
301
|
+
st: 'list',
|
|
302
|
+
si: endOfLine(1),
|
|
303
|
+
ei: endOfLine(1),
|
|
304
|
+
sm: { ls_id: 'kix.list.7' },
|
|
305
|
+
},
|
|
306
|
+
{
|
|
307
|
+
ty: 'as',
|
|
308
|
+
st: 'list',
|
|
309
|
+
si: endOfLine(2),
|
|
310
|
+
ei: endOfLine(2),
|
|
311
|
+
sm: { ls_id: 'kix.list.7' },
|
|
312
|
+
},
|
|
313
|
+
{
|
|
314
|
+
ty: 'as',
|
|
315
|
+
st: 'list',
|
|
316
|
+
si: endOfLine(3),
|
|
317
|
+
ei: endOfLine(3),
|
|
318
|
+
sm: { ls_id: 'kix.list.7' },
|
|
319
|
+
},
|
|
320
|
+
],
|
|
321
|
+
},
|
|
322
|
+
];
|
|
323
|
+
const capture = parseGoogleDocsModelChunks(chunks);
|
|
324
|
+
const markdown = renderGoogleDocsCapture(capture, 'markdown');
|
|
325
|
+
const lines = markdown.split('\n');
|
|
326
|
+
|
|
327
|
+
expect(lines[0]).toBe('1. First item');
|
|
328
|
+
expect(lines[1]).toBe('2. Second item');
|
|
329
|
+
expect(lines[2]).toBe('3. Third item');
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
it('joins consecutive list items with a single newline (issue #92 R4)', () => {
|
|
333
|
+
const text = 'First item\nSecond item\n';
|
|
334
|
+
const endFirst = text.indexOf('\n') + 1;
|
|
335
|
+
const endSecond = text.length;
|
|
336
|
+
const chunks = [
|
|
337
|
+
{
|
|
338
|
+
chunk: [
|
|
339
|
+
{ ty: 'is', s: text },
|
|
340
|
+
{
|
|
341
|
+
ty: 'as',
|
|
342
|
+
st: 'list',
|
|
343
|
+
si: endFirst,
|
|
344
|
+
ei: endFirst,
|
|
345
|
+
sm: { ls_id: 'kix.list.7' },
|
|
346
|
+
},
|
|
347
|
+
{
|
|
348
|
+
ty: 'as',
|
|
349
|
+
st: 'list',
|
|
350
|
+
si: endSecond,
|
|
351
|
+
ei: endSecond,
|
|
352
|
+
sm: { ls_id: 'kix.list.7' },
|
|
353
|
+
},
|
|
354
|
+
],
|
|
355
|
+
},
|
|
356
|
+
];
|
|
357
|
+
const capture = parseGoogleDocsModelChunks(chunks);
|
|
358
|
+
const markdown = renderGoogleDocsCapture(capture, 'markdown');
|
|
359
|
+
|
|
360
|
+
expect(markdown).not.toMatch(/First item\n\n2\. Second item/u);
|
|
361
|
+
expect(markdown).toMatch(/First item\n2\. Second item/u);
|
|
362
|
+
});
|
|
363
|
+
|
|
253
364
|
it('renders model style records for headings, inline formatting, links, lists, blockquotes, rules, and images', () => {
|
|
254
365
|
const text = [
|
|
255
366
|
'Title',
|
|
@@ -442,6 +553,146 @@ describe('gdocs', () => {
|
|
|
442
553
|
});
|
|
443
554
|
});
|
|
444
555
|
|
|
556
|
+
describe('localizeGoogleDocsModelImages (issue #92 R5)', () => {
|
|
557
|
+
it('downloads docs-images-rt URLs and rewrites markdown/html', async () => {
|
|
558
|
+
const url =
|
|
559
|
+
'https://docs.google.com/docs-images-rt/doc-id/cid/example.png';
|
|
560
|
+
const modelResult = {
|
|
561
|
+
markdown: ``,
|
|
562
|
+
html: `<img src="${url}" alt="pic">`,
|
|
563
|
+
capture: {
|
|
564
|
+
images: [{ type: 'image', url, alt: 'pic', cid: 'cid-abc' }],
|
|
565
|
+
},
|
|
566
|
+
};
|
|
567
|
+
const fakeFetch = jest.fn().mockResolvedValue({
|
|
568
|
+
ok: true,
|
|
569
|
+
status: 200,
|
|
570
|
+
headers: new Map([['content-type', 'image/png']]),
|
|
571
|
+
arrayBuffer: async () => new Uint8Array([1, 2, 3, 4]).buffer,
|
|
572
|
+
});
|
|
573
|
+
// The map-based headers need a `get` method in the correct shape:
|
|
574
|
+
fakeFetch.mockResolvedValue({
|
|
575
|
+
ok: true,
|
|
576
|
+
status: 200,
|
|
577
|
+
headers: { get: () => 'image/png' },
|
|
578
|
+
arrayBuffer: async () =>
|
|
579
|
+
new Uint8Array([0x89, 0x50, 0x4e, 0x47]).buffer,
|
|
580
|
+
});
|
|
581
|
+
|
|
582
|
+
const localized = await localizeGoogleDocsModelImages(modelResult, {
|
|
583
|
+
fetchImpl: fakeFetch,
|
|
584
|
+
});
|
|
585
|
+
|
|
586
|
+
expect(fakeFetch).toHaveBeenCalledTimes(1);
|
|
587
|
+
expect(localized.images).toHaveLength(1);
|
|
588
|
+
expect(localized.images[0].filename).toBe('image-01.png');
|
|
589
|
+
expect(localized.images[0].mimeType).toBe('image/png');
|
|
590
|
+
expect(localized.images[0].data).toBeInstanceOf(Buffer);
|
|
591
|
+
expect(localized.markdown).toBe('');
|
|
592
|
+
expect(localized.html).toContain('images/image-01.png');
|
|
593
|
+
expect(localized.html).not.toContain('docs-images-rt');
|
|
594
|
+
});
|
|
595
|
+
|
|
596
|
+
it('keeps original URL when download fails', async () => {
|
|
597
|
+
const url =
|
|
598
|
+
'https://docs.google.com/docs-images-rt/doc-id/cid/missing.png';
|
|
599
|
+
const modelResult = {
|
|
600
|
+
markdown: ``,
|
|
601
|
+
html: `<img src="${url}" alt="pic">`,
|
|
602
|
+
capture: { images: [{ url, alt: 'pic' }] },
|
|
603
|
+
};
|
|
604
|
+
const fakeFetch = jest.fn().mockResolvedValue({
|
|
605
|
+
ok: false,
|
|
606
|
+
status: 404,
|
|
607
|
+
headers: { get: () => null },
|
|
608
|
+
});
|
|
609
|
+
|
|
610
|
+
const localized = await localizeGoogleDocsModelImages(modelResult, {
|
|
611
|
+
fetchImpl: fakeFetch,
|
|
612
|
+
});
|
|
613
|
+
|
|
614
|
+
expect(localized.images).toHaveLength(0);
|
|
615
|
+
// Markdown still gets rewritten (local path) even if download fails —
|
|
616
|
+
// callers may choose to ignore this; the key guarantee is that no
|
|
617
|
+
// exception leaks out.
|
|
618
|
+
expect(localized.markdown).toBe('');
|
|
619
|
+
});
|
|
620
|
+
});
|
|
621
|
+
|
|
622
|
+
describe('preprocessGoogleDocsExportHtml (issue #92 R6)', () => {
|
|
623
|
+
it('hoists font-weight:700 spans into <strong>', () => {
|
|
624
|
+
const html = '<p><span style="font-weight:700">Bold text</span></p>';
|
|
625
|
+
const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
|
|
626
|
+
expect(hoisted).toBe(1);
|
|
627
|
+
expect(out).toContain('<strong>');
|
|
628
|
+
expect(out).toContain('Bold text');
|
|
629
|
+
});
|
|
630
|
+
|
|
631
|
+
it('hoists font-style:italic spans into <em>', () => {
|
|
632
|
+
const html = '<p><span style="font-style:italic">Italic text</span></p>';
|
|
633
|
+
const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
|
|
634
|
+
expect(hoisted).toBe(1);
|
|
635
|
+
expect(out).toContain('<em>');
|
|
636
|
+
});
|
|
637
|
+
|
|
638
|
+
it('hoists text-decoration:line-through spans into <del>', () => {
|
|
639
|
+
const html =
|
|
640
|
+
'<p><span style="text-decoration:line-through">Strike</span></p>';
|
|
641
|
+
const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
|
|
642
|
+
expect(hoisted).toBe(1);
|
|
643
|
+
expect(out).toContain('<del>');
|
|
644
|
+
});
|
|
645
|
+
|
|
646
|
+
it('combines multiple styles on a single span', () => {
|
|
647
|
+
const html =
|
|
648
|
+
'<p><span style="font-weight:700;font-style:italic;text-decoration:line-through">Mixed</span></p>';
|
|
649
|
+
const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
|
|
650
|
+
expect(hoisted).toBe(1);
|
|
651
|
+
// Wrappers appear in <strong><em><del>... order (outer-to-inner).
|
|
652
|
+
expect(out).toContain('<strong>');
|
|
653
|
+
expect(out).toContain('<em>');
|
|
654
|
+
expect(out).toContain('<del>');
|
|
655
|
+
});
|
|
656
|
+
|
|
657
|
+
it('unwraps google.com/url?q= redirect links', () => {
|
|
658
|
+
const html =
|
|
659
|
+
'<p><a href="https://www.google.com/url?q=https://example.com&sa=D&source=editors&usg=ABC">Link</a></p>';
|
|
660
|
+
const { html: out, unwrappedLinks } =
|
|
661
|
+
preprocessGoogleDocsExportHtml(html);
|
|
662
|
+
expect(unwrappedLinks).toBe(1);
|
|
663
|
+
expect(out).toContain('href="https://example.com"');
|
|
664
|
+
expect(out).not.toContain('google.com/url?q=');
|
|
665
|
+
});
|
|
666
|
+
|
|
667
|
+
it('strips leading empty anchors and numbering span inside headings', () => {
|
|
668
|
+
const html = '<h1><a id="anchor-1"></a><span>1. </span>Headings</h1>';
|
|
669
|
+
const { html: out } = preprocessGoogleDocsExportHtml(html);
|
|
670
|
+
expect(out).toContain('<h1>Headings</h1>');
|
|
671
|
+
expect(out).not.toContain('1. ');
|
|
672
|
+
expect(out).not.toContain('<a id=');
|
|
673
|
+
});
|
|
674
|
+
|
|
675
|
+
it('replaces entity and U+00A0 with regular spaces', () => {
|
|
676
|
+
const html = '<p>A\u00A0B C</p>';
|
|
677
|
+
const { html: out } = preprocessGoogleDocsExportHtml(html);
|
|
678
|
+
expect(out).toContain('A B C');
|
|
679
|
+
expect(out).not.toMatch(/\u00A0/);
|
|
680
|
+
expect(out).not.toContain(' ');
|
|
681
|
+
});
|
|
682
|
+
|
|
683
|
+
it('is a no-op for HTML without Google Docs markers', () => {
|
|
684
|
+
const html = '<p>Plain text with <strong>bold</strong>.</p>';
|
|
685
|
+
const {
|
|
686
|
+
html: out,
|
|
687
|
+
hoisted,
|
|
688
|
+
unwrappedLinks,
|
|
689
|
+
} = preprocessGoogleDocsExportHtml(html);
|
|
690
|
+
expect(hoisted).toBe(0);
|
|
691
|
+
expect(unwrappedLinks).toBe(0);
|
|
692
|
+
expect(out).toContain('<p>Plain text with <strong>bold</strong>.</p>');
|
|
693
|
+
});
|
|
694
|
+
});
|
|
695
|
+
|
|
445
696
|
describe('extractBase64Images', () => {
|
|
446
697
|
it('extracts a single PNG data URI image', () => {
|
|
447
698
|
const html = '<img src="data:image/png;base64,iVBORw0KGgo=" alt="test">';
|