@link-assistant/web-capture 1.7.5 → 1.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +7 -4
- package/bin/web-capture.js +369 -115
- package/package.json +2 -1
- package/src/browser.js +15 -0
- package/src/gdocs.js +827 -12
- package/src/lib.js +9 -1
- package/src/localize-images.js +17 -2
- package/tests/unit/gdocs.test.js +161 -0
- package/tests/unit/html2md.test.js +45 -1
- package/tests/unit/localize-images.test.js +36 -0
package/src/gdocs.js
CHANGED
|
@@ -1,16 +1,23 @@
|
|
|
1
|
-
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
/* global document, window */
|
|
2
|
+
// Google Docs capture module.
|
|
3
|
+
//
|
|
4
|
+
// Public export capture uses:
|
|
4
5
|
// https://docs.google.com/document/d/{DOCUMENT_ID}/export?format={FORMAT}
|
|
5
|
-
//
|
|
6
|
+
// Docs API capture uses:
|
|
7
|
+
// https://docs.googleapis.com/v1/documents/{DOCUMENT_ID}
|
|
8
|
+
// Browser-model capture loads /edit and extracts DOCS_modelChunk data.
|
|
6
9
|
|
|
7
10
|
import fetch from 'node-fetch';
|
|
8
11
|
import he from 'he';
|
|
9
12
|
import { convertHtmlToMarkdown } from './lib.js';
|
|
13
|
+
import { createBrowser as defaultCreateBrowser } from './browser.js';
|
|
10
14
|
|
|
11
15
|
const GDOCS_URL_PATTERN = /docs\.google\.com\/document\/d\/([a-zA-Z0-9_-]+)/;
|
|
12
16
|
|
|
13
17
|
const GDOCS_EXPORT_BASE = 'https://docs.google.com/document/d';
|
|
18
|
+
const GDOCS_API_BASE = 'https://docs.googleapis.com/v1/documents';
|
|
19
|
+
const DEFAULT_USER_AGENT =
|
|
20
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
|
14
21
|
|
|
15
22
|
/**
|
|
16
23
|
* Supported Google Docs export formats.
|
|
@@ -66,7 +73,47 @@ export function buildExportUrl(documentId, format = 'html') {
|
|
|
66
73
|
}
|
|
67
74
|
|
|
68
75
|
/**
|
|
69
|
-
*
|
|
76
|
+
* Build the Google Docs editor URL used for browser-model capture.
|
|
77
|
+
*
|
|
78
|
+
* @param {string} documentId - Google Docs document ID
|
|
79
|
+
* @returns {string} Edit URL
|
|
80
|
+
*/
|
|
81
|
+
export function buildEditUrl(documentId) {
|
|
82
|
+
return `${GDOCS_EXPORT_BASE}/${documentId}/edit`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Build the Google Docs REST API URL.
|
|
87
|
+
*
|
|
88
|
+
* @param {string} documentId - Google Docs document ID
|
|
89
|
+
* @returns {string} Docs API URL
|
|
90
|
+
*/
|
|
91
|
+
export function buildDocsApiUrl(documentId) {
|
|
92
|
+
return `${GDOCS_API_BASE}/${documentId}`;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Select the Google Docs capture backend for CLI --capture behavior.
|
|
97
|
+
*
|
|
98
|
+
* @param {string} [capture='browser'] - Capture flag value
|
|
99
|
+
* @param {string} [apiToken] - Optional token
|
|
100
|
+
* @returns {'browser-model'|'public-export'|'docs-api'}
|
|
101
|
+
*/
|
|
102
|
+
export function selectGoogleDocsCaptureMethod(capture = 'browser', apiToken) {
|
|
103
|
+
const normalized = (capture || 'browser').toLowerCase();
|
|
104
|
+
if (normalized === 'browser') {
|
|
105
|
+
return 'browser-model';
|
|
106
|
+
}
|
|
107
|
+
if (normalized === 'api') {
|
|
108
|
+
return apiToken ? 'docs-api' : 'public-export';
|
|
109
|
+
}
|
|
110
|
+
throw new Error(
|
|
111
|
+
`Unsupported Google Docs capture method "${capture}". Use "browser" or "api".`
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Fetch a Google Docs document via the public export URL.
|
|
70
117
|
*
|
|
71
118
|
* For public documents, no authentication is needed.
|
|
72
119
|
* For private documents, pass an API token via the apiToken option.
|
|
@@ -78,7 +125,7 @@ export function buildExportUrl(documentId, format = 'html') {
|
|
|
78
125
|
* @returns {Promise<{content: string, format: string, documentId: string, exportUrl: string}>}
|
|
79
126
|
*/
|
|
80
127
|
export async function fetchGoogleDoc(url, options = {}) {
|
|
81
|
-
const { format = 'html', apiToken } = options;
|
|
128
|
+
const { format = 'html', apiToken, log } = options;
|
|
82
129
|
|
|
83
130
|
const documentId = extractDocumentId(url);
|
|
84
131
|
if (!documentId) {
|
|
@@ -86,22 +133,35 @@ export async function fetchGoogleDoc(url, options = {}) {
|
|
|
86
133
|
}
|
|
87
134
|
|
|
88
135
|
const exportUrl = buildExportUrl(documentId, format);
|
|
136
|
+
log?.debug?.(() => ({
|
|
137
|
+
event: 'gdocs.public-export.request',
|
|
138
|
+
documentId,
|
|
139
|
+
format,
|
|
140
|
+
exportUrl,
|
|
141
|
+
hasApiToken: Boolean(apiToken),
|
|
142
|
+
}));
|
|
89
143
|
|
|
90
144
|
const headers = {
|
|
91
|
-
'User-Agent':
|
|
92
|
-
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
145
|
+
'User-Agent': DEFAULT_USER_AGENT,
|
|
93
146
|
'Accept-Charset': 'utf-8',
|
|
94
147
|
'Accept-Language': 'en-US,en;q=0.9',
|
|
95
148
|
};
|
|
96
149
|
|
|
97
150
|
if (apiToken) {
|
|
98
|
-
headers
|
|
151
|
+
headers.Authorization = `Bearer ${apiToken}`;
|
|
99
152
|
}
|
|
100
153
|
|
|
101
154
|
const response = await fetch(exportUrl, {
|
|
102
155
|
headers,
|
|
103
156
|
redirect: 'follow',
|
|
104
157
|
});
|
|
158
|
+
log?.debug?.(() => ({
|
|
159
|
+
event: 'gdocs.public-export.response',
|
|
160
|
+
documentId,
|
|
161
|
+
status: response.status,
|
|
162
|
+
ok: response.ok,
|
|
163
|
+
contentType: response.headers.get('content-type'),
|
|
164
|
+
}));
|
|
105
165
|
|
|
106
166
|
if (!response.ok) {
|
|
107
167
|
const statusText = response.statusText || 'Unknown error';
|
|
@@ -111,6 +171,11 @@ export async function fetchGoogleDoc(url, options = {}) {
|
|
|
111
171
|
}
|
|
112
172
|
|
|
113
173
|
const rawContent = await response.text();
|
|
174
|
+
log?.debug?.(() => ({
|
|
175
|
+
event: 'gdocs.public-export.body',
|
|
176
|
+
documentId,
|
|
177
|
+
bytes: Buffer.byteLength(rawContent),
|
|
178
|
+
}));
|
|
114
179
|
|
|
115
180
|
// Decode HTML entities to unicode for text-based formats
|
|
116
181
|
const content =
|
|
@@ -126,6 +191,77 @@ export async function fetchGoogleDoc(url, options = {}) {
|
|
|
126
191
|
};
|
|
127
192
|
}
|
|
128
193
|
|
|
194
|
+
/**
|
|
195
|
+
* Fetch a Google Docs document using the Google Docs REST API.
|
|
196
|
+
*
|
|
197
|
+
* @param {string} url - Google Docs URL
|
|
198
|
+
* @param {Object} options - Options
|
|
199
|
+
* @param {string} options.apiToken - OAuth bearer token
|
|
200
|
+
* @returns {Promise<{content: string, markdown: string, html: string, text: string, document: Object, documentId: string, exportUrl: string}>}
|
|
201
|
+
*/
|
|
202
|
+
export async function fetchGoogleDocFromDocsApi(url, options = {}) {
|
|
203
|
+
const { apiToken, log } = options;
|
|
204
|
+
if (!apiToken) {
|
|
205
|
+
throw new Error('Google Docs REST API capture requires --apiToken');
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const documentId = extractDocumentId(url);
|
|
209
|
+
if (!documentId) {
|
|
210
|
+
throw new Error(`Not a valid Google Docs URL: ${url}`);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const apiUrl = buildDocsApiUrl(documentId);
|
|
214
|
+
log?.debug?.(() => ({
|
|
215
|
+
event: 'gdocs.docs-api.request',
|
|
216
|
+
documentId,
|
|
217
|
+
apiUrl,
|
|
218
|
+
hasApiToken: Boolean(apiToken),
|
|
219
|
+
}));
|
|
220
|
+
const response = await fetch(apiUrl, {
|
|
221
|
+
headers: {
|
|
222
|
+
Authorization: `Bearer ${apiToken}`,
|
|
223
|
+
Accept: 'application/json',
|
|
224
|
+
'User-Agent': DEFAULT_USER_AGENT,
|
|
225
|
+
},
|
|
226
|
+
redirect: 'follow',
|
|
227
|
+
});
|
|
228
|
+
log?.debug?.(() => ({
|
|
229
|
+
event: 'gdocs.docs-api.response',
|
|
230
|
+
documentId,
|
|
231
|
+
status: response.status,
|
|
232
|
+
ok: response.ok,
|
|
233
|
+
contentType: response.headers.get('content-type'),
|
|
234
|
+
}));
|
|
235
|
+
|
|
236
|
+
if (!response.ok) {
|
|
237
|
+
const statusText = response.statusText || 'Unknown error';
|
|
238
|
+
throw new Error(
|
|
239
|
+
`Failed to fetch Google Doc via Docs API (${response.status} ${statusText}): ${apiUrl}`
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
const document = await response.json();
|
|
244
|
+
const rendered = renderDocsApiDocument(document);
|
|
245
|
+
log?.debug?.(() => ({
|
|
246
|
+
event: 'gdocs.docs-api.rendered',
|
|
247
|
+
documentId,
|
|
248
|
+
title: document.title,
|
|
249
|
+
bodyElements: document.body?.content?.length || 0,
|
|
250
|
+
inlineObjects: Object.keys(document.inlineObjects || {}).length,
|
|
251
|
+
markdownBytes: Buffer.byteLength(rendered.markdown),
|
|
252
|
+
htmlBytes: Buffer.byteLength(rendered.html),
|
|
253
|
+
textBytes: Buffer.byteLength(rendered.text),
|
|
254
|
+
}));
|
|
255
|
+
|
|
256
|
+
return {
|
|
257
|
+
...rendered,
|
|
258
|
+
content: rendered.markdown,
|
|
259
|
+
document,
|
|
260
|
+
documentId,
|
|
261
|
+
exportUrl: apiUrl,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
|
|
129
265
|
/**
|
|
130
266
|
* Fetch a Google Docs document and convert to Markdown.
|
|
131
267
|
*
|
|
@@ -138,15 +274,21 @@ export async function fetchGoogleDoc(url, options = {}) {
|
|
|
138
274
|
* @returns {Promise<{markdown: string, documentId: string, exportUrl: string}>}
|
|
139
275
|
*/
|
|
140
276
|
export async function fetchGoogleDocAsMarkdown(url, options = {}) {
|
|
141
|
-
const { apiToken } = options;
|
|
277
|
+
const { apiToken, log } = options;
|
|
142
278
|
|
|
143
279
|
// Always fetch as HTML first, then convert to Markdown
|
|
144
280
|
const result = await fetchGoogleDoc(url, {
|
|
145
281
|
format: 'html',
|
|
146
282
|
apiToken,
|
|
283
|
+
log,
|
|
147
284
|
});
|
|
148
285
|
|
|
149
286
|
const markdown = convertHtmlToMarkdown(result.content, result.exportUrl);
|
|
287
|
+
log?.debug?.(() => ({
|
|
288
|
+
event: 'gdocs.public-export.markdown',
|
|
289
|
+
documentId: result.documentId,
|
|
290
|
+
markdownBytes: Buffer.byteLength(markdown),
|
|
291
|
+
}));
|
|
150
292
|
|
|
151
293
|
return {
|
|
152
294
|
markdown,
|
|
@@ -155,6 +297,672 @@ export async function fetchGoogleDocAsMarkdown(url, options = {}) {
|
|
|
155
297
|
};
|
|
156
298
|
}
|
|
157
299
|
|
|
300
|
+
/**
|
|
301
|
+
* Capture a Google Doc from the editor page model (`DOCS_modelChunk`).
|
|
302
|
+
*
|
|
303
|
+
* @param {string} url - Google Docs URL
|
|
304
|
+
* @param {Object} [options] - Capture options
|
|
305
|
+
* @param {string} [options.engine='playwright'] - Browser engine
|
|
306
|
+
* @param {string} [options.apiToken] - Optional Authorization header token
|
|
307
|
+
* @param {number} [options.waitMs=8000] - Post-load wait for chunks
|
|
308
|
+
* @param {Function} [options.createBrowser] - Browser factory for tests
|
|
309
|
+
* @returns {Promise<{capture: Object, markdown: string, html: string, text: string, documentId: string, exportUrl: string}>}
|
|
310
|
+
*/
|
|
311
|
+
export async function captureGoogleDocWithBrowser(url, options = {}) {
|
|
312
|
+
const {
|
|
313
|
+
engine = 'playwright',
|
|
314
|
+
apiToken,
|
|
315
|
+
waitMs = 8000,
|
|
316
|
+
createBrowser = defaultCreateBrowser,
|
|
317
|
+
log,
|
|
318
|
+
} = options;
|
|
319
|
+
const documentId = extractDocumentId(url);
|
|
320
|
+
if (!documentId) {
|
|
321
|
+
throw new Error(`Not a valid Google Docs URL: ${url}`);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
const editUrl = buildEditUrl(documentId);
|
|
325
|
+
log?.debug?.(() => ({
|
|
326
|
+
event: 'gdocs.browser-model.start',
|
|
327
|
+
documentId,
|
|
328
|
+
editUrl,
|
|
329
|
+
engine,
|
|
330
|
+
waitMs,
|
|
331
|
+
hasApiToken: Boolean(apiToken),
|
|
332
|
+
}));
|
|
333
|
+
const browser = await createBrowser(engine);
|
|
334
|
+
let page;
|
|
335
|
+
try {
|
|
336
|
+
page = await browser.newPage();
|
|
337
|
+
await installDocsModelCapture(page);
|
|
338
|
+
await page.setUserAgent(DEFAULT_USER_AGENT);
|
|
339
|
+
await page.setExtraHTTPHeaders({
|
|
340
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
341
|
+
'Accept-Charset': 'utf-8',
|
|
342
|
+
...(apiToken ? { Authorization: `Bearer ${apiToken}` } : {}),
|
|
343
|
+
});
|
|
344
|
+
await page.setViewport({ width: 1280, height: 800 });
|
|
345
|
+
await page.goto(editUrl, {
|
|
346
|
+
waitUntil: 'domcontentloaded',
|
|
347
|
+
timeout: 60000,
|
|
348
|
+
});
|
|
349
|
+
log?.debug?.(() => ({
|
|
350
|
+
event: 'gdocs.browser-model.loaded',
|
|
351
|
+
documentId,
|
|
352
|
+
editUrl,
|
|
353
|
+
}));
|
|
354
|
+
await waitForPage(page, waitMs);
|
|
355
|
+
|
|
356
|
+
const modelData = await evaluateOnPage(page, () => {
|
|
357
|
+
const chunks = window.__captured_chunks || [];
|
|
358
|
+
const cidUrlMap = {};
|
|
359
|
+
const scripts = document.querySelectorAll('script');
|
|
360
|
+
for (const script of scripts) {
|
|
361
|
+
const text = script.textContent || '';
|
|
362
|
+
if (!text.includes('docs-images-rt')) {
|
|
363
|
+
continue;
|
|
364
|
+
}
|
|
365
|
+
const regex =
|
|
366
|
+
/"([A-Za-z0-9_-]{20,})"\s*:\s*"(https:\/\/docs\.google\.com\/docs-images-rt\/[^"]+)"/g;
|
|
367
|
+
let match;
|
|
368
|
+
while ((match = regex.exec(text)) !== null) {
|
|
369
|
+
cidUrlMap[match[1]] = match[2]
|
|
370
|
+
.replace(/\\u003d/g, '=')
|
|
371
|
+
.replace(/\\u0026/g, '&')
|
|
372
|
+
.replace(/\\\//g, '/');
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
return { chunks, cidUrlMap };
|
|
376
|
+
});
|
|
377
|
+
|
|
378
|
+
const capture = parseGoogleDocsModelChunks(
|
|
379
|
+
modelData.chunks,
|
|
380
|
+
modelData.cidUrlMap
|
|
381
|
+
);
|
|
382
|
+
log?.debug?.(() => ({
|
|
383
|
+
event: 'gdocs.browser-model.parsed',
|
|
384
|
+
documentId,
|
|
385
|
+
chunks: modelData.chunks?.length || 0,
|
|
386
|
+
cidUrls: Object.keys(modelData.cidUrlMap || {}).length,
|
|
387
|
+
blocks: capture.blocks.length,
|
|
388
|
+
tables: capture.tables.length,
|
|
389
|
+
images: capture.images.length,
|
|
390
|
+
textBytes: Buffer.byteLength(capture.text || ''),
|
|
391
|
+
}));
|
|
392
|
+
if (capture.blocks.length === 0) {
|
|
393
|
+
throw new Error(
|
|
394
|
+
'Google Docs editor page did not expose DOCS_modelChunk data'
|
|
395
|
+
);
|
|
396
|
+
}
|
|
397
|
+
return {
|
|
398
|
+
capture,
|
|
399
|
+
markdown: renderGoogleDocsCapture(capture, 'markdown'),
|
|
400
|
+
html: renderGoogleDocsCapture(capture, 'html'),
|
|
401
|
+
text: renderGoogleDocsCapture(capture, 'txt'),
|
|
402
|
+
documentId,
|
|
403
|
+
exportUrl: editUrl,
|
|
404
|
+
};
|
|
405
|
+
} finally {
|
|
406
|
+
if (page) {
|
|
407
|
+
await page.close();
|
|
408
|
+
}
|
|
409
|
+
await browser.close();
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
async function installDocsModelCapture(page) {
|
|
414
|
+
const initScript = () => {
|
|
415
|
+
window.__captured_chunks = [];
|
|
416
|
+
Object.defineProperty(window, 'DOCS_modelChunk', {
|
|
417
|
+
set(value) {
|
|
418
|
+
try {
|
|
419
|
+
window.__captured_chunks.push(JSON.parse(JSON.stringify(value)));
|
|
420
|
+
} catch {
|
|
421
|
+
window.__captured_chunks.push(value);
|
|
422
|
+
}
|
|
423
|
+
window.__DOCS_modelChunk_latest = value;
|
|
424
|
+
},
|
|
425
|
+
get() {
|
|
426
|
+
return window.__DOCS_modelChunk_latest;
|
|
427
|
+
},
|
|
428
|
+
configurable: true,
|
|
429
|
+
});
|
|
430
|
+
};
|
|
431
|
+
|
|
432
|
+
const rawPage = page.rawPage || page;
|
|
433
|
+
if (typeof rawPage.addInitScript === 'function') {
|
|
434
|
+
await rawPage.addInitScript(initScript);
|
|
435
|
+
} else if (typeof rawPage.evaluateOnNewDocument === 'function') {
|
|
436
|
+
await rawPage.evaluateOnNewDocument(initScript);
|
|
437
|
+
} else if (typeof page.addInitScript === 'function') {
|
|
438
|
+
await page.addInitScript(initScript);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
async function waitForPage(page, waitMs) {
|
|
443
|
+
if (waitMs <= 0) {
|
|
444
|
+
return;
|
|
445
|
+
}
|
|
446
|
+
const rawPage = page.rawPage || page;
|
|
447
|
+
if (typeof rawPage.waitForTimeout === 'function') {
|
|
448
|
+
await rawPage.waitForTimeout(waitMs);
|
|
449
|
+
return;
|
|
450
|
+
}
|
|
451
|
+
if (typeof page.waitForTimeout === 'function') {
|
|
452
|
+
await page.waitForTimeout(waitMs);
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
await new Promise((resolve) => setTimeout(resolve, waitMs));
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
async function evaluateOnPage(page, fn) {
|
|
459
|
+
const rawPage = page.rawPage || page;
|
|
460
|
+
if (rawPage !== page && typeof rawPage.evaluate === 'function') {
|
|
461
|
+
return await rawPage.evaluate(fn);
|
|
462
|
+
}
|
|
463
|
+
return await page.evaluate(fn);
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Extract a CID-to-image URL map from Google Docs editor HTML.
|
|
468
|
+
*
|
|
469
|
+
* @param {string} html - Editor HTML
|
|
470
|
+
* @returns {Object<string,string>} CID URL map
|
|
471
|
+
*/
|
|
472
|
+
export function extractCidUrlMapFromHtml(html) {
|
|
473
|
+
const cidUrlMap = {};
|
|
474
|
+
const regex =
|
|
475
|
+
/"([A-Za-z0-9_-]{20,})"\s*:\s*"(https:\/\/docs\.google\.com\/docs-images-rt\/[^"]+)"/g;
|
|
476
|
+
let match;
|
|
477
|
+
while ((match = regex.exec(html)) !== null) {
|
|
478
|
+
cidUrlMap[match[1]] = match[2]
|
|
479
|
+
.replace(/\\u003d/g, '=')
|
|
480
|
+
.replace(/\\u0026/g, '&')
|
|
481
|
+
.replace(/\\\//g, '/');
|
|
482
|
+
}
|
|
483
|
+
return cidUrlMap;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/**
|
|
487
|
+
* Parse captured Google Docs model chunks into blocks, tables, and text.
|
|
488
|
+
*
|
|
489
|
+
* @param {Array} chunks - Captured DOCS_modelChunk values
|
|
490
|
+
* @param {Object<string,string>} [cidUrlMap={}] - CID-to-docs-images-rt URL map
|
|
491
|
+
* @returns {{blocks: Array, tables: Array, images: Array, text: string}}
|
|
492
|
+
*/
|
|
493
|
+
export function parseGoogleDocsModelChunks(chunks = [], cidUrlMap = {}) {
|
|
494
|
+
const items = collectModelItems(chunks);
|
|
495
|
+
const fullText = items
|
|
496
|
+
.filter((item) => item.ty === 'is' || item.ty === 'iss')
|
|
497
|
+
.map((item) => item.s || '')
|
|
498
|
+
.join('');
|
|
499
|
+
|
|
500
|
+
const positions = new Map();
|
|
501
|
+
for (const item of items) {
|
|
502
|
+
if ((item.ty === 'te' || item.ty === 'ste') && item.id) {
|
|
503
|
+
positions.set(item.id, Number(item.spi));
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
const imagesByPos = new Map();
|
|
508
|
+
const images = [];
|
|
509
|
+
for (const item of items) {
|
|
510
|
+
if (item.ty !== 'ae' && item.ty !== 'ase') {
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
const pos = positions.get(item.id);
|
|
514
|
+
if (!Number.isFinite(pos)) {
|
|
515
|
+
continue;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
const cid = item.epm?.ee_eo?.i_cid || null;
|
|
519
|
+
const image = {
|
|
520
|
+
type: 'image',
|
|
521
|
+
cid,
|
|
522
|
+
url: cid ? cidUrlMap[cid] || null : null,
|
|
523
|
+
width: item.epm?.ee_eo?.i_wth,
|
|
524
|
+
height: item.epm?.ee_eo?.i_ht,
|
|
525
|
+
isSuggestion: item.ty === 'ase',
|
|
526
|
+
alt: item.ty === 'ase' ? 'suggested image' : 'image',
|
|
527
|
+
};
|
|
528
|
+
imagesByPos.set(pos, image);
|
|
529
|
+
images.push(image);
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
const blocks = [];
|
|
533
|
+
const tables = [];
|
|
534
|
+
let paragraph = [];
|
|
535
|
+
let table = null;
|
|
536
|
+
let row = null;
|
|
537
|
+
let cell = null;
|
|
538
|
+
|
|
539
|
+
const flushParagraph = () => {
|
|
540
|
+
const text = contentToText(paragraph).trim();
|
|
541
|
+
if (text || paragraph.some((node) => node.type === 'image')) {
|
|
542
|
+
blocks.push({ type: 'paragraph', content: paragraph });
|
|
543
|
+
}
|
|
544
|
+
paragraph = [];
|
|
545
|
+
};
|
|
546
|
+
|
|
547
|
+
const flushCell = () => {
|
|
548
|
+
if (cell && row) {
|
|
549
|
+
row.cells.push(cell);
|
|
550
|
+
}
|
|
551
|
+
cell = null;
|
|
552
|
+
};
|
|
553
|
+
|
|
554
|
+
const flushRow = () => {
|
|
555
|
+
flushCell();
|
|
556
|
+
if (row && table) {
|
|
557
|
+
table.rows.push(row);
|
|
558
|
+
}
|
|
559
|
+
row = null;
|
|
560
|
+
};
|
|
561
|
+
|
|
562
|
+
const flushTable = () => {
|
|
563
|
+
flushRow();
|
|
564
|
+
if (table) {
|
|
565
|
+
tables.push(table);
|
|
566
|
+
blocks.push(table);
|
|
567
|
+
}
|
|
568
|
+
table = null;
|
|
569
|
+
};
|
|
570
|
+
|
|
571
|
+
const targetContent = () => {
|
|
572
|
+
if (table) {
|
|
573
|
+
if (!row) {
|
|
574
|
+
row = { cells: [] };
|
|
575
|
+
}
|
|
576
|
+
if (!cell) {
|
|
577
|
+
cell = { content: [] };
|
|
578
|
+
}
|
|
579
|
+
return cell.content;
|
|
580
|
+
}
|
|
581
|
+
return paragraph;
|
|
582
|
+
};
|
|
583
|
+
|
|
584
|
+
for (let i = 0; i < fullText.length; i++) {
|
|
585
|
+
const code = fullText.charCodeAt(i);
|
|
586
|
+
if (code === 0x10) {
|
|
587
|
+
flushParagraph();
|
|
588
|
+
table = { type: 'table', rows: [] };
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
if (code === 0x11) {
|
|
592
|
+
flushTable();
|
|
593
|
+
continue;
|
|
594
|
+
}
|
|
595
|
+
if (code === 0x12) {
|
|
596
|
+
flushRow();
|
|
597
|
+
row = { cells: [] };
|
|
598
|
+
continue;
|
|
599
|
+
}
|
|
600
|
+
if (code === 0x1c) {
|
|
601
|
+
flushCell();
|
|
602
|
+
if (!row) {
|
|
603
|
+
row = { cells: [] };
|
|
604
|
+
}
|
|
605
|
+
cell = { content: [] };
|
|
606
|
+
continue;
|
|
607
|
+
}
|
|
608
|
+
if (code === 0x0a) {
|
|
609
|
+
if (table) {
|
|
610
|
+
flushRow();
|
|
611
|
+
} else {
|
|
612
|
+
flushParagraph();
|
|
613
|
+
}
|
|
614
|
+
continue;
|
|
615
|
+
}
|
|
616
|
+
if (code === 0x0b) {
|
|
617
|
+
appendText(targetContent(), '\n');
|
|
618
|
+
continue;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
const image = imagesByPos.get(i);
|
|
622
|
+
if (image) {
|
|
623
|
+
targetContent().push(image);
|
|
624
|
+
if (fullText[i] === '*') {
|
|
625
|
+
continue;
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
appendText(targetContent(), fullText[i]);
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
if (table) {
|
|
633
|
+
flushTable();
|
|
634
|
+
}
|
|
635
|
+
flushParagraph();
|
|
636
|
+
|
|
637
|
+
return {
|
|
638
|
+
blocks,
|
|
639
|
+
tables,
|
|
640
|
+
images,
|
|
641
|
+
text: contentBlocksToText(blocks),
|
|
642
|
+
};
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
function collectModelItems(chunks) {
|
|
646
|
+
const items = [];
|
|
647
|
+
for (const chunk of chunks || []) {
|
|
648
|
+
if (Array.isArray(chunk)) {
|
|
649
|
+
items.push(...chunk);
|
|
650
|
+
} else if (Array.isArray(chunk?.chunk)) {
|
|
651
|
+
items.push(...chunk.chunk);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
return items;
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
/**
|
|
658
|
+
* Render a parsed Google Docs model capture.
|
|
659
|
+
*
|
|
660
|
+
* @param {Object} capture - Parsed capture
|
|
661
|
+
* @param {string} [format='markdown'] - markdown, html, or txt
|
|
662
|
+
* @returns {string} Rendered output
|
|
663
|
+
*/
|
|
664
|
+
export function renderGoogleDocsCapture(capture, format = 'markdown') {
|
|
665
|
+
const normalized = (format || 'markdown').toLowerCase();
|
|
666
|
+
if (normalized === 'html') {
|
|
667
|
+
return renderBlocksHtml(capture.blocks);
|
|
668
|
+
}
|
|
669
|
+
if (normalized === 'txt' || normalized === 'text') {
|
|
670
|
+
return contentBlocksToText(capture.blocks);
|
|
671
|
+
}
|
|
672
|
+
return renderBlocksMarkdown(capture.blocks);
|
|
673
|
+
}
|
|
674
|
+
|
|
675
|
+
/**
|
|
676
|
+
* Render a Google Docs REST API document as Markdown, HTML, and text.
|
|
677
|
+
*
|
|
678
|
+
* @param {Object} document - docs.googleapis.com v1 documents.get response
|
|
679
|
+
* @returns {{markdown: string, html: string, text: string}}
|
|
680
|
+
*/
|
|
681
|
+
export function renderDocsApiDocument(document) {
|
|
682
|
+
const inlineObjects = document.inlineObjects || {};
|
|
683
|
+
const blocks = structuralElementsToBlocks(
|
|
684
|
+
document.body?.content || [],
|
|
685
|
+
inlineObjects
|
|
686
|
+
);
|
|
687
|
+
return {
|
|
688
|
+
markdown: renderBlocksMarkdown(blocks),
|
|
689
|
+
html: renderBlocksHtml(blocks),
|
|
690
|
+
text: contentBlocksToText(blocks),
|
|
691
|
+
};
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
function structuralElementsToBlocks(elements, inlineObjects) {
|
|
695
|
+
const blocks = [];
|
|
696
|
+
for (const element of elements || []) {
|
|
697
|
+
if (element.paragraph) {
|
|
698
|
+
const content = paragraphToContent(element.paragraph, inlineObjects);
|
|
699
|
+
if (
|
|
700
|
+
contentToText(content).trim() ||
|
|
701
|
+
content.some((n) => n.type === 'image')
|
|
702
|
+
) {
|
|
703
|
+
blocks.push({
|
|
704
|
+
type: 'paragraph',
|
|
705
|
+
style: element.paragraph.paragraphStyle?.namedStyleType,
|
|
706
|
+
content,
|
|
707
|
+
});
|
|
708
|
+
}
|
|
709
|
+
} else if (element.table) {
|
|
710
|
+
blocks.push(tableToBlock(element.table, inlineObjects));
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
return blocks;
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
function tableToBlock(table, inlineObjects) {
|
|
717
|
+
return {
|
|
718
|
+
type: 'table',
|
|
719
|
+
rows: (table.tableRows || []).map((tableRow) => ({
|
|
720
|
+
cells: (tableRow.tableCells || []).map((tableCell) => ({
|
|
721
|
+
content: structuralElementsToInlineContent(
|
|
722
|
+
tableCell.content || [],
|
|
723
|
+
inlineObjects
|
|
724
|
+
),
|
|
725
|
+
})),
|
|
726
|
+
})),
|
|
727
|
+
};
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
function structuralElementsToInlineContent(elements, inlineObjects) {
|
|
731
|
+
const content = [];
|
|
732
|
+
for (const element of elements || []) {
|
|
733
|
+
if (element.paragraph) {
|
|
734
|
+
const paragraphContent = paragraphToContent(
|
|
735
|
+
element.paragraph,
|
|
736
|
+
inlineObjects
|
|
737
|
+
);
|
|
738
|
+
if (content.length > 0 && paragraphContent.length > 0) {
|
|
739
|
+
appendText(content, '\n');
|
|
740
|
+
}
|
|
741
|
+
content.push(...paragraphContent);
|
|
742
|
+
} else if (element.table) {
|
|
743
|
+
appendText(
|
|
744
|
+
content,
|
|
745
|
+
renderBlocksMarkdown([tableToBlock(element.table, inlineObjects)])
|
|
746
|
+
);
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
return content;
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
function paragraphToContent(paragraph, inlineObjects) {
|
|
753
|
+
const content = [];
|
|
754
|
+
for (const element of paragraph.elements || []) {
|
|
755
|
+
if (element.textRun) {
|
|
756
|
+
appendText(content, element.textRun.content.replace(/\n$/u, ''));
|
|
757
|
+
} else if (element.inlineObjectElement) {
|
|
758
|
+
const image = inlineObjectToImage(
|
|
759
|
+
element.inlineObjectElement.inlineObjectId,
|
|
760
|
+
inlineObjects
|
|
761
|
+
);
|
|
762
|
+
if (image) {
|
|
763
|
+
content.push(image);
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
return content;
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
function inlineObjectToImage(inlineObjectId, inlineObjects) {
|
|
771
|
+
const embedded =
|
|
772
|
+
inlineObjects[inlineObjectId]?.inlineObjectProperties?.embeddedObject;
|
|
773
|
+
const url =
|
|
774
|
+
embedded?.imageProperties?.contentUri ||
|
|
775
|
+
embedded?.imageProperties?.sourceUri ||
|
|
776
|
+
null;
|
|
777
|
+
if (!url) {
|
|
778
|
+
return null;
|
|
779
|
+
}
|
|
780
|
+
return {
|
|
781
|
+
type: 'image',
|
|
782
|
+
url,
|
|
783
|
+
alt: embedded.title || embedded.description || 'image',
|
|
784
|
+
isSuggestion: false,
|
|
785
|
+
};
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
function appendText(content, text) {
|
|
789
|
+
if (!text) {
|
|
790
|
+
return;
|
|
791
|
+
}
|
|
792
|
+
const last = content[content.length - 1];
|
|
793
|
+
if (last?.type === 'text') {
|
|
794
|
+
last.text += text;
|
|
795
|
+
} else {
|
|
796
|
+
content.push({ type: 'text', text });
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
function renderBlocksMarkdown(blocks) {
|
|
801
|
+
return blocks
|
|
802
|
+
.map((block) => {
|
|
803
|
+
if (block.type === 'table') {
|
|
804
|
+
return renderTableMarkdown(block);
|
|
805
|
+
}
|
|
806
|
+
return renderParagraphMarkdown(block);
|
|
807
|
+
})
|
|
808
|
+
.filter(Boolean)
|
|
809
|
+
.join('\n\n')
|
|
810
|
+
.trimEnd();
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
function renderParagraphMarkdown(block) {
|
|
814
|
+
const text = renderContentMarkdown(block.content).trim();
|
|
815
|
+
const style = block.style || '';
|
|
816
|
+
const headingMatch = style.match(/^HEADING_(\d)$/u);
|
|
817
|
+
if (headingMatch) {
|
|
818
|
+
return `${'#'.repeat(Number(headingMatch[1]))} ${text}`;
|
|
819
|
+
}
|
|
820
|
+
if (style === 'TITLE') {
|
|
821
|
+
return `# ${text}`;
|
|
822
|
+
}
|
|
823
|
+
if (style === 'SUBTITLE') {
|
|
824
|
+
return `## ${text}`;
|
|
825
|
+
}
|
|
826
|
+
return text;
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
function renderTableMarkdown(table) {
|
|
830
|
+
if (!table.rows.length) {
|
|
831
|
+
return '';
|
|
832
|
+
}
|
|
833
|
+
const width = Math.max(...table.rows.map((row) => row.cells.length), 1);
|
|
834
|
+
const rows = table.rows.map((row) =>
|
|
835
|
+
Array.from({ length: width }, (_, idx) =>
|
|
836
|
+
escapeMarkdownTableCell(
|
|
837
|
+
renderContentMarkdown(row.cells[idx]?.content || [])
|
|
838
|
+
)
|
|
839
|
+
)
|
|
840
|
+
);
|
|
841
|
+
const separator = Array.from({ length: width }, () => '---');
|
|
842
|
+
return [rows[0], separator, ...rows.slice(1)]
|
|
843
|
+
.map((row) => `| ${row.join(' | ')} |`)
|
|
844
|
+
.join('\n');
|
|
845
|
+
}
|
|
846
|
+
|
|
847
|
+
function renderContentMarkdown(content = []) {
|
|
848
|
+
return content
|
|
849
|
+
.map((node) => {
|
|
850
|
+
if (node.type === 'image') {
|
|
851
|
+
return node.url ? `` : '';
|
|
852
|
+
}
|
|
853
|
+
return node.text || '';
|
|
854
|
+
})
|
|
855
|
+
.join('');
|
|
856
|
+
}
|
|
857
|
+
|
|
858
|
+
function renderBlocksHtml(blocks) {
|
|
859
|
+
return `<!doctype html><html><body>${blocks
|
|
860
|
+
.map((block) => {
|
|
861
|
+
if (block.type === 'table') {
|
|
862
|
+
return renderTableHtml(block);
|
|
863
|
+
}
|
|
864
|
+
return renderParagraphHtml(block);
|
|
865
|
+
})
|
|
866
|
+
.join('')}</body></html>`;
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
function renderParagraphHtml(block) {
|
|
870
|
+
const tag = paragraphTag(block.style);
|
|
871
|
+
return `<${tag}>${renderContentHtml(block.content)}</${tag}>`;
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
function renderTableHtml(table) {
|
|
875
|
+
return `<table>${table.rows
|
|
876
|
+
.map(
|
|
877
|
+
(row) =>
|
|
878
|
+
`<tr>${row.cells
|
|
879
|
+
.map((cell) => `<td>${renderContentHtml(cell.content)}</td>`)
|
|
880
|
+
.join('')}</tr>`
|
|
881
|
+
)
|
|
882
|
+
.join('')}</table>`;
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
function renderContentHtml(content = []) {
|
|
886
|
+
return content
|
|
887
|
+
.map((node) => {
|
|
888
|
+
if (node.type === 'image') {
|
|
889
|
+
if (!node.url) {
|
|
890
|
+
return '';
|
|
891
|
+
}
|
|
892
|
+
const width = node.width ? ` width="${escapeHtml(node.width)}"` : '';
|
|
893
|
+
const height = node.height
|
|
894
|
+
? ` height="${escapeHtml(node.height)}"`
|
|
895
|
+
: '';
|
|
896
|
+
return `<img src="${escapeHtml(node.url)}" alt="${escapeHtml(node.alt || 'image')}"${width}${height}>`;
|
|
897
|
+
}
|
|
898
|
+
return escapeHtml(node.text || '').replace(/\n/g, '<br>');
|
|
899
|
+
})
|
|
900
|
+
.join('');
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
function paragraphTag(style = '') {
|
|
904
|
+
const headingMatch = style.match(/^HEADING_([1-6])$/u);
|
|
905
|
+
if (headingMatch) {
|
|
906
|
+
return `h${headingMatch[1]}`;
|
|
907
|
+
}
|
|
908
|
+
if (style === 'TITLE') {
|
|
909
|
+
return 'h1';
|
|
910
|
+
}
|
|
911
|
+
if (style === 'SUBTITLE') {
|
|
912
|
+
return 'h2';
|
|
913
|
+
}
|
|
914
|
+
return 'p';
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
function contentBlocksToText(blocks) {
|
|
918
|
+
return blocks
|
|
919
|
+
.map((block) => {
|
|
920
|
+
if (block.type === 'table') {
|
|
921
|
+
return block.rows
|
|
922
|
+
.map((row) =>
|
|
923
|
+
row.cells.map((cell) => contentToText(cell.content)).join('\t')
|
|
924
|
+
)
|
|
925
|
+
.join('\n');
|
|
926
|
+
}
|
|
927
|
+
return contentToText(block.content);
|
|
928
|
+
})
|
|
929
|
+
.filter(Boolean)
|
|
930
|
+
.join('\n')
|
|
931
|
+
.trimEnd();
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
function contentToText(content = []) {
|
|
935
|
+
return content
|
|
936
|
+
.map((node) => {
|
|
937
|
+
if (node.type === 'image') {
|
|
938
|
+
return node.url ? `[${node.alt || 'image'}]` : '';
|
|
939
|
+
}
|
|
940
|
+
return node.text || '';
|
|
941
|
+
})
|
|
942
|
+
.join('');
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
function escapeHtml(value) {
|
|
946
|
+
return String(value).replace(/[&<>"']/g, (char) => {
|
|
947
|
+
switch (char) {
|
|
948
|
+
case '&':
|
|
949
|
+
return '&';
|
|
950
|
+
case '<':
|
|
951
|
+
return '<';
|
|
952
|
+
case '>':
|
|
953
|
+
return '>';
|
|
954
|
+
case '"':
|
|
955
|
+
return '"';
|
|
956
|
+
default:
|
|
957
|
+
return ''';
|
|
958
|
+
}
|
|
959
|
+
});
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
function escapeMarkdownTableCell(value) {
|
|
963
|
+
return String(value).replace(/\|/g, '\\|').replace(/\n/g, '<br>');
|
|
964
|
+
}
|
|
965
|
+
|
|
158
966
|
/**
|
|
159
967
|
* Extract base64 data URI images from HTML content.
|
|
160
968
|
*
|
|
@@ -203,16 +1011,23 @@ export function extractBase64Images(html) {
|
|
|
203
1011
|
* @returns {Promise<{documentId: string, exportUrl: string, createArchive: function}>}
|
|
204
1012
|
*/
|
|
205
1013
|
export async function fetchGoogleDocAsArchive(url, options = {}) {
|
|
206
|
-
const { apiToken } = options;
|
|
1014
|
+
const { apiToken, log } = options;
|
|
207
1015
|
|
|
208
1016
|
// Fetch HTML with embedded base64 images
|
|
209
|
-
const result = await fetchGoogleDoc(url, { format: 'html', apiToken });
|
|
1017
|
+
const result = await fetchGoogleDoc(url, { format: 'html', apiToken, log });
|
|
210
1018
|
|
|
211
1019
|
// Extract base64 images from HTML
|
|
212
1020
|
const { html: localHtml, images } = extractBase64Images(result.content);
|
|
213
1021
|
|
|
214
1022
|
// Convert the localized HTML to Markdown
|
|
215
1023
|
const markdown = convertHtmlToMarkdown(localHtml);
|
|
1024
|
+
log?.debug?.(() => ({
|
|
1025
|
+
event: 'gdocs.public-export.archive',
|
|
1026
|
+
documentId: result.documentId,
|
|
1027
|
+
images: images.length,
|
|
1028
|
+
htmlBytes: Buffer.byteLength(localHtml),
|
|
1029
|
+
markdownBytes: Buffer.byteLength(markdown),
|
|
1030
|
+
}));
|
|
216
1031
|
|
|
217
1032
|
return {
|
|
218
1033
|
documentId: result.documentId,
|