@link-assistant/web-capture 2.0.0 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/bin/web-capture.js +2 -5
  3. package/package.json +36 -23
  4. package/src/archive.js +0 -1
  5. package/src/binary-download.js +0 -1
  6. package/src/docx.js +13 -10
  7. package/src/fetch.js +0 -1
  8. package/src/gdocs-images.js +0 -1
  9. package/src/gdocs.js +0 -1
  10. package/src/github.js +0 -1
  11. package/src/lib.js +0 -1
  12. package/src/markdown.js +0 -1
  13. package/src/shared-dialog.js +0 -1
  14. package/src/stream.js +0 -1
  15. package/src/transport.js +23 -5
  16. package/src/txt.js +0 -1
  17. package/.changeset/README.md +0 -8
  18. package/.changeset/config.json +0 -11
  19. package/.dockerignore +0 -6
  20. package/.jscpd.json +0 -19
  21. package/.lenv +0 -8
  22. package/.prettierignore +0 -8
  23. package/.prettierrc +0 -10
  24. package/Dockerfile +0 -52
  25. package/docker-compose.yml +0 -11
  26. package/eslint.config.js +0 -112
  27. package/examples/js/capture_cache.js +0 -12
  28. package/examples/js/engine_comparison.js +0 -39
  29. package/examples/js/html_download.js +0 -23
  30. package/examples/js/image_download.js +0 -36
  31. package/examples/js/markdown_download.js +0 -23
  32. package/examples/js/playwright_example.js +0 -23
  33. package/examples/python/capture_client.py +0 -20
  34. package/examples/python/html_download.py +0 -3
  35. package/examples/python/image_download.py +0 -16
  36. package/examples/python/markdown_download.py +0 -3
  37. package/examples/python/playwright_example.py +0 -22
  38. package/jest.config.mjs +0 -17
  39. package/tests/ci/dependency-freshness.test.js +0 -74
  40. package/tests/ci/detect-code-changes.test.js +0 -187
  41. package/tests/ci/docker-install.test.js +0 -31
  42. package/tests/ci/rust-dockerfile.test.js +0 -54
  43. package/tests/ci/workflow-policy.test.js +0 -79
  44. package/tests/e2e/docker.test.js +0 -186
  45. package/tests/e2e/process.test.js +0 -215
  46. package/tests/fixtures/bold-coalesce-and-leakage.html +0 -27
  47. package/tests/fixtures/deepwiki/deepwiki-full-page.png +0 -0
  48. package/tests/fixtures/deepwiki/index.md +0 -1500
  49. package/tests/fixtures/deepwiki/part-1.md +0 -776
  50. package/tests/fixtures/habr-articles.js +0 -45
  51. package/tests/fixtures/heading-numbering.html +0 -19
  52. package/tests/fixtures/list-item-with-br.html +0 -8
  53. package/tests/fixtures/ol-continuous-numbering.html +0 -12
  54. package/tests/fixtures/paragraph-vs-line-break.html +0 -11
  55. package/tests/helpers/nock.js +0 -10
  56. package/tests/helpers/screenshot.js +0 -10
  57. package/tests/helpers/search-fixture.cjs +0 -17
  58. package/tests/helpers/temporary-directory.js +0 -12
  59. package/tests/integration/api-endpoints.test.js +0 -418
  60. package/tests/integration/archive-default-layout.test.js +0 -37
  61. package/tests/integration/browser-engines.test.js +0 -80
  62. package/tests/integration/browser-packaged-path.test.js +0 -45
  63. package/tests/integration/deepwiki-download.test.js +0 -144
  64. package/tests/integration/deepwiki-rendered-page.mock.test.js +0 -121
  65. package/tests/integration/formalai-contract.test.js +0 -214
  66. package/tests/integration/gdocs-public-doc.test.js +0 -315
  67. package/tests/integration/github-readme.test.js +0 -133
  68. package/tests/integration/google-drive-image.test.js +0 -246
  69. package/tests/integration/habr-article.test.js +0 -337
  70. package/tests/integration/search-endpoint.test.js +0 -88
  71. package/tests/integration/stackoverflow-download.test.js +0 -173
  72. package/tests/integration/wikipedia-download.test.js +0 -134
  73. package/tests/jest.setup.mjs +0 -7
  74. package/tests/mock/index.test.js +0 -225
  75. package/tests/unit/animation.test.js +0 -33
  76. package/tests/unit/batch.test.js +0 -114
  77. package/tests/unit/bold-coalesce-and-leakage.test.js +0 -54
  78. package/tests/unit/browser-launch-options.test.js +0 -42
  79. package/tests/unit/browser.test.js +0 -172
  80. package/tests/unit/cache.test.js +0 -182
  81. package/tests/unit/cli.test.js +0 -399
  82. package/tests/unit/convertGoogleDriveUrl.test.js +0 -89
  83. package/tests/unit/convertRelativeUrls.test.js +0 -34
  84. package/tests/unit/extract-images.test.js +0 -259
  85. package/tests/unit/fetch-html.test.js +0 -54
  86. package/tests/unit/figures.test.js +0 -74
  87. package/tests/unit/gdocs-export-headers.test.js +0 -28
  88. package/tests/unit/gdocs-preprocess.test.js +0 -36
  89. package/tests/unit/gdocs.test.js +0 -1490
  90. package/tests/unit/github.test.js +0 -156
  91. package/tests/unit/heading-image.test.js +0 -49
  92. package/tests/unit/heading-numbering.test.js +0 -28
  93. package/tests/unit/html2md-br-in-list-item.test.js +0 -23
  94. package/tests/unit/html2md-ol-numbering.test.js +0 -20
  95. package/tests/unit/html2md.test.js +0 -695
  96. package/tests/unit/image-mode-defaults.test.js +0 -54
  97. package/tests/unit/kreuzberg.test.js +0 -249
  98. package/tests/unit/latex.test.js +0 -123
  99. package/tests/unit/localize-images.test.js +0 -114
  100. package/tests/unit/markdown.test.js +0 -36
  101. package/tests/unit/metadata.test.js +0 -140
  102. package/tests/unit/package-runtime-dependencies.test.js +0 -74
  103. package/tests/unit/paragraph-vs-line-break.test.js +0 -56
  104. package/tests/unit/postprocess.test.js +0 -110
  105. package/tests/unit/proxy-headers.test.js +0 -100
  106. package/tests/unit/publish-verification.test.js +0 -54
  107. package/tests/unit/retry.test.js +0 -111
  108. package/tests/unit/search.test.js +0 -339
  109. package/tests/unit/shared-dialog.test.js +0 -173
  110. package/tests/unit/stackoverflow.test.js +0 -32
  111. package/tests/unit/stream.test.js +0 -83
  112. package/tests/unit/verify.test.js +0 -182
  113. package/tests/xpaste/xpaste-screenshot-markdown.mock.test.js +0 -375
  114. package/tests/xpaste/xpaste.e2e.test.js +0 -36
  115. package/tests/xpaste/xpaste.integration.test.js +0 -41
  116. package/tests/xpaste/xpaste.mock.test.js +0 -261
  117. package/tests/xpaste/xpaste.unit.test.js +0 -119
@@ -1,1490 +0,0 @@
1
- import { jest } from '@jest/globals';
2
- import {
3
- isGoogleDocsUrl,
4
- extractDocumentId,
5
- buildExportUrl,
6
- buildEditUrl,
7
- buildDocsApiUrl,
8
- extractBase64Images,
9
- captureGoogleDocWithBrowserOrFallback,
10
- captureGoogleDocWithBrowser,
11
- isGoogleDocsBrowserModelUnavailableError,
12
- parseGoogleDocsModelChunks,
13
- renderGoogleDocsCapture,
14
- renderDocsApiDocument,
15
- selectGoogleDocsCaptureMethod,
16
- localizeGoogleDocsModelImages,
17
- normalizeGoogleDocsExportMarkdown,
18
- preprocessGoogleDocsExportHtml,
19
- GDOCS_EXPORT_FORMATS,
20
- } from '../../src/gdocs.js';
21
- import { convertHtmlToMarkdown } from '../../src/lib.js';
22
- import {
23
- extractAndSaveImages,
24
- extractBase64ToBuffers,
25
- stripBase64Images,
26
- } from '../../src/extract-images.js';
27
- import fs from 'fs';
28
- import os from 'os';
29
- import path from 'path';
30
- import { fileURLToPath } from 'url';
31
- import nock from '../helpers/nock.js';
32
-
33
- jest.setTimeout(30000);
34
-
35
- const __filename = fileURLToPath(import.meta.url);
36
- const __dirname = path.dirname(__filename);
37
- const REPO_ROOT = path.resolve(__dirname, '..', '..', '..');
38
- const ISSUE_104_FIXTURE_DIR = path.join(
39
- REPO_ROOT,
40
- 'docs',
41
- 'case-studies',
42
- 'issue-104',
43
- 'fixtures'
44
- );
45
-
46
- function readIssue104Fixture(filename) {
47
- return fs.readFileSync(path.join(ISSUE_104_FIXTURE_DIR, filename), 'utf-8');
48
- }
49
-
50
- describe('gdocs', () => {
51
- describe('isGoogleDocsUrl', () => {
52
- it('returns true for standard Google Docs edit URL', () => {
53
- expect(
54
- isGoogleDocsUrl(
55
- 'https://docs.google.com/document/d/1yn5AWdus0Rh5xRr6voNy-g2wNzdC4bM0CV7SQWch9H4/edit'
56
- )
57
- ).toBe(true);
58
- });
59
-
60
- it('returns true for URL with query parameters', () => {
61
- expect(
62
- isGoogleDocsUrl(
63
- 'https://docs.google.com/document/d/abc123/edit?tab=t.0'
64
- )
65
- ).toBe(true);
66
- });
67
-
68
- it('returns true for URL without /edit', () => {
69
- expect(
70
- isGoogleDocsUrl('https://docs.google.com/document/d/abc-123_XYZ/')
71
- ).toBe(true);
72
- });
73
-
74
- it('returns false for non-Google Docs URL', () => {
75
- expect(isGoogleDocsUrl('https://example.com')).toBe(false);
76
- });
77
-
78
- it('returns false for Google Sheets URL', () => {
79
- expect(
80
- isGoogleDocsUrl('https://docs.google.com/spreadsheets/d/abc123/edit')
81
- ).toBe(false);
82
- });
83
-
84
- it('returns false for empty string', () => {
85
- expect(isGoogleDocsUrl('')).toBe(false);
86
- });
87
-
88
- it('returns false for null', () => {
89
- expect(isGoogleDocsUrl(null)).toBe(false);
90
- });
91
-
92
- it('returns false for undefined', () => {
93
- expect(isGoogleDocsUrl(undefined)).toBe(false);
94
- });
95
-
96
- it('returns false for non-string input', () => {
97
- expect(isGoogleDocsUrl(123)).toBe(false);
98
- });
99
- });
100
-
101
- describe('extractDocumentId', () => {
102
- it('extracts document ID from standard URL', () => {
103
- expect(
104
- extractDocumentId(
105
- 'https://docs.google.com/document/d/1yn5AWdus0Rh5xRr6voNy-g2wNzdC4bM0CV7SQWch9H4/edit'
106
- )
107
- ).toBe('1yn5AWdus0Rh5xRr6voNy-g2wNzdC4bM0CV7SQWch9H4');
108
- });
109
-
110
- it('extracts document ID from URL with query params', () => {
111
- expect(
112
- extractDocumentId(
113
- 'https://docs.google.com/document/d/abc123/edit?tab=t.0'
114
- )
115
- ).toBe('abc123');
116
- });
117
-
118
- it('extracts document ID with hyphens and underscores', () => {
119
- expect(
120
- extractDocumentId('https://docs.google.com/document/d/abc-123_XYZ/')
121
- ).toBe('abc-123_XYZ');
122
- });
123
-
124
- it('returns null for non-Google Docs URL', () => {
125
- expect(extractDocumentId('https://example.com')).toBeNull();
126
- });
127
-
128
- it('returns null for empty string', () => {
129
- expect(extractDocumentId('')).toBeNull();
130
- });
131
-
132
- it('returns null for null input', () => {
133
- expect(extractDocumentId(null)).toBeNull();
134
- });
135
- });
136
-
137
- describe('buildExportUrl', () => {
138
- it('builds HTML export URL', () => {
139
- expect(buildExportUrl('abc123', 'html')).toBe(
140
- 'https://docs.google.com/document/d/abc123/export?format=html'
141
- );
142
- });
143
-
144
- it('builds Markdown export URL', () => {
145
- expect(buildExportUrl('abc123', 'md')).toBe(
146
- 'https://docs.google.com/document/d/abc123/export?format=md'
147
- );
148
- });
149
-
150
- it('builds PDF export URL', () => {
151
- expect(buildExportUrl('abc123', 'pdf')).toBe(
152
- 'https://docs.google.com/document/d/abc123/export?format=pdf'
153
- );
154
- });
155
-
156
- it('builds TXT export URL', () => {
157
- expect(buildExportUrl('abc123', 'txt')).toBe(
158
- 'https://docs.google.com/document/d/abc123/export?format=txt'
159
- );
160
- });
161
-
162
- it('defaults to html for unknown format', () => {
163
- expect(buildExportUrl('abc123', 'invalid')).toBe(
164
- 'https://docs.google.com/document/d/abc123/export?format=html'
165
- );
166
- });
167
-
168
- it('defaults to html when format is omitted', () => {
169
- expect(buildExportUrl('abc123')).toBe(
170
- 'https://docs.google.com/document/d/abc123/export?format=html'
171
- );
172
- });
173
-
174
- it('uses the real document ID from test document', () => {
175
- const docId = '1yn5AWdus0Rh5xRr6voNy-g2wNzdC4bM0CV7SQWch9H4';
176
- expect(buildExportUrl(docId, 'html')).toBe(
177
- `https://docs.google.com/document/d/${docId}/export?format=html`
178
- );
179
- });
180
- });
181
-
182
- describe('capture method selection (issue #72)', () => {
183
- it('honors explicit browser capture for Google Docs URLs', () => {
184
- expect(selectGoogleDocsCaptureMethod('browser')).toBe('browser-model');
185
- });
186
-
187
- it('uses public export for API capture without token', () => {
188
- expect(selectGoogleDocsCaptureMethod('api')).toBe('public-export');
189
- });
190
-
191
- it('uses Docs REST API for API capture with token', () => {
192
- expect(selectGoogleDocsCaptureMethod('api', 'token-123')).toBe(
193
- 'docs-api'
194
- );
195
- });
196
- });
197
-
198
- describe('buildEditUrl', () => {
199
- it('builds the fresh Google Docs editor URL', () => {
200
- expect(buildEditUrl('abc123')).toBe(
201
- 'https://docs.google.com/document/d/abc123/edit'
202
- );
203
- });
204
- });
205
-
206
- describe('buildDocsApiUrl', () => {
207
- it('builds the Google Docs REST API URL', () => {
208
- expect(buildDocsApiUrl('abc123')).toBe(
209
- 'https://docs.googleapis.com/v1/documents/abc123'
210
- );
211
- });
212
- });
213
-
214
- describe('parseGoogleDocsModelChunks (issue #72)', () => {
215
- it('includes regular and suggested text from DOCS_modelChunk data', () => {
216
- const capture = parseGoogleDocsModelChunks([
217
- {
218
- chunk: [
219
- { ty: 'is', s: 'Stable ' },
220
- { ty: 'iss', s: 'suggested\n' },
221
- ],
222
- },
223
- ]);
224
-
225
- expect(capture.text).toContain('Stable suggested');
226
- expect(renderGoogleDocsCapture(capture, 'markdown')).toContain(
227
- 'Stable suggested'
228
- );
229
- });
230
-
231
- it('accepts individual DOCS_modelChunk items captured from push calls', () => {
232
- const capture = parseGoogleDocsModelChunks([
233
- { ty: 'is', s: 'Pushed item\n' },
234
- ]);
235
-
236
- expect(capture.text).toContain('Pushed item');
237
- expect(renderGoogleDocsCapture(capture, 'markdown')).toContain(
238
- 'Pushed item'
239
- );
240
- });
241
-
242
- it('extracts table cells and suggested images from DOCS_modelChunk data', () => {
243
- const chunks = [
244
- {
245
- chunk: [
246
- { ty: 'is', s: String.fromCharCode(0x10) },
247
- { ty: 'is', s: String.fromCharCode(0x12) },
248
- { ty: 'is', s: String.fromCharCode(0x1c) },
249
- { ty: 'is', s: 'Cell A' },
250
- { ty: 'is', s: String.fromCharCode(0x1c) },
251
- { ty: 'is', s: '*' },
252
- { ty: 'is', s: '\n' },
253
- { ty: 'is', s: String.fromCharCode(0x11) },
254
- {
255
- ty: 'ase',
256
- id: 'suggested-image',
257
- epm: { ee_eo: { i_cid: 'cid_12345678901234567890' } },
258
- },
259
- { ty: 'ste', id: 'suggested-image', spi: 11 },
260
- ],
261
- },
262
- ];
263
- const capture = parseGoogleDocsModelChunks(chunks, {
264
- cid_12345678901234567890:
265
- 'https://docs.google.com/docs-images-rt/image-id',
266
- });
267
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
268
-
269
- expect(capture.tables).toHaveLength(1);
270
- expect(capture.tables[0].rows[0].cells).toHaveLength(2);
271
- expect(markdown).toContain('Cell A');
272
- expect(markdown).toContain(
273
- '![suggested image](https://docs.google.com/docs-images-rt/image-id)'
274
- );
275
- });
276
-
277
- it('parses multi-column tables when rows are separated by 0x0a (issue #92 R2)', () => {
278
- // Google Docs sometimes emits cell boundaries as `\n` (0x0a) alongside
279
- // the 0x12 new-row marker. The parser must keep all cells on the same
280
- // row rather than collapsing the table to a single column.
281
- const chunks = [
282
- {
283
- chunk: [
284
- {
285
- ty: 'is',
286
- s: `${
287
- String.fromCharCode(0x10) + String.fromCharCode(0x12)
288
- }A\nB\nC\n${String.fromCharCode(
289
- 0x12
290
- )}D\nE\nF\n${String.fromCharCode(0x11)}`,
291
- },
292
- ],
293
- },
294
- ];
295
- const capture = parseGoogleDocsModelChunks(chunks);
296
-
297
- expect(capture.tables).toHaveLength(1);
298
- expect(capture.tables[0].rows).toHaveLength(2);
299
- expect(capture.tables[0].rows[0].cells).toHaveLength(3);
300
- expect(capture.tables[0].rows[1].cells).toHaveLength(3);
301
-
302
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
303
- expect(markdown).toContain('| A | B | C |');
304
- expect(markdown).toContain('| D | E | F |');
305
- });
306
-
307
- it('does not create empty columns when Docs emits both 0x1c and 0x0a cell separators (issue #96)', () => {
308
- const chunks = [
309
- {
310
- chunk: [
311
- {
312
- ty: 'is',
313
- s: `${
314
- String.fromCharCode(0x10) + String.fromCharCode(0x12)
315
- }A${String.fromCharCode(0x1c)}\nB${String.fromCharCode(
316
- 0x1c
317
- )}\nC\n${String.fromCharCode(0x11)}`,
318
- },
319
- ],
320
- },
321
- ];
322
- const capture = parseGoogleDocsModelChunks(chunks);
323
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
324
-
325
- expect(capture.tables[0].rows[0].cells).toHaveLength(3);
326
- expect(markdown).toContain('| A | B | C |');
327
- expect(markdown).not.toContain('| A | | B |');
328
- });
329
-
330
- it('does not create ghost columns from the live Google Docs 0x0a+0x1c table separator pattern (issue #100)', () => {
331
- const chunks = [
332
- {
333
- chunk: [
334
- {
335
- ty: 'is',
336
- s: `${String.fromCharCode(0x10)}${String.fromCharCode(
337
- 0x12
338
- )}${String.fromCharCode(0x1c)}Feature\n${String.fromCharCode(
339
- 0x1c
340
- )}Supported\n${String.fromCharCode(0x1c)}Notes\n${String.fromCharCode(
341
- 0x12
342
- )}${String.fromCharCode(0x1c)}Bold\n${String.fromCharCode(
343
- 0x1c
344
- )}Yes\n${String.fromCharCode(
345
- 0x1c
346
- )}Using double asterisks\n${String.fromCharCode(0x11)}`,
347
- },
348
- ],
349
- },
350
- ];
351
- const capture = parseGoogleDocsModelChunks(chunks);
352
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
353
-
354
- expect(capture.tables[0].rows.map((row) => row.cells.length)).toEqual([
355
- 3, 3,
356
- ]);
357
- expect(markdown).toContain('| Feature | Supported | Notes |');
358
- expect(markdown).toContain('| Bold | Yes | Using double asterisks |');
359
- expect(markdown).not.toContain('| Feature | | Supported |');
360
- });
361
-
362
- it('preserves intentionally empty table cells without shifting following cells (issue #100)', () => {
363
- const chunks = [
364
- {
365
- chunk: [
366
- {
367
- ty: 'is',
368
- s: `${String.fromCharCode(0x10)}${String.fromCharCode(
369
- 0x12
370
- )}${String.fromCharCode(0x1c)}A\n${String.fromCharCode(
371
- 0x1c
372
- )}B\n${String.fromCharCode(0x1c)}C\n${String.fromCharCode(
373
- 0x12
374
- )}${String.fromCharCode(0x1c)}\n${String.fromCharCode(
375
- 0x1c
376
- )}x\n${String.fromCharCode(0x1c)}\n${String.fromCharCode(
377
- 0x12
378
- )}${String.fromCharCode(0x1c)}y\n${String.fromCharCode(
379
- 0x1c
380
- )}\n${String.fromCharCode(0x1c)}z\n${String.fromCharCode(0x11)}`,
381
- },
382
- ],
383
- },
384
- ];
385
- const capture = parseGoogleDocsModelChunks(chunks);
386
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
387
-
388
- expect(capture.tables[0].rows.map((row) => row.cells.length)).toEqual([
389
- 3, 3, 3,
390
- ]);
391
- expect(markdown).toContain('| A | B | C |');
392
- expect(markdown).toContain('| | x | |');
393
- expect(markdown).toContain('| y | | z |');
394
- });
395
-
396
- it('numbers ordered list items sequentially (issue #92 R3)', () => {
397
- // Three list items sharing the same list id should render as 1. 2. 3.
398
- const text = 'First item\nSecond item\nThird item\n';
399
- const endOfLine = (n) => {
400
- let idx = -1;
401
- for (let k = 0; k < n; k++) {
402
- idx = text.indexOf('\n', idx + 1);
403
- }
404
- return idx + 1;
405
- };
406
- const chunks = [
407
- {
408
- chunk: [
409
- { ty: 'is', s: text },
410
- {
411
- ty: 'as',
412
- st: 'list',
413
- si: endOfLine(1),
414
- ei: endOfLine(1),
415
- sm: { ls_id: 'kix.list.7' },
416
- },
417
- {
418
- ty: 'as',
419
- st: 'list',
420
- si: endOfLine(2),
421
- ei: endOfLine(2),
422
- sm: { ls_id: 'kix.list.7' },
423
- },
424
- {
425
- ty: 'as',
426
- st: 'list',
427
- si: endOfLine(3),
428
- ei: endOfLine(3),
429
- sm: { ls_id: 'kix.list.7' },
430
- },
431
- ],
432
- },
433
- ];
434
- const exportHtml = `
435
- <html><body>
436
- <ol>
437
- <li>First item</li>
438
- <li>Second item</li>
439
- <li>Third item</li>
440
- </ol>
441
- </body></html>
442
- `;
443
- const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
444
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
445
- const lines = markdown.split('\n');
446
-
447
- expect(lines[0]).toBe('1. First item');
448
- expect(lines[1]).toBe('2. Second item');
449
- expect(lines[2]).toBe('3. Third item');
450
- });
451
-
452
- it('joins consecutive list items with a single newline (issue #92 R4)', () => {
453
- const text = 'First item\nSecond item\n';
454
- const endFirst = text.indexOf('\n') + 1;
455
- const endSecond = text.length;
456
- const chunks = [
457
- {
458
- chunk: [
459
- { ty: 'is', s: text },
460
- {
461
- ty: 'as',
462
- st: 'list',
463
- si: endFirst,
464
- ei: endFirst,
465
- sm: { ls_id: 'kix.list.7' },
466
- },
467
- {
468
- ty: 'as',
469
- st: 'list',
470
- si: endSecond,
471
- ei: endSecond,
472
- sm: { ls_id: 'kix.list.7' },
473
- },
474
- ],
475
- },
476
- ];
477
- const exportHtml = `
478
- <html><body>
479
- <ol>
480
- <li>First item</li>
481
- <li>Second item</li>
482
- </ol>
483
- </body></html>
484
- `;
485
- const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
486
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
487
-
488
- expect(markdown).not.toMatch(/First item\n\n2\. Second item/u);
489
- expect(markdown).toMatch(/First item\n2\. Second item/u);
490
- });
491
-
492
- it('keeps nested ordered lists ordered and tight when Google Docs uses separate list ids per level (issue #100)', () => {
493
- const text = [
494
- 'Parent item 1',
495
- 'Child item 1.1',
496
- 'Child item 1.2',
497
- 'Grandchild item 1.2.1',
498
- 'Grandchild item 1.2.2',
499
- 'Child item 1.3',
500
- 'Parent item 2',
501
- '',
502
- ].join('\n');
503
- const lineEnd = (needle) => text.indexOf('\n', text.indexOf(needle)) + 1;
504
- const chunks = [
505
- {
506
- chunk: [
507
- { ty: 'is', s: text },
508
- {
509
- ty: 'as',
510
- st: 'list',
511
- si: lineEnd('Parent item 1'),
512
- ei: lineEnd('Parent item 1'),
513
- sm: { ls_id: 'kix.list.8' },
514
- },
515
- {
516
- ty: 'as',
517
- st: 'list',
518
- si: lineEnd('Child item 1.1'),
519
- ei: lineEnd('Child item 1.1'),
520
- sm: { ls_id: 'kix.list.9', ls_nest: 1 },
521
- },
522
- {
523
- ty: 'as',
524
- st: 'list',
525
- si: lineEnd('Child item 1.2'),
526
- ei: lineEnd('Child item 1.2'),
527
- sm: { ls_id: 'kix.list.9', ls_nest: 1 },
528
- },
529
- {
530
- ty: 'as',
531
- st: 'list',
532
- si: lineEnd('Grandchild item 1.2.1'),
533
- ei: lineEnd('Grandchild item 1.2.1'),
534
- sm: { ls_id: 'kix.list.10', ls_nest: 2 },
535
- },
536
- {
537
- ty: 'as',
538
- st: 'list',
539
- si: lineEnd('Grandchild item 1.2.2'),
540
- ei: lineEnd('Grandchild item 1.2.2'),
541
- sm: { ls_id: 'kix.list.10', ls_nest: 2 },
542
- },
543
- {
544
- ty: 'as',
545
- st: 'list',
546
- si: lineEnd('Child item 1.3'),
547
- ei: lineEnd('Child item 1.3'),
548
- sm: { ls_id: 'kix.list.9', ls_nest: 1 },
549
- },
550
- {
551
- ty: 'as',
552
- st: 'list',
553
- si: lineEnd('Parent item 2'),
554
- ei: lineEnd('Parent item 2'),
555
- sm: { ls_id: 'kix.list.8' },
556
- },
557
- ],
558
- },
559
- ];
560
- const exportHtml = `
561
- <html><body><ol><li>Parent item 1<ol><li>Child item 1.1</li><li>Child item 1.2<ol><li>Grandchild item 1.2.1</li><li>Grandchild item 1.2.2</li></ol></li><li>Child item 1.3</li></ol></li><li>Parent item 2</li></ol></body></html>
562
- `;
563
- const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
564
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
565
-
566
- expect(markdown).toContain(
567
- [
568
- '1. Parent item 1',
569
- ' 1. Child item 1.1',
570
- ' 2. Child item 1.2',
571
- ' 1. Grandchild item 1.2.1',
572
- ' 2. Grandchild item 1.2.2',
573
- ' 3. Child item 1.3',
574
- '2. Parent item 2',
575
- ].join('\n')
576
- );
577
- expect(markdown).not.toContain('Parent item 1\n\n');
578
- expect(markdown).not.toContain('- Child item 1.1');
579
- });
580
-
581
- it('uses export semantics for ambiguous browser-model lists and continuation paragraphs (issue #108)', () => {
582
- const text = [
583
- 'Apple',
584
- 'Banana',
585
- 'Cherry',
586
- 'Step one',
587
- 'Continuation paragraph that is not a list item. Same indent as Step one.',
588
- 'Step two',
589
- 'Red',
590
- 'Green',
591
- 'Blue',
592
- '',
593
- ].join('\n');
594
- const lineEnd = (needle) => text.indexOf('\n', text.indexOf(needle)) + 1;
595
- const listStyle = (needle, id) => ({
596
- ty: 'as',
597
- st: 'list',
598
- si: lineEnd(needle),
599
- ei: lineEnd(needle),
600
- sm: { ls_id: id },
601
- });
602
- const paragraphStyle = (needle, indentStart, indentFirstLine) => ({
603
- ty: 'as',
604
- st: 'paragraph',
605
- si: lineEnd(needle),
606
- ei: lineEnd(needle),
607
- sm: { ps_il: indentStart, ps_ifl: indentFirstLine },
608
- });
609
- const chunks = [
610
- {
611
- chunk: [
612
- { ty: 'is', s: text },
613
- listStyle('Apple', 'kix.ordered'),
614
- listStyle('Banana', 'kix.ordered'),
615
- listStyle('Cherry', 'kix.ordered'),
616
- listStyle('Step one', 'kix.step-one'),
617
- paragraphStyle(
618
- 'Continuation paragraph that is not a list item',
619
- 36,
620
- 36
621
- ),
622
- listStyle('Step two', 'kix.step-two'),
623
- listStyle('Red', 'kix.bullets'),
624
- listStyle('Green', 'kix.bullets'),
625
- listStyle('Blue', 'kix.bullets'),
626
- {
627
- ty: 'as',
628
- st: 'text',
629
- si: 1,
630
- ei: text.length - 1,
631
- sm: { ts_it: true, ts_it_i: true },
632
- },
633
- ],
634
- },
635
- ];
636
- const exportHtml = `
637
- <html><body>
638
- <ol><li>Apple</li><li>Banana</li><li>Cherry</li></ol>
639
- <ol><li>Step one</li></ol>
640
- <p>Continuation paragraph that is not a list item. Same indent as Step one.</p>
641
- <ol><li>Step two</li></ol>
642
- <ul><li>Red</li><li>Green</li><li>Blue</li></ul>
643
- </body></html>
644
- `;
645
- const capture = parseGoogleDocsModelChunks(chunks, {}, { exportHtml });
646
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
647
- const html = renderGoogleDocsCapture(capture, 'html');
648
-
649
- expect(markdown).toContain('1. Apple\n2. Banana\n3. Cherry');
650
- expect(markdown).not.toContain('- Apple');
651
- expect(markdown).toContain(
652
- 'Continuation paragraph that is not a list item. Same indent as Step one.'
653
- );
654
- expect(markdown).not.toContain(
655
- '> Continuation paragraph that is not a list item.'
656
- );
657
- expect(markdown).toContain('- Red\n- Green\n- Blue');
658
- expect(markdown).not.toContain('*Apple*');
659
- expect(html).toContain('<ol><li>Apple</li></ol>');
660
- expect(html).toContain('<ul><li>Red</li></ul>');
661
- expect(html).not.toContain(
662
- '<blockquote>Continuation paragraph that is not a list item.'
663
- );
664
- });
665
-
666
- it('renders model style records for headings, inline formatting, links, lists, blockquotes, rules, and images', () => {
667
- const text = [
668
- 'Title',
669
- 'This is bold, italic, strike, and link',
670
- '-',
671
- 'Item',
672
- 'Quote',
673
- '*',
674
- '',
675
- ].join('\n');
676
- const startOf = (needle) => text.indexOf(needle) + 1;
677
- const endOf = (needle) => startOf(needle) + needle.length - 1;
678
- const lineEnd = (needle) => text.indexOf('\n', text.indexOf(needle)) + 1;
679
- const chunks = [
680
- {
681
- chunk: [
682
- { ty: 'is', s: text },
683
- {
684
- ty: 'as',
685
- st: 'paragraph',
686
- si: lineEnd('Title'),
687
- ei: lineEnd('Title'),
688
- sm: { ps_hd: 1 },
689
- },
690
- {
691
- ty: 'as',
692
- st: 'text',
693
- si: startOf('bold'),
694
- ei: endOf('bold'),
695
- sm: { ts_bd: true },
696
- },
697
- {
698
- ty: 'as',
699
- st: 'text',
700
- si: startOf('italic'),
701
- ei: endOf('italic'),
702
- sm: { ts_it: true },
703
- },
704
- {
705
- ty: 'as',
706
- st: 'text',
707
- si: startOf('strike'),
708
- ei: endOf('strike'),
709
- sm: { ts_st: true },
710
- },
711
- {
712
- ty: 'as',
713
- st: 'link',
714
- si: startOf('link'),
715
- ei: endOf('link'),
716
- sm: { lnks_link: { ulnk_url: 'https://example.com' } },
717
- },
718
- {
719
- ty: 'as',
720
- st: 'horizontal_rule',
721
- si: startOf('-'),
722
- ei: startOf('-'),
723
- sm: {},
724
- },
725
- {
726
- ty: 'as',
727
- st: 'list',
728
- si: lineEnd('Item'),
729
- ei: lineEnd('Item'),
730
- sm: { ls_id: 'kix.list.3' },
731
- },
732
- {
733
- ty: 'as',
734
- st: 'paragraph',
735
- si: lineEnd('Quote'),
736
- ei: lineEnd('Quote'),
737
- sm: { ps_il: 24, ps_ifl: 24 },
738
- },
739
- {
740
- ty: 'ae',
741
- et: 'inline',
742
- id: 'image-1',
743
- epm: {
744
- ee_eo: {
745
- i_cid: 'cid_12345678901234567890',
746
- eo_ad: 'Blue rectangle',
747
- },
748
- },
749
- },
750
- { ty: 'te', id: 'image-1', spi: startOf('*') },
751
- ],
752
- },
753
- ];
754
-
755
- const capture = parseGoogleDocsModelChunks(chunks, {
756
- cid_12345678901234567890:
757
- 'https://docs.google.com/docs-images-rt/image-id',
758
- });
759
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
760
-
761
- expect(markdown).toContain('# Title');
762
- expect(markdown).toContain(
763
- 'This is **bold**, *italic*, ~~strike~~, and [link](https://example.com)'
764
- );
765
- expect(markdown).toContain('---');
766
- expect(markdown).toContain('- Item');
767
- expect(markdown).toContain('> Quote');
768
- expect(markdown).toContain(
769
- '![Blue rectangle](https://docs.google.com/docs-images-rt/image-id)'
770
- );
771
- });
772
-
773
- it('renders soft breaks outside marks and preserves image dimensions (issue #104)', () => {
774
- const fixture = JSON.parse(
775
- readIssue104Fixture('multiline-marked-inline-image-model.json')
776
- );
777
- const capture = parseGoogleDocsModelChunks(
778
- fixture.chunks,
779
- fixture.cidUrlMap
780
- );
781
-
782
- expect(capture.blocks[0].content).toEqual([
783
- expect.objectContaining({
784
- type: 'text',
785
- text: 'Line one of bold text.',
786
- bold: true,
787
- }),
788
- expect.objectContaining({ type: 'text', text: '\n', bold: false }),
789
- expect.objectContaining({
790
- type: 'text',
791
- text: 'Line two of bold text.',
792
- bold: true,
793
- }),
794
- expect.objectContaining({
795
- type: 'image',
796
- alt: 'Inline diagram',
797
- width: 320,
798
- height: 180,
799
- }),
800
- expect.objectContaining({ type: 'text', text: '\n\n', bold: false }),
801
- expect.objectContaining({
802
- type: 'text',
803
- text: 'Line three of bold text.',
804
- bold: true,
805
- }),
806
- ]);
807
- expect(renderGoogleDocsCapture(capture, 'html')).toBe(
808
- readIssue104Fixture(
809
- 'multiline-marked-inline-image.expected.html'
810
- ).trimEnd()
811
- );
812
- expect(renderGoogleDocsCapture(capture, 'markdown')).toBe(
813
- readIssue104Fixture('multiline-marked-inline-image.expected.md')
814
- );
815
- });
816
-
817
- it('does not keep inline marks open across embedded newlines (issue #104)', () => {
818
- const capture = {
819
- blocks: [
820
- {
821
- type: 'paragraph',
822
- content: [
823
- {
824
- type: 'text',
825
- text: 'Alpha\nBeta',
826
- bold: true,
827
- },
828
- ],
829
- },
830
- ],
831
- };
832
-
833
- expect(renderGoogleDocsCapture(capture, 'html')).toBe(
834
- '<!doctype html><html><body><p><strong>Alpha</strong><br><strong>Beta</strong></p></body></html>'
835
- );
836
- expect(renderGoogleDocsCapture(capture, 'markdown')).toBe(
837
- '**Alpha**\n**Beta**\n'
838
- );
839
- });
840
-
841
- it('keeps nested bold and italic markers balanced (issue #96)', () => {
842
- const text = 'Bold text with italic inside and back to bold\n';
843
- const chunks = [
844
- {
845
- chunk: [
846
- { ty: 'is', s: text },
847
- {
848
- ty: 'as',
849
- st: 'text',
850
- si: 1,
851
- ei: text.length - 1,
852
- sm: { ts_bd: true },
853
- },
854
- {
855
- ty: 'as',
856
- st: 'text',
857
- si: text.indexOf('italic') + 1,
858
- ei: text.indexOf('italic') + 'italic inside'.length,
859
- sm: { ts_it: true },
860
- },
861
- ],
862
- },
863
- ];
864
- const capture = parseGoogleDocsModelChunks(chunks);
865
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
866
-
867
- expect(markdown).toBe(
868
- '**Bold text with *italic inside* and back to bold**\n'
869
- );
870
- });
871
-
872
- it('keeps styled same-target link segments in one link label (issue #96)', () => {
873
- const text = 'Link with bold text\n';
874
- const startOf = (needle) => text.indexOf(needle) + 1;
875
- const endOf = (needle) => startOf(needle) + needle.length - 1;
876
- const chunks = [
877
- {
878
- chunk: [
879
- { ty: 'is', s: text },
880
- {
881
- ty: 'as',
882
- st: 'link',
883
- si: 1,
884
- ei: text.length - 1,
885
- sm: { lnks_link: { ulnk_url: 'https://example.com' } },
886
- },
887
- {
888
- ty: 'as',
889
- st: 'text',
890
- si: startOf('bold'),
891
- ei: endOf('bold'),
892
- sm: { ts_bd: true },
893
- },
894
- ],
895
- },
896
- ];
897
- const capture = parseGoogleDocsModelChunks(chunks);
898
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
899
-
900
- expect(markdown).toBe('[Link with **bold** text](https://example.com)\n');
901
- });
902
-
903
- it('keeps consecutive blockquote paragraphs in the same quote block (issue #96)', () => {
904
- const text = 'Quote paragraph one\nQuote paragraph two\n';
905
- const lineEnd = (needle) => text.indexOf(needle) + needle.length + 1;
906
- const chunks = [
907
- {
908
- chunk: [
909
- { ty: 'is', s: text },
910
- {
911
- ty: 'as',
912
- st: 'paragraph',
913
- si: lineEnd('Quote paragraph one'),
914
- ei: lineEnd('Quote paragraph one'),
915
- sm: { ps_il: 24, ps_ifl: 24 },
916
- },
917
- {
918
- ty: 'as',
919
- st: 'paragraph',
920
- si: lineEnd('Quote paragraph two'),
921
- ei: lineEnd('Quote paragraph two'),
922
- sm: { ps_il: 24, ps_ifl: 24 },
923
- },
924
- ],
925
- },
926
- ];
927
- const capture = parseGoogleDocsModelChunks(chunks);
928
- const markdown = renderGoogleDocsCapture(capture, 'markdown');
929
-
930
- expect(markdown).toBe(
931
- '> Quote paragraph one\n>\n> Quote paragraph two\n'
932
- );
933
- });
934
-
935
- it('ends rendered markdown with a newline (issue #96)', () => {
936
- const capture = parseGoogleDocsModelChunks([
937
- { ty: 'is', s: 'Last line\n' },
938
- ]);
939
-
940
- expect(renderGoogleDocsCapture(capture, 'markdown')).toBe('Last line\n');
941
- });
942
- });
943
-
944
- describe('renderDocsApiDocument (issue #72)', () => {
945
- it('renders Google Docs REST API paragraphs, tables, and inline images', () => {
946
- const apiDocument = {
947
- title: 'API Doc',
948
- body: {
949
- content: [
950
- {
951
- paragraph: {
952
- elements: [{ textRun: { content: 'Intro paragraph\n' } }],
953
- },
954
- },
955
- {
956
- table: {
957
- tableRows: [
958
- {
959
- tableCells: [
960
- {
961
- content: [
962
- {
963
- paragraph: {
964
- elements: [{ textRun: { content: 'Name\n' } }],
965
- },
966
- },
967
- ],
968
- },
969
- {
970
- content: [
971
- {
972
- paragraph: {
973
- elements: [
974
- {
975
- inlineObjectElement: {
976
- inlineObjectId: 'image-1',
977
- },
978
- },
979
- ],
980
- },
981
- },
982
- ],
983
- },
984
- ],
985
- },
986
- ],
987
- },
988
- },
989
- ],
990
- },
991
- inlineObjects: {
992
- 'image-1': {
993
- inlineObjectProperties: {
994
- embeddedObject: {
995
- title: 'Diagram',
996
- imageProperties: {
997
- contentUri: 'https://example.com/diagram.png',
998
- },
999
- },
1000
- },
1001
- },
1002
- },
1003
- };
1004
-
1005
- const rendered = renderDocsApiDocument(apiDocument);
1006
-
1007
- expect(rendered.markdown).toContain('Intro paragraph');
1008
- expect(rendered.markdown).toContain('| Name | ![Diagram]');
1009
- expect(rendered.html).toContain('<table>');
1010
- expect(rendered.html).toContain('src="https://example.com/diagram.png"');
1011
- });
1012
- });
1013
-
1014
- describe('captureGoogleDocWithBrowserOrFallback (issue #81)', () => {
1015
- afterEach(() => {
1016
- nock.cleanAll();
1017
- });
1018
-
1019
- it('falls back to public export archive when the editor exposes no model chunks', async () => {
1020
- const tinyPng =
1021
- 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
1022
- nock('https://docs.google.com')
1023
- .get('/document/d/fallback-doc/export')
1024
- .query({ format: 'html' })
1025
- .reply(
1026
- 200,
1027
- `<html><body><h1>Fallback Doc</h1><p>Complete text</p><img src="data:image/png;base64,${tinyPng}" alt="diagram"></body></html>`,
1028
- { 'content-type': 'text/html; charset=utf-8' }
1029
- );
1030
-
1031
- const fallbackErrors = [];
1032
- const result = await captureGoogleDocWithBrowserOrFallback(
1033
- 'https://docs.google.com/document/d/fallback-doc/edit',
1034
- {
1035
- format: 'archive',
1036
- waitMs: 0,
1037
- createBrowser: async () =>
1038
- fakeBrowserReturningModel({ chunks: [], cidUrlMap: {} }),
1039
- onFallback: (err) => fallbackErrors.push(err),
1040
- }
1041
- );
1042
-
1043
- expect(result.method).toBe('public-export');
1044
- expect(result.fallback).toBe(true);
1045
- expect(result.markdown).toContain('Fallback Doc');
1046
- expect(result.markdown).toContain('Complete text');
1047
- expect(result.images).toHaveLength(1);
1048
- expect(result.html).toContain('images/image-01.png');
1049
- expect(fallbackErrors).toHaveLength(1);
1050
- expect(isGoogleDocsBrowserModelUnavailableError(fallbackErrors[0])).toBe(
1051
- true
1052
- );
1053
- expect(nock.isDone()).toBe(true);
1054
- });
1055
- });
1056
-
1057
- describe('captureGoogleDocWithBrowser (issue #106)', () => {
1058
- it('waits for DOCS_modelChunk data to stop changing before parsing', async () => {
1059
- const browser = fakeBrowserReturningModelSequence([
1060
- {
1061
- chunks: [{ chunk: [{ ty: 'is', s: 'First paragraph\n' }] }],
1062
- cidUrlMap: {},
1063
- },
1064
- {
1065
- chunks: [{ chunk: [{ ty: 'is', s: 'First paragraph\n' }] }],
1066
- cidUrlMap: {},
1067
- },
1068
- {
1069
- chunks: [
1070
- { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
1071
- { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
1072
- ],
1073
- cidUrlMap: {},
1074
- },
1075
- {
1076
- chunks: [
1077
- { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
1078
- { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
1079
- ],
1080
- cidUrlMap: {},
1081
- },
1082
- {
1083
- chunks: [
1084
- { chunk: [{ ty: 'is', s: 'First paragraph\n' }] },
1085
- { chunk: [{ ty: 'is', s: 'Second paragraph\n' }] },
1086
- ],
1087
- cidUrlMap: {},
1088
- },
1089
- ]);
1090
-
1091
- const result = await captureGoogleDocWithBrowser(
1092
- 'https://docs.google.com/document/d/quiescence-doc/edit',
1093
- {
1094
- waitMs: 0,
1095
- modelMaxWaitMs: 200,
1096
- modelPollMs: 2,
1097
- modelStabilityMs: 2,
1098
- createBrowser: async () => browser,
1099
- }
1100
- );
1101
-
1102
- expect(result.markdown).toContain('First paragraph');
1103
- expect(result.markdown).toContain('Second paragraph');
1104
- expect(browser.evaluateCalls).toBeGreaterThan(2);
1105
- });
1106
- });
1107
-
1108
- describe('GDOCS_EXPORT_FORMATS', () => {
1109
- it('contains all expected formats', () => {
1110
- expect(GDOCS_EXPORT_FORMATS).toEqual({
1111
- html: 'html',
1112
- txt: 'txt',
1113
- md: 'md',
1114
- pdf: 'pdf',
1115
- docx: 'docx',
1116
- epub: 'epub',
1117
- zip: 'zip',
1118
- });
1119
- });
1120
- });
1121
-
1122
- describe('localizeGoogleDocsModelImages (issue #92 R5)', () => {
1123
- it('downloads docs-images-rt URLs and rewrites markdown/html', async () => {
1124
- const url =
1125
- 'https://docs.google.com/docs-images-rt/doc-id/cid/example.png';
1126
- const modelResult = {
1127
- markdown: `![pic](${url})`,
1128
- html: `<img src="${url}" alt="pic">`,
1129
- capture: {
1130
- images: [{ type: 'image', url, alt: 'pic', cid: 'cid-abc' }],
1131
- },
1132
- };
1133
- const fakeFetch = jest.fn().mockResolvedValue({
1134
- ok: true,
1135
- status: 200,
1136
- headers: new Map([['content-type', 'image/png']]),
1137
- arrayBuffer: async () => new Uint8Array([1, 2, 3, 4]).buffer,
1138
- });
1139
- // The map-based headers need a `get` method in the correct shape:
1140
- fakeFetch.mockResolvedValue({
1141
- ok: true,
1142
- status: 200,
1143
- headers: { get: () => 'image/png' },
1144
- arrayBuffer: async () =>
1145
- new Uint8Array([0x89, 0x50, 0x4e, 0x47]).buffer,
1146
- });
1147
-
1148
- const localized = await localizeGoogleDocsModelImages(modelResult, {
1149
- fetchImpl: fakeFetch,
1150
- });
1151
-
1152
- expect(fakeFetch).toHaveBeenCalledTimes(1);
1153
- expect(localized.images).toHaveLength(1);
1154
- expect(localized.images[0].filename).toBe('image-01.png');
1155
- expect(localized.images[0].mimeType).toBe('image/png');
1156
- expect(localized.images[0].data).toBeInstanceOf(Buffer);
1157
- expect(localized.markdown).toBe('![pic](images/image-01.png)');
1158
- expect(localized.html).toContain('images/image-01.png');
1159
- expect(localized.html).not.toContain('docs-images-rt');
1160
- });
1161
-
1162
- it('keeps original URL when download fails', async () => {
1163
- const url =
1164
- 'https://docs.google.com/docs-images-rt/doc-id/cid/missing.png';
1165
- const modelResult = {
1166
- markdown: `![pic](${url})`,
1167
- html: `<img src="${url}" alt="pic">`,
1168
- capture: { images: [{ url, alt: 'pic' }] },
1169
- };
1170
- const fakeFetch = jest.fn().mockResolvedValue({
1171
- ok: false,
1172
- status: 404,
1173
- headers: { get: () => null },
1174
- });
1175
-
1176
- const localized = await localizeGoogleDocsModelImages(modelResult, {
1177
- fetchImpl: fakeFetch,
1178
- });
1179
-
1180
- expect(localized.images).toHaveLength(0);
1181
- // Markdown still gets rewritten (local path) even if download fails —
1182
- // callers may choose to ignore this; the key guarantee is that no
1183
- // exception leaks out.
1184
- expect(localized.markdown).toBe('![pic](images/image-01.png)');
1185
- });
1186
- });
1187
-
1188
- describe('preprocessGoogleDocsExportHtml (issue #92 R6)', () => {
1189
- it('hoists font-weight:700 spans into <strong>', () => {
1190
- const html = '<p><span style="font-weight:700">Bold text</span></p>';
1191
- const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
1192
- expect(hoisted).toBe(1);
1193
- expect(out).toContain('<strong>');
1194
- expect(out).toContain('Bold text');
1195
- });
1196
-
1197
- it('hoists font-style:italic spans into <em>', () => {
1198
- const html = '<p><span style="font-style:italic">Italic text</span></p>';
1199
- const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
1200
- expect(hoisted).toBe(1);
1201
- expect(out).toContain('<em>');
1202
- });
1203
-
1204
- it('hoists text-decoration:line-through spans into <del>', () => {
1205
- const html =
1206
- '<p><span style="text-decoration:line-through">Strike</span></p>';
1207
- const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
1208
- expect(hoisted).toBe(1);
1209
- expect(out).toContain('<del>');
1210
- });
1211
-
1212
- it('hoists Google Docs CSS class styles into semantic inline tags', () => {
1213
- const html =
1214
- '<style>.c7{font-weight:700}.c19{font-style:italic}.c21{text-decoration:line-through}</style>' +
1215
- '<p><span class="c7">Bold</span> <span class="c19">Italic</span> <span class="c21">Strike</span></p>';
1216
- const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
1217
-
1218
- expect(hoisted).toBe(3);
1219
- expect(out).toContain('<strong>Bold</strong>');
1220
- expect(out).toContain('<em>Italic</em>');
1221
- expect(out).toContain('<del>Strike</del>');
1222
- });
1223
-
1224
- it('combines multiple styles on a single span', () => {
1225
- const html =
1226
- '<p><span style="font-weight:700;font-style:italic;text-decoration:line-through">Mixed</span></p>';
1227
- const { html: out, hoisted } = preprocessGoogleDocsExportHtml(html);
1228
- expect(hoisted).toBe(1);
1229
- // Wrappers appear in <strong><em><del>... order (outer-to-inner).
1230
- expect(out).toContain('<strong>');
1231
- expect(out).toContain('<em>');
1232
- expect(out).toContain('<del>');
1233
- });
1234
-
1235
- it('unwraps google.com/url?q= redirect links', () => {
1236
- const html =
1237
- '<p><a href="https://www.google.com/url?q=https://example.com&sa=D&source=editors&usg=ABC">Link</a></p>';
1238
- const { html: out, unwrappedLinks } =
1239
- preprocessGoogleDocsExportHtml(html);
1240
- expect(unwrappedLinks).toBe(1);
1241
- expect(out).toContain('href="https://example.com"');
1242
- expect(out).not.toContain('google.com/url?q=');
1243
- });
1244
-
1245
- it('strips leading empty anchors and numbering span inside headings', () => {
1246
- const html = '<h1><a id="anchor-1"></a><span>1. </span>Headings</h1>';
1247
- const { html: out } = preprocessGoogleDocsExportHtml(html);
1248
- expect(out).toContain('<h1>Headings</h1>');
1249
- expect(out).not.toContain('1. ');
1250
- expect(out).not.toContain('<a id=');
1251
- });
1252
-
1253
- it('strips standalone empty anchors before headings', () => {
1254
- const html = '<a id="anchor-1"></a><h2>Headings</h2>';
1255
- const { html: out } = preprocessGoogleDocsExportHtml(html);
1256
-
1257
- expect(out).toContain('<h2>Headings</h2>');
1258
- expect(out).not.toContain('<a id=');
1259
- });
1260
-
1261
- it('turns class-indented Google Docs paragraphs into blockquotes', () => {
1262
- const html =
1263
- '<style>.c18{margin-left:24pt;margin-right:24pt}</style><p class="c18">Quote</p>';
1264
- const { html: out } = preprocessGoogleDocsExportHtml(html);
1265
-
1266
- expect(out).toContain('<blockquote><p>Quote</p></blockquote>');
1267
- });
1268
-
1269
- it('replaces &nbsp; entity and U+00A0 with regular spaces', () => {
1270
- const html = '<p>A\u00A0B&nbsp;C</p>';
1271
- const { html: out } = preprocessGoogleDocsExportHtml(html);
1272
- expect(out).toContain('A B C');
1273
- expect(out).not.toMatch(/\u00A0/);
1274
- expect(out).not.toContain('&nbsp;');
1275
- });
1276
-
1277
- it('is a no-op for HTML without Google Docs markers', () => {
1278
- const html = '<p>Plain text with <strong>bold</strong>.</p>';
1279
- const {
1280
- html: out,
1281
- hoisted,
1282
- unwrappedLinks,
1283
- } = preprocessGoogleDocsExportHtml(html);
1284
- expect(hoisted).toBe(0);
1285
- expect(unwrappedLinks).toBe(0);
1286
- expect(out).toContain('<p>Plain text with <strong>bold</strong>.</p>');
1287
- });
1288
-
1289
- it('recovers public-export markdown structure for issue #102', () => {
1290
- const html = `
1291
- <style>
1292
- .c5{margin-left:36pt}
1293
- .c8{margin-left:72pt}
1294
- .c19{margin-left:108pt}
1295
- .q{margin-left:24pt;margin-right:24pt}
1296
- .i{font-style:italic}
1297
- .s{text-decoration:line-through}
1298
- </style>
1299
- <h2><span class="i">1. Headings</span></h2>
1300
- <p class="q">Quote one.</p>
1301
- <p class="q">Quote two.</p>
1302
- <ol><li class="c5">Parent</li></ol>
1303
- <ol><li class="c8">Child</li></ol>
1304
- <ol><li class="c19">Grandchild</li></ol>
1305
- <ol><li class="c5">Parent 2</li></ol>
1306
- <table>
1307
- <thead>
1308
- <tr><td><p>Feature</p></td><td><p>Supported</p></td><tbody></tbody></tr>
1309
- <tr><td><p><span class="s">Strike</span></p></td><td><p>Yes</p></td></tr>
1310
- </thead>
1311
- </table>
1312
- `;
1313
-
1314
- const preprocessed = preprocessGoogleDocsExportHtml(html);
1315
- const markdown = normalizeGoogleDocsExportMarkdown(
1316
- convertHtmlToMarkdown(preprocessed.html)
1317
- );
1318
-
1319
- expect(markdown).toContain('## 1. Headings');
1320
- expect(markdown).not.toContain('*1. Headings*');
1321
- expect(markdown).toContain('> Quote one.\n> \n> Quote two.');
1322
- expect(markdown).toMatch(
1323
- /1\.\s+Parent\n\s+1\.\s+Child\n\s+1\.\s+Grandchild\n2\.\s+Parent 2/
1324
- );
1325
- expect(markdown).toContain('| Feature | Supported |');
1326
- expect(markdown).toContain('| ~~Strike~~ | Yes |');
1327
- });
1328
- });
1329
-
1330
- describe('extractBase64Images', () => {
1331
- it('extracts a single PNG data URI image', () => {
1332
- const html = '<img src="data:image/png;base64,iVBORw0KGgo=" alt="test">';
1333
- const { html: updated, images } = extractBase64Images(html);
1334
-
1335
- expect(images).toHaveLength(1);
1336
- expect(images[0].filename).toBe('image-01.png');
1337
- expect(images[0].mimeType).toBe('image/png');
1338
- expect(images[0].data).toBeInstanceOf(Buffer);
1339
- expect(updated).toContain('src="images/image-01.png"');
1340
- expect(updated).not.toContain('data:image');
1341
- });
1342
-
1343
- it('extracts multiple images with correct numbering and extensions', () => {
1344
- const html =
1345
- '<img src="data:image/png;base64,AAAA" alt="a"><img src="data:image/jpeg;base64,BBBB" alt="b">';
1346
- const { html: updated, images } = extractBase64Images(html);
1347
-
1348
- expect(images).toHaveLength(2);
1349
- expect(images[0].filename).toBe('image-01.png');
1350
- expect(images[1].filename).toBe('image-02.jpg');
1351
- expect(images[1].mimeType).toBe('image/jpeg');
1352
- expect(updated).toContain('images/image-01.png');
1353
- expect(updated).toContain('images/image-02.jpg');
1354
- });
1355
-
1356
- it('returns empty array when no data URI images', () => {
1357
- const html = '<p>No images here</p>';
1358
- const { html: updated, images } = extractBase64Images(html);
1359
-
1360
- expect(images).toHaveLength(0);
1361
- expect(updated).toBe(html);
1362
- });
1363
-
1364
- it('preserves non-data-URI images', () => {
1365
- const html =
1366
- '<img src="https://example.com/photo.png" alt="remote"><img src="data:image/gif;base64,R0lG" alt="local">';
1367
- const { html: updated, images } = extractBase64Images(html);
1368
-
1369
- expect(images).toHaveLength(1);
1370
- expect(images[0].filename).toBe('image-01.gif');
1371
- expect(updated).toContain('https://example.com/photo.png');
1372
- expect(updated).toContain('images/image-01.gif');
1373
- });
1374
- });
1375
-
1376
- describe('image extraction pipeline (issue #53)', () => {
1377
- const TINY_PNG =
1378
- 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==';
1379
- let tmpDir;
1380
-
1381
- beforeEach(() => {
1382
- tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gdocs-issue53-'));
1383
- });
1384
-
1385
- afterEach(() => {
1386
- fs.rmSync(tmpDir, { recursive: true, force: true });
1387
- });
1388
-
1389
- it('extractBase64Images from HTML produces images with data buffers', () => {
1390
- const html = `<html><body><img src="data:image/png;base64,${TINY_PNG}" alt="photo"><p>Hello</p></body></html>`;
1391
- const { html: localHtml, images } = extractBase64Images(html);
1392
-
1393
- expect(images).toHaveLength(1);
1394
- expect(images[0].filename).toBe('image-01.png');
1395
- expect(images[0].data.length).toBeGreaterThan(0);
1396
- // Verify PNG magic bytes in extracted data
1397
- expect(images[0].data[0]).toBe(0x89);
1398
- expect(images[0].data[1]).toBe(0x50); // P
1399
- expect(localHtml).toContain('images/image-01.png');
1400
- expect(localHtml).not.toContain('data:image');
1401
- });
1402
-
1403
- it('default markdown mode extracts base64 images to disk (not strip)', () => {
1404
- const markdown = `# Hello\n\n![photo](data:image/png;base64,${TINY_PNG})\n\nEnd.`;
1405
- const result = extractAndSaveImages(markdown, tmpDir);
1406
-
1407
- expect(result.extracted).toBe(1);
1408
- expect(result.markdown).not.toContain('data:image');
1409
- expect(result.markdown).toContain('images/image-');
1410
- expect(result.markdown).toContain('.png');
1411
-
1412
- const imagesDir = path.join(tmpDir, 'images');
1413
- expect(fs.existsSync(imagesDir)).toBe(true);
1414
- const files = fs.readdirSync(imagesDir);
1415
- expect(files).toHaveLength(1);
1416
- expect(files[0]).toMatch(/^image-.*\.png$/);
1417
-
1418
- const imgBuf = fs.readFileSync(path.join(imagesDir, files[0]));
1419
- expect(imgBuf[0]).toBe(0x89); // PNG magic
1420
- });
1421
-
1422
- it('keepOriginalLinks strips base64 images', () => {
1423
- const markdown = `![photo](data:image/png;base64,${TINY_PNG})`;
1424
- const result = stripBase64Images(markdown);
1425
-
1426
- expect(result.stripped).toBe(1);
1427
- expect(result.markdown).toBe('*[image: photo]*');
1428
- expect(result.markdown).not.toContain('data:image');
1429
- });
1430
-
1431
- it('extractBase64ToBuffers produces in-memory buffers for archive', () => {
1432
- const markdown = `![test](data:image/png;base64,${TINY_PNG})`;
1433
- const result = extractBase64ToBuffers(markdown);
1434
-
1435
- expect(result.images).toHaveLength(1);
1436
- expect(result.images[0].buffer.length).toBeGreaterThan(0);
1437
- expect(result.images[0].buffer[0]).toBe(0x89); // PNG magic
1438
- expect(result.markdown).toContain('images/');
1439
- expect(result.markdown).not.toContain('data:image');
1440
- });
1441
- });
1442
- });
1443
-
1444
- function fakeBrowserReturningModel(modelData) {
1445
- return {
1446
- async newPage() {
1447
- return {
1448
- async addInitScript() {},
1449
- async setUserAgent() {},
1450
- async setExtraHTTPHeaders() {},
1451
- async setViewport() {},
1452
- async goto() {},
1453
- async waitForTimeout() {},
1454
- async evaluate() {
1455
- return modelData;
1456
- },
1457
- async close() {},
1458
- };
1459
- },
1460
- async close() {},
1461
- };
1462
- }
1463
-
1464
- function fakeBrowserReturningModelSequence(modelDataSequence) {
1465
- let evaluateCalls = 0;
1466
- return {
1467
- get evaluateCalls() {
1468
- return evaluateCalls;
1469
- },
1470
- async newPage() {
1471
- return {
1472
- async addInitScript() {},
1473
- async setUserAgent() {},
1474
- async setExtraHTTPHeaders() {},
1475
- async setViewport() {},
1476
- async goto() {},
1477
- async waitForTimeout(waitMs) {
1478
- await new Promise((resolve) => setTimeout(resolve, waitMs));
1479
- },
1480
- async evaluate() {
1481
- const index = Math.min(evaluateCalls, modelDataSequence.length - 1);
1482
- evaluateCalls += 1;
1483
- return JSON.parse(JSON.stringify(modelDataSequence[index]));
1484
- },
1485
- async close() {},
1486
- };
1487
- },
1488
- async close() {},
1489
- };
1490
- }