@link-assistant/web-capture 1.12.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/README.md +2 -1
  3. package/bin/web-capture.js +5 -7
  4. package/package.json +32 -18
  5. package/src/archive.js +1 -2
  6. package/src/binary-download.js +13 -0
  7. package/src/docx.js +14 -11
  8. package/src/fetch.js +1 -2
  9. package/src/figures.js +2 -12
  10. package/src/gdocs-images.js +0 -1
  11. package/src/gdocs.js +0 -1
  12. package/src/github.js +0 -1
  13. package/src/lib.js +1 -2
  14. package/src/localize-images.js +2 -12
  15. package/src/markdown.js +0 -1
  16. package/src/shared-dialog.js +0 -1
  17. package/src/stream.js +0 -1
  18. package/src/transport.js +23 -5
  19. package/src/txt.js +0 -1
  20. package/.changeset/README.md +0 -8
  21. package/.changeset/config.json +0 -11
  22. package/.dockerignore +0 -6
  23. package/.jscpd.json +0 -19
  24. package/.lenv +0 -8
  25. package/.prettierignore +0 -8
  26. package/.prettierrc +0 -10
  27. package/Dockerfile +0 -52
  28. package/babel.config.cjs +0 -13
  29. package/docker-compose.yml +0 -11
  30. package/eslint.config.js +0 -112
  31. package/examples/js/capture_cache.js +0 -12
  32. package/examples/js/engine_comparison.js +0 -39
  33. package/examples/js/html_download.js +0 -23
  34. package/examples/js/image_download.js +0 -36
  35. package/examples/js/markdown_download.js +0 -23
  36. package/examples/js/playwright_example.js +0 -23
  37. package/examples/python/html_download.py +0 -15
  38. package/examples/python/image_download.py +0 -20
  39. package/examples/python/markdown_download.py +0 -15
  40. package/examples/python/playwright_example.py +0 -22
  41. package/jest.config.mjs +0 -20
  42. package/tests/ci/dependency-freshness.test.js +0 -74
  43. package/tests/ci/detect-code-changes.test.js +0 -174
  44. package/tests/ci/docker-install.test.js +0 -31
  45. package/tests/ci/rust-dockerfile.test.js +0 -54
  46. package/tests/ci/workflow-policy.test.js +0 -79
  47. package/tests/e2e/docker.test.js +0 -186
  48. package/tests/e2e/process.test.js +0 -215
  49. package/tests/fixtures/bold-coalesce-and-leakage.html +0 -27
  50. package/tests/fixtures/deepwiki/deepwiki-full-page.png +0 -0
  51. package/tests/fixtures/deepwiki/index.md +0 -1500
  52. package/tests/fixtures/deepwiki/part-1.md +0 -776
  53. package/tests/fixtures/habr-articles.js +0 -45
  54. package/tests/fixtures/heading-numbering.html +0 -19
  55. package/tests/fixtures/list-item-with-br.html +0 -8
  56. package/tests/fixtures/ol-continuous-numbering.html +0 -12
  57. package/tests/fixtures/paragraph-vs-line-break.html +0 -11
  58. package/tests/helpers/nock.js +0 -10
  59. package/tests/helpers/search-fixture.cjs +0 -17
  60. package/tests/helpers/temporary-directory.js +0 -12
  61. package/tests/integration/api-endpoints.test.js +0 -418
  62. package/tests/integration/archive-default-layout.test.js +0 -37
  63. package/tests/integration/browser-engines.test.js +0 -128
  64. package/tests/integration/browser-packaged-path.test.js +0 -45
  65. package/tests/integration/deepwiki-download.test.js +0 -144
  66. package/tests/integration/deepwiki-rendered-page.mock.test.js +0 -121
  67. package/tests/integration/formalai-contract.test.js +0 -214
  68. package/tests/integration/gdocs-public-doc.test.js +0 -315
  69. package/tests/integration/github-readme.test.js +0 -137
  70. package/tests/integration/google-drive-image.test.js +0 -246
  71. package/tests/integration/habr-article.test.js +0 -337
  72. package/tests/integration/search-endpoint.test.js +0 -88
  73. package/tests/integration/stackoverflow-download.test.js +0 -173
  74. package/tests/integration/wikipedia-download.test.js +0 -142
  75. package/tests/jest.setup.mjs +0 -7
  76. package/tests/mock/index.test.js +0 -225
  77. package/tests/unit/animation.test.js +0 -33
  78. package/tests/unit/batch.test.js +0 -114
  79. package/tests/unit/bold-coalesce-and-leakage.test.js +0 -54
  80. package/tests/unit/browser-launch-options.test.js +0 -42
  81. package/tests/unit/browser.test.js +0 -172
  82. package/tests/unit/cache.test.js +0 -182
  83. package/tests/unit/cli.test.js +0 -399
  84. package/tests/unit/convertGoogleDriveUrl.test.js +0 -89
  85. package/tests/unit/convertRelativeUrls.test.js +0 -34
  86. package/tests/unit/extract-images.test.js +0 -259
  87. package/tests/unit/fetch-html.test.js +0 -54
  88. package/tests/unit/figures.test.js +0 -74
  89. package/tests/unit/gdocs-export-headers.test.js +0 -28
  90. package/tests/unit/gdocs-preprocess.test.js +0 -36
  91. package/tests/unit/gdocs.test.js +0 -1490
  92. package/tests/unit/github.test.js +0 -156
  93. package/tests/unit/heading-image.test.js +0 -49
  94. package/tests/unit/heading-numbering.test.js +0 -28
  95. package/tests/unit/html2md-br-in-list-item.test.js +0 -23
  96. package/tests/unit/html2md-ol-numbering.test.js +0 -20
  97. package/tests/unit/html2md.test.js +0 -695
  98. package/tests/unit/image-mode-defaults.test.js +0 -54
  99. package/tests/unit/kreuzberg.test.js +0 -249
  100. package/tests/unit/latex.test.js +0 -123
  101. package/tests/unit/localize-images.test.js +0 -114
  102. package/tests/unit/markdown.test.js +0 -36
  103. package/tests/unit/metadata.test.js +0 -140
  104. package/tests/unit/package-runtime-dependencies.test.js +0 -74
  105. package/tests/unit/paragraph-vs-line-break.test.js +0 -56
  106. package/tests/unit/postprocess.test.js +0 -110
  107. package/tests/unit/proxy-headers.test.js +0 -100
  108. package/tests/unit/publish-verification.test.js +0 -57
  109. package/tests/unit/retry.test.js +0 -111
  110. package/tests/unit/search.test.js +0 -339
  111. package/tests/unit/shared-dialog.test.js +0 -173
  112. package/tests/unit/stackoverflow.test.js +0 -32
  113. package/tests/unit/stream.test.js +0 -85
  114. package/tests/unit/verify.test.js +0 -182
  115. package/tests/xpaste/xpaste-screenshot-markdown.mock.test.js +0 -375
  116. package/tests/xpaste/xpaste.e2e.test.js +0 -36
  117. package/tests/xpaste/xpaste.integration.test.js +0 -41
  118. package/tests/xpaste/xpaste.mock.test.js +0 -261
  119. package/tests/xpaste/xpaste.unit.test.js +0 -119
@@ -1,315 +0,0 @@
1
- /**
2
- * Integration tests for the Google Docs markdown round-trip reference document.
3
- *
4
- * The document at
5
- * https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit
6
- * is a public document created specifically for validating web-capture's Google
7
- * Docs support across every URL variation and every Markdown feature category
8
- * documented in issue #90. The ground-truth markdown and authoring archive are
9
- * preserved under docs/case-studies/issue-90/reference/.
10
- *
11
- * Live capture tests are gated behind GDOCS_INTEGRATION=true so normal PR CI
12
- * stays hermetic. The URL-variation, document-id extraction and feature
13
- * checklist tests always run so regressions in the parsing surface are caught
14
- * even without network access.
15
- */
16
-
17
- import { jest } from '@jest/globals';
18
- import fs from 'fs';
19
- import path from 'path';
20
- import { fileURLToPath } from 'url';
21
-
22
- import {
23
- isGoogleDocsUrl,
24
- extractDocumentId,
25
- buildExportUrl,
26
- buildEditUrl,
27
- buildDocsApiUrl,
28
- selectGoogleDocsCaptureMethod,
29
- fetchGoogleDocAsMarkdown,
30
- captureGoogleDocWithBrowser,
31
- } from '../../src/gdocs.js';
32
- import { retry } from '../../src/retry.js';
33
-
34
- const LIVE =
35
- process.env.GDOCS_INTEGRATION === 'true' ||
36
- process.env.GDOCS_INTEGRATION === '1';
37
- const describeIfLive = LIVE ? describe : describe.skip;
38
-
39
- jest.setTimeout(120000);
40
-
41
- const __filename = fileURLToPath(import.meta.url);
42
- const __dirname = path.dirname(__filename);
43
- const REFERENCE_DIR = path.resolve(
44
- __dirname,
45
- '..',
46
- '..',
47
- '..',
48
- 'docs',
49
- 'case-studies',
50
- 'issue-90',
51
- 'reference'
52
- );
53
-
54
- // The public document referenced in issue #90.
55
- // https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit
56
- export const PUBLIC_TEST_DOCUMENT = {
57
- id: '1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM',
58
- canonicalUrl:
59
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit',
60
- urlVariations: [
61
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM',
62
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit',
63
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit?usp=sharing',
64
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit?usp=sharing&ouid=102030405060708090100&rtpof=true&sd=true',
65
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit?tab=t.0',
66
- ],
67
- // Section headings from the reference markdown. Each one should be present
68
- // in any Markdown produced from the document, regardless of capture mode.
69
- // See docs/case-studies/issue-90/reference/markdown-test-document.md.
70
- sections: [
71
- 'Markdown Feature Test Document',
72
- '1. Headings',
73
- '2. Inline Formatting',
74
- '3. Paragraphs',
75
- '4. Blockquotes',
76
- '5. Unordered Lists',
77
- '6. Ordered Lists',
78
- '7. Mixed Lists',
79
- '8. Tables',
80
- '9. Links',
81
- '10. Images',
82
- '11. Horizontal Rules',
83
- '12. Special Characters',
84
- '13. Nested Formatting Edge Cases',
85
- '14. Empty and Minimal Table Content',
86
- ],
87
- };
88
-
89
- // The issue #108 v2 document extends the public reference fixture with Section
90
- // 15, covering browser-model list type and continuation-paragraph regressions.
91
- export const PUBLIC_TEST_DOCUMENT_V2 = {
92
- id: '1Rvaod_u2wgkAUNdXG-e29yQem-P36KAQ',
93
- canonicalUrl:
94
- 'https://docs.google.com/document/d/1Rvaod_u2wgkAUNdXG-e29yQem-P36KAQ/edit',
95
- };
96
-
97
- function expectIssue108Markdown(markdown) {
98
- expect(markdown).toContain(
99
- '15. List & Blockquote Edge Cases (browser-mode reproducers)'
100
- );
101
- expect(markdown).toMatch(/1\.\s+Apple/u);
102
- expect(markdown).toMatch(/2\.\s+Banana/u);
103
- expect(markdown).toMatch(/3\.\s+Cherry/u);
104
- expect(markdown).not.toMatch(/^-\s+Apple$/mu);
105
- expect(markdown).toContain(
106
- 'Continuation paragraph that is not a list item. Same indent as Step one.'
107
- );
108
- expect(markdown).not.toContain(
109
- '> Continuation paragraph that is not a list item.'
110
- );
111
- expect(markdown).toMatch(/^-\s+Red$/mu);
112
- expect(markdown).toMatch(/^-\s+Green$/mu);
113
- expect(markdown).toMatch(/^-\s+Blue$/mu);
114
- }
115
-
116
- describe('Google Docs public test document (issue #90)', () => {
117
- describe('URL variations', () => {
118
- it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
119
- 'accepts %s as a Google Docs URL',
120
- (url) => {
121
- expect(isGoogleDocsUrl(url)).toBe(true);
122
- }
123
- );
124
-
125
- it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
126
- 'extracts the public document ID from %s',
127
- (url) => {
128
- expect(extractDocumentId(url)).toBe(PUBLIC_TEST_DOCUMENT.id);
129
- }
130
- );
131
-
132
- it('builds the expected export URL for the public document', () => {
133
- expect(buildExportUrl(PUBLIC_TEST_DOCUMENT.id, 'html')).toBe(
134
- `https://docs.google.com/document/d/${PUBLIC_TEST_DOCUMENT.id}/export?format=html`
135
- );
136
- });
137
-
138
- it('builds the expected editor URL for the public document', () => {
139
- expect(buildEditUrl(PUBLIC_TEST_DOCUMENT.id)).toBe(
140
- `https://docs.google.com/document/d/${PUBLIC_TEST_DOCUMENT.id}/edit`
141
- );
142
- });
143
-
144
- it('builds the expected Docs API URL for the public document', () => {
145
- expect(buildDocsApiUrl(PUBLIC_TEST_DOCUMENT.id)).toBe(
146
- `https://docs.googleapis.com/v1/documents/${PUBLIC_TEST_DOCUMENT.id}`
147
- );
148
- });
149
- });
150
-
151
- describe('Capture method selection (issue #72 regression guard)', () => {
152
- it('routes --capture browser to the editor-model backend', () => {
153
- expect(selectGoogleDocsCaptureMethod('browser')).toBe('browser-model');
154
- });
155
-
156
- it('routes --capture api to the public-export backend when no token', () => {
157
- expect(selectGoogleDocsCaptureMethod('api')).toBe('public-export');
158
- });
159
-
160
- it('routes --capture api with a token to the Docs REST API backend', () => {
161
- expect(selectGoogleDocsCaptureMethod('api', 'token-123')).toBe(
162
- 'docs-api'
163
- );
164
- });
165
- });
166
-
167
- describe('Reference markdown fixture', () => {
168
- const referenceMarkdownPath = path.join(
169
- REFERENCE_DIR,
170
- 'markdown-test-document.md'
171
- );
172
- const referenceMarkdown = fs.readFileSync(referenceMarkdownPath, 'utf-8');
173
-
174
- it('contains every feature category documented in issue #90', () => {
175
- for (const section of PUBLIC_TEST_DOCUMENT.sections) {
176
- expect(referenceMarkdown).toContain(section);
177
- }
178
- });
179
-
180
- it('embeds the four test images referenced by the issue', () => {
181
- expect(referenceMarkdown).toMatch(
182
- /!\[Blue rectangle]\(media\/image1\.png\)/
183
- );
184
- expect(referenceMarkdown).toMatch(
185
- /!\[Red rectangle]\(media\/image2\.png\)/
186
- );
187
- expect(referenceMarkdown).toMatch(
188
- /!\[Green square]\(media\/image3\.png\)/
189
- );
190
- expect(referenceMarkdown).toMatch(
191
- /!\[Yellow square]\(media\/image4\.png\)/
192
- );
193
- });
194
- });
195
-
196
- describeIfLive('Live capture against the public document', () => {
197
- // The primary deliverable of issue #90: the reference document is
198
- // downloaded from Google Docs and every feature section can be found in
199
- // the captured markdown. This is our regression net for any future
200
- // changes to the public-export or HTML-to-markdown pipeline.
201
- //
202
- // Google Docs occasionally returns transient 500s on the public-export
203
- // endpoint. The Habr integration suite has the same flake and solves it
204
- // with retry + exponential backoff, so we reuse the shared helper here.
205
- const fetchWithRetry = (url) =>
206
- retry(() => fetchGoogleDocAsMarkdown(url), {
207
- retries: 3,
208
- baseDelay: 2000,
209
- });
210
-
211
- const browserEngine =
212
- process.env.GDOCS_BROWSER_ENGINE || process.env.BROWSER_ENGINE;
213
- const captureBrowserWithRetry = (url) =>
214
- retry(
215
- () =>
216
- captureGoogleDocWithBrowser(url, {
217
- ...(browserEngine ? { engine: browserEngine } : {}),
218
- }),
219
- {
220
- retries: 2,
221
- baseDelay: 2000,
222
- }
223
- );
224
-
225
- it('fetches the public document via --capture api and returns markdown', async () => {
226
- const { markdown, documentId, exportUrl } = await fetchWithRetry(
227
- PUBLIC_TEST_DOCUMENT.canonicalUrl
228
- );
229
-
230
- expect(documentId).toBe(PUBLIC_TEST_DOCUMENT.id);
231
- expect(exportUrl).toBe(buildExportUrl(PUBLIC_TEST_DOCUMENT.id, 'html'));
232
- expect(markdown.length).toBeGreaterThan(1000);
233
-
234
- for (const section of PUBLIC_TEST_DOCUMENT.sections) {
235
- expect(markdown).toContain(section);
236
- }
237
- expect(markdown).toContain('## 1. Headings');
238
- expect(markdown).toContain('**This text is bold**');
239
- expect(markdown).toContain('*This text is italic*');
240
- expect(markdown).toContain('~~This text has strikethrough~~');
241
- expect(markdown).toContain('> This is a single-level blockquote');
242
- expect(markdown).toContain('| Feature | Supported | Notes |');
243
- expect(markdown).not.toContain('| Feature | | Supported |');
244
- expect(markdown).toContain('| | x | |');
245
- expect(markdown).toContain('1. Parent item 1');
246
- expect(markdown).toContain(' 1. Child item 1.1');
247
- expect(markdown).toContain(' 1. Grandchild item 1.2.1');
248
- expect(markdown).not.toContain('- Child item 1.1');
249
- });
250
-
251
- it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
252
- 'resolves %s to the same document when capturing',
253
- async (url) => {
254
- const { documentId } = await fetchWithRetry(url);
255
- expect(documentId).toBe(PUBLIC_TEST_DOCUMENT.id);
256
- }
257
- );
258
-
259
- it('captures the public document via --capture browser and renders DOCS_modelChunk markdown', async () => {
260
- const { markdown, documentId, exportUrl, capture } =
261
- await captureBrowserWithRetry(PUBLIC_TEST_DOCUMENT.canonicalUrl);
262
-
263
- expect(documentId).toBe(PUBLIC_TEST_DOCUMENT.id);
264
- expect(exportUrl).toBe(buildEditUrl(PUBLIC_TEST_DOCUMENT.id));
265
- expect(capture.blocks.length).toBeGreaterThan(20);
266
- expect(capture.images.length).toBeGreaterThanOrEqual(4);
267
- expect(markdown.length).toBeGreaterThan(2500);
268
- expect(markdown).toContain('# Markdown Feature Test Document');
269
- expect(markdown).toContain('## 1. Headings');
270
- expect(markdown).toContain('**This text is bold**');
271
- expect(markdown).toContain('*This text is italic*');
272
- expect(markdown).toContain('~~This text has strikethrough~~');
273
- expect(markdown).toContain('> This is a single-level blockquote');
274
- expect(markdown).toContain('[Regular link](https://example.com)');
275
- expect(markdown).toContain('| Feature | Supported | Notes |');
276
- expect(markdown).not.toContain('| Feature | | Supported |');
277
- expect(markdown).toContain('| | x | |');
278
- expect(markdown).toContain('1. Parent item 1');
279
- expect(markdown).toContain(' 1. Child item 1.1');
280
- expect(markdown).toContain(' 1. Grandchild item 1.2.1');
281
- expect(markdown).not.toContain('- Child item 1.1');
282
- expect(markdown).not.toContain('1. Parent item 1\n\n');
283
- expect(markdown).toContain('![Blue rectangle](');
284
- expect(markdown).toContain('![Red rectangle](');
285
- expect(markdown).toContain('![Green square](');
286
- expect(markdown).toContain('![Yellow square](');
287
- expect(markdown).toContain('docs-images-rt/');
288
- expect(markdown).toContain('---');
289
- });
290
-
291
- it('fetches the issue #108 v2 document via --capture api and preserves Section 15 semantics', async () => {
292
- const { markdown, documentId, exportUrl } = await fetchWithRetry(
293
- PUBLIC_TEST_DOCUMENT_V2.canonicalUrl
294
- );
295
-
296
- expect(documentId).toBe(PUBLIC_TEST_DOCUMENT_V2.id);
297
- expect(exportUrl).toBe(
298
- buildExportUrl(PUBLIC_TEST_DOCUMENT_V2.id, 'html')
299
- );
300
- expectIssue108Markdown(markdown);
301
- });
302
-
303
- it('captures the issue #108 v2 document via --capture browser and preserves Section 15 semantics', async () => {
304
- const { markdown, documentId, exportUrl, capture } =
305
- await captureBrowserWithRetry(PUBLIC_TEST_DOCUMENT_V2.canonicalUrl);
306
-
307
- expect(documentId).toBe(PUBLIC_TEST_DOCUMENT_V2.id);
308
- expect(exportUrl).toBe(buildEditUrl(PUBLIC_TEST_DOCUMENT_V2.id));
309
- expect(capture.blocks.length).toBeGreaterThan(30);
310
- expectIssue108Markdown(markdown);
311
- expect(markdown).not.toContain('*Apple*');
312
- expect(markdown).not.toContain('*Red*');
313
- });
314
- });
315
- });
@@ -1,137 +0,0 @@
1
- /**
2
- * Integration tests for capturing GitHub repository pages (issue #5).
3
- *
4
- * Live tests are gated behind GITHUB_REPOSITORY_INTEGRATION so default/offline
5
- * runs stay deterministic. The mocked API endpoint tests cover the default
6
- * behavior without network access; this suite proves the real GitHub repository
7
- * page can still be captured as compact text/markdown, original HTML, and PNG
8
- * screenshots in every supported JavaScript browser engine.
9
- */
10
-
11
- import { jest } from '@jest/globals';
12
- import { createBrowser } from '../../src/browser.js';
13
- import { fetchHtml } from '../../src/lib.js';
14
- import { retry } from '../../src/retry.js';
15
- import {
16
- fetchGithubRepositorySnapshot,
17
- formatGithubRepositoryMarkdown,
18
- formatGithubRepositoryText,
19
- parseGithubRepositoryUrl,
20
- } from '../../src/github.js';
21
-
22
- const GITHUB_REPOSITORY_URL =
23
- process.env.GITHUB_REPOSITORY_URL ||
24
- 'https://github.com/link-assistant/web-capture';
25
- const PNG_SIGNATURE = Buffer.from([
26
- 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
27
- ]);
28
-
29
- const SKIP_LIVE =
30
- !process.env.GITHUB_REPOSITORY_INTEGRATION ||
31
- process.env.GITHUB_REPOSITORY_INTEGRATION === 'false';
32
- const describeIfLive = SKIP_LIVE ? describe.skip : describe;
33
-
34
- jest.setTimeout(120000);
35
-
36
- async function navigateWithRetry(page, url) {
37
- await retry(
38
- async () => {
39
- await page.goto(url, {
40
- waitUntil: 'domcontentloaded',
41
- timeout: 30000,
42
- });
43
- },
44
- {
45
- retries: 3,
46
- baseDelay: 2000,
47
- onRetry: (err, attempt, delay) => {
48
- console.log(
49
- `Navigation retry ${attempt} for ${url} after ${delay}ms: ${err.message}`
50
- );
51
- },
52
- }
53
- );
54
- await page.waitForTimeout(5000);
55
- }
56
-
57
- describeIfLive('GitHub repository capture', () => {
58
- it('downloads compact repository text and markdown with README plus file tree', async () => {
59
- const parsed = parseGithubRepositoryUrl(GITHUB_REPOSITORY_URL);
60
- expect(parsed).not.toBeNull();
61
-
62
- const snapshot = await retry(
63
- () => fetchGithubRepositorySnapshot(GITHUB_REPOSITORY_URL),
64
- {
65
- retries: 3,
66
- baseDelay: 2000,
67
- }
68
- );
69
-
70
- const text = formatGithubRepositoryText(snapshot);
71
- const markdown = formatGithubRepositoryMarkdown(snapshot);
72
-
73
- expect(text).toContain(`Repository: ${parsed.fullName}`);
74
- expect(text).toContain('Files:');
75
- expect(text).toMatch(/README/i);
76
- expect(text.length).toBeGreaterThan(500);
77
-
78
- expect(markdown).toContain(`# ${parsed.fullName}`);
79
- expect(markdown).toContain('## Files');
80
- expect(markdown).toMatch(/## .*README/i);
81
- expect(markdown.length).toBeGreaterThan(500);
82
- expect(markdown).not.toMatch(/<html/i);
83
- });
84
-
85
- it('downloads original GitHub repository HTML', async () => {
86
- const parsed = parseGithubRepositoryUrl(GITHUB_REPOSITORY_URL);
87
- expect(parsed).not.toBeNull();
88
-
89
- const html = await retry(() => fetchHtml(GITHUB_REPOSITORY_URL), {
90
- retries: 3,
91
- baseDelay: 2000,
92
- });
93
-
94
- expect(html).toMatch(/<html/i);
95
- expect(html).toContain(parsed.owner);
96
- expect(html).toContain(parsed.repo);
97
- expect(html.length).toBeGreaterThan(1000);
98
- });
99
- });
100
-
101
- describe.each(['puppeteer', 'playwright'])(
102
- 'GitHub repository screenshot (%s engine)',
103
- (engine) => {
104
- describeIfLive(`${engine}`, () => {
105
- let browser;
106
-
107
- beforeAll(async () => {
108
- browser = await createBrowser(engine);
109
- });
110
-
111
- afterAll(async () => {
112
- if (browser) {
113
- await browser.close();
114
- }
115
- });
116
-
117
- it('captures the GitHub repository page as a PNG screenshot', async () => {
118
- const page = await browser.newPage();
119
- await page.setExtraHTTPHeaders({
120
- 'Accept-Language': 'en-US,en;q=0.9',
121
- 'Accept-Charset': 'utf-8',
122
- });
123
- await page.setUserAgent(
124
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
125
- );
126
- await page.setViewport({ width: 1280, height: 800 });
127
- await navigateWithRetry(page, GITHUB_REPOSITORY_URL);
128
-
129
- const screenshot = await page.screenshot({ type: 'png' });
130
-
131
- expect(screenshot).toBeInstanceOf(Buffer);
132
- expect(screenshot.length).toBeGreaterThan(1000);
133
- expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
134
- });
135
- });
136
- }
137
- );
@@ -1,246 +0,0 @@
1
- import request from 'supertest';
2
- import nock from '../helpers/nock.js';
3
- import unzipper from 'unzipper';
4
- import { app } from '../../src/index.js';
5
-
6
- const FIRST_FILE_ID = '1Cxkx6-428EQAX0-eiaq66H829ohnPp7q';
7
- const SECOND_FILE_ID = '1fgJaftjv53xCN7vgiJOaQlbfWkUqPgyd';
8
- const JPEG_BYTES = Buffer.from(
9
- '/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////2wBDAf//////////////////////////////////////////////////////////////////////////////////////wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAX/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIQAxAAAAH/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECAQE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Ap//xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAE/IV//2gAMAwEAAgADAAAAEP/EFBQRAQAAAAAAAAAAAAAAAAAAARD/2gAIAQMBAT8QH//EFBQRAQAAAAAAAAAAAAAAAAAAARD/2gAIAQIBAT8QH//EFBABAQAAAAAAAAAAAAAAAAAAARD/2gAIAQEAAT8QH//Z',
10
- 'base64'
11
- );
12
-
13
- const DRIVE_URLS = [
14
- {
15
- name: 'first issue URL with /view',
16
- fileId: FIRST_FILE_ID,
17
- url: `https://drive.google.com/file/d/${FIRST_FILE_ID}/view`,
18
- },
19
- {
20
- name: 'first issue URL without /view',
21
- fileId: FIRST_FILE_ID,
22
- url: `https://drive.google.com/file/d/${FIRST_FILE_ID}`,
23
- },
24
- {
25
- name: 'second issue URL with /view',
26
- fileId: SECOND_FILE_ID,
27
- url: `https://drive.google.com/file/d/${SECOND_FILE_ID}/view`,
28
- },
29
- {
30
- name: 'second issue URL without /view',
31
- fileId: SECOND_FILE_ID,
32
- url: `https://drive.google.com/file/d/${SECOND_FILE_ID}`,
33
- },
34
- ];
35
-
36
- function directDownloadUrl(fileId) {
37
- return `https://drive.google.com/uc?export=download&id=${fileId}`;
38
- }
39
-
40
- function mockDriveHead(fileId) {
41
- return nock('https://drive.google.com')
42
- .head('/uc')
43
- .query({ export: 'download', id: fileId })
44
- .reply(200, '', { 'content-type': 'image/jpeg' });
45
- }
46
-
47
- function mockDriveGet(fileId) {
48
- return nock('https://drive.google.com')
49
- .get('/uc')
50
- .query({ export: 'download', id: fileId })
51
- .reply(200, JPEG_BYTES, {
52
- 'content-type': 'image/jpeg',
53
- 'content-length': String(JPEG_BYTES.length),
54
- });
55
- }
56
-
57
- function bufferParser(res, callback) {
58
- const chunks = [];
59
- res.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
60
- res.on('end', () => callback(null, Buffer.concat(chunks)));
61
- }
62
-
63
- afterEach(() => {
64
- nock.cleanAll();
65
- });
66
-
67
- describe('Google Drive public image links', () => {
68
- it.each(DRIVE_URLS)(
69
- '/image downloads image bytes for $name',
70
- async (item) => {
71
- mockDriveGet(item.fileId);
72
-
73
- const res = await request(app)
74
- .get('/image')
75
- .query({ url: item.url })
76
- .buffer(true)
77
- .parse(bufferParser)
78
- .expect(200);
79
-
80
- expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
81
- expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
82
- expect(nock.isDone()).toBe(true);
83
- }
84
- );
85
-
86
- it.each(DRIVE_URLS)(
87
- '/fetch proxies direct image bytes for $name',
88
- async (item) => {
89
- mockDriveGet(item.fileId);
90
-
91
- const res = await request(app)
92
- .get('/fetch')
93
- .query({ url: item.url })
94
- .buffer(true)
95
- .parse(bufferParser)
96
- .expect(200);
97
-
98
- expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
99
- expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
100
- expect(nock.isDone()).toBe(true);
101
- }
102
- );
103
-
104
- it.each(DRIVE_URLS)(
105
- '/stream proxies direct image bytes for $name',
106
- async (item) => {
107
- mockDriveGet(item.fileId);
108
-
109
- const res = await request(app)
110
- .get('/stream')
111
- .query({ url: item.url })
112
- .buffer(true)
113
- .parse(bufferParser)
114
- .expect(200);
115
-
116
- expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
117
- expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
118
- expect(nock.isDone()).toBe(true);
119
- }
120
- );
121
-
122
- it.each(DRIVE_URLS)(
123
- '/markdown includes a direct image link for $name',
124
- async (item) => {
125
- mockDriveHead(item.fileId);
126
-
127
- const res = await request(app)
128
- .get('/markdown')
129
- .query({ url: item.url })
130
- .expect(200);
131
-
132
- expect(res.headers['content-type']).toMatch(/text\/markdown/);
133
- expect(res.text).toContain(
134
- `![Google Drive image](${directDownloadUrl(item.fileId)})`
135
- );
136
- expect(nock.isDone()).toBe(true);
137
- }
138
- );
139
-
140
- it.each(DRIVE_URLS)(
141
- '/txt includes the original and direct image links for $name',
142
- async (item) => {
143
- mockDriveHead(item.fileId);
144
-
145
- const res = await request(app)
146
- .get('/txt')
147
- .query({ url: item.url })
148
- .expect(200);
149
-
150
- expect(res.headers['content-type']).toMatch(/text\/plain/);
151
- expect(res.text).toContain('Google Drive image');
152
- expect(res.text).toContain(`Original Google Drive file: ${item.url}`);
153
- expect(res.text).toContain(`Image: ${directDownloadUrl(item.fileId)}`);
154
- expect(nock.isDone()).toBe(true);
155
- }
156
- );
157
-
158
- it.each(DRIVE_URLS)(
159
- '/html includes a direct image tag for $name',
160
- async (item) => {
161
- mockDriveHead(item.fileId);
162
-
163
- const res = await request(app)
164
- .get('/html')
165
- .query({ url: item.url })
166
- .expect(200);
167
-
168
- expect(res.headers['content-type']).toMatch(/text\/html/);
169
- expect(res.text).toContain(
170
- `src="${directDownloadUrl(item.fileId).replace('&', '&amp;')}"`
171
- );
172
- expect(nock.isDone()).toBe(true);
173
- }
174
- );
175
-
176
- it('archive includes markdown and the downloaded Drive image', async () => {
177
- const item = DRIVE_URLS[0];
178
- mockDriveHead(item.fileId);
179
- mockDriveGet(item.fileId);
180
-
181
- const res = await request(app)
182
- .get('/archive')
183
- .query({ url: item.url })
184
- .buffer(true)
185
- .parse(bufferParser)
186
- .expect(200);
187
-
188
- const zip = await unzipper.Open.buffer(res.body);
189
- const document = zip.files.find((file) => file.path === 'document.md');
190
- const image = zip.files.find((file) => file.path.startsWith('images/'));
191
-
192
- expect(document).toBeDefined();
193
- expect(image).toBeDefined();
194
- expect((await document.buffer()).toString('utf8')).toContain(
195
- '![Google Drive image](images/image-1.jpg)'
196
- );
197
- expect((await image.buffer()).subarray(0, 3)).toEqual(
198
- Buffer.from([0xff, 0xd8, 0xff])
199
- );
200
- expect(nock.isDone()).toBe(true);
201
- });
202
-
203
- it('docx includes the downloaded Drive image', async () => {
204
- const item = DRIVE_URLS[0];
205
- mockDriveHead(item.fileId);
206
- mockDriveGet(item.fileId);
207
-
208
- const res = await request(app)
209
- .get('/docx')
210
- .query({ url: item.url })
211
- .buffer(true)
212
- .parse(bufferParser)
213
- .expect(200);
214
-
215
- expect(res.headers['content-type']).toMatch(
216
- /application\/vnd\.openxmlformats-officedocument\.wordprocessingml\.document/
217
- );
218
- expect(res.body.subarray(0, 2)).toEqual(Buffer.from('PK'));
219
-
220
- const docx = await unzipper.Open.buffer(res.body);
221
- const mediaEntry = docx.files.find(
222
- (file) => file.path.startsWith('word/media/') && !file.path.endsWith('/')
223
- );
224
- expect(mediaEntry).toBeDefined();
225
- expect((await mediaEntry.buffer()).subarray(0, 3)).toEqual(
226
- Buffer.from([0xff, 0xd8, 0xff])
227
- );
228
- expect(nock.isDone()).toBe(true);
229
- });
230
-
231
- it('pdf renders the downloaded Drive image document', async () => {
232
- const item = DRIVE_URLS[0];
233
- mockDriveGet(item.fileId);
234
-
235
- const res = await request(app)
236
- .get('/pdf')
237
- .query({ url: item.url })
238
- .buffer(true)
239
- .parse(bufferParser)
240
- .expect(200);
241
-
242
- expect(res.headers['content-type']).toMatch(/application\/pdf/);
243
- expect(res.body.subarray(0, 4).toString('utf8')).toBe('%PDF');
244
- expect(nock.isDone()).toBe(true);
245
- });
246
- });