@link-assistant/web-capture 2.0.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/bin/web-capture.js +2 -5
  3. package/package.json +22 -11
  4. package/src/archive.js +0 -1
  5. package/src/binary-download.js +0 -1
  6. package/src/docx.js +13 -10
  7. package/src/fetch.js +0 -1
  8. package/src/gdocs-images.js +0 -1
  9. package/src/gdocs.js +0 -1
  10. package/src/github.js +0 -1
  11. package/src/lib.js +0 -1
  12. package/src/markdown.js +0 -1
  13. package/src/shared-dialog.js +0 -1
  14. package/src/stream.js +0 -1
  15. package/src/transport.js +23 -5
  16. package/src/txt.js +0 -1
  17. package/.changeset/README.md +0 -8
  18. package/.changeset/config.json +0 -11
  19. package/.dockerignore +0 -6
  20. package/.jscpd.json +0 -19
  21. package/.lenv +0 -8
  22. package/.prettierignore +0 -8
  23. package/.prettierrc +0 -10
  24. package/Dockerfile +0 -52
  25. package/docker-compose.yml +0 -11
  26. package/eslint.config.js +0 -112
  27. package/examples/js/capture_cache.js +0 -12
  28. package/examples/js/engine_comparison.js +0 -39
  29. package/examples/js/html_download.js +0 -23
  30. package/examples/js/image_download.js +0 -36
  31. package/examples/js/markdown_download.js +0 -23
  32. package/examples/js/playwright_example.js +0 -23
  33. package/examples/python/capture_client.py +0 -20
  34. package/examples/python/html_download.py +0 -3
  35. package/examples/python/image_download.py +0 -16
  36. package/examples/python/markdown_download.py +0 -3
  37. package/examples/python/playwright_example.py +0 -22
  38. package/jest.config.mjs +0 -17
  39. package/tests/ci/dependency-freshness.test.js +0 -74
  40. package/tests/ci/detect-code-changes.test.js +0 -187
  41. package/tests/ci/docker-install.test.js +0 -31
  42. package/tests/ci/rust-dockerfile.test.js +0 -54
  43. package/tests/ci/workflow-policy.test.js +0 -79
  44. package/tests/e2e/docker.test.js +0 -186
  45. package/tests/e2e/process.test.js +0 -215
  46. package/tests/fixtures/bold-coalesce-and-leakage.html +0 -27
  47. package/tests/fixtures/deepwiki/deepwiki-full-page.png +0 -0
  48. package/tests/fixtures/deepwiki/index.md +0 -1500
  49. package/tests/fixtures/deepwiki/part-1.md +0 -776
  50. package/tests/fixtures/habr-articles.js +0 -45
  51. package/tests/fixtures/heading-numbering.html +0 -19
  52. package/tests/fixtures/list-item-with-br.html +0 -8
  53. package/tests/fixtures/ol-continuous-numbering.html +0 -12
  54. package/tests/fixtures/paragraph-vs-line-break.html +0 -11
  55. package/tests/helpers/nock.js +0 -10
  56. package/tests/helpers/screenshot.js +0 -10
  57. package/tests/helpers/search-fixture.cjs +0 -17
  58. package/tests/helpers/temporary-directory.js +0 -12
  59. package/tests/integration/api-endpoints.test.js +0 -418
  60. package/tests/integration/archive-default-layout.test.js +0 -37
  61. package/tests/integration/browser-engines.test.js +0 -80
  62. package/tests/integration/browser-packaged-path.test.js +0 -45
  63. package/tests/integration/deepwiki-download.test.js +0 -144
  64. package/tests/integration/deepwiki-rendered-page.mock.test.js +0 -121
  65. package/tests/integration/formalai-contract.test.js +0 -214
  66. package/tests/integration/gdocs-public-doc.test.js +0 -315
  67. package/tests/integration/github-readme.test.js +0 -133
  68. package/tests/integration/google-drive-image.test.js +0 -246
  69. package/tests/integration/habr-article.test.js +0 -337
  70. package/tests/integration/search-endpoint.test.js +0 -88
  71. package/tests/integration/stackoverflow-download.test.js +0 -173
  72. package/tests/integration/wikipedia-download.test.js +0 -134
  73. package/tests/jest.setup.mjs +0 -7
  74. package/tests/mock/index.test.js +0 -225
  75. package/tests/unit/animation.test.js +0 -33
  76. package/tests/unit/batch.test.js +0 -114
  77. package/tests/unit/bold-coalesce-and-leakage.test.js +0 -54
  78. package/tests/unit/browser-launch-options.test.js +0 -42
  79. package/tests/unit/browser.test.js +0 -172
  80. package/tests/unit/cache.test.js +0 -182
  81. package/tests/unit/cli.test.js +0 -399
  82. package/tests/unit/convertGoogleDriveUrl.test.js +0 -89
  83. package/tests/unit/convertRelativeUrls.test.js +0 -34
  84. package/tests/unit/extract-images.test.js +0 -259
  85. package/tests/unit/fetch-html.test.js +0 -54
  86. package/tests/unit/figures.test.js +0 -74
  87. package/tests/unit/gdocs-export-headers.test.js +0 -28
  88. package/tests/unit/gdocs-preprocess.test.js +0 -36
  89. package/tests/unit/gdocs.test.js +0 -1490
  90. package/tests/unit/github.test.js +0 -156
  91. package/tests/unit/heading-image.test.js +0 -49
  92. package/tests/unit/heading-numbering.test.js +0 -28
  93. package/tests/unit/html2md-br-in-list-item.test.js +0 -23
  94. package/tests/unit/html2md-ol-numbering.test.js +0 -20
  95. package/tests/unit/html2md.test.js +0 -695
  96. package/tests/unit/image-mode-defaults.test.js +0 -54
  97. package/tests/unit/kreuzberg.test.js +0 -249
  98. package/tests/unit/latex.test.js +0 -123
  99. package/tests/unit/localize-images.test.js +0 -114
  100. package/tests/unit/markdown.test.js +0 -36
  101. package/tests/unit/metadata.test.js +0 -140
  102. package/tests/unit/package-runtime-dependencies.test.js +0 -74
  103. package/tests/unit/paragraph-vs-line-break.test.js +0 -56
  104. package/tests/unit/postprocess.test.js +0 -110
  105. package/tests/unit/proxy-headers.test.js +0 -100
  106. package/tests/unit/publish-verification.test.js +0 -54
  107. package/tests/unit/retry.test.js +0 -111
  108. package/tests/unit/search.test.js +0 -339
  109. package/tests/unit/shared-dialog.test.js +0 -173
  110. package/tests/unit/stackoverflow.test.js +0 -32
  111. package/tests/unit/stream.test.js +0 -83
  112. package/tests/unit/verify.test.js +0 -182
  113. package/tests/xpaste/xpaste-screenshot-markdown.mock.test.js +0 -375
  114. package/tests/xpaste/xpaste.e2e.test.js +0 -36
  115. package/tests/xpaste/xpaste.integration.test.js +0 -41
  116. package/tests/xpaste/xpaste.mock.test.js +0 -261
  117. package/tests/xpaste/xpaste.unit.test.js +0 -119
@@ -1,315 +0,0 @@
1
- /**
2
- * Integration tests for the Google Docs markdown round-trip reference document.
3
- *
4
- * The document at
5
- * https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit
6
- * is a public document created specifically for validating web-capture's Google
7
- * Docs support across every URL variation and every Markdown feature category
8
- * documented in issue #90. The ground-truth markdown and authoring archive are
9
- * preserved under docs/case-studies/issue-90/reference/.
10
- *
11
- * Live capture tests are gated behind GDOCS_INTEGRATION=true so normal PR CI
12
- * stays hermetic. The URL-variation, document-id extraction and feature
13
- * checklist tests always run so regressions in the parsing surface are caught
14
- * even without network access.
15
- */
16
-
17
- import { jest } from '@jest/globals';
18
- import fs from 'fs';
19
- import path from 'path';
20
- import { fileURLToPath } from 'url';
21
-
22
- import {
23
- isGoogleDocsUrl,
24
- extractDocumentId,
25
- buildExportUrl,
26
- buildEditUrl,
27
- buildDocsApiUrl,
28
- selectGoogleDocsCaptureMethod,
29
- fetchGoogleDocAsMarkdown,
30
- captureGoogleDocWithBrowser,
31
- } from '../../src/gdocs.js';
32
- import { retry } from '../../src/retry.js';
33
-
34
- const LIVE =
35
- process.env.GDOCS_INTEGRATION === 'true' ||
36
- process.env.GDOCS_INTEGRATION === '1';
37
- const describeIfLive = LIVE ? describe : describe.skip;
38
-
39
- jest.setTimeout(120000);
40
-
41
- const __filename = fileURLToPath(import.meta.url);
42
- const __dirname = path.dirname(__filename);
43
- const REFERENCE_DIR = path.resolve(
44
- __dirname,
45
- '..',
46
- '..',
47
- '..',
48
- 'docs',
49
- 'case-studies',
50
- 'issue-90',
51
- 'reference'
52
- );
53
-
54
- // The public document referenced in issue #90.
55
- // https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit
56
- export const PUBLIC_TEST_DOCUMENT = {
57
- id: '1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM',
58
- canonicalUrl:
59
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit',
60
- urlVariations: [
61
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM',
62
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit',
63
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit?usp=sharing',
64
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit?usp=sharing&ouid=102030405060708090100&rtpof=true&sd=true',
65
- 'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit?tab=t.0',
66
- ],
67
- // Section headings from the reference markdown. Each one should be present
68
- // in any Markdown produced from the document, regardless of capture mode.
69
- // See docs/case-studies/issue-90/reference/markdown-test-document.md.
70
- sections: [
71
- 'Markdown Feature Test Document',
72
- '1. Headings',
73
- '2. Inline Formatting',
74
- '3. Paragraphs',
75
- '4. Blockquotes',
76
- '5. Unordered Lists',
77
- '6. Ordered Lists',
78
- '7. Mixed Lists',
79
- '8. Tables',
80
- '9. Links',
81
- '10. Images',
82
- '11. Horizontal Rules',
83
- '12. Special Characters',
84
- '13. Nested Formatting Edge Cases',
85
- '14. Empty and Minimal Table Content',
86
- ],
87
- };
88
-
89
- // The issue #108 v2 document extends the public reference fixture with Section
90
- // 15, covering browser-model list type and continuation-paragraph regressions.
91
- export const PUBLIC_TEST_DOCUMENT_V2 = {
92
- id: '1Rvaod_u2wgkAUNdXG-e29yQem-P36KAQ',
93
- canonicalUrl:
94
- 'https://docs.google.com/document/d/1Rvaod_u2wgkAUNdXG-e29yQem-P36KAQ/edit',
95
- };
96
-
97
- function expectIssue108Markdown(markdown) {
98
- expect(markdown).toContain(
99
- '15. List & Blockquote Edge Cases (browser-mode reproducers)'
100
- );
101
- expect(markdown).toMatch(/1\.\s+Apple/u);
102
- expect(markdown).toMatch(/2\.\s+Banana/u);
103
- expect(markdown).toMatch(/3\.\s+Cherry/u);
104
- expect(markdown).not.toMatch(/^-\s+Apple$/mu);
105
- expect(markdown).toContain(
106
- 'Continuation paragraph that is not a list item. Same indent as Step one.'
107
- );
108
- expect(markdown).not.toContain(
109
- '> Continuation paragraph that is not a list item.'
110
- );
111
- expect(markdown).toMatch(/^-\s+Red$/mu);
112
- expect(markdown).toMatch(/^-\s+Green$/mu);
113
- expect(markdown).toMatch(/^-\s+Blue$/mu);
114
- }
115
-
116
- describe('Google Docs public test document (issue #90)', () => {
117
- describe('URL variations', () => {
118
- it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
119
- 'accepts %s as a Google Docs URL',
120
- (url) => {
121
- expect(isGoogleDocsUrl(url)).toBe(true);
122
- }
123
- );
124
-
125
- it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
126
- 'extracts the public document ID from %s',
127
- (url) => {
128
- expect(extractDocumentId(url)).toBe(PUBLIC_TEST_DOCUMENT.id);
129
- }
130
- );
131
-
132
- it('builds the expected export URL for the public document', () => {
133
- expect(buildExportUrl(PUBLIC_TEST_DOCUMENT.id, 'html')).toBe(
134
- `https://docs.google.com/document/d/${PUBLIC_TEST_DOCUMENT.id}/export?format=html`
135
- );
136
- });
137
-
138
- it('builds the expected editor URL for the public document', () => {
139
- expect(buildEditUrl(PUBLIC_TEST_DOCUMENT.id)).toBe(
140
- `https://docs.google.com/document/d/${PUBLIC_TEST_DOCUMENT.id}/edit`
141
- );
142
- });
143
-
144
- it('builds the expected Docs API URL for the public document', () => {
145
- expect(buildDocsApiUrl(PUBLIC_TEST_DOCUMENT.id)).toBe(
146
- `https://docs.googleapis.com/v1/documents/${PUBLIC_TEST_DOCUMENT.id}`
147
- );
148
- });
149
- });
150
-
151
- describe('Capture method selection (issue #72 regression guard)', () => {
152
- it('routes --capture browser to the editor-model backend', () => {
153
- expect(selectGoogleDocsCaptureMethod('browser')).toBe('browser-model');
154
- });
155
-
156
- it('routes --capture api to the public-export backend when no token', () => {
157
- expect(selectGoogleDocsCaptureMethod('api')).toBe('public-export');
158
- });
159
-
160
- it('routes --capture api with a token to the Docs REST API backend', () => {
161
- expect(selectGoogleDocsCaptureMethod('api', 'token-123')).toBe(
162
- 'docs-api'
163
- );
164
- });
165
- });
166
-
167
- describe('Reference markdown fixture', () => {
168
- const referenceMarkdownPath = path.join(
169
- REFERENCE_DIR,
170
- 'markdown-test-document.md'
171
- );
172
- const referenceMarkdown = fs.readFileSync(referenceMarkdownPath, 'utf-8');
173
-
174
- it('contains every feature category documented in issue #90', () => {
175
- for (const section of PUBLIC_TEST_DOCUMENT.sections) {
176
- expect(referenceMarkdown).toContain(section);
177
- }
178
- });
179
-
180
- it('embeds the four test images referenced by the issue', () => {
181
- expect(referenceMarkdown).toMatch(
182
- /!\[Blue rectangle]\(media\/image1\.png\)/
183
- );
184
- expect(referenceMarkdown).toMatch(
185
- /!\[Red rectangle]\(media\/image2\.png\)/
186
- );
187
- expect(referenceMarkdown).toMatch(
188
- /!\[Green square]\(media\/image3\.png\)/
189
- );
190
- expect(referenceMarkdown).toMatch(
191
- /!\[Yellow square]\(media\/image4\.png\)/
192
- );
193
- });
194
- });
195
-
196
- describeIfLive('Live capture against the public document', () => {
197
- // The primary deliverable of issue #90: the reference document is
198
- // downloaded from Google Docs and every feature section can be found in
199
- // the captured markdown. This is our regression net for any future
200
- // changes to the public-export or HTML-to-markdown pipeline.
201
- //
202
- // Google Docs occasionally returns transient 500s on the public-export
203
- // endpoint. The Habr integration suite has the same flake and solves it
204
- // with retry + exponential backoff, so we reuse the shared helper here.
205
- const fetchWithRetry = (url) =>
206
- retry(() => fetchGoogleDocAsMarkdown(url), {
207
- retries: 3,
208
- baseDelay: 2000,
209
- });
210
-
211
- const browserEngine =
212
- process.env.GDOCS_BROWSER_ENGINE || process.env.BROWSER_ENGINE;
213
- const captureBrowserWithRetry = (url) =>
214
- retry(
215
- () =>
216
- captureGoogleDocWithBrowser(url, {
217
- ...(browserEngine ? { engine: browserEngine } : {}),
218
- }),
219
- {
220
- retries: 2,
221
- baseDelay: 2000,
222
- }
223
- );
224
-
225
- it('fetches the public document via --capture api and returns markdown', async () => {
226
- const { markdown, documentId, exportUrl } = await fetchWithRetry(
227
- PUBLIC_TEST_DOCUMENT.canonicalUrl
228
- );
229
-
230
- expect(documentId).toBe(PUBLIC_TEST_DOCUMENT.id);
231
- expect(exportUrl).toBe(buildExportUrl(PUBLIC_TEST_DOCUMENT.id, 'html'));
232
- expect(markdown.length).toBeGreaterThan(1000);
233
-
234
- for (const section of PUBLIC_TEST_DOCUMENT.sections) {
235
- expect(markdown).toContain(section);
236
- }
237
- expect(markdown).toContain('## 1. Headings');
238
- expect(markdown).toContain('**This text is bold**');
239
- expect(markdown).toContain('*This text is italic*');
240
- expect(markdown).toContain('~~This text has strikethrough~~');
241
- expect(markdown).toContain('> This is a single-level blockquote');
242
- expect(markdown).toContain('| Feature | Supported | Notes |');
243
- expect(markdown).not.toContain('| Feature | | Supported |');
244
- expect(markdown).toContain('| | x | |');
245
- expect(markdown).toContain('1. Parent item 1');
246
- expect(markdown).toContain(' 1. Child item 1.1');
247
- expect(markdown).toContain(' 1. Grandchild item 1.2.1');
248
- expect(markdown).not.toContain('- Child item 1.1');
249
- });
250
-
251
- it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
252
- 'resolves %s to the same document when capturing',
253
- async (url) => {
254
- const { documentId } = await fetchWithRetry(url);
255
- expect(documentId).toBe(PUBLIC_TEST_DOCUMENT.id);
256
- }
257
- );
258
-
259
- it('captures the public document via --capture browser and renders DOCS_modelChunk markdown', async () => {
260
- const { markdown, documentId, exportUrl, capture } =
261
- await captureBrowserWithRetry(PUBLIC_TEST_DOCUMENT.canonicalUrl);
262
-
263
- expect(documentId).toBe(PUBLIC_TEST_DOCUMENT.id);
264
- expect(exportUrl).toBe(buildEditUrl(PUBLIC_TEST_DOCUMENT.id));
265
- expect(capture.blocks.length).toBeGreaterThan(20);
266
- expect(capture.images.length).toBeGreaterThanOrEqual(4);
267
- expect(markdown.length).toBeGreaterThan(2500);
268
- expect(markdown).toContain('# Markdown Feature Test Document');
269
- expect(markdown).toContain('## 1. Headings');
270
- expect(markdown).toContain('**This text is bold**');
271
- expect(markdown).toContain('*This text is italic*');
272
- expect(markdown).toContain('~~This text has strikethrough~~');
273
- expect(markdown).toContain('> This is a single-level blockquote');
274
- expect(markdown).toContain('[Regular link](https://example.com)');
275
- expect(markdown).toContain('| Feature | Supported | Notes |');
276
- expect(markdown).not.toContain('| Feature | | Supported |');
277
- expect(markdown).toContain('| | x | |');
278
- expect(markdown).toContain('1. Parent item 1');
279
- expect(markdown).toContain(' 1. Child item 1.1');
280
- expect(markdown).toContain(' 1. Grandchild item 1.2.1');
281
- expect(markdown).not.toContain('- Child item 1.1');
282
- expect(markdown).not.toContain('1. Parent item 1\n\n');
283
- expect(markdown).toContain('![Blue rectangle](');
284
- expect(markdown).toContain('![Red rectangle](');
285
- expect(markdown).toContain('![Green square](');
286
- expect(markdown).toContain('![Yellow square](');
287
- expect(markdown).toContain('docs-images-rt/');
288
- expect(markdown).toContain('---');
289
- });
290
-
291
- it('fetches the issue #108 v2 document via --capture api and preserves Section 15 semantics', async () => {
292
- const { markdown, documentId, exportUrl } = await fetchWithRetry(
293
- PUBLIC_TEST_DOCUMENT_V2.canonicalUrl
294
- );
295
-
296
- expect(documentId).toBe(PUBLIC_TEST_DOCUMENT_V2.id);
297
- expect(exportUrl).toBe(
298
- buildExportUrl(PUBLIC_TEST_DOCUMENT_V2.id, 'html')
299
- );
300
- expectIssue108Markdown(markdown);
301
- });
302
-
303
- it('captures the issue #108 v2 document via --capture browser and preserves Section 15 semantics', async () => {
304
- const { markdown, documentId, exportUrl, capture } =
305
- await captureBrowserWithRetry(PUBLIC_TEST_DOCUMENT_V2.canonicalUrl);
306
-
307
- expect(documentId).toBe(PUBLIC_TEST_DOCUMENT_V2.id);
308
- expect(exportUrl).toBe(buildEditUrl(PUBLIC_TEST_DOCUMENT_V2.id));
309
- expect(capture.blocks.length).toBeGreaterThan(30);
310
- expectIssue108Markdown(markdown);
311
- expect(markdown).not.toContain('*Apple*');
312
- expect(markdown).not.toContain('*Red*');
313
- });
314
- });
315
- });
@@ -1,133 +0,0 @@
1
- /**
2
- * Integration tests for capturing GitHub repository pages (issue #5).
3
- *
4
- * Live tests are gated behind GITHUB_REPOSITORY_INTEGRATION so default/offline
5
- * runs stay deterministic. The mocked API endpoint tests cover the default
6
- * behavior without network access; this suite proves the real GitHub repository
7
- * page can still be captured as compact text/markdown, original HTML, and PNG
8
- * screenshots in every supported JavaScript browser engine.
9
- */
10
-
11
- import { jest } from '@jest/globals';
12
- import { createBrowser } from '../../src/browser.js';
13
- import { fetchHtml } from '../../src/lib.js';
14
- import { retry } from '../../src/retry.js';
15
- import { expectPngScreenshot } from '../helpers/screenshot.js';
16
- import {
17
- fetchGithubRepositorySnapshot,
18
- formatGithubRepositoryMarkdown,
19
- formatGithubRepositoryText,
20
- parseGithubRepositoryUrl,
21
- } from '../../src/github.js';
22
-
23
- const GITHUB_REPOSITORY_URL =
24
- process.env.GITHUB_REPOSITORY_URL ||
25
- 'https://github.com/link-assistant/web-capture';
26
-
27
- const SKIP_LIVE =
28
- !process.env.GITHUB_REPOSITORY_INTEGRATION ||
29
- process.env.GITHUB_REPOSITORY_INTEGRATION === 'false';
30
- const describeIfLive = SKIP_LIVE ? describe.skip : describe;
31
-
32
- jest.setTimeout(120000);
33
-
34
- async function navigateWithRetry(page, url) {
35
- await retry(
36
- async () => {
37
- await page.goto(url, {
38
- waitUntil: 'domcontentloaded',
39
- timeout: 30000,
40
- });
41
- },
42
- {
43
- retries: 3,
44
- baseDelay: 2000,
45
- onRetry: (err, attempt, delay) => {
46
- console.log(
47
- `Navigation retry ${attempt} for ${url} after ${delay}ms: ${err.message}`
48
- );
49
- },
50
- }
51
- );
52
- await page.waitForTimeout(5000);
53
- }
54
-
55
- describeIfLive('GitHub repository capture', () => {
56
- it('downloads compact repository text and markdown with README plus file tree', async () => {
57
- const parsed = parseGithubRepositoryUrl(GITHUB_REPOSITORY_URL);
58
- expect(parsed).not.toBeNull();
59
-
60
- const snapshot = await retry(
61
- () => fetchGithubRepositorySnapshot(GITHUB_REPOSITORY_URL),
62
- {
63
- retries: 3,
64
- baseDelay: 2000,
65
- }
66
- );
67
-
68
- const text = formatGithubRepositoryText(snapshot);
69
- const markdown = formatGithubRepositoryMarkdown(snapshot);
70
-
71
- expect(text).toContain(`Repository: ${parsed.fullName}`);
72
- expect(text).toContain('Files:');
73
- expect(text).toMatch(/README/i);
74
- expect(text.length).toBeGreaterThan(500);
75
-
76
- expect(markdown).toContain(`# ${parsed.fullName}`);
77
- expect(markdown).toContain('## Files');
78
- expect(markdown).toMatch(/## .*README/i);
79
- expect(markdown.length).toBeGreaterThan(500);
80
- expect(markdown).not.toMatch(/<html/i);
81
- });
82
-
83
- it('downloads original GitHub repository HTML', async () => {
84
- const parsed = parseGithubRepositoryUrl(GITHUB_REPOSITORY_URL);
85
- expect(parsed).not.toBeNull();
86
-
87
- const html = await retry(() => fetchHtml(GITHUB_REPOSITORY_URL), {
88
- retries: 3,
89
- baseDelay: 2000,
90
- });
91
-
92
- expect(html).toMatch(/<html/i);
93
- expect(html).toContain(parsed.owner);
94
- expect(html).toContain(parsed.repo);
95
- expect(html.length).toBeGreaterThan(1000);
96
- });
97
- });
98
-
99
- describe.each(['puppeteer', 'playwright'])(
100
- 'GitHub repository screenshot (%s engine)',
101
- (engine) => {
102
- describeIfLive(`${engine}`, () => {
103
- let browser;
104
-
105
- beforeAll(async () => {
106
- browser = await createBrowser(engine);
107
- });
108
-
109
- afterAll(async () => {
110
- if (browser) {
111
- await browser.close();
112
- }
113
- });
114
-
115
- it('captures the GitHub repository page as a PNG screenshot', async () => {
116
- const page = await browser.newPage();
117
- await page.setExtraHTTPHeaders({
118
- 'Accept-Language': 'en-US,en;q=0.9',
119
- 'Accept-Charset': 'utf-8',
120
- });
121
- await page.setUserAgent(
122
- 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
123
- );
124
- await page.setViewport({ width: 1280, height: 800 });
125
- await navigateWithRetry(page, GITHUB_REPOSITORY_URL);
126
-
127
- const screenshot = await page.screenshot({ type: 'png' });
128
-
129
- expectPngScreenshot(screenshot);
130
- });
131
- });
132
- }
133
- );
@@ -1,246 +0,0 @@
1
- import request from 'supertest';
2
- import nock from '../helpers/nock.js';
3
- import unzipper from 'unzipper';
4
- import { app } from '../../src/index.js';
5
-
6
- const FIRST_FILE_ID = '1Cxkx6-428EQAX0-eiaq66H829ohnPp7q';
7
- const SECOND_FILE_ID = '1fgJaftjv53xCN7vgiJOaQlbfWkUqPgyd';
8
- const JPEG_BYTES = Buffer.from(
9
- '/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////2wBDAf//////////////////////////////////////////////////////////////////////////////////////wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAX/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIQAxAAAAH/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECAQE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Ap//xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAE/IV//2gAMAwEAAgADAAAAEP/EFBQRAQAAAAAAAAAAAAAAAAAAARD/2gAIAQMBAT8QH//EFBQRAQAAAAAAAAAAAAAAAAAAARD/2gAIAQIBAT8QH//EFBABAQAAAAAAAAAAAAAAAAAAARD/2gAIAQEAAT8QH//Z',
10
- 'base64'
11
- );
12
-
13
- const DRIVE_URLS = [
14
- {
15
- name: 'first issue URL with /view',
16
- fileId: FIRST_FILE_ID,
17
- url: `https://drive.google.com/file/d/${FIRST_FILE_ID}/view`,
18
- },
19
- {
20
- name: 'first issue URL without /view',
21
- fileId: FIRST_FILE_ID,
22
- url: `https://drive.google.com/file/d/${FIRST_FILE_ID}`,
23
- },
24
- {
25
- name: 'second issue URL with /view',
26
- fileId: SECOND_FILE_ID,
27
- url: `https://drive.google.com/file/d/${SECOND_FILE_ID}/view`,
28
- },
29
- {
30
- name: 'second issue URL without /view',
31
- fileId: SECOND_FILE_ID,
32
- url: `https://drive.google.com/file/d/${SECOND_FILE_ID}`,
33
- },
34
- ];
35
-
36
- function directDownloadUrl(fileId) {
37
- return `https://drive.google.com/uc?export=download&id=${fileId}`;
38
- }
39
-
40
- function mockDriveHead(fileId) {
41
- return nock('https://drive.google.com')
42
- .head('/uc')
43
- .query({ export: 'download', id: fileId })
44
- .reply(200, '', { 'content-type': 'image/jpeg' });
45
- }
46
-
47
- function mockDriveGet(fileId) {
48
- return nock('https://drive.google.com')
49
- .get('/uc')
50
- .query({ export: 'download', id: fileId })
51
- .reply(200, JPEG_BYTES, {
52
- 'content-type': 'image/jpeg',
53
- 'content-length': String(JPEG_BYTES.length),
54
- });
55
- }
56
-
57
- function bufferParser(res, callback) {
58
- const chunks = [];
59
- res.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
60
- res.on('end', () => callback(null, Buffer.concat(chunks)));
61
- }
62
-
63
- afterEach(() => {
64
- nock.cleanAll();
65
- });
66
-
67
- describe('Google Drive public image links', () => {
68
- it.each(DRIVE_URLS)(
69
- '/image downloads image bytes for $name',
70
- async (item) => {
71
- mockDriveGet(item.fileId);
72
-
73
- const res = await request(app)
74
- .get('/image')
75
- .query({ url: item.url })
76
- .buffer(true)
77
- .parse(bufferParser)
78
- .expect(200);
79
-
80
- expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
81
- expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
82
- expect(nock.isDone()).toBe(true);
83
- }
84
- );
85
-
86
- it.each(DRIVE_URLS)(
87
- '/fetch proxies direct image bytes for $name',
88
- async (item) => {
89
- mockDriveGet(item.fileId);
90
-
91
- const res = await request(app)
92
- .get('/fetch')
93
- .query({ url: item.url })
94
- .buffer(true)
95
- .parse(bufferParser)
96
- .expect(200);
97
-
98
- expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
99
- expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
100
- expect(nock.isDone()).toBe(true);
101
- }
102
- );
103
-
104
- it.each(DRIVE_URLS)(
105
- '/stream proxies direct image bytes for $name',
106
- async (item) => {
107
- mockDriveGet(item.fileId);
108
-
109
- const res = await request(app)
110
- .get('/stream')
111
- .query({ url: item.url })
112
- .buffer(true)
113
- .parse(bufferParser)
114
- .expect(200);
115
-
116
- expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
117
- expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
118
- expect(nock.isDone()).toBe(true);
119
- }
120
- );
121
-
122
- it.each(DRIVE_URLS)(
123
- '/markdown includes a direct image link for $name',
124
- async (item) => {
125
- mockDriveHead(item.fileId);
126
-
127
- const res = await request(app)
128
- .get('/markdown')
129
- .query({ url: item.url })
130
- .expect(200);
131
-
132
- expect(res.headers['content-type']).toMatch(/text\/markdown/);
133
- expect(res.text).toContain(
134
- `![Google Drive image](${directDownloadUrl(item.fileId)})`
135
- );
136
- expect(nock.isDone()).toBe(true);
137
- }
138
- );
139
-
140
- it.each(DRIVE_URLS)(
141
- '/txt includes the original and direct image links for $name',
142
- async (item) => {
143
- mockDriveHead(item.fileId);
144
-
145
- const res = await request(app)
146
- .get('/txt')
147
- .query({ url: item.url })
148
- .expect(200);
149
-
150
- expect(res.headers['content-type']).toMatch(/text\/plain/);
151
- expect(res.text).toContain('Google Drive image');
152
- expect(res.text).toContain(`Original Google Drive file: ${item.url}`);
153
- expect(res.text).toContain(`Image: ${directDownloadUrl(item.fileId)}`);
154
- expect(nock.isDone()).toBe(true);
155
- }
156
- );
157
-
158
- it.each(DRIVE_URLS)(
159
- '/html includes a direct image tag for $name',
160
- async (item) => {
161
- mockDriveHead(item.fileId);
162
-
163
- const res = await request(app)
164
- .get('/html')
165
- .query({ url: item.url })
166
- .expect(200);
167
-
168
- expect(res.headers['content-type']).toMatch(/text\/html/);
169
- expect(res.text).toContain(
170
- `src="${directDownloadUrl(item.fileId).replace('&', '&amp;')}"`
171
- );
172
- expect(nock.isDone()).toBe(true);
173
- }
174
- );
175
-
176
- it('archive includes markdown and the downloaded Drive image', async () => {
177
- const item = DRIVE_URLS[0];
178
- mockDriveHead(item.fileId);
179
- mockDriveGet(item.fileId);
180
-
181
- const res = await request(app)
182
- .get('/archive')
183
- .query({ url: item.url })
184
- .buffer(true)
185
- .parse(bufferParser)
186
- .expect(200);
187
-
188
- const zip = await unzipper.Open.buffer(res.body);
189
- const document = zip.files.find((file) => file.path === 'document.md');
190
- const image = zip.files.find((file) => file.path.startsWith('images/'));
191
-
192
- expect(document).toBeDefined();
193
- expect(image).toBeDefined();
194
- expect((await document.buffer()).toString('utf8')).toContain(
195
- '![Google Drive image](images/image-1.jpg)'
196
- );
197
- expect((await image.buffer()).subarray(0, 3)).toEqual(
198
- Buffer.from([0xff, 0xd8, 0xff])
199
- );
200
- expect(nock.isDone()).toBe(true);
201
- });
202
-
203
- it('docx includes the downloaded Drive image', async () => {
204
- const item = DRIVE_URLS[0];
205
- mockDriveHead(item.fileId);
206
- mockDriveGet(item.fileId);
207
-
208
- const res = await request(app)
209
- .get('/docx')
210
- .query({ url: item.url })
211
- .buffer(true)
212
- .parse(bufferParser)
213
- .expect(200);
214
-
215
- expect(res.headers['content-type']).toMatch(
216
- /application\/vnd\.openxmlformats-officedocument\.wordprocessingml\.document/
217
- );
218
- expect(res.body.subarray(0, 2)).toEqual(Buffer.from('PK'));
219
-
220
- const docx = await unzipper.Open.buffer(res.body);
221
- const mediaEntry = docx.files.find(
222
- (file) => file.path.startsWith('word/media/') && !file.path.endsWith('/')
223
- );
224
- expect(mediaEntry).toBeDefined();
225
- expect((await mediaEntry.buffer()).subarray(0, 3)).toEqual(
226
- Buffer.from([0xff, 0xd8, 0xff])
227
- );
228
- expect(nock.isDone()).toBe(true);
229
- });
230
-
231
- it('pdf renders the downloaded Drive image document', async () => {
232
- const item = DRIVE_URLS[0];
233
- mockDriveGet(item.fileId);
234
-
235
- const res = await request(app)
236
- .get('/pdf')
237
- .query({ url: item.url })
238
- .buffer(true)
239
- .parse(bufferParser)
240
- .expect(200);
241
-
242
- expect(res.headers['content-type']).toMatch(/application\/pdf/);
243
- expect(res.body.subarray(0, 4).toString('utf8')).toBe('%PDF');
244
- expect(nock.isDone()).toBe(true);
245
- });
246
- });