@link-assistant/web-capture 1.12.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/README.md +2 -1
- package/bin/web-capture.js +5 -7
- package/package.json +32 -18
- package/src/archive.js +1 -2
- package/src/binary-download.js +13 -0
- package/src/docx.js +14 -11
- package/src/fetch.js +1 -2
- package/src/figures.js +2 -12
- package/src/gdocs-images.js +0 -1
- package/src/gdocs.js +0 -1
- package/src/github.js +0 -1
- package/src/lib.js +1 -2
- package/src/localize-images.js +2 -12
- package/src/markdown.js +0 -1
- package/src/shared-dialog.js +0 -1
- package/src/stream.js +0 -1
- package/src/transport.js +23 -5
- package/src/txt.js +0 -1
- package/.changeset/README.md +0 -8
- package/.changeset/config.json +0 -11
- package/.dockerignore +0 -6
- package/.jscpd.json +0 -19
- package/.lenv +0 -8
- package/.prettierignore +0 -8
- package/.prettierrc +0 -10
- package/Dockerfile +0 -52
- package/babel.config.cjs +0 -13
- package/docker-compose.yml +0 -11
- package/eslint.config.js +0 -112
- package/examples/js/capture_cache.js +0 -12
- package/examples/js/engine_comparison.js +0 -39
- package/examples/js/html_download.js +0 -23
- package/examples/js/image_download.js +0 -36
- package/examples/js/markdown_download.js +0 -23
- package/examples/js/playwright_example.js +0 -23
- package/examples/python/html_download.py +0 -15
- package/examples/python/image_download.py +0 -20
- package/examples/python/markdown_download.py +0 -15
- package/examples/python/playwright_example.py +0 -22
- package/jest.config.mjs +0 -20
- package/tests/ci/dependency-freshness.test.js +0 -74
- package/tests/ci/detect-code-changes.test.js +0 -174
- package/tests/ci/docker-install.test.js +0 -31
- package/tests/ci/rust-dockerfile.test.js +0 -54
- package/tests/ci/workflow-policy.test.js +0 -79
- package/tests/e2e/docker.test.js +0 -186
- package/tests/e2e/process.test.js +0 -215
- package/tests/fixtures/bold-coalesce-and-leakage.html +0 -27
- package/tests/fixtures/deepwiki/deepwiki-full-page.png +0 -0
- package/tests/fixtures/deepwiki/index.md +0 -1500
- package/tests/fixtures/deepwiki/part-1.md +0 -776
- package/tests/fixtures/habr-articles.js +0 -45
- package/tests/fixtures/heading-numbering.html +0 -19
- package/tests/fixtures/list-item-with-br.html +0 -8
- package/tests/fixtures/ol-continuous-numbering.html +0 -12
- package/tests/fixtures/paragraph-vs-line-break.html +0 -11
- package/tests/helpers/nock.js +0 -10
- package/tests/helpers/search-fixture.cjs +0 -17
- package/tests/helpers/temporary-directory.js +0 -12
- package/tests/integration/api-endpoints.test.js +0 -418
- package/tests/integration/archive-default-layout.test.js +0 -37
- package/tests/integration/browser-engines.test.js +0 -128
- package/tests/integration/browser-packaged-path.test.js +0 -45
- package/tests/integration/deepwiki-download.test.js +0 -144
- package/tests/integration/deepwiki-rendered-page.mock.test.js +0 -121
- package/tests/integration/formalai-contract.test.js +0 -214
- package/tests/integration/gdocs-public-doc.test.js +0 -315
- package/tests/integration/github-readme.test.js +0 -137
- package/tests/integration/google-drive-image.test.js +0 -246
- package/tests/integration/habr-article.test.js +0 -337
- package/tests/integration/search-endpoint.test.js +0 -88
- package/tests/integration/stackoverflow-download.test.js +0 -173
- package/tests/integration/wikipedia-download.test.js +0 -142
- package/tests/jest.setup.mjs +0 -7
- package/tests/mock/index.test.js +0 -225
- package/tests/unit/animation.test.js +0 -33
- package/tests/unit/batch.test.js +0 -114
- package/tests/unit/bold-coalesce-and-leakage.test.js +0 -54
- package/tests/unit/browser-launch-options.test.js +0 -42
- package/tests/unit/browser.test.js +0 -172
- package/tests/unit/cache.test.js +0 -182
- package/tests/unit/cli.test.js +0 -399
- package/tests/unit/convertGoogleDriveUrl.test.js +0 -89
- package/tests/unit/convertRelativeUrls.test.js +0 -34
- package/tests/unit/extract-images.test.js +0 -259
- package/tests/unit/fetch-html.test.js +0 -54
- package/tests/unit/figures.test.js +0 -74
- package/tests/unit/gdocs-export-headers.test.js +0 -28
- package/tests/unit/gdocs-preprocess.test.js +0 -36
- package/tests/unit/gdocs.test.js +0 -1490
- package/tests/unit/github.test.js +0 -156
- package/tests/unit/heading-image.test.js +0 -49
- package/tests/unit/heading-numbering.test.js +0 -28
- package/tests/unit/html2md-br-in-list-item.test.js +0 -23
- package/tests/unit/html2md-ol-numbering.test.js +0 -20
- package/tests/unit/html2md.test.js +0 -695
- package/tests/unit/image-mode-defaults.test.js +0 -54
- package/tests/unit/kreuzberg.test.js +0 -249
- package/tests/unit/latex.test.js +0 -123
- package/tests/unit/localize-images.test.js +0 -114
- package/tests/unit/markdown.test.js +0 -36
- package/tests/unit/metadata.test.js +0 -140
- package/tests/unit/package-runtime-dependencies.test.js +0 -74
- package/tests/unit/paragraph-vs-line-break.test.js +0 -56
- package/tests/unit/postprocess.test.js +0 -110
- package/tests/unit/proxy-headers.test.js +0 -100
- package/tests/unit/publish-verification.test.js +0 -57
- package/tests/unit/retry.test.js +0 -111
- package/tests/unit/search.test.js +0 -339
- package/tests/unit/shared-dialog.test.js +0 -173
- package/tests/unit/stackoverflow.test.js +0 -32
- package/tests/unit/stream.test.js +0 -85
- package/tests/unit/verify.test.js +0 -182
- package/tests/xpaste/xpaste-screenshot-markdown.mock.test.js +0 -375
- package/tests/xpaste/xpaste.e2e.test.js +0 -36
- package/tests/xpaste/xpaste.integration.test.js +0 -41
- package/tests/xpaste/xpaste.mock.test.js +0 -261
- package/tests/xpaste/xpaste.unit.test.js +0 -119
|
@@ -1,315 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Integration tests for the Google Docs markdown round-trip reference document.
|
|
3
|
-
*
|
|
4
|
-
* The document at
|
|
5
|
-
* https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit
|
|
6
|
-
* is a public document created specifically for validating web-capture's Google
|
|
7
|
-
* Docs support across every URL variation and every Markdown feature category
|
|
8
|
-
* documented in issue #90. The ground-truth markdown and authoring archive are
|
|
9
|
-
* preserved under docs/case-studies/issue-90/reference/.
|
|
10
|
-
*
|
|
11
|
-
* Live capture tests are gated behind GDOCS_INTEGRATION=true so normal PR CI
|
|
12
|
-
* stays hermetic. The URL-variation, document-id extraction and feature
|
|
13
|
-
* checklist tests always run so regressions in the parsing surface are caught
|
|
14
|
-
* even without network access.
|
|
15
|
-
*/
|
|
16
|
-
|
|
17
|
-
import { jest } from '@jest/globals';
|
|
18
|
-
import fs from 'fs';
|
|
19
|
-
import path from 'path';
|
|
20
|
-
import { fileURLToPath } from 'url';
|
|
21
|
-
|
|
22
|
-
import {
|
|
23
|
-
isGoogleDocsUrl,
|
|
24
|
-
extractDocumentId,
|
|
25
|
-
buildExportUrl,
|
|
26
|
-
buildEditUrl,
|
|
27
|
-
buildDocsApiUrl,
|
|
28
|
-
selectGoogleDocsCaptureMethod,
|
|
29
|
-
fetchGoogleDocAsMarkdown,
|
|
30
|
-
captureGoogleDocWithBrowser,
|
|
31
|
-
} from '../../src/gdocs.js';
|
|
32
|
-
import { retry } from '../../src/retry.js';
|
|
33
|
-
|
|
34
|
-
const LIVE =
|
|
35
|
-
process.env.GDOCS_INTEGRATION === 'true' ||
|
|
36
|
-
process.env.GDOCS_INTEGRATION === '1';
|
|
37
|
-
const describeIfLive = LIVE ? describe : describe.skip;
|
|
38
|
-
|
|
39
|
-
jest.setTimeout(120000);
|
|
40
|
-
|
|
41
|
-
const __filename = fileURLToPath(import.meta.url);
|
|
42
|
-
const __dirname = path.dirname(__filename);
|
|
43
|
-
const REFERENCE_DIR = path.resolve(
|
|
44
|
-
__dirname,
|
|
45
|
-
'..',
|
|
46
|
-
'..',
|
|
47
|
-
'..',
|
|
48
|
-
'docs',
|
|
49
|
-
'case-studies',
|
|
50
|
-
'issue-90',
|
|
51
|
-
'reference'
|
|
52
|
-
);
|
|
53
|
-
|
|
54
|
-
// The public document referenced in issue #90.
|
|
55
|
-
// https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit
|
|
56
|
-
export const PUBLIC_TEST_DOCUMENT = {
|
|
57
|
-
id: '1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM',
|
|
58
|
-
canonicalUrl:
|
|
59
|
-
'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit',
|
|
60
|
-
urlVariations: [
|
|
61
|
-
'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM',
|
|
62
|
-
'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit',
|
|
63
|
-
'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit?usp=sharing',
|
|
64
|
-
'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit?usp=sharing&ouid=102030405060708090100&rtpof=true&sd=true',
|
|
65
|
-
'https://docs.google.com/document/d/1f5zI2xOFpKa90v0GjamO_t7lqSdzMlaM/edit?tab=t.0',
|
|
66
|
-
],
|
|
67
|
-
// Section headings from the reference markdown. Each one should be present
|
|
68
|
-
// in any Markdown produced from the document, regardless of capture mode.
|
|
69
|
-
// See docs/case-studies/issue-90/reference/markdown-test-document.md.
|
|
70
|
-
sections: [
|
|
71
|
-
'Markdown Feature Test Document',
|
|
72
|
-
'1. Headings',
|
|
73
|
-
'2. Inline Formatting',
|
|
74
|
-
'3. Paragraphs',
|
|
75
|
-
'4. Blockquotes',
|
|
76
|
-
'5. Unordered Lists',
|
|
77
|
-
'6. Ordered Lists',
|
|
78
|
-
'7. Mixed Lists',
|
|
79
|
-
'8. Tables',
|
|
80
|
-
'9. Links',
|
|
81
|
-
'10. Images',
|
|
82
|
-
'11. Horizontal Rules',
|
|
83
|
-
'12. Special Characters',
|
|
84
|
-
'13. Nested Formatting Edge Cases',
|
|
85
|
-
'14. Empty and Minimal Table Content',
|
|
86
|
-
],
|
|
87
|
-
};
|
|
88
|
-
|
|
89
|
-
// The issue #108 v2 document extends the public reference fixture with Section
|
|
90
|
-
// 15, covering browser-model list type and continuation-paragraph regressions.
|
|
91
|
-
export const PUBLIC_TEST_DOCUMENT_V2 = {
|
|
92
|
-
id: '1Rvaod_u2wgkAUNdXG-e29yQem-P36KAQ',
|
|
93
|
-
canonicalUrl:
|
|
94
|
-
'https://docs.google.com/document/d/1Rvaod_u2wgkAUNdXG-e29yQem-P36KAQ/edit',
|
|
95
|
-
};
|
|
96
|
-
|
|
97
|
-
function expectIssue108Markdown(markdown) {
|
|
98
|
-
expect(markdown).toContain(
|
|
99
|
-
'15. List & Blockquote Edge Cases (browser-mode reproducers)'
|
|
100
|
-
);
|
|
101
|
-
expect(markdown).toMatch(/1\.\s+Apple/u);
|
|
102
|
-
expect(markdown).toMatch(/2\.\s+Banana/u);
|
|
103
|
-
expect(markdown).toMatch(/3\.\s+Cherry/u);
|
|
104
|
-
expect(markdown).not.toMatch(/^-\s+Apple$/mu);
|
|
105
|
-
expect(markdown).toContain(
|
|
106
|
-
'Continuation paragraph that is not a list item. Same indent as Step one.'
|
|
107
|
-
);
|
|
108
|
-
expect(markdown).not.toContain(
|
|
109
|
-
'> Continuation paragraph that is not a list item.'
|
|
110
|
-
);
|
|
111
|
-
expect(markdown).toMatch(/^-\s+Red$/mu);
|
|
112
|
-
expect(markdown).toMatch(/^-\s+Green$/mu);
|
|
113
|
-
expect(markdown).toMatch(/^-\s+Blue$/mu);
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
describe('Google Docs public test document (issue #90)', () => {
|
|
117
|
-
describe('URL variations', () => {
|
|
118
|
-
it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
|
|
119
|
-
'accepts %s as a Google Docs URL',
|
|
120
|
-
(url) => {
|
|
121
|
-
expect(isGoogleDocsUrl(url)).toBe(true);
|
|
122
|
-
}
|
|
123
|
-
);
|
|
124
|
-
|
|
125
|
-
it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
|
|
126
|
-
'extracts the public document ID from %s',
|
|
127
|
-
(url) => {
|
|
128
|
-
expect(extractDocumentId(url)).toBe(PUBLIC_TEST_DOCUMENT.id);
|
|
129
|
-
}
|
|
130
|
-
);
|
|
131
|
-
|
|
132
|
-
it('builds the expected export URL for the public document', () => {
|
|
133
|
-
expect(buildExportUrl(PUBLIC_TEST_DOCUMENT.id, 'html')).toBe(
|
|
134
|
-
`https://docs.google.com/document/d/${PUBLIC_TEST_DOCUMENT.id}/export?format=html`
|
|
135
|
-
);
|
|
136
|
-
});
|
|
137
|
-
|
|
138
|
-
it('builds the expected editor URL for the public document', () => {
|
|
139
|
-
expect(buildEditUrl(PUBLIC_TEST_DOCUMENT.id)).toBe(
|
|
140
|
-
`https://docs.google.com/document/d/${PUBLIC_TEST_DOCUMENT.id}/edit`
|
|
141
|
-
);
|
|
142
|
-
});
|
|
143
|
-
|
|
144
|
-
it('builds the expected Docs API URL for the public document', () => {
|
|
145
|
-
expect(buildDocsApiUrl(PUBLIC_TEST_DOCUMENT.id)).toBe(
|
|
146
|
-
`https://docs.googleapis.com/v1/documents/${PUBLIC_TEST_DOCUMENT.id}`
|
|
147
|
-
);
|
|
148
|
-
});
|
|
149
|
-
});
|
|
150
|
-
|
|
151
|
-
describe('Capture method selection (issue #72 regression guard)', () => {
|
|
152
|
-
it('routes --capture browser to the editor-model backend', () => {
|
|
153
|
-
expect(selectGoogleDocsCaptureMethod('browser')).toBe('browser-model');
|
|
154
|
-
});
|
|
155
|
-
|
|
156
|
-
it('routes --capture api to the public-export backend when no token', () => {
|
|
157
|
-
expect(selectGoogleDocsCaptureMethod('api')).toBe('public-export');
|
|
158
|
-
});
|
|
159
|
-
|
|
160
|
-
it('routes --capture api with a token to the Docs REST API backend', () => {
|
|
161
|
-
expect(selectGoogleDocsCaptureMethod('api', 'token-123')).toBe(
|
|
162
|
-
'docs-api'
|
|
163
|
-
);
|
|
164
|
-
});
|
|
165
|
-
});
|
|
166
|
-
|
|
167
|
-
describe('Reference markdown fixture', () => {
|
|
168
|
-
const referenceMarkdownPath = path.join(
|
|
169
|
-
REFERENCE_DIR,
|
|
170
|
-
'markdown-test-document.md'
|
|
171
|
-
);
|
|
172
|
-
const referenceMarkdown = fs.readFileSync(referenceMarkdownPath, 'utf-8');
|
|
173
|
-
|
|
174
|
-
it('contains every feature category documented in issue #90', () => {
|
|
175
|
-
for (const section of PUBLIC_TEST_DOCUMENT.sections) {
|
|
176
|
-
expect(referenceMarkdown).toContain(section);
|
|
177
|
-
}
|
|
178
|
-
});
|
|
179
|
-
|
|
180
|
-
it('embeds the four test images referenced by the issue', () => {
|
|
181
|
-
expect(referenceMarkdown).toMatch(
|
|
182
|
-
/!\[Blue rectangle]\(media\/image1\.png\)/
|
|
183
|
-
);
|
|
184
|
-
expect(referenceMarkdown).toMatch(
|
|
185
|
-
/!\[Red rectangle]\(media\/image2\.png\)/
|
|
186
|
-
);
|
|
187
|
-
expect(referenceMarkdown).toMatch(
|
|
188
|
-
/!\[Green square]\(media\/image3\.png\)/
|
|
189
|
-
);
|
|
190
|
-
expect(referenceMarkdown).toMatch(
|
|
191
|
-
/!\[Yellow square]\(media\/image4\.png\)/
|
|
192
|
-
);
|
|
193
|
-
});
|
|
194
|
-
});
|
|
195
|
-
|
|
196
|
-
describeIfLive('Live capture against the public document', () => {
|
|
197
|
-
// The primary deliverable of issue #90: the reference document is
|
|
198
|
-
// downloaded from Google Docs and every feature section can be found in
|
|
199
|
-
// the captured markdown. This is our regression net for any future
|
|
200
|
-
// changes to the public-export or HTML-to-markdown pipeline.
|
|
201
|
-
//
|
|
202
|
-
// Google Docs occasionally returns transient 500s on the public-export
|
|
203
|
-
// endpoint. The Habr integration suite has the same flake and solves it
|
|
204
|
-
// with retry + exponential backoff, so we reuse the shared helper here.
|
|
205
|
-
const fetchWithRetry = (url) =>
|
|
206
|
-
retry(() => fetchGoogleDocAsMarkdown(url), {
|
|
207
|
-
retries: 3,
|
|
208
|
-
baseDelay: 2000,
|
|
209
|
-
});
|
|
210
|
-
|
|
211
|
-
const browserEngine =
|
|
212
|
-
process.env.GDOCS_BROWSER_ENGINE || process.env.BROWSER_ENGINE;
|
|
213
|
-
const captureBrowserWithRetry = (url) =>
|
|
214
|
-
retry(
|
|
215
|
-
() =>
|
|
216
|
-
captureGoogleDocWithBrowser(url, {
|
|
217
|
-
...(browserEngine ? { engine: browserEngine } : {}),
|
|
218
|
-
}),
|
|
219
|
-
{
|
|
220
|
-
retries: 2,
|
|
221
|
-
baseDelay: 2000,
|
|
222
|
-
}
|
|
223
|
-
);
|
|
224
|
-
|
|
225
|
-
it('fetches the public document via --capture api and returns markdown', async () => {
|
|
226
|
-
const { markdown, documentId, exportUrl } = await fetchWithRetry(
|
|
227
|
-
PUBLIC_TEST_DOCUMENT.canonicalUrl
|
|
228
|
-
);
|
|
229
|
-
|
|
230
|
-
expect(documentId).toBe(PUBLIC_TEST_DOCUMENT.id);
|
|
231
|
-
expect(exportUrl).toBe(buildExportUrl(PUBLIC_TEST_DOCUMENT.id, 'html'));
|
|
232
|
-
expect(markdown.length).toBeGreaterThan(1000);
|
|
233
|
-
|
|
234
|
-
for (const section of PUBLIC_TEST_DOCUMENT.sections) {
|
|
235
|
-
expect(markdown).toContain(section);
|
|
236
|
-
}
|
|
237
|
-
expect(markdown).toContain('## 1. Headings');
|
|
238
|
-
expect(markdown).toContain('**This text is bold**');
|
|
239
|
-
expect(markdown).toContain('*This text is italic*');
|
|
240
|
-
expect(markdown).toContain('~~This text has strikethrough~~');
|
|
241
|
-
expect(markdown).toContain('> This is a single-level blockquote');
|
|
242
|
-
expect(markdown).toContain('| Feature | Supported | Notes |');
|
|
243
|
-
expect(markdown).not.toContain('| Feature | | Supported |');
|
|
244
|
-
expect(markdown).toContain('| | x | |');
|
|
245
|
-
expect(markdown).toContain('1. Parent item 1');
|
|
246
|
-
expect(markdown).toContain(' 1. Child item 1.1');
|
|
247
|
-
expect(markdown).toContain(' 1. Grandchild item 1.2.1');
|
|
248
|
-
expect(markdown).not.toContain('- Child item 1.1');
|
|
249
|
-
});
|
|
250
|
-
|
|
251
|
-
it.each(PUBLIC_TEST_DOCUMENT.urlVariations)(
|
|
252
|
-
'resolves %s to the same document when capturing',
|
|
253
|
-
async (url) => {
|
|
254
|
-
const { documentId } = await fetchWithRetry(url);
|
|
255
|
-
expect(documentId).toBe(PUBLIC_TEST_DOCUMENT.id);
|
|
256
|
-
}
|
|
257
|
-
);
|
|
258
|
-
|
|
259
|
-
it('captures the public document via --capture browser and renders DOCS_modelChunk markdown', async () => {
|
|
260
|
-
const { markdown, documentId, exportUrl, capture } =
|
|
261
|
-
await captureBrowserWithRetry(PUBLIC_TEST_DOCUMENT.canonicalUrl);
|
|
262
|
-
|
|
263
|
-
expect(documentId).toBe(PUBLIC_TEST_DOCUMENT.id);
|
|
264
|
-
expect(exportUrl).toBe(buildEditUrl(PUBLIC_TEST_DOCUMENT.id));
|
|
265
|
-
expect(capture.blocks.length).toBeGreaterThan(20);
|
|
266
|
-
expect(capture.images.length).toBeGreaterThanOrEqual(4);
|
|
267
|
-
expect(markdown.length).toBeGreaterThan(2500);
|
|
268
|
-
expect(markdown).toContain('# Markdown Feature Test Document');
|
|
269
|
-
expect(markdown).toContain('## 1. Headings');
|
|
270
|
-
expect(markdown).toContain('**This text is bold**');
|
|
271
|
-
expect(markdown).toContain('*This text is italic*');
|
|
272
|
-
expect(markdown).toContain('~~This text has strikethrough~~');
|
|
273
|
-
expect(markdown).toContain('> This is a single-level blockquote');
|
|
274
|
-
expect(markdown).toContain('[Regular link](https://example.com)');
|
|
275
|
-
expect(markdown).toContain('| Feature | Supported | Notes |');
|
|
276
|
-
expect(markdown).not.toContain('| Feature | | Supported |');
|
|
277
|
-
expect(markdown).toContain('| | x | |');
|
|
278
|
-
expect(markdown).toContain('1. Parent item 1');
|
|
279
|
-
expect(markdown).toContain(' 1. Child item 1.1');
|
|
280
|
-
expect(markdown).toContain(' 1. Grandchild item 1.2.1');
|
|
281
|
-
expect(markdown).not.toContain('- Child item 1.1');
|
|
282
|
-
expect(markdown).not.toContain('1. Parent item 1\n\n');
|
|
283
|
-
expect(markdown).toContain(';
|
|
284
|
-
expect(markdown).toContain(';
|
|
285
|
-
expect(markdown).toContain(';
|
|
286
|
-
expect(markdown).toContain(';
|
|
287
|
-
expect(markdown).toContain('docs-images-rt/');
|
|
288
|
-
expect(markdown).toContain('---');
|
|
289
|
-
});
|
|
290
|
-
|
|
291
|
-
it('fetches the issue #108 v2 document via --capture api and preserves Section 15 semantics', async () => {
|
|
292
|
-
const { markdown, documentId, exportUrl } = await fetchWithRetry(
|
|
293
|
-
PUBLIC_TEST_DOCUMENT_V2.canonicalUrl
|
|
294
|
-
);
|
|
295
|
-
|
|
296
|
-
expect(documentId).toBe(PUBLIC_TEST_DOCUMENT_V2.id);
|
|
297
|
-
expect(exportUrl).toBe(
|
|
298
|
-
buildExportUrl(PUBLIC_TEST_DOCUMENT_V2.id, 'html')
|
|
299
|
-
);
|
|
300
|
-
expectIssue108Markdown(markdown);
|
|
301
|
-
});
|
|
302
|
-
|
|
303
|
-
it('captures the issue #108 v2 document via --capture browser and preserves Section 15 semantics', async () => {
|
|
304
|
-
const { markdown, documentId, exportUrl, capture } =
|
|
305
|
-
await captureBrowserWithRetry(PUBLIC_TEST_DOCUMENT_V2.canonicalUrl);
|
|
306
|
-
|
|
307
|
-
expect(documentId).toBe(PUBLIC_TEST_DOCUMENT_V2.id);
|
|
308
|
-
expect(exportUrl).toBe(buildEditUrl(PUBLIC_TEST_DOCUMENT_V2.id));
|
|
309
|
-
expect(capture.blocks.length).toBeGreaterThan(30);
|
|
310
|
-
expectIssue108Markdown(markdown);
|
|
311
|
-
expect(markdown).not.toContain('*Apple*');
|
|
312
|
-
expect(markdown).not.toContain('*Red*');
|
|
313
|
-
});
|
|
314
|
-
});
|
|
315
|
-
});
|
|
@@ -1,137 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Integration tests for capturing GitHub repository pages (issue #5).
|
|
3
|
-
*
|
|
4
|
-
* Live tests are gated behind GITHUB_REPOSITORY_INTEGRATION so default/offline
|
|
5
|
-
* runs stay deterministic. The mocked API endpoint tests cover the default
|
|
6
|
-
* behavior without network access; this suite proves the real GitHub repository
|
|
7
|
-
* page can still be captured as compact text/markdown, original HTML, and PNG
|
|
8
|
-
* screenshots in every supported JavaScript browser engine.
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
import { jest } from '@jest/globals';
|
|
12
|
-
import { createBrowser } from '../../src/browser.js';
|
|
13
|
-
import { fetchHtml } from '../../src/lib.js';
|
|
14
|
-
import { retry } from '../../src/retry.js';
|
|
15
|
-
import {
|
|
16
|
-
fetchGithubRepositorySnapshot,
|
|
17
|
-
formatGithubRepositoryMarkdown,
|
|
18
|
-
formatGithubRepositoryText,
|
|
19
|
-
parseGithubRepositoryUrl,
|
|
20
|
-
} from '../../src/github.js';
|
|
21
|
-
|
|
22
|
-
const GITHUB_REPOSITORY_URL =
|
|
23
|
-
process.env.GITHUB_REPOSITORY_URL ||
|
|
24
|
-
'https://github.com/link-assistant/web-capture';
|
|
25
|
-
const PNG_SIGNATURE = Buffer.from([
|
|
26
|
-
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
|
|
27
|
-
]);
|
|
28
|
-
|
|
29
|
-
const SKIP_LIVE =
|
|
30
|
-
!process.env.GITHUB_REPOSITORY_INTEGRATION ||
|
|
31
|
-
process.env.GITHUB_REPOSITORY_INTEGRATION === 'false';
|
|
32
|
-
const describeIfLive = SKIP_LIVE ? describe.skip : describe;
|
|
33
|
-
|
|
34
|
-
jest.setTimeout(120000);
|
|
35
|
-
|
|
36
|
-
async function navigateWithRetry(page, url) {
|
|
37
|
-
await retry(
|
|
38
|
-
async () => {
|
|
39
|
-
await page.goto(url, {
|
|
40
|
-
waitUntil: 'domcontentloaded',
|
|
41
|
-
timeout: 30000,
|
|
42
|
-
});
|
|
43
|
-
},
|
|
44
|
-
{
|
|
45
|
-
retries: 3,
|
|
46
|
-
baseDelay: 2000,
|
|
47
|
-
onRetry: (err, attempt, delay) => {
|
|
48
|
-
console.log(
|
|
49
|
-
`Navigation retry ${attempt} for ${url} after ${delay}ms: ${err.message}`
|
|
50
|
-
);
|
|
51
|
-
},
|
|
52
|
-
}
|
|
53
|
-
);
|
|
54
|
-
await page.waitForTimeout(5000);
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
describeIfLive('GitHub repository capture', () => {
|
|
58
|
-
it('downloads compact repository text and markdown with README plus file tree', async () => {
|
|
59
|
-
const parsed = parseGithubRepositoryUrl(GITHUB_REPOSITORY_URL);
|
|
60
|
-
expect(parsed).not.toBeNull();
|
|
61
|
-
|
|
62
|
-
const snapshot = await retry(
|
|
63
|
-
() => fetchGithubRepositorySnapshot(GITHUB_REPOSITORY_URL),
|
|
64
|
-
{
|
|
65
|
-
retries: 3,
|
|
66
|
-
baseDelay: 2000,
|
|
67
|
-
}
|
|
68
|
-
);
|
|
69
|
-
|
|
70
|
-
const text = formatGithubRepositoryText(snapshot);
|
|
71
|
-
const markdown = formatGithubRepositoryMarkdown(snapshot);
|
|
72
|
-
|
|
73
|
-
expect(text).toContain(`Repository: ${parsed.fullName}`);
|
|
74
|
-
expect(text).toContain('Files:');
|
|
75
|
-
expect(text).toMatch(/README/i);
|
|
76
|
-
expect(text.length).toBeGreaterThan(500);
|
|
77
|
-
|
|
78
|
-
expect(markdown).toContain(`# ${parsed.fullName}`);
|
|
79
|
-
expect(markdown).toContain('## Files');
|
|
80
|
-
expect(markdown).toMatch(/## .*README/i);
|
|
81
|
-
expect(markdown.length).toBeGreaterThan(500);
|
|
82
|
-
expect(markdown).not.toMatch(/<html/i);
|
|
83
|
-
});
|
|
84
|
-
|
|
85
|
-
it('downloads original GitHub repository HTML', async () => {
|
|
86
|
-
const parsed = parseGithubRepositoryUrl(GITHUB_REPOSITORY_URL);
|
|
87
|
-
expect(parsed).not.toBeNull();
|
|
88
|
-
|
|
89
|
-
const html = await retry(() => fetchHtml(GITHUB_REPOSITORY_URL), {
|
|
90
|
-
retries: 3,
|
|
91
|
-
baseDelay: 2000,
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
expect(html).toMatch(/<html/i);
|
|
95
|
-
expect(html).toContain(parsed.owner);
|
|
96
|
-
expect(html).toContain(parsed.repo);
|
|
97
|
-
expect(html.length).toBeGreaterThan(1000);
|
|
98
|
-
});
|
|
99
|
-
});
|
|
100
|
-
|
|
101
|
-
describe.each(['puppeteer', 'playwright'])(
|
|
102
|
-
'GitHub repository screenshot (%s engine)',
|
|
103
|
-
(engine) => {
|
|
104
|
-
describeIfLive(`${engine}`, () => {
|
|
105
|
-
let browser;
|
|
106
|
-
|
|
107
|
-
beforeAll(async () => {
|
|
108
|
-
browser = await createBrowser(engine);
|
|
109
|
-
});
|
|
110
|
-
|
|
111
|
-
afterAll(async () => {
|
|
112
|
-
if (browser) {
|
|
113
|
-
await browser.close();
|
|
114
|
-
}
|
|
115
|
-
});
|
|
116
|
-
|
|
117
|
-
it('captures the GitHub repository page as a PNG screenshot', async () => {
|
|
118
|
-
const page = await browser.newPage();
|
|
119
|
-
await page.setExtraHTTPHeaders({
|
|
120
|
-
'Accept-Language': 'en-US,en;q=0.9',
|
|
121
|
-
'Accept-Charset': 'utf-8',
|
|
122
|
-
});
|
|
123
|
-
await page.setUserAgent(
|
|
124
|
-
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
125
|
-
);
|
|
126
|
-
await page.setViewport({ width: 1280, height: 800 });
|
|
127
|
-
await navigateWithRetry(page, GITHUB_REPOSITORY_URL);
|
|
128
|
-
|
|
129
|
-
const screenshot = await page.screenshot({ type: 'png' });
|
|
130
|
-
|
|
131
|
-
expect(screenshot).toBeInstanceOf(Buffer);
|
|
132
|
-
expect(screenshot.length).toBeGreaterThan(1000);
|
|
133
|
-
expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
|
|
134
|
-
});
|
|
135
|
-
});
|
|
136
|
-
}
|
|
137
|
-
);
|
|
@@ -1,246 +0,0 @@
|
|
|
1
|
-
import request from 'supertest';
|
|
2
|
-
import nock from '../helpers/nock.js';
|
|
3
|
-
import unzipper from 'unzipper';
|
|
4
|
-
import { app } from '../../src/index.js';
|
|
5
|
-
|
|
6
|
-
const FIRST_FILE_ID = '1Cxkx6-428EQAX0-eiaq66H829ohnPp7q';
|
|
7
|
-
const SECOND_FILE_ID = '1fgJaftjv53xCN7vgiJOaQlbfWkUqPgyd';
|
|
8
|
-
const JPEG_BYTES = Buffer.from(
|
|
9
|
-
'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////2wBDAf//////////////////////////////////////////////////////////////////////////////////////wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAX/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIQAxAAAAH/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECAQE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Ap//xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAE/IV//2gAMAwEAAgADAAAAEP/EFBQRAQAAAAAAAAAAAAAAAAAAARD/2gAIAQMBAT8QH//EFBQRAQAAAAAAAAAAAAAAAAAAARD/2gAIAQIBAT8QH//EFBABAQAAAAAAAAAAAAAAAAAAARD/2gAIAQEAAT8QH//Z',
|
|
10
|
-
'base64'
|
|
11
|
-
);
|
|
12
|
-
|
|
13
|
-
const DRIVE_URLS = [
|
|
14
|
-
{
|
|
15
|
-
name: 'first issue URL with /view',
|
|
16
|
-
fileId: FIRST_FILE_ID,
|
|
17
|
-
url: `https://drive.google.com/file/d/${FIRST_FILE_ID}/view`,
|
|
18
|
-
},
|
|
19
|
-
{
|
|
20
|
-
name: 'first issue URL without /view',
|
|
21
|
-
fileId: FIRST_FILE_ID,
|
|
22
|
-
url: `https://drive.google.com/file/d/${FIRST_FILE_ID}`,
|
|
23
|
-
},
|
|
24
|
-
{
|
|
25
|
-
name: 'second issue URL with /view',
|
|
26
|
-
fileId: SECOND_FILE_ID,
|
|
27
|
-
url: `https://drive.google.com/file/d/${SECOND_FILE_ID}/view`,
|
|
28
|
-
},
|
|
29
|
-
{
|
|
30
|
-
name: 'second issue URL without /view',
|
|
31
|
-
fileId: SECOND_FILE_ID,
|
|
32
|
-
url: `https://drive.google.com/file/d/${SECOND_FILE_ID}`,
|
|
33
|
-
},
|
|
34
|
-
];
|
|
35
|
-
|
|
36
|
-
function directDownloadUrl(fileId) {
|
|
37
|
-
return `https://drive.google.com/uc?export=download&id=${fileId}`;
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
function mockDriveHead(fileId) {
|
|
41
|
-
return nock('https://drive.google.com')
|
|
42
|
-
.head('/uc')
|
|
43
|
-
.query({ export: 'download', id: fileId })
|
|
44
|
-
.reply(200, '', { 'content-type': 'image/jpeg' });
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
function mockDriveGet(fileId) {
|
|
48
|
-
return nock('https://drive.google.com')
|
|
49
|
-
.get('/uc')
|
|
50
|
-
.query({ export: 'download', id: fileId })
|
|
51
|
-
.reply(200, JPEG_BYTES, {
|
|
52
|
-
'content-type': 'image/jpeg',
|
|
53
|
-
'content-length': String(JPEG_BYTES.length),
|
|
54
|
-
});
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
function bufferParser(res, callback) {
|
|
58
|
-
const chunks = [];
|
|
59
|
-
res.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
|
|
60
|
-
res.on('end', () => callback(null, Buffer.concat(chunks)));
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
afterEach(() => {
|
|
64
|
-
nock.cleanAll();
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
describe('Google Drive public image links', () => {
|
|
68
|
-
it.each(DRIVE_URLS)(
|
|
69
|
-
'/image downloads image bytes for $name',
|
|
70
|
-
async (item) => {
|
|
71
|
-
mockDriveGet(item.fileId);
|
|
72
|
-
|
|
73
|
-
const res = await request(app)
|
|
74
|
-
.get('/image')
|
|
75
|
-
.query({ url: item.url })
|
|
76
|
-
.buffer(true)
|
|
77
|
-
.parse(bufferParser)
|
|
78
|
-
.expect(200);
|
|
79
|
-
|
|
80
|
-
expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
|
|
81
|
-
expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
|
|
82
|
-
expect(nock.isDone()).toBe(true);
|
|
83
|
-
}
|
|
84
|
-
);
|
|
85
|
-
|
|
86
|
-
it.each(DRIVE_URLS)(
|
|
87
|
-
'/fetch proxies direct image bytes for $name',
|
|
88
|
-
async (item) => {
|
|
89
|
-
mockDriveGet(item.fileId);
|
|
90
|
-
|
|
91
|
-
const res = await request(app)
|
|
92
|
-
.get('/fetch')
|
|
93
|
-
.query({ url: item.url })
|
|
94
|
-
.buffer(true)
|
|
95
|
-
.parse(bufferParser)
|
|
96
|
-
.expect(200);
|
|
97
|
-
|
|
98
|
-
expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
|
|
99
|
-
expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
|
|
100
|
-
expect(nock.isDone()).toBe(true);
|
|
101
|
-
}
|
|
102
|
-
);
|
|
103
|
-
|
|
104
|
-
it.each(DRIVE_URLS)(
|
|
105
|
-
'/stream proxies direct image bytes for $name',
|
|
106
|
-
async (item) => {
|
|
107
|
-
mockDriveGet(item.fileId);
|
|
108
|
-
|
|
109
|
-
const res = await request(app)
|
|
110
|
-
.get('/stream')
|
|
111
|
-
.query({ url: item.url })
|
|
112
|
-
.buffer(true)
|
|
113
|
-
.parse(bufferParser)
|
|
114
|
-
.expect(200);
|
|
115
|
-
|
|
116
|
-
expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
|
|
117
|
-
expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
|
|
118
|
-
expect(nock.isDone()).toBe(true);
|
|
119
|
-
}
|
|
120
|
-
);
|
|
121
|
-
|
|
122
|
-
it.each(DRIVE_URLS)(
|
|
123
|
-
'/markdown includes a direct image link for $name',
|
|
124
|
-
async (item) => {
|
|
125
|
-
mockDriveHead(item.fileId);
|
|
126
|
-
|
|
127
|
-
const res = await request(app)
|
|
128
|
-
.get('/markdown')
|
|
129
|
-
.query({ url: item.url })
|
|
130
|
-
.expect(200);
|
|
131
|
-
|
|
132
|
-
expect(res.headers['content-type']).toMatch(/text\/markdown/);
|
|
133
|
-
expect(res.text).toContain(
|
|
134
|
-
`})`
|
|
135
|
-
);
|
|
136
|
-
expect(nock.isDone()).toBe(true);
|
|
137
|
-
}
|
|
138
|
-
);
|
|
139
|
-
|
|
140
|
-
it.each(DRIVE_URLS)(
|
|
141
|
-
'/txt includes the original and direct image links for $name',
|
|
142
|
-
async (item) => {
|
|
143
|
-
mockDriveHead(item.fileId);
|
|
144
|
-
|
|
145
|
-
const res = await request(app)
|
|
146
|
-
.get('/txt')
|
|
147
|
-
.query({ url: item.url })
|
|
148
|
-
.expect(200);
|
|
149
|
-
|
|
150
|
-
expect(res.headers['content-type']).toMatch(/text\/plain/);
|
|
151
|
-
expect(res.text).toContain('Google Drive image');
|
|
152
|
-
expect(res.text).toContain(`Original Google Drive file: ${item.url}`);
|
|
153
|
-
expect(res.text).toContain(`Image: ${directDownloadUrl(item.fileId)}`);
|
|
154
|
-
expect(nock.isDone()).toBe(true);
|
|
155
|
-
}
|
|
156
|
-
);
|
|
157
|
-
|
|
158
|
-
it.each(DRIVE_URLS)(
|
|
159
|
-
'/html includes a direct image tag for $name',
|
|
160
|
-
async (item) => {
|
|
161
|
-
mockDriveHead(item.fileId);
|
|
162
|
-
|
|
163
|
-
const res = await request(app)
|
|
164
|
-
.get('/html')
|
|
165
|
-
.query({ url: item.url })
|
|
166
|
-
.expect(200);
|
|
167
|
-
|
|
168
|
-
expect(res.headers['content-type']).toMatch(/text\/html/);
|
|
169
|
-
expect(res.text).toContain(
|
|
170
|
-
`src="${directDownloadUrl(item.fileId).replace('&', '&')}"`
|
|
171
|
-
);
|
|
172
|
-
expect(nock.isDone()).toBe(true);
|
|
173
|
-
}
|
|
174
|
-
);
|
|
175
|
-
|
|
176
|
-
it('archive includes markdown and the downloaded Drive image', async () => {
|
|
177
|
-
const item = DRIVE_URLS[0];
|
|
178
|
-
mockDriveHead(item.fileId);
|
|
179
|
-
mockDriveGet(item.fileId);
|
|
180
|
-
|
|
181
|
-
const res = await request(app)
|
|
182
|
-
.get('/archive')
|
|
183
|
-
.query({ url: item.url })
|
|
184
|
-
.buffer(true)
|
|
185
|
-
.parse(bufferParser)
|
|
186
|
-
.expect(200);
|
|
187
|
-
|
|
188
|
-
const zip = await unzipper.Open.buffer(res.body);
|
|
189
|
-
const document = zip.files.find((file) => file.path === 'document.md');
|
|
190
|
-
const image = zip.files.find((file) => file.path.startsWith('images/'));
|
|
191
|
-
|
|
192
|
-
expect(document).toBeDefined();
|
|
193
|
-
expect(image).toBeDefined();
|
|
194
|
-
expect((await document.buffer()).toString('utf8')).toContain(
|
|
195
|
-
''
|
|
196
|
-
);
|
|
197
|
-
expect((await image.buffer()).subarray(0, 3)).toEqual(
|
|
198
|
-
Buffer.from([0xff, 0xd8, 0xff])
|
|
199
|
-
);
|
|
200
|
-
expect(nock.isDone()).toBe(true);
|
|
201
|
-
});
|
|
202
|
-
|
|
203
|
-
it('docx includes the downloaded Drive image', async () => {
|
|
204
|
-
const item = DRIVE_URLS[0];
|
|
205
|
-
mockDriveHead(item.fileId);
|
|
206
|
-
mockDriveGet(item.fileId);
|
|
207
|
-
|
|
208
|
-
const res = await request(app)
|
|
209
|
-
.get('/docx')
|
|
210
|
-
.query({ url: item.url })
|
|
211
|
-
.buffer(true)
|
|
212
|
-
.parse(bufferParser)
|
|
213
|
-
.expect(200);
|
|
214
|
-
|
|
215
|
-
expect(res.headers['content-type']).toMatch(
|
|
216
|
-
/application\/vnd\.openxmlformats-officedocument\.wordprocessingml\.document/
|
|
217
|
-
);
|
|
218
|
-
expect(res.body.subarray(0, 2)).toEqual(Buffer.from('PK'));
|
|
219
|
-
|
|
220
|
-
const docx = await unzipper.Open.buffer(res.body);
|
|
221
|
-
const mediaEntry = docx.files.find(
|
|
222
|
-
(file) => file.path.startsWith('word/media/') && !file.path.endsWith('/')
|
|
223
|
-
);
|
|
224
|
-
expect(mediaEntry).toBeDefined();
|
|
225
|
-
expect((await mediaEntry.buffer()).subarray(0, 3)).toEqual(
|
|
226
|
-
Buffer.from([0xff, 0xd8, 0xff])
|
|
227
|
-
);
|
|
228
|
-
expect(nock.isDone()).toBe(true);
|
|
229
|
-
});
|
|
230
|
-
|
|
231
|
-
it('pdf renders the downloaded Drive image document', async () => {
|
|
232
|
-
const item = DRIVE_URLS[0];
|
|
233
|
-
mockDriveGet(item.fileId);
|
|
234
|
-
|
|
235
|
-
const res = await request(app)
|
|
236
|
-
.get('/pdf')
|
|
237
|
-
.query({ url: item.url })
|
|
238
|
-
.buffer(true)
|
|
239
|
-
.parse(bufferParser)
|
|
240
|
-
.expect(200);
|
|
241
|
-
|
|
242
|
-
expect(res.headers['content-type']).toMatch(/application\/pdf/);
|
|
243
|
-
expect(res.body.subarray(0, 4).toString('utf8')).toBe('%PDF');
|
|
244
|
-
expect(nock.isDone()).toBe(true);
|
|
245
|
-
});
|
|
246
|
-
});
|