@link-assistant/web-capture 1.12.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/README.md +2 -1
  3. package/bin/web-capture.js +5 -7
  4. package/package.json +32 -18
  5. package/src/archive.js +1 -2
  6. package/src/binary-download.js +13 -0
  7. package/src/docx.js +14 -11
  8. package/src/fetch.js +1 -2
  9. package/src/figures.js +2 -12
  10. package/src/gdocs-images.js +0 -1
  11. package/src/gdocs.js +0 -1
  12. package/src/github.js +0 -1
  13. package/src/lib.js +1 -2
  14. package/src/localize-images.js +2 -12
  15. package/src/markdown.js +0 -1
  16. package/src/shared-dialog.js +0 -1
  17. package/src/stream.js +0 -1
  18. package/src/transport.js +23 -5
  19. package/src/txt.js +0 -1
  20. package/.changeset/README.md +0 -8
  21. package/.changeset/config.json +0 -11
  22. package/.dockerignore +0 -6
  23. package/.jscpd.json +0 -19
  24. package/.lenv +0 -8
  25. package/.prettierignore +0 -8
  26. package/.prettierrc +0 -10
  27. package/Dockerfile +0 -52
  28. package/babel.config.cjs +0 -13
  29. package/docker-compose.yml +0 -11
  30. package/eslint.config.js +0 -112
  31. package/examples/js/capture_cache.js +0 -12
  32. package/examples/js/engine_comparison.js +0 -39
  33. package/examples/js/html_download.js +0 -23
  34. package/examples/js/image_download.js +0 -36
  35. package/examples/js/markdown_download.js +0 -23
  36. package/examples/js/playwright_example.js +0 -23
  37. package/examples/python/html_download.py +0 -15
  38. package/examples/python/image_download.py +0 -20
  39. package/examples/python/markdown_download.py +0 -15
  40. package/examples/python/playwright_example.py +0 -22
  41. package/jest.config.mjs +0 -20
  42. package/tests/ci/dependency-freshness.test.js +0 -74
  43. package/tests/ci/detect-code-changes.test.js +0 -174
  44. package/tests/ci/docker-install.test.js +0 -31
  45. package/tests/ci/rust-dockerfile.test.js +0 -54
  46. package/tests/ci/workflow-policy.test.js +0 -79
  47. package/tests/e2e/docker.test.js +0 -186
  48. package/tests/e2e/process.test.js +0 -215
  49. package/tests/fixtures/bold-coalesce-and-leakage.html +0 -27
  50. package/tests/fixtures/deepwiki/deepwiki-full-page.png +0 -0
  51. package/tests/fixtures/deepwiki/index.md +0 -1500
  52. package/tests/fixtures/deepwiki/part-1.md +0 -776
  53. package/tests/fixtures/habr-articles.js +0 -45
  54. package/tests/fixtures/heading-numbering.html +0 -19
  55. package/tests/fixtures/list-item-with-br.html +0 -8
  56. package/tests/fixtures/ol-continuous-numbering.html +0 -12
  57. package/tests/fixtures/paragraph-vs-line-break.html +0 -11
  58. package/tests/helpers/nock.js +0 -10
  59. package/tests/helpers/search-fixture.cjs +0 -17
  60. package/tests/helpers/temporary-directory.js +0 -12
  61. package/tests/integration/api-endpoints.test.js +0 -418
  62. package/tests/integration/archive-default-layout.test.js +0 -37
  63. package/tests/integration/browser-engines.test.js +0 -128
  64. package/tests/integration/browser-packaged-path.test.js +0 -45
  65. package/tests/integration/deepwiki-download.test.js +0 -144
  66. package/tests/integration/deepwiki-rendered-page.mock.test.js +0 -121
  67. package/tests/integration/formalai-contract.test.js +0 -214
  68. package/tests/integration/gdocs-public-doc.test.js +0 -315
  69. package/tests/integration/github-readme.test.js +0 -137
  70. package/tests/integration/google-drive-image.test.js +0 -246
  71. package/tests/integration/habr-article.test.js +0 -337
  72. package/tests/integration/search-endpoint.test.js +0 -88
  73. package/tests/integration/stackoverflow-download.test.js +0 -173
  74. package/tests/integration/wikipedia-download.test.js +0 -142
  75. package/tests/jest.setup.mjs +0 -7
  76. package/tests/mock/index.test.js +0 -225
  77. package/tests/unit/animation.test.js +0 -33
  78. package/tests/unit/batch.test.js +0 -114
  79. package/tests/unit/bold-coalesce-and-leakage.test.js +0 -54
  80. package/tests/unit/browser-launch-options.test.js +0 -42
  81. package/tests/unit/browser.test.js +0 -172
  82. package/tests/unit/cache.test.js +0 -182
  83. package/tests/unit/cli.test.js +0 -399
  84. package/tests/unit/convertGoogleDriveUrl.test.js +0 -89
  85. package/tests/unit/convertRelativeUrls.test.js +0 -34
  86. package/tests/unit/extract-images.test.js +0 -259
  87. package/tests/unit/fetch-html.test.js +0 -54
  88. package/tests/unit/figures.test.js +0 -74
  89. package/tests/unit/gdocs-export-headers.test.js +0 -28
  90. package/tests/unit/gdocs-preprocess.test.js +0 -36
  91. package/tests/unit/gdocs.test.js +0 -1490
  92. package/tests/unit/github.test.js +0 -156
  93. package/tests/unit/heading-image.test.js +0 -49
  94. package/tests/unit/heading-numbering.test.js +0 -28
  95. package/tests/unit/html2md-br-in-list-item.test.js +0 -23
  96. package/tests/unit/html2md-ol-numbering.test.js +0 -20
  97. package/tests/unit/html2md.test.js +0 -695
  98. package/tests/unit/image-mode-defaults.test.js +0 -54
  99. package/tests/unit/kreuzberg.test.js +0 -249
  100. package/tests/unit/latex.test.js +0 -123
  101. package/tests/unit/localize-images.test.js +0 -114
  102. package/tests/unit/markdown.test.js +0 -36
  103. package/tests/unit/metadata.test.js +0 -140
  104. package/tests/unit/package-runtime-dependencies.test.js +0 -74
  105. package/tests/unit/paragraph-vs-line-break.test.js +0 -56
  106. package/tests/unit/postprocess.test.js +0 -110
  107. package/tests/unit/proxy-headers.test.js +0 -100
  108. package/tests/unit/publish-verification.test.js +0 -57
  109. package/tests/unit/retry.test.js +0 -111
  110. package/tests/unit/search.test.js +0 -339
  111. package/tests/unit/shared-dialog.test.js +0 -173
  112. package/tests/unit/stackoverflow.test.js +0 -32
  113. package/tests/unit/stream.test.js +0 -85
  114. package/tests/unit/verify.test.js +0 -182
  115. package/tests/xpaste/xpaste-screenshot-markdown.mock.test.js +0 -375
  116. package/tests/xpaste/xpaste.e2e.test.js +0 -36
  117. package/tests/xpaste/xpaste.integration.test.js +0 -41
  118. package/tests/xpaste/xpaste.mock.test.js +0 -261
  119. package/tests/xpaste/xpaste.unit.test.js +0 -119
@@ -1,156 +0,0 @@
1
- import nock from '../helpers/nock.js';
2
- import {
3
- fetchGithubRepositorySnapshot,
4
- formatGithubRepositoryMarkdown,
5
- formatGithubRepositoryText,
6
- getGithubRepositoryTextFilename,
7
- isGithubRepositoryUrl,
8
- parseGithubRepositoryUrl,
9
- } from '../../src/github.js';
10
-
11
- const SNAPSHOT = {
12
- sourceUrl: 'https://github.com/octocat/Hello-World',
13
- defaultBranch: 'master',
14
- repository: {
15
- full_name: 'octocat/Hello-World',
16
- description: 'A friendly test repository',
17
- html_url: 'https://github.com/octocat/Hello-World',
18
- default_branch: 'master',
19
- language: 'JavaScript',
20
- stargazers_count: 42,
21
- forks_count: 7,
22
- open_issues_count: 3,
23
- license: { spdx_id: 'MIT' },
24
- topics: ['demo', 'capture'],
25
- },
26
- tree: [
27
- {
28
- name: 'src',
29
- path: 'src',
30
- type: 'dir',
31
- html_url: 'https://github.com/octocat/Hello-World/tree/master/src',
32
- },
33
- {
34
- name: 'README.md',
35
- path: 'README.md',
36
- type: 'file',
37
- size: 37,
38
- html_url: 'https://github.com/octocat/Hello-World/blob/master/README.md',
39
- },
40
- ],
41
- readme: {
42
- path: 'README.md',
43
- content: '# Hello World\n\nThis is the README.',
44
- },
45
- };
46
-
47
- afterEach(() => {
48
- nock.cleanAll();
49
- });
50
-
51
- describe('GitHub repository URLs', () => {
52
- it('detects plain GitHub repository pages', () => {
53
- expect(
54
- parseGithubRepositoryUrl('https://github.com/octocat/Hello-World')
55
- ).toEqual({
56
- owner: 'octocat',
57
- repo: 'Hello-World',
58
- fullName: 'octocat/Hello-World',
59
- htmlUrl: 'https://github.com/octocat/Hello-World',
60
- });
61
- expect(
62
- isGithubRepositoryUrl('https://github.com/octocat/Hello-World')
63
- ).toBe(true);
64
- expect(
65
- isGithubRepositoryUrl('https://github.com/octocat/Hello-World/issues')
66
- ).toBe(false);
67
- expect(
68
- isGithubRepositoryUrl('https://example.com/octocat/Hello-World')
69
- ).toBe(false);
70
- });
71
-
72
- it('derives repository text filenames', () => {
73
- expect(
74
- getGithubRepositoryTextFilename('https://github.com/octocat/Hello-World')
75
- ).toBe('octocat-Hello-World.txt');
76
- });
77
- });
78
-
79
- describe('GitHub repository snapshot fetching', () => {
80
- it('requests identity encoding from GitHub JSON and raw README fetches', async () => {
81
- const owner = 'octocat';
82
- const repo = 'Hello-World';
83
- const readme = '# Hello World\n\nRaw README content.';
84
-
85
- const githubApi = nock('https://api.github.com', {
86
- reqheaders: {
87
- 'accept-encoding': 'identity',
88
- },
89
- })
90
- .get(`/repos/${owner}/${repo}`)
91
- .reply(200, {
92
- full_name: `${owner}/${repo}`,
93
- html_url: `https://github.com/${owner}/${repo}`,
94
- default_branch: 'master',
95
- })
96
- .get(`/repos/${owner}/${repo}/readme`)
97
- .query({ ref: 'master' })
98
- .reply(200, {
99
- name: 'README.md',
100
- path: 'README.md',
101
- download_url: `https://raw.githubusercontent.com/${owner}/${repo}/master/README.md`,
102
- })
103
- .get(`/repos/${owner}/${repo}/contents`)
104
- .query({ ref: 'master' })
105
- .reply(200, [
106
- {
107
- name: 'README.md',
108
- path: 'README.md',
109
- type: 'file',
110
- size: readme.length,
111
- },
112
- ]);
113
-
114
- const rawGithub = nock('https://raw.githubusercontent.com', {
115
- reqheaders: {
116
- 'accept-encoding': 'identity',
117
- },
118
- })
119
- .get(`/${owner}/${repo}/master/README.md`)
120
- .reply(200, readme);
121
-
122
- const snapshot = await fetchGithubRepositorySnapshot(
123
- `https://github.com/${owner}/${repo}`
124
- );
125
-
126
- expect(snapshot.readme.content).toBe(readme);
127
- expect(githubApi.isDone()).toBe(true);
128
- expect(rawGithub.isDone()).toBe(true);
129
- });
130
- });
131
-
132
- describe('GitHub repository snapshot formatting', () => {
133
- it('formats a compact markdown snapshot with metadata, file tree, and README', () => {
134
- const markdown = formatGithubRepositoryMarkdown(SNAPSHOT);
135
-
136
- expect(markdown).toContain('# octocat/Hello-World');
137
- expect(markdown).toContain('> A friendly test repository');
138
- expect(markdown).toContain('- Default branch: `master`');
139
- expect(markdown).toContain('- [src/](');
140
- expect(markdown).toContain('- [README.md](');
141
- expect(markdown).toContain('## README.md');
142
- expect(markdown).toContain('# Hello World');
143
- });
144
-
145
- it('formats a compact plain-text snapshot with metadata, file tree, and README', () => {
146
- const text = formatGithubRepositoryText(SNAPSHOT);
147
-
148
- expect(text).toContain('Repository: octocat/Hello-World');
149
- expect(text).toContain('Description: A friendly test repository');
150
- expect(text).toContain('Files:');
151
- expect(text).toContain('- src/');
152
- expect(text).toContain('- README.md');
153
- expect(text).toContain('README.md:');
154
- expect(text).toContain('This is the README.');
155
- });
156
- });
@@ -1,49 +0,0 @@
1
- import { convertHtmlToMarkdown } from '../../src/lib.js';
2
- import { extractBase64Images } from '../../src/gdocs.js';
3
-
4
- const PNG_B64 =
5
- 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNgYAAAAAMAASsJTYQAAAAASUVORK5CYII=';
6
-
7
- const IMG_REF_PATTERN = /!\[[^\]]*\]\([^)]*images\/[^)]+\)/g;
8
-
9
- describe('heading image parity', () => {
10
- it('<img> inside <h1> survives Turndown and stays in sync with extraction', () => {
11
- const html = `<html><body><h1>Title <img src="data:image/png;base64,${PNG_B64}" alt="x"></h1><p>Body</p></body></html>`;
12
-
13
- const { html: localHtml, images } = extractBase64Images(html);
14
- const md = convertHtmlToMarkdown(localHtml);
15
- const mdImgCount = (md.match(IMG_REF_PATTERN) || []).length;
16
-
17
- expect(images).toHaveLength(1);
18
- expect(mdImgCount).toBe(images.length);
19
- });
20
-
21
- it('<img> inside all heading levels is preserved', () => {
22
- for (let level = 1; level <= 6; level++) {
23
- const tag = `h${level}`;
24
- const html = `<html><body><${tag}><img src="data:image/png;base64,${PNG_B64}" alt="icon"> Heading</${tag}></body></html>`;
25
-
26
- const { html: localHtml, images } = extractBase64Images(html);
27
- const md = convertHtmlToMarkdown(localHtml);
28
- const refs = (md.match(IMG_REF_PATTERN) || []).length;
29
-
30
- expect(refs).toBe(images.length);
31
- }
32
- });
33
-
34
- it('image count parity across full pipeline', () => {
35
- const html = `<html><body>
36
- <h1><img src="data:image/png;base64,${PNG_B64}" alt="h1"> Chapter</h1>
37
- <p>Text <img src="data:image/png;base64,${PNG_B64}" alt="p1"> here.</p>
38
- <h2><img src="data:image/png;base64,${PNG_B64}" alt="h2"> Section</h2>
39
- <p><img src="data:image/png;base64,${PNG_B64}" alt="p2"></p>
40
- </body></html>`;
41
-
42
- const { html: localHtml, images } = extractBase64Images(html);
43
- const md = convertHtmlToMarkdown(localHtml);
44
- const mdRefCount = (md.match(IMG_REF_PATTERN) || []).length;
45
-
46
- expect(images).toHaveLength(4);
47
- expect(mdRefCount).toBe(images.length);
48
- });
49
- });
@@ -1,28 +0,0 @@
1
- import fs from 'fs';
2
- import path from 'path';
3
- import { fileURLToPath } from 'url';
4
- import { convertHtmlToMarkdown } from '../../src/lib.js';
5
-
6
- const __filename = fileURLToPath(import.meta.url);
7
- const __dirname = path.dirname(__filename);
8
- const HTML = fs.readFileSync(
9
- path.join(__dirname, '..', 'fixtures', 'heading-numbering.html'),
10
- 'utf8'
11
- );
12
-
13
- it('preserves hierarchical heading numbering (13, 13.1, 13.2, 13.3)', () => {
14
- const md = convertHtmlToMarkdown(HTML);
15
-
16
- // Parent number 13 must survive — either as Markdown heading or as numbered-list item.
17
- expect(md).toMatch(/(^|\n)#{1,6}\s+13\.|(^|\n)13\.\s/);
18
-
19
- // Sub-numbers 13.1, 13.2, 13.3 must each appear in heading-like context (line start, not buried mid-paragraph).
20
- for (const sub of ['13.1', '13.2', '13.3']) {
21
- expect(md).toMatch(
22
- new RegExp(`(^|\\n)\\s*\\*{0,3}\\s*${sub.replace('.', '\\.')}`, 'm')
23
- );
24
- }
25
-
26
- // Subsections must NOT be wrapped in markdown blockquotes — there is no <blockquote> in the fixture.
27
- expect(md).not.toMatch(/^>\s+\*{1,3}13\.\d/m);
28
- });
@@ -1,23 +0,0 @@
1
- import fs from 'fs';
2
- import path from 'path';
3
- import { fileURLToPath } from 'url';
4
- import { convertHtmlToMarkdown } from '../../src/lib.js';
5
-
6
- const __filename = fileURLToPath(import.meta.url);
7
- const __dirname = path.dirname(__filename);
8
- const HTML = fs.readFileSync(
9
- path.join(__dirname, '..', 'fixtures', 'list-item-with-br.html'),
10
- 'utf8'
11
- );
12
-
13
- it('preserves <br> as a line separator inside list items', () => {
14
- const md = convertHtmlToMarkdown(HTML);
15
- for (const tag of ['TAG1', 'TAG2', 'TAG3']) {
16
- expect(md).toMatch(
17
- new RegExp(`(^|\\n)\\s*\\*{0,2}${tag}\\*{0,2}\\s*–`, 'm')
18
- );
19
- }
20
- // Definitions must not be glued on the same line (no <br> dropped).
21
- expect(md).not.toMatch(/Definition one\.[ \t]*\*{0,2}TAG2/);
22
- expect(md).not.toMatch(/Definition two\.[ \t]*\*{0,2}TAG3/);
23
- });
@@ -1,20 +0,0 @@
1
- import fs from 'fs';
2
- import path from 'path';
3
- import { fileURLToPath } from 'url';
4
- import { convertHtmlToMarkdown } from '../../src/lib.js';
5
-
6
- const __filename = fileURLToPath(import.meta.url);
7
- const __dirname = path.dirname(__filename);
8
- const HTML = fs.readFileSync(
9
- path.join(__dirname, '..', 'fixtures', 'ol-continuous-numbering.html'),
10
- 'utf8'
11
- );
12
-
13
- it('numbers consecutive top-level ordered lists continuously', () => {
14
- const md = convertHtmlToMarkdown(HTML);
15
- expect(md).toMatch(/^\s*1\.\s+\*\*First\*\*/m);
16
- expect(md).toMatch(/^\s*2\.\s+\*\*Second\*\*/m);
17
- expect(md).toMatch(/^\s*3\.\s+\*\*Third\*\*/m);
18
- expect(md).toMatch(/^\s*4\.\s+\*\*Fourth\*\*/m);
19
- expect(md).toMatch(/^\s*13\.\s+\*\*Top-level/m);
20
- });