@link-assistant/web-capture 1.12.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/README.md +2 -1
  3. package/bin/web-capture.js +5 -7
  4. package/package.json +32 -18
  5. package/src/archive.js +1 -2
  6. package/src/binary-download.js +13 -0
  7. package/src/docx.js +14 -11
  8. package/src/fetch.js +1 -2
  9. package/src/figures.js +2 -12
  10. package/src/gdocs-images.js +0 -1
  11. package/src/gdocs.js +0 -1
  12. package/src/github.js +0 -1
  13. package/src/lib.js +1 -2
  14. package/src/localize-images.js +2 -12
  15. package/src/markdown.js +0 -1
  16. package/src/shared-dialog.js +0 -1
  17. package/src/stream.js +0 -1
  18. package/src/transport.js +23 -5
  19. package/src/txt.js +0 -1
  20. package/.changeset/README.md +0 -8
  21. package/.changeset/config.json +0 -11
  22. package/.dockerignore +0 -6
  23. package/.jscpd.json +0 -19
  24. package/.lenv +0 -8
  25. package/.prettierignore +0 -8
  26. package/.prettierrc +0 -10
  27. package/Dockerfile +0 -52
  28. package/babel.config.cjs +0 -13
  29. package/docker-compose.yml +0 -11
  30. package/eslint.config.js +0 -112
  31. package/examples/js/capture_cache.js +0 -12
  32. package/examples/js/engine_comparison.js +0 -39
  33. package/examples/js/html_download.js +0 -23
  34. package/examples/js/image_download.js +0 -36
  35. package/examples/js/markdown_download.js +0 -23
  36. package/examples/js/playwright_example.js +0 -23
  37. package/examples/python/html_download.py +0 -15
  38. package/examples/python/image_download.py +0 -20
  39. package/examples/python/markdown_download.py +0 -15
  40. package/examples/python/playwright_example.py +0 -22
  41. package/jest.config.mjs +0 -20
  42. package/tests/ci/dependency-freshness.test.js +0 -74
  43. package/tests/ci/detect-code-changes.test.js +0 -174
  44. package/tests/ci/docker-install.test.js +0 -31
  45. package/tests/ci/rust-dockerfile.test.js +0 -54
  46. package/tests/ci/workflow-policy.test.js +0 -79
  47. package/tests/e2e/docker.test.js +0 -186
  48. package/tests/e2e/process.test.js +0 -215
  49. package/tests/fixtures/bold-coalesce-and-leakage.html +0 -27
  50. package/tests/fixtures/deepwiki/deepwiki-full-page.png +0 -0
  51. package/tests/fixtures/deepwiki/index.md +0 -1500
  52. package/tests/fixtures/deepwiki/part-1.md +0 -776
  53. package/tests/fixtures/habr-articles.js +0 -45
  54. package/tests/fixtures/heading-numbering.html +0 -19
  55. package/tests/fixtures/list-item-with-br.html +0 -8
  56. package/tests/fixtures/ol-continuous-numbering.html +0 -12
  57. package/tests/fixtures/paragraph-vs-line-break.html +0 -11
  58. package/tests/helpers/nock.js +0 -10
  59. package/tests/helpers/search-fixture.cjs +0 -17
  60. package/tests/helpers/temporary-directory.js +0 -12
  61. package/tests/integration/api-endpoints.test.js +0 -418
  62. package/tests/integration/archive-default-layout.test.js +0 -37
  63. package/tests/integration/browser-engines.test.js +0 -128
  64. package/tests/integration/browser-packaged-path.test.js +0 -45
  65. package/tests/integration/deepwiki-download.test.js +0 -144
  66. package/tests/integration/deepwiki-rendered-page.mock.test.js +0 -121
  67. package/tests/integration/formalai-contract.test.js +0 -214
  68. package/tests/integration/gdocs-public-doc.test.js +0 -315
  69. package/tests/integration/github-readme.test.js +0 -137
  70. package/tests/integration/google-drive-image.test.js +0 -246
  71. package/tests/integration/habr-article.test.js +0 -337
  72. package/tests/integration/search-endpoint.test.js +0 -88
  73. package/tests/integration/stackoverflow-download.test.js +0 -173
  74. package/tests/integration/wikipedia-download.test.js +0 -142
  75. package/tests/jest.setup.mjs +0 -7
  76. package/tests/mock/index.test.js +0 -225
  77. package/tests/unit/animation.test.js +0 -33
  78. package/tests/unit/batch.test.js +0 -114
  79. package/tests/unit/bold-coalesce-and-leakage.test.js +0 -54
  80. package/tests/unit/browser-launch-options.test.js +0 -42
  81. package/tests/unit/browser.test.js +0 -172
  82. package/tests/unit/cache.test.js +0 -182
  83. package/tests/unit/cli.test.js +0 -399
  84. package/tests/unit/convertGoogleDriveUrl.test.js +0 -89
  85. package/tests/unit/convertRelativeUrls.test.js +0 -34
  86. package/tests/unit/extract-images.test.js +0 -259
  87. package/tests/unit/fetch-html.test.js +0 -54
  88. package/tests/unit/figures.test.js +0 -74
  89. package/tests/unit/gdocs-export-headers.test.js +0 -28
  90. package/tests/unit/gdocs-preprocess.test.js +0 -36
  91. package/tests/unit/gdocs.test.js +0 -1490
  92. package/tests/unit/github.test.js +0 -156
  93. package/tests/unit/heading-image.test.js +0 -49
  94. package/tests/unit/heading-numbering.test.js +0 -28
  95. package/tests/unit/html2md-br-in-list-item.test.js +0 -23
  96. package/tests/unit/html2md-ol-numbering.test.js +0 -20
  97. package/tests/unit/html2md.test.js +0 -695
  98. package/tests/unit/image-mode-defaults.test.js +0 -54
  99. package/tests/unit/kreuzberg.test.js +0 -249
  100. package/tests/unit/latex.test.js +0 -123
  101. package/tests/unit/localize-images.test.js +0 -114
  102. package/tests/unit/markdown.test.js +0 -36
  103. package/tests/unit/metadata.test.js +0 -140
  104. package/tests/unit/package-runtime-dependencies.test.js +0 -74
  105. package/tests/unit/paragraph-vs-line-break.test.js +0 -56
  106. package/tests/unit/postprocess.test.js +0 -110
  107. package/tests/unit/proxy-headers.test.js +0 -100
  108. package/tests/unit/publish-verification.test.js +0 -57
  109. package/tests/unit/retry.test.js +0 -111
  110. package/tests/unit/search.test.js +0 -339
  111. package/tests/unit/shared-dialog.test.js +0 -173
  112. package/tests/unit/stackoverflow.test.js +0 -32
  113. package/tests/unit/stream.test.js +0 -85
  114. package/tests/unit/verify.test.js +0 -182
  115. package/tests/xpaste/xpaste-screenshot-markdown.mock.test.js +0 -375
  116. package/tests/xpaste/xpaste.e2e.test.js +0 -36
  117. package/tests/xpaste/xpaste.integration.test.js +0 -41
  118. package/tests/xpaste/xpaste.mock.test.js +0 -261
  119. package/tests/xpaste/xpaste.unit.test.js +0 -119
@@ -1,172 +0,0 @@
1
- import { createBrowser, getBrowserEngine } from '../../src/browser.js';
2
-
3
- describe('Browser Abstraction Layer', () => {
4
- describe('getBrowserEngine', () => {
5
- it('returns puppeteer by default', () => {
6
- const req = { query: {} };
7
- expect(getBrowserEngine(req)).toBe('puppeteer');
8
- });
9
-
10
- it('returns playwright when engine=playwright query param is provided', () => {
11
- const req = { query: { engine: 'playwright' } };
12
- expect(getBrowserEngine(req)).toBe('playwright');
13
- });
14
-
15
- it('returns playwright when engine=pw query param is provided', () => {
16
- const req = { query: { engine: 'pw' } };
17
- expect(getBrowserEngine(req)).toBe('playwright');
18
- });
19
-
20
- it('returns puppeteer when engine=puppeteer query param is provided', () => {
21
- const req = { query: { engine: 'puppeteer' } };
22
- expect(getBrowserEngine(req)).toBe('puppeteer');
23
- });
24
-
25
- it('returns puppeteer when engine=pptr query param is provided', () => {
26
- const req = { query: { engine: 'pptr' } };
27
- expect(getBrowserEngine(req)).toBe('puppeteer');
28
- });
29
-
30
- it('returns playwright when browser=playwright query param is provided', () => {
31
- const req = { query: { browser: 'playwright' } };
32
- expect(getBrowserEngine(req)).toBe('playwright');
33
- });
34
-
35
- it('is case insensitive', () => {
36
- const req1 = { query: { engine: 'PLAYWRIGHT' } };
37
- const req2 = { query: { engine: 'PlAyWrIgHt' } };
38
- expect(getBrowserEngine(req1)).toBe('playwright');
39
- expect(getBrowserEngine(req2)).toBe('playwright');
40
- });
41
-
42
- it('returns playwright from BROWSER_ENGINE env var', () => {
43
- const originalEnv = process.env.BROWSER_ENGINE;
44
- process.env.BROWSER_ENGINE = 'playwright';
45
- const req = { query: {} };
46
- expect(getBrowserEngine(req)).toBe('playwright');
47
- if (originalEnv) {
48
- process.env.BROWSER_ENGINE = originalEnv;
49
- } else {
50
- delete process.env.BROWSER_ENGINE;
51
- }
52
- });
53
-
54
- it('prefers query param over env var', () => {
55
- const originalEnv = process.env.BROWSER_ENGINE;
56
- process.env.BROWSER_ENGINE = 'playwright';
57
- const req = { query: { engine: 'puppeteer' } };
58
- expect(getBrowserEngine(req)).toBe('puppeteer');
59
- if (originalEnv) {
60
- process.env.BROWSER_ENGINE = originalEnv;
61
- } else {
62
- delete process.env.BROWSER_ENGINE;
63
- }
64
- });
65
- });
66
-
67
- describe('createBrowser - Puppeteer', () => {
68
- it('creates a puppeteer browser with type field', async () => {
69
- const browser = await createBrowser('puppeteer');
70
- expect(browser.type).toBe('puppeteer');
71
- expect(browser.newPage).toBeDefined();
72
- expect(browser.close).toBeDefined();
73
- await browser.close();
74
- });
75
-
76
- it('can create and use a page', async () => {
77
- const browser = await createBrowser('puppeteer');
78
- const page = await browser.newPage();
79
- expect(page.setViewport).toBeDefined();
80
- expect(page.goto).toBeDefined();
81
- expect(page.content).toBeDefined();
82
- expect(page.screenshot).toBeDefined();
83
- expect(page.type).toBe('puppeteer');
84
- await browser.close();
85
- });
86
- });
87
-
88
- describe('createBrowser - Playwright', () => {
89
- it('creates a playwright browser with type field', async () => {
90
- const browser = await createBrowser('playwright');
91
- expect(browser.type).toBe('playwright');
92
- expect(browser.newPage).toBeDefined();
93
- expect(browser.close).toBeDefined();
94
- await browser.close();
95
- });
96
-
97
- it('can create and use a page', async () => {
98
- const browser = await createBrowser('playwright');
99
- const page = await browser.newPage();
100
- expect(page.setViewport).toBeDefined();
101
- expect(page.goto).toBeDefined();
102
- expect(page.content).toBeDefined();
103
- expect(page.screenshot).toBeDefined();
104
- expect(page.type).toBe('playwright');
105
- await browser.close();
106
- });
107
- });
108
-
109
- describe('Page Adapter Compatibility', () => {
110
- it('both adapters support the same interface', async () => {
111
- const puppeteerBrowser = await createBrowser('puppeteer');
112
- const playwrightBrowser = await createBrowser('playwright');
113
-
114
- const puppeteerPage = await puppeteerBrowser.newPage();
115
- const playwrightPage = await playwrightBrowser.newPage();
116
-
117
- // Check that both have the same methods
118
- const methods = [
119
- 'setExtraHTTPHeaders',
120
- 'setUserAgent',
121
- 'setViewport',
122
- 'goto',
123
- 'content',
124
- 'screenshot',
125
- 'close',
126
- ];
127
- for (const method of methods) {
128
- expect(puppeteerPage[method]).toBeDefined();
129
- expect(playwrightPage[method]).toBeDefined();
130
- }
131
-
132
- await puppeteerBrowser.close();
133
- await playwrightBrowser.close();
134
- });
135
-
136
- it('both adapters expose browser-commander v0.7.0+ APIs', async () => {
137
- const puppeteerBrowser = await createBrowser('puppeteer');
138
- const playwrightBrowser = await createBrowser('playwright');
139
-
140
- const puppeteerPage = await puppeteerBrowser.newPage();
141
- const playwrightPage = await playwrightBrowser.newPage();
142
-
143
- // browser-commander v0.7.0: keyboard interaction
144
- expect(puppeteerPage.keyboard).toBeDefined();
145
- expect(puppeteerPage.keyboard.press).toBeDefined();
146
- expect(puppeteerPage.keyboard.type).toBeDefined();
147
- expect(playwrightPage.keyboard).toBeDefined();
148
- expect(playwrightPage.keyboard.press).toBeDefined();
149
-
150
- // browser-commander v0.7.0: dialog handling
151
- expect(puppeteerPage.onDialog).toBeDefined();
152
- expect(puppeteerPage.offDialog).toBeDefined();
153
- expect(puppeteerPage.clearDialogHandlers).toBeDefined();
154
- expect(playwrightPage.onDialog).toBeDefined();
155
-
156
- // browser-commander v0.7.0: media emulation
157
- expect(puppeteerPage.emulateMedia).toBeDefined();
158
- expect(playwrightPage.emulateMedia).toBeDefined();
159
-
160
- // browser-commander v0.8.0: PDF generation
161
- expect(puppeteerPage.pdf).toBeDefined();
162
- expect(playwrightPage.pdf).toBeDefined();
163
-
164
- // browser-commander: evaluate
165
- expect(puppeteerPage.evaluate).toBeDefined();
166
- expect(playwrightPage.evaluate).toBeDefined();
167
-
168
- await puppeteerBrowser.close();
169
- await playwrightBrowser.close();
170
- });
171
- });
172
- });
@@ -1,182 +0,0 @@
1
- import { jest } from '@jest/globals';
2
- import { mkdtemp, readFile, rm, writeFile, readdir } from 'node:fs/promises';
3
- import { tmpdir } from 'node:os';
4
- import path from 'node:path';
5
- import { CaptureStore, CachedTransport } from '../../src/cache.js';
6
- import { captureResponse } from '../../src/transport.js';
7
-
8
- const request = {
9
- url: 'https://example.test/source',
10
- method: 'GET',
11
- headers: { accept: 'application/octet-stream' },
12
- };
13
- const receipt = {
14
- body: Buffer.from([0, 255, 65]),
15
- finalUrl: 'https://example.test/final',
16
- status: 206,
17
- headers: { etag: 'v1' },
18
- diagnostics: { outcome: 'response' },
19
- };
20
- let directory, store;
21
- beforeEach(async () => {
22
- directory = await mkdtemp(path.join(tmpdir(), 'capture-cache-'));
23
- store = new CaptureStore(directory);
24
- });
25
- afterEach(async () => {
26
- await rm(directory, { recursive: true, force: true });
27
- });
28
-
29
- test('exact receipts survive cache hits, offline stale replay and online refresh', async () => {
30
- let time = 1000;
31
- const inner = jest.fn(async () => receipt);
32
- const options = { ttl: 100, now: () => time };
33
- const online = new CachedTransport(inner, store, options);
34
- const first = await online.capture(request);
35
- expect(first).toMatchObject({ cached: false, stale: false, receipt });
36
- expect(
37
- await captureResponse(request.url, {
38
- transport: online.execute,
39
- headers: request.headers,
40
- })
41
- ).toEqual(receipt);
42
- expect(inner).toHaveBeenCalledTimes(1);
43
- const forbidden = jest.fn(() => {
44
- throw new Error('offline network access');
45
- });
46
- const offline = new CachedTransport(forbidden, store, {
47
- ...options,
48
- offline: true,
49
- });
50
- expect(await offline.capture(request)).toMatchObject({
51
- cached: true,
52
- stale: false,
53
- receipt,
54
- });
55
- time = 1100;
56
- expect(await offline.capture(request)).toMatchObject({
57
- cached: true,
58
- stale: true,
59
- receipt,
60
- });
61
- expect(forbidden).not.toHaveBeenCalled();
62
- expect(await online.capture(request)).toMatchObject({
63
- cached: false,
64
- stale: false,
65
- });
66
- expect(inner).toHaveBeenCalledTimes(2);
67
- expect(await store.list()).toHaveLength(1);
68
- expect(await store.prune(1200, 100)).toBe(1);
69
- await expect(offline.capture(request)).rejects.toMatchObject({
70
- kind: 'offline_cache_miss',
71
- });
72
- });
73
-
74
- test('portable fixture imports/exports, request isolation and body deduplication', async () => {
75
- const fixture = await readFile(
76
- new globalThis.URL(
77
- '../../../rust/tests/fixtures/capture-v1.json',
78
- import.meta.url
79
- ),
80
- 'utf8'
81
- );
82
- await store.import(fixture);
83
- expect((await store.load(request)).receipt).toEqual(receipt);
84
- expect(JSON.parse(await store.export(request))).toEqual(JSON.parse(fixture));
85
- for (const different of [
86
- { ...request, method: 'POST' },
87
- { ...request, url: `${request.url}?query=1` },
88
- { ...request, headers: { authorization: 'different' } },
89
- ]) {
90
- expect(await store.load(different)).toBeNull();
91
- }
92
- await store.store(
93
- { ...request, url: `${request.url}/duplicate` },
94
- receipt,
95
- 2000
96
- );
97
- expect(await readdir(path.join(directory, 'bodies'))).toHaveLength(1);
98
- expect(await store.prune(1100, 100)).toBe(1);
99
- expect(await readdir(path.join(directory, 'bodies'))).toHaveLength(1);
100
- await expect(
101
- store.import(fixture.replace('255', '254'))
102
- ).rejects.toMatchObject({ kind: 'cache' });
103
- await expect(
104
- store.import(fixture.replace('"version": 1', '"version": 999'))
105
- ).rejects.toMatchObject({ kind: 'cache' });
106
- });
107
-
108
- test('corrupt bodies are rejected and failed transports are never cached', async () => {
109
- const stored = await store.store(request, receipt, 1000);
110
- await writeFile(path.join(directory, 'bodies', stored.sha256), 'corrupt');
111
- await expect(store.load(request)).rejects.toMatchObject({ kind: 'cache' });
112
- const failure = new Error('transport failed');
113
- const cache = new CachedTransport(async () => {
114
- throw failure;
115
- }, store);
116
- await expect(
117
- cache.capture({ ...request, url: 'https://other.test' })
118
- ).rejects.toThrow('transport failed');
119
- expect(await store.list()).toHaveLength(1);
120
- });
121
-
122
- test('Unicode header ordering matches the shared Rust replay format', async () => {
123
- const fixture = await readFile(
124
- new globalThis.URL(
125
- '../../../rust/tests/fixtures/capture-v1-unicode.json',
126
- import.meta.url
127
- ),
128
- 'utf8'
129
- );
130
- const replay = JSON.parse(fixture);
131
- await store.import(fixture);
132
- expect((await store.load(replay.request)).receipt).toEqual(receipt);
133
- expect(JSON.parse(await store.export(replay.request))).toEqual(replay);
134
- });
135
-
136
- test('failed online refresh retains the expired capture for offline replay', async () => {
137
- await store.store(request, receipt, 800);
138
- const cache = new CachedTransport(
139
- async () => {
140
- throw new Error('offline upstream');
141
- },
142
- store,
143
- { ttl: 100, now: () => 1000 }
144
- );
145
- await expect(cache.capture(request)).rejects.toThrow('offline upstream');
146
- const offline = new CachedTransport(null, store, {
147
- offline: true,
148
- ttl: 100,
149
- now: () => 1000,
150
- });
151
- expect(await offline.capture(request)).toMatchObject({
152
- stale: true,
153
- receipt,
154
- });
155
- for (const replay of ['{', 'null', '{}']) {
156
- await expect(store.import(replay)).rejects.toMatchObject({ kind: 'cache' });
157
- }
158
- });
159
-
160
- test('headers are sorted without mutating the caller; TTL zero always refreshes', async () => {
161
- const a = { ...request, headers: { z: '1', a: '2' } };
162
- const b = { ...request, headers: { a: '2', z: '1' } };
163
- await store.store(a, receipt, 1000);
164
- expect((await store.load(b)).receipt).toEqual(receipt);
165
- const inner = jest.fn(async () => receipt);
166
- const cache = new CachedTransport(inner, store, { ttl: 0, now: () => 1000 });
167
- await cache.capture(a);
168
- await cache.capture(a);
169
- expect(inner).toHaveBeenCalledTimes(2);
170
- });
171
-
172
- test('caller mutation during a fetch cannot change the stored request identity', async () => {
173
- const mutable = { ...request, headers: { ...request.headers } };
174
- const cache = new CachedTransport(async () => {
175
- mutable.url = 'https://mutated.test';
176
- mutable.headers.accept = 'text/plain';
177
- return receipt;
178
- }, store);
179
- await cache.capture(mutable);
180
- expect((await store.load(request)).receipt).toEqual(receipt);
181
- expect(await store.load(mutable)).toBeNull();
182
- });