@link-assistant/web-capture 2.0.0 → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +11 -0
  2. package/bin/web-capture.js +2 -5
  3. package/package.json +22 -11
  4. package/src/archive.js +0 -1
  5. package/src/binary-download.js +0 -1
  6. package/src/docx.js +13 -10
  7. package/src/fetch.js +0 -1
  8. package/src/gdocs-images.js +0 -1
  9. package/src/gdocs.js +0 -1
  10. package/src/github.js +0 -1
  11. package/src/lib.js +0 -1
  12. package/src/markdown.js +0 -1
  13. package/src/shared-dialog.js +0 -1
  14. package/src/stream.js +0 -1
  15. package/src/transport.js +23 -5
  16. package/src/txt.js +0 -1
  17. package/.changeset/README.md +0 -8
  18. package/.changeset/config.json +0 -11
  19. package/.dockerignore +0 -6
  20. package/.jscpd.json +0 -19
  21. package/.lenv +0 -8
  22. package/.prettierignore +0 -8
  23. package/.prettierrc +0 -10
  24. package/Dockerfile +0 -52
  25. package/docker-compose.yml +0 -11
  26. package/eslint.config.js +0 -112
  27. package/examples/js/capture_cache.js +0 -12
  28. package/examples/js/engine_comparison.js +0 -39
  29. package/examples/js/html_download.js +0 -23
  30. package/examples/js/image_download.js +0 -36
  31. package/examples/js/markdown_download.js +0 -23
  32. package/examples/js/playwright_example.js +0 -23
  33. package/examples/python/capture_client.py +0 -20
  34. package/examples/python/html_download.py +0 -3
  35. package/examples/python/image_download.py +0 -16
  36. package/examples/python/markdown_download.py +0 -3
  37. package/examples/python/playwright_example.py +0 -22
  38. package/jest.config.mjs +0 -17
  39. package/tests/ci/dependency-freshness.test.js +0 -74
  40. package/tests/ci/detect-code-changes.test.js +0 -187
  41. package/tests/ci/docker-install.test.js +0 -31
  42. package/tests/ci/rust-dockerfile.test.js +0 -54
  43. package/tests/ci/workflow-policy.test.js +0 -79
  44. package/tests/e2e/docker.test.js +0 -186
  45. package/tests/e2e/process.test.js +0 -215
  46. package/tests/fixtures/bold-coalesce-and-leakage.html +0 -27
  47. package/tests/fixtures/deepwiki/deepwiki-full-page.png +0 -0
  48. package/tests/fixtures/deepwiki/index.md +0 -1500
  49. package/tests/fixtures/deepwiki/part-1.md +0 -776
  50. package/tests/fixtures/habr-articles.js +0 -45
  51. package/tests/fixtures/heading-numbering.html +0 -19
  52. package/tests/fixtures/list-item-with-br.html +0 -8
  53. package/tests/fixtures/ol-continuous-numbering.html +0 -12
  54. package/tests/fixtures/paragraph-vs-line-break.html +0 -11
  55. package/tests/helpers/nock.js +0 -10
  56. package/tests/helpers/screenshot.js +0 -10
  57. package/tests/helpers/search-fixture.cjs +0 -17
  58. package/tests/helpers/temporary-directory.js +0 -12
  59. package/tests/integration/api-endpoints.test.js +0 -418
  60. package/tests/integration/archive-default-layout.test.js +0 -37
  61. package/tests/integration/browser-engines.test.js +0 -80
  62. package/tests/integration/browser-packaged-path.test.js +0 -45
  63. package/tests/integration/deepwiki-download.test.js +0 -144
  64. package/tests/integration/deepwiki-rendered-page.mock.test.js +0 -121
  65. package/tests/integration/formalai-contract.test.js +0 -214
  66. package/tests/integration/gdocs-public-doc.test.js +0 -315
  67. package/tests/integration/github-readme.test.js +0 -133
  68. package/tests/integration/google-drive-image.test.js +0 -246
  69. package/tests/integration/habr-article.test.js +0 -337
  70. package/tests/integration/search-endpoint.test.js +0 -88
  71. package/tests/integration/stackoverflow-download.test.js +0 -173
  72. package/tests/integration/wikipedia-download.test.js +0 -134
  73. package/tests/jest.setup.mjs +0 -7
  74. package/tests/mock/index.test.js +0 -225
  75. package/tests/unit/animation.test.js +0 -33
  76. package/tests/unit/batch.test.js +0 -114
  77. package/tests/unit/bold-coalesce-and-leakage.test.js +0 -54
  78. package/tests/unit/browser-launch-options.test.js +0 -42
  79. package/tests/unit/browser.test.js +0 -172
  80. package/tests/unit/cache.test.js +0 -182
  81. package/tests/unit/cli.test.js +0 -399
  82. package/tests/unit/convertGoogleDriveUrl.test.js +0 -89
  83. package/tests/unit/convertRelativeUrls.test.js +0 -34
  84. package/tests/unit/extract-images.test.js +0 -259
  85. package/tests/unit/fetch-html.test.js +0 -54
  86. package/tests/unit/figures.test.js +0 -74
  87. package/tests/unit/gdocs-export-headers.test.js +0 -28
  88. package/tests/unit/gdocs-preprocess.test.js +0 -36
  89. package/tests/unit/gdocs.test.js +0 -1490
  90. package/tests/unit/github.test.js +0 -156
  91. package/tests/unit/heading-image.test.js +0 -49
  92. package/tests/unit/heading-numbering.test.js +0 -28
  93. package/tests/unit/html2md-br-in-list-item.test.js +0 -23
  94. package/tests/unit/html2md-ol-numbering.test.js +0 -20
  95. package/tests/unit/html2md.test.js +0 -695
  96. package/tests/unit/image-mode-defaults.test.js +0 -54
  97. package/tests/unit/kreuzberg.test.js +0 -249
  98. package/tests/unit/latex.test.js +0 -123
  99. package/tests/unit/localize-images.test.js +0 -114
  100. package/tests/unit/markdown.test.js +0 -36
  101. package/tests/unit/metadata.test.js +0 -140
  102. package/tests/unit/package-runtime-dependencies.test.js +0 -74
  103. package/tests/unit/paragraph-vs-line-break.test.js +0 -56
  104. package/tests/unit/postprocess.test.js +0 -110
  105. package/tests/unit/proxy-headers.test.js +0 -100
  106. package/tests/unit/publish-verification.test.js +0 -54
  107. package/tests/unit/retry.test.js +0 -111
  108. package/tests/unit/search.test.js +0 -339
  109. package/tests/unit/shared-dialog.test.js +0 -173
  110. package/tests/unit/stackoverflow.test.js +0 -32
  111. package/tests/unit/stream.test.js +0 -83
  112. package/tests/unit/verify.test.js +0 -182
  113. package/tests/xpaste/xpaste-screenshot-markdown.mock.test.js +0 -375
  114. package/tests/xpaste/xpaste.e2e.test.js +0 -36
  115. package/tests/xpaste/xpaste.integration.test.js +0 -41
  116. package/tests/xpaste/xpaste.mock.test.js +0 -261
  117. package/tests/xpaste/xpaste.unit.test.js +0 -119
@@ -1,172 +0,0 @@
1
- import { createBrowser, getBrowserEngine } from '../../src/browser.js';
2
-
3
- describe('Browser Abstraction Layer', () => {
4
- describe('getBrowserEngine', () => {
5
- it('returns puppeteer by default', () => {
6
- const req = { query: {} };
7
- expect(getBrowserEngine(req)).toBe('puppeteer');
8
- });
9
-
10
- it('returns playwright when engine=playwright query param is provided', () => {
11
- const req = { query: { engine: 'playwright' } };
12
- expect(getBrowserEngine(req)).toBe('playwright');
13
- });
14
-
15
- it('returns playwright when engine=pw query param is provided', () => {
16
- const req = { query: { engine: 'pw' } };
17
- expect(getBrowserEngine(req)).toBe('playwright');
18
- });
19
-
20
- it('returns puppeteer when engine=puppeteer query param is provided', () => {
21
- const req = { query: { engine: 'puppeteer' } };
22
- expect(getBrowserEngine(req)).toBe('puppeteer');
23
- });
24
-
25
- it('returns puppeteer when engine=pptr query param is provided', () => {
26
- const req = { query: { engine: 'pptr' } };
27
- expect(getBrowserEngine(req)).toBe('puppeteer');
28
- });
29
-
30
- it('returns playwright when browser=playwright query param is provided', () => {
31
- const req = { query: { browser: 'playwright' } };
32
- expect(getBrowserEngine(req)).toBe('playwright');
33
- });
34
-
35
- it('is case insensitive', () => {
36
- const req1 = { query: { engine: 'PLAYWRIGHT' } };
37
- const req2 = { query: { engine: 'PlAyWrIgHt' } };
38
- expect(getBrowserEngine(req1)).toBe('playwright');
39
- expect(getBrowserEngine(req2)).toBe('playwright');
40
- });
41
-
42
- it('returns playwright from BROWSER_ENGINE env var', () => {
43
- const originalEnv = process.env.BROWSER_ENGINE;
44
- process.env.BROWSER_ENGINE = 'playwright';
45
- const req = { query: {} };
46
- expect(getBrowserEngine(req)).toBe('playwright');
47
- if (originalEnv) {
48
- process.env.BROWSER_ENGINE = originalEnv;
49
- } else {
50
- delete process.env.BROWSER_ENGINE;
51
- }
52
- });
53
-
54
- it('prefers query param over env var', () => {
55
- const originalEnv = process.env.BROWSER_ENGINE;
56
- process.env.BROWSER_ENGINE = 'playwright';
57
- const req = { query: { engine: 'puppeteer' } };
58
- expect(getBrowserEngine(req)).toBe('puppeteer');
59
- if (originalEnv) {
60
- process.env.BROWSER_ENGINE = originalEnv;
61
- } else {
62
- delete process.env.BROWSER_ENGINE;
63
- }
64
- });
65
- });
66
-
67
- describe('createBrowser - Puppeteer', () => {
68
- it('creates a puppeteer browser with type field', async () => {
69
- const browser = await createBrowser('puppeteer');
70
- expect(browser.type).toBe('puppeteer');
71
- expect(browser.newPage).toBeDefined();
72
- expect(browser.close).toBeDefined();
73
- await browser.close();
74
- });
75
-
76
- it('can create and use a page', async () => {
77
- const browser = await createBrowser('puppeteer');
78
- const page = await browser.newPage();
79
- expect(page.setViewport).toBeDefined();
80
- expect(page.goto).toBeDefined();
81
- expect(page.content).toBeDefined();
82
- expect(page.screenshot).toBeDefined();
83
- expect(page.type).toBe('puppeteer');
84
- await browser.close();
85
- });
86
- });
87
-
88
- describe('createBrowser - Playwright', () => {
89
- it('creates a playwright browser with type field', async () => {
90
- const browser = await createBrowser('playwright');
91
- expect(browser.type).toBe('playwright');
92
- expect(browser.newPage).toBeDefined();
93
- expect(browser.close).toBeDefined();
94
- await browser.close();
95
- });
96
-
97
- it('can create and use a page', async () => {
98
- const browser = await createBrowser('playwright');
99
- const page = await browser.newPage();
100
- expect(page.setViewport).toBeDefined();
101
- expect(page.goto).toBeDefined();
102
- expect(page.content).toBeDefined();
103
- expect(page.screenshot).toBeDefined();
104
- expect(page.type).toBe('playwright');
105
- await browser.close();
106
- });
107
- });
108
-
109
- describe('Page Adapter Compatibility', () => {
110
- it('both adapters support the same interface', async () => {
111
- const puppeteerBrowser = await createBrowser('puppeteer');
112
- const playwrightBrowser = await createBrowser('playwright');
113
-
114
- const puppeteerPage = await puppeteerBrowser.newPage();
115
- const playwrightPage = await playwrightBrowser.newPage();
116
-
117
- // Check that both have the same methods
118
- const methods = [
119
- 'setExtraHTTPHeaders',
120
- 'setUserAgent',
121
- 'setViewport',
122
- 'goto',
123
- 'content',
124
- 'screenshot',
125
- 'close',
126
- ];
127
- for (const method of methods) {
128
- expect(puppeteerPage[method]).toBeDefined();
129
- expect(playwrightPage[method]).toBeDefined();
130
- }
131
-
132
- await puppeteerBrowser.close();
133
- await playwrightBrowser.close();
134
- });
135
-
136
- it('both adapters expose browser-commander v0.7.0+ APIs', async () => {
137
- const puppeteerBrowser = await createBrowser('puppeteer');
138
- const playwrightBrowser = await createBrowser('playwright');
139
-
140
- const puppeteerPage = await puppeteerBrowser.newPage();
141
- const playwrightPage = await playwrightBrowser.newPage();
142
-
143
- // browser-commander v0.7.0: keyboard interaction
144
- expect(puppeteerPage.keyboard).toBeDefined();
145
- expect(puppeteerPage.keyboard.press).toBeDefined();
146
- expect(puppeteerPage.keyboard.type).toBeDefined();
147
- expect(playwrightPage.keyboard).toBeDefined();
148
- expect(playwrightPage.keyboard.press).toBeDefined();
149
-
150
- // browser-commander v0.7.0: dialog handling
151
- expect(puppeteerPage.onDialog).toBeDefined();
152
- expect(puppeteerPage.offDialog).toBeDefined();
153
- expect(puppeteerPage.clearDialogHandlers).toBeDefined();
154
- expect(playwrightPage.onDialog).toBeDefined();
155
-
156
- // browser-commander v0.7.0: media emulation
157
- expect(puppeteerPage.emulateMedia).toBeDefined();
158
- expect(playwrightPage.emulateMedia).toBeDefined();
159
-
160
- // browser-commander v0.8.0: PDF generation
161
- expect(puppeteerPage.pdf).toBeDefined();
162
- expect(playwrightPage.pdf).toBeDefined();
163
-
164
- // browser-commander: evaluate
165
- expect(puppeteerPage.evaluate).toBeDefined();
166
- expect(playwrightPage.evaluate).toBeDefined();
167
-
168
- await puppeteerBrowser.close();
169
- await playwrightBrowser.close();
170
- });
171
- });
172
- });
@@ -1,182 +0,0 @@
1
- import { jest } from '@jest/globals';
2
- import { mkdtemp, readFile, rm, writeFile, readdir } from 'node:fs/promises';
3
- import { tmpdir } from 'node:os';
4
- import path from 'node:path';
5
- import { CaptureStore, CachedTransport } from '../../src/cache.js';
6
- import { captureResponse } from '../../src/transport.js';
7
-
8
- const request = {
9
- url: 'https://example.test/source',
10
- method: 'GET',
11
- headers: { accept: 'application/octet-stream' },
12
- };
13
- const receipt = {
14
- body: Buffer.from([0, 255, 65]),
15
- finalUrl: 'https://example.test/final',
16
- status: 206,
17
- headers: { etag: 'v1' },
18
- diagnostics: { outcome: 'response' },
19
- };
20
- let directory, store;
21
- beforeEach(async () => {
22
- directory = await mkdtemp(path.join(tmpdir(), 'capture-cache-'));
23
- store = new CaptureStore(directory);
24
- });
25
- afterEach(async () => {
26
- await rm(directory, { recursive: true, force: true });
27
- });
28
-
29
- test('exact receipts survive cache hits, offline stale replay and online refresh', async () => {
30
- let time = 1000;
31
- const inner = jest.fn(async () => receipt);
32
- const options = { ttl: 100, now: () => time };
33
- const online = new CachedTransport(inner, store, options);
34
- const first = await online.capture(request);
35
- expect(first).toMatchObject({ cached: false, stale: false, receipt });
36
- expect(
37
- await captureResponse(request.url, {
38
- transport: online.execute,
39
- headers: request.headers,
40
- })
41
- ).toEqual(receipt);
42
- expect(inner).toHaveBeenCalledTimes(1);
43
- const forbidden = jest.fn(() => {
44
- throw new Error('offline network access');
45
- });
46
- const offline = new CachedTransport(forbidden, store, {
47
- ...options,
48
- offline: true,
49
- });
50
- expect(await offline.capture(request)).toMatchObject({
51
- cached: true,
52
- stale: false,
53
- receipt,
54
- });
55
- time = 1100;
56
- expect(await offline.capture(request)).toMatchObject({
57
- cached: true,
58
- stale: true,
59
- receipt,
60
- });
61
- expect(forbidden).not.toHaveBeenCalled();
62
- expect(await online.capture(request)).toMatchObject({
63
- cached: false,
64
- stale: false,
65
- });
66
- expect(inner).toHaveBeenCalledTimes(2);
67
- expect(await store.list()).toHaveLength(1);
68
- expect(await store.prune(1200, 100)).toBe(1);
69
- await expect(offline.capture(request)).rejects.toMatchObject({
70
- kind: 'offline_cache_miss',
71
- });
72
- });
73
-
74
- test('portable fixture imports/exports, request isolation and body deduplication', async () => {
75
- const fixture = await readFile(
76
- new globalThis.URL(
77
- '../../../rust/tests/fixtures/capture-v1.json',
78
- import.meta.url
79
- ),
80
- 'utf8'
81
- );
82
- await store.import(fixture);
83
- expect((await store.load(request)).receipt).toEqual(receipt);
84
- expect(JSON.parse(await store.export(request))).toEqual(JSON.parse(fixture));
85
- for (const different of [
86
- { ...request, method: 'POST' },
87
- { ...request, url: `${request.url}?query=1` },
88
- { ...request, headers: { authorization: 'different' } },
89
- ]) {
90
- expect(await store.load(different)).toBeNull();
91
- }
92
- await store.store(
93
- { ...request, url: `${request.url}/duplicate` },
94
- receipt,
95
- 2000
96
- );
97
- expect(await readdir(path.join(directory, 'bodies'))).toHaveLength(1);
98
- expect(await store.prune(1100, 100)).toBe(1);
99
- expect(await readdir(path.join(directory, 'bodies'))).toHaveLength(1);
100
- await expect(
101
- store.import(fixture.replace('255', '254'))
102
- ).rejects.toMatchObject({ kind: 'cache' });
103
- await expect(
104
- store.import(fixture.replace('"version": 1', '"version": 999'))
105
- ).rejects.toMatchObject({ kind: 'cache' });
106
- });
107
-
108
- test('corrupt bodies are rejected and failed transports are never cached', async () => {
109
- const stored = await store.store(request, receipt, 1000);
110
- await writeFile(path.join(directory, 'bodies', stored.sha256), 'corrupt');
111
- await expect(store.load(request)).rejects.toMatchObject({ kind: 'cache' });
112
- const failure = new Error('transport failed');
113
- const cache = new CachedTransport(async () => {
114
- throw failure;
115
- }, store);
116
- await expect(
117
- cache.capture({ ...request, url: 'https://other.test' })
118
- ).rejects.toThrow('transport failed');
119
- expect(await store.list()).toHaveLength(1);
120
- });
121
-
122
- test('Unicode header ordering matches the shared Rust replay format', async () => {
123
- const fixture = await readFile(
124
- new globalThis.URL(
125
- '../../../rust/tests/fixtures/capture-v1-unicode.json',
126
- import.meta.url
127
- ),
128
- 'utf8'
129
- );
130
- const replay = JSON.parse(fixture);
131
- await store.import(fixture);
132
- expect((await store.load(replay.request)).receipt).toEqual(receipt);
133
- expect(JSON.parse(await store.export(replay.request))).toEqual(replay);
134
- });
135
-
136
- test('failed online refresh retains the expired capture for offline replay', async () => {
137
- await store.store(request, receipt, 800);
138
- const cache = new CachedTransport(
139
- async () => {
140
- throw new Error('offline upstream');
141
- },
142
- store,
143
- { ttl: 100, now: () => 1000 }
144
- );
145
- await expect(cache.capture(request)).rejects.toThrow('offline upstream');
146
- const offline = new CachedTransport(null, store, {
147
- offline: true,
148
- ttl: 100,
149
- now: () => 1000,
150
- });
151
- expect(await offline.capture(request)).toMatchObject({
152
- stale: true,
153
- receipt,
154
- });
155
- for (const replay of ['{', 'null', '{}']) {
156
- await expect(store.import(replay)).rejects.toMatchObject({ kind: 'cache' });
157
- }
158
- });
159
-
160
- test('headers are sorted without mutating the caller; TTL zero always refreshes', async () => {
161
- const a = { ...request, headers: { z: '1', a: '2' } };
162
- const b = { ...request, headers: { a: '2', z: '1' } };
163
- await store.store(a, receipt, 1000);
164
- expect((await store.load(b)).receipt).toEqual(receipt);
165
- const inner = jest.fn(async () => receipt);
166
- const cache = new CachedTransport(inner, store, { ttl: 0, now: () => 1000 });
167
- await cache.capture(a);
168
- await cache.capture(a);
169
- expect(inner).toHaveBeenCalledTimes(2);
170
- });
171
-
172
- test('caller mutation during a fetch cannot change the stored request identity', async () => {
173
- const mutable = { ...request, headers: { ...request.headers } };
174
- const cache = new CachedTransport(async () => {
175
- mutable.url = 'https://mutated.test';
176
- mutable.headers.accept = 'text/plain';
177
- return receipt;
178
- }, store);
179
- await cache.capture(mutable);
180
- expect((await store.load(request)).receipt).toEqual(receipt);
181
- expect(await store.load(mutable)).toBeNull();
182
- });