@link-assistant/web-capture 1.10.5 → 1.10.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,17 @@
1
+ const nock = require('nock');
2
+
3
+ nock('https://en.wikipedia.org')
4
+ .persist()
5
+ .get('/w/rest.php/v1/search/page')
6
+ .query(true)
7
+ .reply(200, {
8
+ pages: [
9
+ {
10
+ id: 1,
11
+ key: 'Formal_methods',
12
+ title: 'Formal methods',
13
+ excerpt: 'the study of <b>formal</b> methods',
14
+ description: 'rigorous techniques',
15
+ },
16
+ ],
17
+ });
@@ -0,0 +1,144 @@
1
+ /**
2
+ * Integration test for converting the DeepWiki search page from issue #17.
3
+ *
4
+ * The page is JavaScript-rendered, so this suite verifies the /markdown
5
+ * endpoint captures rendered content instead of the initial Next.js shell.
6
+ *
7
+ * Set DEEPWIKI_INTEGRATION=true to run it. The suite is skipped by default so
8
+ * local/offline test runs remain deterministic.
9
+ */
10
+
11
+ import { jest } from '@jest/globals';
12
+ import fs from 'fs';
13
+ import path from 'path';
14
+ import request from 'supertest';
15
+ import { fileURLToPath } from 'url';
16
+ import { app } from '../../src/index.js';
17
+ import { retry } from '../../src/retry.js';
18
+
19
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
20
+ const DEEPWIKI_URL =
21
+ 'https://deepwiki.com/search/-57-4-23-57_0e4aa687-7a9d-4591-8c6f-67c4b2d732f6';
22
+ const FIXTURE_DIR = path.join(__dirname, '..', 'fixtures', 'deepwiki');
23
+ const INLINE_MARKDOWN_LINE_LIMIT = 1500;
24
+ const PNG_SIGNATURE = Buffer.from([
25
+ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
26
+ ]);
27
+ const ORDERED_CONTENT_MARKERS = [
28
+ 'Search | DeepWiki',
29
+ 'deep-assistant/hive-mind',
30
+ 'расскажи пожалуйста на английском',
31
+ 'Deep\n\nThought Process',
32
+ '# Hive Mind: A Comprehensive Overview of an AI Agent System for Software Development',
33
+ '## Executive Summary',
34
+ '## I. Foundational Philosophy: Human-AI Collaboration Model',
35
+ '## II. Architectural Layers: The Three-Tier Design',
36
+ '## III. Original Ideas and Innovations',
37
+ 'Innovation 1: The Task Clarification System',
38
+ 'Innovation 2: Multi-Dimensional Feedback Detection',
39
+ 'Innovation 3: Thinking Depth Control',
40
+ '## IV. The Workflow Logic: How Everything Connects',
41
+ '## XI. Practical Implications: What This Enables',
42
+ ];
43
+
44
+ const SKIP_LIVE =
45
+ !process.env.DEEPWIKI_INTEGRATION ||
46
+ process.env.DEEPWIKI_INTEGRATION === 'false';
47
+ const describeIfLive = SKIP_LIVE ? describe.skip : describe;
48
+
49
+ jest.setTimeout(180000);
50
+
51
+ function readFixtureChunks() {
52
+ return ['index.md', 'part-1.md'].map((filename) =>
53
+ fs.readFileSync(path.join(FIXTURE_DIR, filename), 'utf8')
54
+ );
55
+ }
56
+
57
+ function splitMarkdown(markdown) {
58
+ const lines = markdown.split('\n');
59
+ const chunks = [];
60
+ for (
61
+ let index = 0;
62
+ index < lines.length;
63
+ index += INLINE_MARKDOWN_LINE_LIMIT
64
+ ) {
65
+ chunks.push(
66
+ lines.slice(index, index + INLINE_MARKDOWN_LINE_LIMIT).join('\n')
67
+ );
68
+ }
69
+ return chunks;
70
+ }
71
+
72
+ function expectMarkersInOrder(markdown, markers) {
73
+ let previousIndex = -1;
74
+ for (const marker of markers) {
75
+ const index = markdown.indexOf(marker);
76
+ expect(index).toBeGreaterThan(previousIndex);
77
+ previousIndex = index;
78
+ }
79
+ }
80
+
81
+ async function getDeepwikiMarkdown() {
82
+ const response = await retry(
83
+ () =>
84
+ request(app)
85
+ .get('/markdown')
86
+ .query({ url: DEEPWIKI_URL, engine: 'playwright' })
87
+ .expect(200),
88
+ {
89
+ retries: 2,
90
+ baseDelay: 3000,
91
+ onRetry: (err, attempt, delay) => {
92
+ console.log(
93
+ `Retry ${attempt} for DeepWiki markdown after ${delay}ms: ${err.message}`
94
+ );
95
+ },
96
+ }
97
+ );
98
+
99
+ expect(response.headers['content-type']).toMatch(/text\/markdown/);
100
+ return response.text;
101
+ }
102
+
103
+ describe('DeepWiki committed reference fixtures (issue #17)', () => {
104
+ it('stores markdown split into inline-sized reference documents', () => {
105
+ const fixtureChunks = readFixtureChunks();
106
+ const fixtureMarkdown = fixtureChunks.join('\n');
107
+
108
+ expect(fixtureChunks).toHaveLength(2);
109
+ expect(fixtureChunks[0].split('\n').length).toBeLessThanOrEqual(
110
+ INLINE_MARKDOWN_LINE_LIMIT
111
+ );
112
+ expect(fixtureChunks[1].split('\n').length).toBeGreaterThan(0);
113
+ expect(fixtureMarkdown.split('\n').length).toBeGreaterThan(
114
+ INLINE_MARKDOWN_LINE_LIMIT
115
+ );
116
+ expectMarkersInOrder(fixtureMarkdown, ORDERED_CONTENT_MARKERS);
117
+ });
118
+
119
+ it('keeps the full-page screenshot fixture as a PNG visual reference', () => {
120
+ const screenshot = fs.readFileSync(
121
+ path.join(FIXTURE_DIR, 'deepwiki-full-page.png')
122
+ );
123
+
124
+ expect(screenshot.subarray(0, PNG_SIGNATURE.length)).toEqual(PNG_SIGNATURE);
125
+ expect(screenshot.length).toBeGreaterThan(100000);
126
+ });
127
+ });
128
+
129
+ describeIfLive('DeepWiki search page markdown capture (issue #17)', () => {
130
+ it('matches the committed markdown fixture split from the rendered page', async () => {
131
+ const markdown = await getDeepwikiMarkdown();
132
+ const fixtureChunks = readFixtureChunks();
133
+ const fixtureMarkdown = fixtureChunks.join('\n');
134
+
135
+ expect(markdown).not.toMatch(/<html/i);
136
+ expect(markdown).not.toContain('__NEXT_DATA__');
137
+ expect(markdown.split('\n').length).toBeGreaterThan(
138
+ INLINE_MARKDOWN_LINE_LIMIT
139
+ );
140
+ expectMarkersInOrder(markdown, ORDERED_CONTENT_MARKERS);
141
+ expectMarkersInOrder(fixtureMarkdown, ORDERED_CONTENT_MARKERS);
142
+ expect(splitMarkdown(markdown)).toEqual(fixtureChunks);
143
+ }, 180000);
144
+ });
@@ -0,0 +1,121 @@
1
+ import { jest } from '@jest/globals';
2
+ import nock from 'nock';
3
+ import request from 'supertest';
4
+ import { URL, fileURLToPath } from 'url';
5
+
6
+ const DEEPWIKI_URL =
7
+ 'https://deepwiki.com/search/-57-4-23-57_0e4aa687-7a9d-4591-8c6f-67c4b2d732f6';
8
+ const deepwikiShellHtml = `<!doctype html>
9
+ <html lang="en">
10
+ <head>
11
+ <title>Search | DeepWiki</title>
12
+ <script src="/_next/static/chunks/app.js"></script>
13
+ <script>self.__next_f = []; self.__next_f.push([1, "shell"])</script>
14
+ </head>
15
+ <body><div id="__next"></div></body>
16
+ </html>`;
17
+ const renderedDeepwikiHtml = `<!doctype html>
18
+ <html lang="en">
19
+ <head><title>Search | DeepWiki</title></head>
20
+ <body>
21
+ <main>
22
+ <p>deep-assistant/hive-mind</p>
23
+ <p>расскажи пожалуйста на английском</p>
24
+ <h1>Hive Mind: A Comprehensive Overview of an AI Agent System for Software Development</h1>
25
+ <h2>Executive Summary</h2>
26
+ <p>Deep analysis of the issue-solver workflow.</p>
27
+ <h2>Thought Process</h2>
28
+ <pre><code>const clarifyPrompt = \`Task: "\${taskDescription}"\`;
29
+ await solver.run(clarifyPrompt);</code></pre>
30
+ </main>
31
+ </body>
32
+ </html>`;
33
+
34
+ const fakePage = {
35
+ type: 'playwright',
36
+ rawPage: {
37
+ evaluate: jest
38
+ .fn()
39
+ .mockRejectedValueOnce(
40
+ new Error(
41
+ 'Execution context was destroyed, most likely because of a navigation'
42
+ )
43
+ )
44
+ .mockResolvedValue(
45
+ renderedDeepwikiHtml.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ')
46
+ .length
47
+ ),
48
+ },
49
+ setViewport: jest.fn(async () => {}),
50
+ setExtraHTTPHeaders: jest.fn(async () => {}),
51
+ setUserAgent: jest.fn(async () => {}),
52
+ goto: jest.fn(async () => {}),
53
+ waitForTimeout: jest.fn(
54
+ (ms) => new Promise((resolve) => setTimeout(resolve, ms))
55
+ ),
56
+ content: jest.fn(async () => renderedDeepwikiHtml),
57
+ };
58
+ const fakeBrowser = {
59
+ newPage: jest.fn(async () => fakePage),
60
+ close: jest.fn(async () => {}),
61
+ };
62
+ const createBrowser = jest.fn(async () => fakeBrowser);
63
+ const browserModulePath = fileURLToPath(
64
+ new URL('../../src/browser.js', import.meta.url)
65
+ );
66
+
67
+ jest.unstable_mockModule(browserModulePath, () => ({
68
+ createBrowser,
69
+ getBrowserEngine: (req) => req.query.engine || 'puppeteer',
70
+ }));
71
+
72
+ const { app } = await import('../../src/index.js');
73
+
74
+ describe('DeepWiki rendered markdown endpoint behavior', () => {
75
+ beforeEach(() => {
76
+ nock.cleanAll();
77
+ jest.clearAllMocks();
78
+ });
79
+
80
+ afterEach(() => {
81
+ nock.cleanAll();
82
+ });
83
+
84
+ it('converts browser-rendered content when the initial fetch is a Next.js shell', async () => {
85
+ nock('https://deepwiki.com')
86
+ .get('/search/-57-4-23-57_0e4aa687-7a9d-4591-8c6f-67c4b2d732f6')
87
+ .reply(200, deepwikiShellHtml);
88
+
89
+ const response = await request(app)
90
+ .get('/markdown')
91
+ .query({ url: DEEPWIKI_URL, engine: 'playwright' })
92
+ .expect(200);
93
+
94
+ expect(createBrowser).toHaveBeenCalledWith(
95
+ 'playwright',
96
+ expect.objectContaining({
97
+ args: expect.arrayContaining([
98
+ expect.stringContaining('--user-agent='),
99
+ '--window-size=1280,800',
100
+ '--disable-blink-features=AutomationControlled',
101
+ ]),
102
+ })
103
+ );
104
+ expect(fakePage.goto).toHaveBeenCalledWith(
105
+ DEEPWIKI_URL,
106
+ expect.objectContaining({ waitUntil: 'networkidle0' })
107
+ );
108
+ expect(fakeBrowser.close).toHaveBeenCalledTimes(1);
109
+ expect(response.headers['content-type']).toMatch(/text\/markdown/);
110
+ expect(response.text).toContain('Search | DeepWiki');
111
+ expect(response.text).toContain('deep-assistant/hive-mind');
112
+ expect(response.text).toContain('расскажи пожалуйста на английском');
113
+ expect(response.text).toContain(
114
+ '# Hive Mind: A Comprehensive Overview of an AI Agent System for Software Development'
115
+ );
116
+ expect(response.text).toContain('## Thought Process');
117
+ expect(response.text).toContain('```');
118
+ expect(response.text).toContain('const clarifyPrompt');
119
+ expect(response.text).not.toContain('__next_f');
120
+ });
121
+ });
@@ -0,0 +1,214 @@
1
+ /**
2
+ * Smoke tests for the FormalAI-facing HTTP contract (issue #135).
3
+ *
4
+ * These tests intentionally assert the stable response shapes documented in
5
+ * docs/formalai-contract.md rather than endpoint internals.
6
+ */
7
+
8
+ import request from 'supertest';
9
+ import nock from 'nock';
10
+ import unzipper from 'unzipper';
11
+ import { app, SEARCH_PROVIDERS } from '../../src/index.js';
12
+
13
+ const HTML = `<!doctype html>
14
+ <html>
15
+ <head><title>FormalAI fixture</title></head>
16
+ <body><h1>FormalAI Fixture</h1><p>Stable HTTP shape.</p></body>
17
+ </html>`;
18
+
19
+ const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
20
+
21
+ const WIKI_JSON = {
22
+ pages: [
23
+ {
24
+ id: 1,
25
+ key: 'Formal_methods',
26
+ title: 'Formal methods',
27
+ excerpt: 'the study of <b>formal</b> methods',
28
+ description: 'rigorous techniques',
29
+ },
30
+ ],
31
+ };
32
+
33
+ function parseBinary(res, callback) {
34
+ const chunks = [];
35
+ res.on('data', (chunk) => chunks.push(chunk));
36
+ res.on('end', () => callback(null, Buffer.concat(chunks)));
37
+ }
38
+
39
+ afterEach(() => {
40
+ nock.cleanAll();
41
+ });
42
+
43
+ describe('FormalAI HTTP contract (#135)', () => {
44
+ it('returns stable text artifact shapes for /html, /txt, and /markdown', async () => {
45
+ nock('https://formalai.example').get('/page').reply(200, HTML, {
46
+ 'content-type': 'text/html; charset=utf-8',
47
+ });
48
+ const html = await request(app)
49
+ .get('/html')
50
+ .query({ url: 'https://formalai.example/page' })
51
+ .expect(200)
52
+ .expect('Content-Type', /text\/html/);
53
+ expect(html.text).toContain('<h1>FormalAI Fixture</h1>');
54
+
55
+ nock('https://formalai.example')
56
+ .get('/text')
57
+ .reply(200, 'FormalAI plain text', {
58
+ 'content-type': 'text/plain; charset=utf-8',
59
+ });
60
+ const text = await request(app)
61
+ .get('/txt')
62
+ .query({ url: 'https://formalai.example/text' })
63
+ .expect(200)
64
+ .expect('Content-Type', /text\/plain/);
65
+ expect(text.headers['content-disposition']).toContain('.txt');
66
+ expect(text.text).toBe('FormalAI plain text');
67
+
68
+ nock('https://formalai.example').get('/markdown').reply(200, HTML, {
69
+ 'content-type': 'text/html; charset=utf-8',
70
+ });
71
+ const markdown = await request(app)
72
+ .get('/markdown')
73
+ .query({ url: 'https://formalai.example/markdown' })
74
+ .expect(200)
75
+ .expect('Content-Type', /text\/markdown/);
76
+ expect(markdown.text).toContain('FormalAI Fixture');
77
+ expect(markdown.text).toContain('Stable HTTP shape.');
78
+ });
79
+
80
+ it('preserves upstream status and content for /fetch and /stream', async () => {
81
+ nock('https://formalai.example')
82
+ .get('/fetch-source')
83
+ .reply(203, 'fetch body', {
84
+ 'content-type': 'text/plain; charset=utf-8',
85
+ });
86
+ const fetched = await request(app)
87
+ .get('/fetch')
88
+ .query({ url: 'https://formalai.example/fetch-source' })
89
+ .expect(203)
90
+ .expect('Content-Type', /text\/plain/);
91
+ expect(fetched.text).toBe('fetch body');
92
+
93
+ nock('https://formalai.example')
94
+ .get('/stream-source')
95
+ .reply(206, 'stream body', {
96
+ 'content-type': 'text/plain; charset=utf-8',
97
+ });
98
+ const streamed = await request(app)
99
+ .get('/stream')
100
+ .query({ url: 'https://formalai.example/stream-source' })
101
+ .expect(206)
102
+ .expect('Content-Type', /text\/plain/);
103
+ expect(streamed.text).toBe('stream body');
104
+ });
105
+
106
+ it('returns stable binary artifact shapes for /image and /archive', async () => {
107
+ nock('https://drive.google.com')
108
+ .get('/uc')
109
+ .query({ export: 'download', id: 'formalaiimage' })
110
+ .reply(200, PNG, { 'content-type': 'image/png' });
111
+ const image = await request(app)
112
+ .get('/image')
113
+ .query({
114
+ url: 'https://drive.google.com/file/d/formalaiimage/view',
115
+ })
116
+ .buffer(true)
117
+ .parse(parseBinary)
118
+ .expect(200)
119
+ .expect('Content-Type', /image\/png/);
120
+ expect(image.headers['content-disposition']).toContain(
121
+ 'google-drive-formalaiimage.png'
122
+ );
123
+ expect(image.body.subarray(0, 8)).toEqual(PNG);
124
+
125
+ nock('https://formalai.example').get('/archive').reply(200, HTML, {
126
+ 'content-type': 'text/html; charset=utf-8',
127
+ });
128
+ const archive = await request(app)
129
+ .get('/archive')
130
+ .query({
131
+ url: 'https://formalai.example/archive',
132
+ localImages: 'false',
133
+ })
134
+ .buffer(true)
135
+ .parse(parseBinary)
136
+ .expect(200)
137
+ .expect('Content-Type', /application\/zip/);
138
+ expect(archive.body[0]).toBe(0x50);
139
+ expect(archive.body[1]).toBe(0x4b);
140
+ const zip = await unzipper.Open.buffer(archive.body);
141
+ expect(zip.files.map((file) => file.path)).toEqual(
142
+ expect.arrayContaining(['document.md', 'document.html'])
143
+ );
144
+ });
145
+
146
+ it('returns normalized /search JSON diagnostics and the documented provider catalog', async () => {
147
+ expect(SEARCH_PROVIDERS).toEqual([
148
+ 'wikipedia',
149
+ 'duckduckgo',
150
+ 'google',
151
+ 'bing',
152
+ 'brave',
153
+ ]);
154
+
155
+ nock('https://en.wikipedia.org')
156
+ .get('/w/rest.php/v1/search/page')
157
+ .query(true)
158
+ .reply(200, WIKI_JSON);
159
+
160
+ const res = await request(app)
161
+ .get('/search')
162
+ .query({ q: 'formal-ai', provider: 'wikipedia', limit: 1 })
163
+ .expect(200)
164
+ .expect('Content-Type', /application\/json/);
165
+
166
+ expect(res.body).toMatchObject({
167
+ query: 'formal-ai',
168
+ provider: 'wikipedia',
169
+ captureMode: 'fetch',
170
+ results: [
171
+ {
172
+ rank: 1,
173
+ title: 'Formal methods',
174
+ url: 'https://en.wikipedia.org/wiki/Formal_methods',
175
+ snippet: 'the study of formal methods',
176
+ },
177
+ ],
178
+ diagnostics: {
179
+ status: 200,
180
+ blockedByCors: false,
181
+ blockedByCaptcha: false,
182
+ },
183
+ });
184
+ expect(res.body.capturedAt).toEqual(expect.any(String));
185
+ expect(res.body.diagnostics.sourceUrl).toContain('en.wikipedia.org');
186
+ });
187
+
188
+ it('keeps normalized /search diagnostics when provider capture fails', async () => {
189
+ nock('https://en.wikipedia.org')
190
+ .get('/w/rest.php/v1/search/page')
191
+ .query(true)
192
+ .replyWithError(new Error('provider offline'));
193
+
194
+ const res = await request(app)
195
+ .get('/search')
196
+ .query({ q: 'formal-ai', provider: 'wikipedia', limit: 1 })
197
+ .expect(200)
198
+ .expect('Content-Type', /application\/json/);
199
+
200
+ expect(res.body).toMatchObject({
201
+ query: 'formal-ai',
202
+ provider: 'wikipedia',
203
+ captureMode: 'fetch',
204
+ results: [],
205
+ diagnostics: {
206
+ status: 0,
207
+ blockedByCors: false,
208
+ blockedByCaptcha: false,
209
+ },
210
+ });
211
+ expect(res.body.diagnostics.sourceUrl).toContain('en.wikipedia.org');
212
+ expect(res.body.diagnostics.error).toContain('provider offline');
213
+ });
214
+ });
@@ -1,7 +1,7 @@
1
1
  // Unit tests for CLI argument parsing and functionality
2
2
  import { spawn } from 'child_process';
3
3
  import http from 'node:http';
4
- import { mkdtempSync, rmSync, writeFileSync } from 'fs';
4
+ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
5
5
  import { fileURLToPath } from 'url';
6
6
  import { dirname, join, resolve } from 'path';
7
7
  import packageJson from '../../package.json' with { type: 'json' };
@@ -9,11 +9,12 @@ import packageJson from '../../package.json' with { type: 'json' };
9
9
  const __filename = fileURLToPath(import.meta.url);
10
10
  const __dirname = dirname(__filename);
11
11
  const cliPath = resolve(__dirname, '../../bin/web-capture.js');
12
+ const searchFixturePath = resolve(__dirname, '../helpers/search-fixture.cjs');
12
13
 
13
14
  // Helper function to run CLI and capture output
14
15
  function runCli(args, options = {}) {
15
16
  return new Promise((resolve, reject) => {
16
- const proc = spawn('node', [cliPath, ...args], {
17
+ const proc = spawn(process.execPath, [cliPath, ...args], {
17
18
  cwd: options.cwd || dirname(cliPath),
18
19
  env: { ...process.env, ...options.env },
19
20
  });
@@ -36,13 +37,14 @@ function runCli(args, options = {}) {
36
37
  });
37
38
  }
38
39
 
39
- function startFixtureServer() {
40
+ function startFixtureServer({
41
+ body = '<!doctype html><html><body><h1>Issue 68</h1><p>Captured from positional URL.</p></body></html>',
42
+ contentType = 'text/html; charset=utf-8',
43
+ } = {}) {
40
44
  return new Promise((resolve, reject) => {
41
45
  const server = http.createServer((req, res) => {
42
- res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
43
- res.end(
44
- '<!doctype html><html><body><h1>Issue 68</h1><p>Captured from positional URL.</p></body></html>'
45
- );
46
+ res.writeHead(200, { 'Content-Type': contentType });
47
+ res.end(body);
46
48
  });
47
49
 
48
50
  server.on('error', reject);
@@ -255,4 +257,99 @@ describe('CLI', () => {
255
257
  expect(result.stdout).toContain('-o');
256
258
  });
257
259
  });
260
+
261
+ describe('FormalAI-compatible CLI contract', () => {
262
+ test('writes HTML, Markdown, and text captures to stdout with -o -', async () => {
263
+ const { server, url } = await startFixtureServer({
264
+ body: '<!doctype html><html><body><h1>FormalAI Fixture</h1><p>Stable CLI shape.</p></body></html>',
265
+ });
266
+
267
+ try {
268
+ const html = await runCli([url, '--format', 'html', '--output', '-']);
269
+ expect(html.code).toBe(0);
270
+ expect(html.stdout).toContain('<h1>FormalAI Fixture</h1>');
271
+
272
+ const markdown = await runCli([
273
+ url,
274
+ '--format',
275
+ 'markdown',
276
+ '--output',
277
+ '-',
278
+ ]);
279
+ expect(markdown.code).toBe(0);
280
+ expect(markdown.stdout).toContain('FormalAI Fixture');
281
+ expect(markdown.stdout).toContain('Stable CLI shape.');
282
+
283
+ const text = await runCli([url, '--format', 'txt', '--output', '-']);
284
+ expect(text.code).toBe(0);
285
+ expect(text.stdout).toContain('FormalAI Fixture');
286
+ } finally {
287
+ await stopFixtureServer(server);
288
+ }
289
+ }, 20000);
290
+
291
+ test('writes archive captures as ZIP files for binary CLI consumers', async () => {
292
+ const outputDir = mkdtempSync(join(process.cwd(), 'formalai-cli-'));
293
+ const outputPath = join(outputDir, 'capture.zip');
294
+ const { server, url } = await startFixtureServer({
295
+ body: '<!doctype html><html><body><h1>FormalAI Archive</h1></body></html>',
296
+ });
297
+
298
+ try {
299
+ const result = await runCli([
300
+ url,
301
+ '--archive',
302
+ 'zip',
303
+ '--output',
304
+ outputPath,
305
+ ]);
306
+
307
+ expect(result.code).toBe(0);
308
+ const zipBytes = readFileSync(outputPath);
309
+ expect(zipBytes.length).toBeGreaterThan(50);
310
+ expect(zipBytes[0]).toBe(0x50);
311
+ expect(zipBytes[1]).toBe(0x4b);
312
+ } finally {
313
+ await stopFixtureServer(server);
314
+ rmSync(outputDir, { recursive: true, force: true });
315
+ }
316
+ }, 20000);
317
+
318
+ test('emits normalized search JSON from the search subcommand', async () => {
319
+ const nodeOptions = [
320
+ process.env.NODE_OPTIONS,
321
+ `--require=${searchFixturePath}`,
322
+ ]
323
+ .filter(Boolean)
324
+ .join(' ');
325
+ const result = await runCli(
326
+ ['search', 'formal-ai', '--provider', 'wikipedia', '--limit', '1'],
327
+ {
328
+ env: { NODE_OPTIONS: nodeOptions },
329
+ }
330
+ );
331
+
332
+ expect(result.code).toBe(0);
333
+ const body = JSON.parse(result.stdout);
334
+ expect(body).toMatchObject({
335
+ query: 'formal-ai',
336
+ provider: 'wikipedia',
337
+ captureMode: 'fetch',
338
+ });
339
+ expect(body.results).toEqual([
340
+ {
341
+ rank: 1,
342
+ title: 'Formal methods',
343
+ url: 'https://en.wikipedia.org/wiki/Formal_methods',
344
+ snippet: 'the study of formal methods',
345
+ },
346
+ ]);
347
+ expect(body.diagnostics).toMatchObject({
348
+ status: 200,
349
+ blockedByCors: false,
350
+ blockedByCaptcha: false,
351
+ });
352
+ expect(body.diagnostics.sourceUrl).toContain('en.wikipedia.org');
353
+ }, 20000);
354
+ });
258
355
  });
@@ -0,0 +1,36 @@
1
+ import { shouldRenderHtmlWithBrowser } from '../../src/markdown.js';
2
+
3
+ describe('markdown rendering decisions', () => {
4
+ it('uses browser rendering for JavaScript-heavy HTML shells', () => {
5
+ const html = `<!doctype html>
6
+ <html>
7
+ <head><script src="/_next/static/app.js"></script></head>
8
+ <body><div id="__next"></div></body>
9
+ </html>`;
10
+
11
+ expect(shouldRenderHtmlWithBrowser(html)).toBe(true);
12
+ });
13
+
14
+ it('keeps plain HTML on the direct fetch path', () => {
15
+ const html = `<!doctype html>
16
+ <html>
17
+ <head><title>Static page</title></head>
18
+ <body><main><h1>Static content</h1></main></body>
19
+ </html>`;
20
+
21
+ expect(shouldRenderHtmlWithBrowser(html)).toBe(false);
22
+ });
23
+
24
+ it('keeps static pages with incidental scripts on the direct fetch path', () => {
25
+ const html = `<!doctype html>
26
+ <html>
27
+ <head>
28
+ <title>Static page</title>
29
+ <script src="/analytics.js"></script>
30
+ </head>
31
+ <body><main><h1>Static content</h1></main></body>
32
+ </html>`;
33
+
34
+ expect(shouldRenderHtmlWithBrowser(html)).toBe(false);
35
+ });
36
+ });