@link-assistant/web-capture 1.10.4 → 1.10.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierignore +1 -0
- package/CHANGELOG.md +12 -0
- package/bin/web-capture.js +77 -18
- package/package.json +1 -1
- package/src/archive.js +20 -1
- package/src/browser.js +14 -3
- package/src/docx.js +20 -2
- package/src/fetch.js +2 -1
- package/src/image.js +12 -1
- package/src/lib.js +163 -0
- package/src/markdown.js +171 -5
- package/src/pdf.js +22 -8
- package/src/stream.js +2 -1
- package/src/txt.js +10 -0
- package/tests/fixtures/deepwiki/deepwiki-full-page.png +0 -0
- package/tests/fixtures/deepwiki/index.md +1500 -0
- package/tests/fixtures/deepwiki/part-1.md +776 -0
- package/tests/integration/deepwiki-download.test.js +144 -0
- package/tests/integration/deepwiki-rendered-page.mock.test.js +121 -0
- package/tests/integration/google-drive-image.test.js +246 -0
- package/tests/unit/convertGoogleDriveUrl.test.js +89 -0
- package/tests/unit/markdown.test.js +36 -0
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Integration test for converting the DeepWiki search page from issue #17.
|
|
3
|
+
*
|
|
4
|
+
* The page is JavaScript-rendered, so this suite verifies the /markdown
|
|
5
|
+
* endpoint captures rendered content instead of the initial Next.js shell.
|
|
6
|
+
*
|
|
7
|
+
* Set DEEPWIKI_INTEGRATION=true to run it. The suite is skipped by default so
|
|
8
|
+
* local/offline test runs remain deterministic.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { jest } from '@jest/globals';
|
|
12
|
+
import fs from 'fs';
|
|
13
|
+
import path from 'path';
|
|
14
|
+
import request from 'supertest';
|
|
15
|
+
import { fileURLToPath } from 'url';
|
|
16
|
+
import { app } from '../../src/index.js';
|
|
17
|
+
import { retry } from '../../src/retry.js';
|
|
18
|
+
|
|
19
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
20
|
+
const DEEPWIKI_URL =
|
|
21
|
+
'https://deepwiki.com/search/-57-4-23-57_0e4aa687-7a9d-4591-8c6f-67c4b2d732f6';
|
|
22
|
+
const FIXTURE_DIR = path.join(__dirname, '..', 'fixtures', 'deepwiki');
|
|
23
|
+
const INLINE_MARKDOWN_LINE_LIMIT = 1500;
|
|
24
|
+
const PNG_SIGNATURE = Buffer.from([
|
|
25
|
+
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
|
|
26
|
+
]);
|
|
27
|
+
const ORDERED_CONTENT_MARKERS = [
|
|
28
|
+
'Search | DeepWiki',
|
|
29
|
+
'deep-assistant/hive-mind',
|
|
30
|
+
'расскажи пожалуйста на английском',
|
|
31
|
+
'Deep\n\nThought Process',
|
|
32
|
+
'# Hive Mind: A Comprehensive Overview of an AI Agent System for Software Development',
|
|
33
|
+
'## Executive Summary',
|
|
34
|
+
'## I. Foundational Philosophy: Human-AI Collaboration Model',
|
|
35
|
+
'## II. Architectural Layers: The Three-Tier Design',
|
|
36
|
+
'## III. Original Ideas and Innovations',
|
|
37
|
+
'Innovation 1: The Task Clarification System',
|
|
38
|
+
'Innovation 2: Multi-Dimensional Feedback Detection',
|
|
39
|
+
'Innovation 3: Thinking Depth Control',
|
|
40
|
+
'## IV. The Workflow Logic: How Everything Connects',
|
|
41
|
+
'## XI. Practical Implications: What This Enables',
|
|
42
|
+
];
|
|
43
|
+
|
|
44
|
+
const SKIP_LIVE =
|
|
45
|
+
!process.env.DEEPWIKI_INTEGRATION ||
|
|
46
|
+
process.env.DEEPWIKI_INTEGRATION === 'false';
|
|
47
|
+
const describeIfLive = SKIP_LIVE ? describe.skip : describe;
|
|
48
|
+
|
|
49
|
+
jest.setTimeout(180000);
|
|
50
|
+
|
|
51
|
+
function readFixtureChunks() {
|
|
52
|
+
return ['index.md', 'part-1.md'].map((filename) =>
|
|
53
|
+
fs.readFileSync(path.join(FIXTURE_DIR, filename), 'utf8')
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function splitMarkdown(markdown) {
|
|
58
|
+
const lines = markdown.split('\n');
|
|
59
|
+
const chunks = [];
|
|
60
|
+
for (
|
|
61
|
+
let index = 0;
|
|
62
|
+
index < lines.length;
|
|
63
|
+
index += INLINE_MARKDOWN_LINE_LIMIT
|
|
64
|
+
) {
|
|
65
|
+
chunks.push(
|
|
66
|
+
lines.slice(index, index + INLINE_MARKDOWN_LINE_LIMIT).join('\n')
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
return chunks;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function expectMarkersInOrder(markdown, markers) {
|
|
73
|
+
let previousIndex = -1;
|
|
74
|
+
for (const marker of markers) {
|
|
75
|
+
const index = markdown.indexOf(marker);
|
|
76
|
+
expect(index).toBeGreaterThan(previousIndex);
|
|
77
|
+
previousIndex = index;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async function getDeepwikiMarkdown() {
|
|
82
|
+
const response = await retry(
|
|
83
|
+
() =>
|
|
84
|
+
request(app)
|
|
85
|
+
.get('/markdown')
|
|
86
|
+
.query({ url: DEEPWIKI_URL, engine: 'playwright' })
|
|
87
|
+
.expect(200),
|
|
88
|
+
{
|
|
89
|
+
retries: 2,
|
|
90
|
+
baseDelay: 3000,
|
|
91
|
+
onRetry: (err, attempt, delay) => {
|
|
92
|
+
console.log(
|
|
93
|
+
`Retry ${attempt} for DeepWiki markdown after ${delay}ms: ${err.message}`
|
|
94
|
+
);
|
|
95
|
+
},
|
|
96
|
+
}
|
|
97
|
+
);
|
|
98
|
+
|
|
99
|
+
expect(response.headers['content-type']).toMatch(/text\/markdown/);
|
|
100
|
+
return response.text;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
describe('DeepWiki committed reference fixtures (issue #17)', () => {
|
|
104
|
+
it('stores markdown split into inline-sized reference documents', () => {
|
|
105
|
+
const fixtureChunks = readFixtureChunks();
|
|
106
|
+
const fixtureMarkdown = fixtureChunks.join('\n');
|
|
107
|
+
|
|
108
|
+
expect(fixtureChunks).toHaveLength(2);
|
|
109
|
+
expect(fixtureChunks[0].split('\n').length).toBeLessThanOrEqual(
|
|
110
|
+
INLINE_MARKDOWN_LINE_LIMIT
|
|
111
|
+
);
|
|
112
|
+
expect(fixtureChunks[1].split('\n').length).toBeGreaterThan(0);
|
|
113
|
+
expect(fixtureMarkdown.split('\n').length).toBeGreaterThan(
|
|
114
|
+
INLINE_MARKDOWN_LINE_LIMIT
|
|
115
|
+
);
|
|
116
|
+
expectMarkersInOrder(fixtureMarkdown, ORDERED_CONTENT_MARKERS);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it('keeps the full-page screenshot fixture as a PNG visual reference', () => {
|
|
120
|
+
const screenshot = fs.readFileSync(
|
|
121
|
+
path.join(FIXTURE_DIR, 'deepwiki-full-page.png')
|
|
122
|
+
);
|
|
123
|
+
|
|
124
|
+
expect(screenshot.subarray(0, PNG_SIGNATURE.length)).toEqual(PNG_SIGNATURE);
|
|
125
|
+
expect(screenshot.length).toBeGreaterThan(100000);
|
|
126
|
+
});
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
describeIfLive('DeepWiki search page markdown capture (issue #17)', () => {
|
|
130
|
+
it('matches the committed markdown fixture split from the rendered page', async () => {
|
|
131
|
+
const markdown = await getDeepwikiMarkdown();
|
|
132
|
+
const fixtureChunks = readFixtureChunks();
|
|
133
|
+
const fixtureMarkdown = fixtureChunks.join('\n');
|
|
134
|
+
|
|
135
|
+
expect(markdown).not.toMatch(/<html/i);
|
|
136
|
+
expect(markdown).not.toContain('__NEXT_DATA__');
|
|
137
|
+
expect(markdown.split('\n').length).toBeGreaterThan(
|
|
138
|
+
INLINE_MARKDOWN_LINE_LIMIT
|
|
139
|
+
);
|
|
140
|
+
expectMarkersInOrder(markdown, ORDERED_CONTENT_MARKERS);
|
|
141
|
+
expectMarkersInOrder(fixtureMarkdown, ORDERED_CONTENT_MARKERS);
|
|
142
|
+
expect(splitMarkdown(markdown)).toEqual(fixtureChunks);
|
|
143
|
+
}, 180000);
|
|
144
|
+
});
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { jest } from '@jest/globals';
|
|
2
|
+
import nock from 'nock';
|
|
3
|
+
import request from 'supertest';
|
|
4
|
+
import { URL, fileURLToPath } from 'url';
|
|
5
|
+
|
|
6
|
+
const DEEPWIKI_URL =
|
|
7
|
+
'https://deepwiki.com/search/-57-4-23-57_0e4aa687-7a9d-4591-8c6f-67c4b2d732f6';
|
|
8
|
+
const deepwikiShellHtml = `<!doctype html>
|
|
9
|
+
<html lang="en">
|
|
10
|
+
<head>
|
|
11
|
+
<title>Search | DeepWiki</title>
|
|
12
|
+
<script src="/_next/static/chunks/app.js"></script>
|
|
13
|
+
<script>self.__next_f = []; self.__next_f.push([1, "shell"])</script>
|
|
14
|
+
</head>
|
|
15
|
+
<body><div id="__next"></div></body>
|
|
16
|
+
</html>`;
|
|
17
|
+
const renderedDeepwikiHtml = `<!doctype html>
|
|
18
|
+
<html lang="en">
|
|
19
|
+
<head><title>Search | DeepWiki</title></head>
|
|
20
|
+
<body>
|
|
21
|
+
<main>
|
|
22
|
+
<p>deep-assistant/hive-mind</p>
|
|
23
|
+
<p>расскажи пожалуйста на английском</p>
|
|
24
|
+
<h1>Hive Mind: A Comprehensive Overview of an AI Agent System for Software Development</h1>
|
|
25
|
+
<h2>Executive Summary</h2>
|
|
26
|
+
<p>Deep analysis of the issue-solver workflow.</p>
|
|
27
|
+
<h2>Thought Process</h2>
|
|
28
|
+
<pre><code>const clarifyPrompt = \`Task: "\${taskDescription}"\`;
|
|
29
|
+
await solver.run(clarifyPrompt);</code></pre>
|
|
30
|
+
</main>
|
|
31
|
+
</body>
|
|
32
|
+
</html>`;
|
|
33
|
+
|
|
34
|
+
const fakePage = {
|
|
35
|
+
type: 'playwright',
|
|
36
|
+
rawPage: {
|
|
37
|
+
evaluate: jest
|
|
38
|
+
.fn()
|
|
39
|
+
.mockRejectedValueOnce(
|
|
40
|
+
new Error(
|
|
41
|
+
'Execution context was destroyed, most likely because of a navigation'
|
|
42
|
+
)
|
|
43
|
+
)
|
|
44
|
+
.mockResolvedValue(
|
|
45
|
+
renderedDeepwikiHtml.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ')
|
|
46
|
+
.length
|
|
47
|
+
),
|
|
48
|
+
},
|
|
49
|
+
setViewport: jest.fn(async () => {}),
|
|
50
|
+
setExtraHTTPHeaders: jest.fn(async () => {}),
|
|
51
|
+
setUserAgent: jest.fn(async () => {}),
|
|
52
|
+
goto: jest.fn(async () => {}),
|
|
53
|
+
waitForTimeout: jest.fn(
|
|
54
|
+
(ms) => new Promise((resolve) => setTimeout(resolve, ms))
|
|
55
|
+
),
|
|
56
|
+
content: jest.fn(async () => renderedDeepwikiHtml),
|
|
57
|
+
};
|
|
58
|
+
const fakeBrowser = {
|
|
59
|
+
newPage: jest.fn(async () => fakePage),
|
|
60
|
+
close: jest.fn(async () => {}),
|
|
61
|
+
};
|
|
62
|
+
const createBrowser = jest.fn(async () => fakeBrowser);
|
|
63
|
+
const browserModulePath = fileURLToPath(
|
|
64
|
+
new URL('../../src/browser.js', import.meta.url)
|
|
65
|
+
);
|
|
66
|
+
|
|
67
|
+
jest.unstable_mockModule(browserModulePath, () => ({
|
|
68
|
+
createBrowser,
|
|
69
|
+
getBrowserEngine: (req) => req.query.engine || 'puppeteer',
|
|
70
|
+
}));
|
|
71
|
+
|
|
72
|
+
const { app } = await import('../../src/index.js');
|
|
73
|
+
|
|
74
|
+
describe('DeepWiki rendered markdown endpoint behavior', () => {
|
|
75
|
+
beforeEach(() => {
|
|
76
|
+
nock.cleanAll();
|
|
77
|
+
jest.clearAllMocks();
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
afterEach(() => {
|
|
81
|
+
nock.cleanAll();
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it('converts browser-rendered content when the initial fetch is a Next.js shell', async () => {
|
|
85
|
+
nock('https://deepwiki.com')
|
|
86
|
+
.get('/search/-57-4-23-57_0e4aa687-7a9d-4591-8c6f-67c4b2d732f6')
|
|
87
|
+
.reply(200, deepwikiShellHtml);
|
|
88
|
+
|
|
89
|
+
const response = await request(app)
|
|
90
|
+
.get('/markdown')
|
|
91
|
+
.query({ url: DEEPWIKI_URL, engine: 'playwright' })
|
|
92
|
+
.expect(200);
|
|
93
|
+
|
|
94
|
+
expect(createBrowser).toHaveBeenCalledWith(
|
|
95
|
+
'playwright',
|
|
96
|
+
expect.objectContaining({
|
|
97
|
+
args: expect.arrayContaining([
|
|
98
|
+
expect.stringContaining('--user-agent='),
|
|
99
|
+
'--window-size=1280,800',
|
|
100
|
+
'--disable-blink-features=AutomationControlled',
|
|
101
|
+
]),
|
|
102
|
+
})
|
|
103
|
+
);
|
|
104
|
+
expect(fakePage.goto).toHaveBeenCalledWith(
|
|
105
|
+
DEEPWIKI_URL,
|
|
106
|
+
expect.objectContaining({ waitUntil: 'networkidle0' })
|
|
107
|
+
);
|
|
108
|
+
expect(fakeBrowser.close).toHaveBeenCalledTimes(1);
|
|
109
|
+
expect(response.headers['content-type']).toMatch(/text\/markdown/);
|
|
110
|
+
expect(response.text).toContain('Search | DeepWiki');
|
|
111
|
+
expect(response.text).toContain('deep-assistant/hive-mind');
|
|
112
|
+
expect(response.text).toContain('расскажи пожалуйста на английском');
|
|
113
|
+
expect(response.text).toContain(
|
|
114
|
+
'# Hive Mind: A Comprehensive Overview of an AI Agent System for Software Development'
|
|
115
|
+
);
|
|
116
|
+
expect(response.text).toContain('## Thought Process');
|
|
117
|
+
expect(response.text).toContain('```');
|
|
118
|
+
expect(response.text).toContain('const clarifyPrompt');
|
|
119
|
+
expect(response.text).not.toContain('__next_f');
|
|
120
|
+
});
|
|
121
|
+
});
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import request from 'supertest';
|
|
2
|
+
import nock from 'nock';
|
|
3
|
+
import unzipper from 'unzipper';
|
|
4
|
+
import { app } from '../../src/index.js';
|
|
5
|
+
|
|
6
|
+
const FIRST_FILE_ID = '1Cxkx6-428EQAX0-eiaq66H829ohnPp7q';
|
|
7
|
+
const SECOND_FILE_ID = '1fgJaftjv53xCN7vgiJOaQlbfWkUqPgyd';
|
|
8
|
+
const JPEG_BYTES = Buffer.from(
|
|
9
|
+
'/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////2wBDAf//////////////////////////////////////////////////////////////////////////////////////wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAX/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIQAxAAAAH/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECAQE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Ap//xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAE/IV//2gAMAwEAAgADAAAAEP/EFBQRAQAAAAAAAAAAAAAAAAAAARD/2gAIAQMBAT8QH//EFBQRAQAAAAAAAAAAAAAAAAAAARD/2gAIAQIBAT8QH//EFBABAQAAAAAAAAAAAAAAAAAAARD/2gAIAQEAAT8QH//Z',
|
|
10
|
+
'base64'
|
|
11
|
+
);
|
|
12
|
+
|
|
13
|
+
const DRIVE_URLS = [
|
|
14
|
+
{
|
|
15
|
+
name: 'first issue URL with /view',
|
|
16
|
+
fileId: FIRST_FILE_ID,
|
|
17
|
+
url: `https://drive.google.com/file/d/${FIRST_FILE_ID}/view`,
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
name: 'first issue URL without /view',
|
|
21
|
+
fileId: FIRST_FILE_ID,
|
|
22
|
+
url: `https://drive.google.com/file/d/${FIRST_FILE_ID}`,
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
name: 'second issue URL with /view',
|
|
26
|
+
fileId: SECOND_FILE_ID,
|
|
27
|
+
url: `https://drive.google.com/file/d/${SECOND_FILE_ID}/view`,
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
name: 'second issue URL without /view',
|
|
31
|
+
fileId: SECOND_FILE_ID,
|
|
32
|
+
url: `https://drive.google.com/file/d/${SECOND_FILE_ID}`,
|
|
33
|
+
},
|
|
34
|
+
];
|
|
35
|
+
|
|
36
|
+
function directDownloadUrl(fileId) {
|
|
37
|
+
return `https://drive.google.com/uc?export=download&id=${fileId}`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function mockDriveHead(fileId) {
|
|
41
|
+
return nock('https://drive.google.com')
|
|
42
|
+
.head('/uc')
|
|
43
|
+
.query({ export: 'download', id: fileId })
|
|
44
|
+
.reply(200, '', { 'content-type': 'image/jpeg' });
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function mockDriveGet(fileId) {
|
|
48
|
+
return nock('https://drive.google.com')
|
|
49
|
+
.get('/uc')
|
|
50
|
+
.query({ export: 'download', id: fileId })
|
|
51
|
+
.reply(200, JPEG_BYTES, {
|
|
52
|
+
'content-type': 'image/jpeg',
|
|
53
|
+
'content-length': String(JPEG_BYTES.length),
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function bufferParser(res, callback) {
|
|
58
|
+
const chunks = [];
|
|
59
|
+
res.on('data', (chunk) => chunks.push(Buffer.from(chunk)));
|
|
60
|
+
res.on('end', () => callback(null, Buffer.concat(chunks)));
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
afterEach(() => {
|
|
64
|
+
nock.cleanAll();
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
describe('Google Drive public image links', () => {
|
|
68
|
+
it.each(DRIVE_URLS)(
|
|
69
|
+
'/image downloads image bytes for $name',
|
|
70
|
+
async (item) => {
|
|
71
|
+
mockDriveGet(item.fileId);
|
|
72
|
+
|
|
73
|
+
const res = await request(app)
|
|
74
|
+
.get('/image')
|
|
75
|
+
.query({ url: item.url })
|
|
76
|
+
.buffer(true)
|
|
77
|
+
.parse(bufferParser)
|
|
78
|
+
.expect(200);
|
|
79
|
+
|
|
80
|
+
expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
|
|
81
|
+
expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
|
|
82
|
+
expect(nock.isDone()).toBe(true);
|
|
83
|
+
}
|
|
84
|
+
);
|
|
85
|
+
|
|
86
|
+
it.each(DRIVE_URLS)(
|
|
87
|
+
'/fetch proxies direct image bytes for $name',
|
|
88
|
+
async (item) => {
|
|
89
|
+
mockDriveGet(item.fileId);
|
|
90
|
+
|
|
91
|
+
const res = await request(app)
|
|
92
|
+
.get('/fetch')
|
|
93
|
+
.query({ url: item.url })
|
|
94
|
+
.buffer(true)
|
|
95
|
+
.parse(bufferParser)
|
|
96
|
+
.expect(200);
|
|
97
|
+
|
|
98
|
+
expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
|
|
99
|
+
expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
|
|
100
|
+
expect(nock.isDone()).toBe(true);
|
|
101
|
+
}
|
|
102
|
+
);
|
|
103
|
+
|
|
104
|
+
it.each(DRIVE_URLS)(
|
|
105
|
+
'/stream proxies direct image bytes for $name',
|
|
106
|
+
async (item) => {
|
|
107
|
+
mockDriveGet(item.fileId);
|
|
108
|
+
|
|
109
|
+
const res = await request(app)
|
|
110
|
+
.get('/stream')
|
|
111
|
+
.query({ url: item.url })
|
|
112
|
+
.buffer(true)
|
|
113
|
+
.parse(bufferParser)
|
|
114
|
+
.expect(200);
|
|
115
|
+
|
|
116
|
+
expect(res.headers['content-type']).toMatch(/^image\/jpeg/);
|
|
117
|
+
expect(res.body.subarray(0, 3)).toEqual(Buffer.from([0xff, 0xd8, 0xff]));
|
|
118
|
+
expect(nock.isDone()).toBe(true);
|
|
119
|
+
}
|
|
120
|
+
);
|
|
121
|
+
|
|
122
|
+
it.each(DRIVE_URLS)(
|
|
123
|
+
'/markdown includes a direct image link for $name',
|
|
124
|
+
async (item) => {
|
|
125
|
+
mockDriveHead(item.fileId);
|
|
126
|
+
|
|
127
|
+
const res = await request(app)
|
|
128
|
+
.get('/markdown')
|
|
129
|
+
.query({ url: item.url })
|
|
130
|
+
.expect(200);
|
|
131
|
+
|
|
132
|
+
expect(res.headers['content-type']).toMatch(/text\/markdown/);
|
|
133
|
+
expect(res.text).toContain(
|
|
134
|
+
`})`
|
|
135
|
+
);
|
|
136
|
+
expect(nock.isDone()).toBe(true);
|
|
137
|
+
}
|
|
138
|
+
);
|
|
139
|
+
|
|
140
|
+
it.each(DRIVE_URLS)(
|
|
141
|
+
'/txt includes the original and direct image links for $name',
|
|
142
|
+
async (item) => {
|
|
143
|
+
mockDriveHead(item.fileId);
|
|
144
|
+
|
|
145
|
+
const res = await request(app)
|
|
146
|
+
.get('/txt')
|
|
147
|
+
.query({ url: item.url })
|
|
148
|
+
.expect(200);
|
|
149
|
+
|
|
150
|
+
expect(res.headers['content-type']).toMatch(/text\/plain/);
|
|
151
|
+
expect(res.text).toContain('Google Drive image');
|
|
152
|
+
expect(res.text).toContain(`Original Google Drive file: ${item.url}`);
|
|
153
|
+
expect(res.text).toContain(`Image: ${directDownloadUrl(item.fileId)}`);
|
|
154
|
+
expect(nock.isDone()).toBe(true);
|
|
155
|
+
}
|
|
156
|
+
);
|
|
157
|
+
|
|
158
|
+
it.each(DRIVE_URLS)(
|
|
159
|
+
'/html includes a direct image tag for $name',
|
|
160
|
+
async (item) => {
|
|
161
|
+
mockDriveHead(item.fileId);
|
|
162
|
+
|
|
163
|
+
const res = await request(app)
|
|
164
|
+
.get('/html')
|
|
165
|
+
.query({ url: item.url })
|
|
166
|
+
.expect(200);
|
|
167
|
+
|
|
168
|
+
expect(res.headers['content-type']).toMatch(/text\/html/);
|
|
169
|
+
expect(res.text).toContain(
|
|
170
|
+
`src="${directDownloadUrl(item.fileId).replace('&', '&')}"`
|
|
171
|
+
);
|
|
172
|
+
expect(nock.isDone()).toBe(true);
|
|
173
|
+
}
|
|
174
|
+
);
|
|
175
|
+
|
|
176
|
+
it('archive includes markdown and the downloaded Drive image', async () => {
|
|
177
|
+
const item = DRIVE_URLS[0];
|
|
178
|
+
mockDriveHead(item.fileId);
|
|
179
|
+
mockDriveGet(item.fileId);
|
|
180
|
+
|
|
181
|
+
const res = await request(app)
|
|
182
|
+
.get('/archive')
|
|
183
|
+
.query({ url: item.url })
|
|
184
|
+
.buffer(true)
|
|
185
|
+
.parse(bufferParser)
|
|
186
|
+
.expect(200);
|
|
187
|
+
|
|
188
|
+
const zip = await unzipper.Open.buffer(res.body);
|
|
189
|
+
const document = zip.files.find((file) => file.path === 'document.md');
|
|
190
|
+
const image = zip.files.find((file) => file.path.startsWith('images/'));
|
|
191
|
+
|
|
192
|
+
expect(document).toBeDefined();
|
|
193
|
+
expect(image).toBeDefined();
|
|
194
|
+
expect((await document.buffer()).toString('utf8')).toContain(
|
|
195
|
+
''
|
|
196
|
+
);
|
|
197
|
+
expect((await image.buffer()).subarray(0, 3)).toEqual(
|
|
198
|
+
Buffer.from([0xff, 0xd8, 0xff])
|
|
199
|
+
);
|
|
200
|
+
expect(nock.isDone()).toBe(true);
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
it('docx includes the downloaded Drive image', async () => {
|
|
204
|
+
const item = DRIVE_URLS[0];
|
|
205
|
+
mockDriveHead(item.fileId);
|
|
206
|
+
mockDriveGet(item.fileId);
|
|
207
|
+
|
|
208
|
+
const res = await request(app)
|
|
209
|
+
.get('/docx')
|
|
210
|
+
.query({ url: item.url })
|
|
211
|
+
.buffer(true)
|
|
212
|
+
.parse(bufferParser)
|
|
213
|
+
.expect(200);
|
|
214
|
+
|
|
215
|
+
expect(res.headers['content-type']).toMatch(
|
|
216
|
+
/application\/vnd\.openxmlformats-officedocument\.wordprocessingml\.document/
|
|
217
|
+
);
|
|
218
|
+
expect(res.body.subarray(0, 2)).toEqual(Buffer.from('PK'));
|
|
219
|
+
|
|
220
|
+
const docx = await unzipper.Open.buffer(res.body);
|
|
221
|
+
const mediaEntry = docx.files.find(
|
|
222
|
+
(file) => file.path.startsWith('word/media/') && !file.path.endsWith('/')
|
|
223
|
+
);
|
|
224
|
+
expect(mediaEntry).toBeDefined();
|
|
225
|
+
expect((await mediaEntry.buffer()).subarray(0, 3)).toEqual(
|
|
226
|
+
Buffer.from([0xff, 0xd8, 0xff])
|
|
227
|
+
);
|
|
228
|
+
expect(nock.isDone()).toBe(true);
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
it('pdf renders the downloaded Drive image document', async () => {
|
|
232
|
+
const item = DRIVE_URLS[0];
|
|
233
|
+
mockDriveGet(item.fileId);
|
|
234
|
+
|
|
235
|
+
const res = await request(app)
|
|
236
|
+
.get('/pdf')
|
|
237
|
+
.query({ url: item.url })
|
|
238
|
+
.buffer(true)
|
|
239
|
+
.parse(bufferParser)
|
|
240
|
+
.expect(200);
|
|
241
|
+
|
|
242
|
+
expect(res.headers['content-type']).toMatch(/application\/pdf/);
|
|
243
|
+
expect(res.body.subarray(0, 4).toString('utf8')).toBe('%PDF');
|
|
244
|
+
expect(nock.isDone()).toBe(true);
|
|
245
|
+
});
|
|
246
|
+
});
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import {
|
|
2
|
+
convertGoogleDriveUrl,
|
|
3
|
+
getGoogleDriveFileId,
|
|
4
|
+
googleDriveImageTextForUrl,
|
|
5
|
+
isGoogleDriveFileUrl,
|
|
6
|
+
} from '../../src/lib.js';
|
|
7
|
+
|
|
8
|
+
const FIRST_FILE_ID = '1Cxkx6-428EQAX0-eiaq66H829ohnPp7q';
|
|
9
|
+
const SECOND_FILE_ID = '1fgJaftjv53xCN7vgiJOaQlbfWkUqPgyd';
|
|
10
|
+
|
|
11
|
+
function directDownloadUrl(fileId) {
|
|
12
|
+
return `https://drive.google.com/uc?export=download&id=${fileId}`;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
describe('Google Drive URL helpers', () => {
|
|
16
|
+
it.each([
|
|
17
|
+
[
|
|
18
|
+
'issue URL with /view',
|
|
19
|
+
`https://drive.google.com/file/d/${FIRST_FILE_ID}/view`,
|
|
20
|
+
FIRST_FILE_ID,
|
|
21
|
+
],
|
|
22
|
+
[
|
|
23
|
+
'issue URL without /view',
|
|
24
|
+
`https://drive.google.com/file/d/${FIRST_FILE_ID}`,
|
|
25
|
+
FIRST_FILE_ID,
|
|
26
|
+
],
|
|
27
|
+
[
|
|
28
|
+
'second issue URL with /view',
|
|
29
|
+
`https://drive.google.com/file/d/${SECOND_FILE_ID}/view`,
|
|
30
|
+
SECOND_FILE_ID,
|
|
31
|
+
],
|
|
32
|
+
[
|
|
33
|
+
'second issue URL without /view',
|
|
34
|
+
`https://drive.google.com/file/d/${SECOND_FILE_ID}`,
|
|
35
|
+
SECOND_FILE_ID,
|
|
36
|
+
],
|
|
37
|
+
[
|
|
38
|
+
'open?id URL',
|
|
39
|
+
`https://drive.google.com/open?id=${FIRST_FILE_ID}`,
|
|
40
|
+
FIRST_FILE_ID,
|
|
41
|
+
],
|
|
42
|
+
[
|
|
43
|
+
'direct uc URL',
|
|
44
|
+
`https://drive.google.com/uc?export=download&id=${FIRST_FILE_ID}`,
|
|
45
|
+
FIRST_FILE_ID,
|
|
46
|
+
],
|
|
47
|
+
[
|
|
48
|
+
'redirected usercontent URL',
|
|
49
|
+
`https://drive.usercontent.google.com/download?id=${FIRST_FILE_ID}&export=download`,
|
|
50
|
+
FIRST_FILE_ID,
|
|
51
|
+
],
|
|
52
|
+
])('extracts and converts %s', (_name, input, fileId) => {
|
|
53
|
+
expect(getGoogleDriveFileId(input)).toBe(fileId);
|
|
54
|
+
expect(isGoogleDriveFileUrl(input)).toBe(true);
|
|
55
|
+
expect(convertGoogleDriveUrl(input)).toBe(directDownloadUrl(fileId));
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it('ignores non-Google Drive URLs and malformed Drive IDs', () => {
|
|
59
|
+
const example = 'https://example.com/image.jpg';
|
|
60
|
+
const malformed = 'https://drive.google.com/file/d/not%20valid/view';
|
|
61
|
+
|
|
62
|
+
expect(getGoogleDriveFileId(example)).toBeNull();
|
|
63
|
+
expect(isGoogleDriveFileUrl(example)).toBe(false);
|
|
64
|
+
expect(convertGoogleDriveUrl(example)).toBe(example);
|
|
65
|
+
expect(getGoogleDriveFileId(malformed)).toBeNull();
|
|
66
|
+
expect(isGoogleDriveFileUrl(malformed)).toBe(false);
|
|
67
|
+
expect(convertGoogleDriveUrl(malformed)).toBe(malformed);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it('keeps empty or missing inputs unchanged when converting', () => {
|
|
71
|
+
expect(convertGoogleDriveUrl(null)).toBeNull();
|
|
72
|
+
expect(convertGoogleDriveUrl(undefined)).toBeUndefined();
|
|
73
|
+
expect(convertGoogleDriveUrl('')).toBe('');
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('renders text output with the original file URL and direct image URL', () => {
|
|
77
|
+
const input = `https://drive.google.com/file/d/${FIRST_FILE_ID}/view`;
|
|
78
|
+
|
|
79
|
+
expect(googleDriveImageTextForUrl(input)).toContain(
|
|
80
|
+
`Original Google Drive file: ${input}`
|
|
81
|
+
);
|
|
82
|
+
expect(googleDriveImageTextForUrl(input)).toContain(
|
|
83
|
+
`Image: ${directDownloadUrl(FIRST_FILE_ID)}`
|
|
84
|
+
);
|
|
85
|
+
expect(
|
|
86
|
+
googleDriveImageTextForUrl('https://example.com/image.jpg')
|
|
87
|
+
).toBeNull();
|
|
88
|
+
});
|
|
89
|
+
});
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { shouldRenderHtmlWithBrowser } from '../../src/markdown.js';
|
|
2
|
+
|
|
3
|
+
describe('markdown rendering decisions', () => {
|
|
4
|
+
it('uses browser rendering for JavaScript-heavy HTML shells', () => {
|
|
5
|
+
const html = `<!doctype html>
|
|
6
|
+
<html>
|
|
7
|
+
<head><script src="/_next/static/app.js"></script></head>
|
|
8
|
+
<body><div id="__next"></div></body>
|
|
9
|
+
</html>`;
|
|
10
|
+
|
|
11
|
+
expect(shouldRenderHtmlWithBrowser(html)).toBe(true);
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
it('keeps plain HTML on the direct fetch path', () => {
|
|
15
|
+
const html = `<!doctype html>
|
|
16
|
+
<html>
|
|
17
|
+
<head><title>Static page</title></head>
|
|
18
|
+
<body><main><h1>Static content</h1></main></body>
|
|
19
|
+
</html>`;
|
|
20
|
+
|
|
21
|
+
expect(shouldRenderHtmlWithBrowser(html)).toBe(false);
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
it('keeps static pages with incidental scripts on the direct fetch path', () => {
|
|
25
|
+
const html = `<!doctype html>
|
|
26
|
+
<html>
|
|
27
|
+
<head>
|
|
28
|
+
<title>Static page</title>
|
|
29
|
+
<script src="/analytics.js"></script>
|
|
30
|
+
</head>
|
|
31
|
+
<body><main><h1>Static content</h1></main></body>
|
|
32
|
+
</html>`;
|
|
33
|
+
|
|
34
|
+
expect(shouldRenderHtmlWithBrowser(html)).toBe(false);
|
|
35
|
+
});
|
|
36
|
+
});
|