@link-assistant/web-capture 1.12.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.changeset/config.json +1 -1
- package/CHANGELOG.md +6 -0
- package/Dockerfile +8 -8
- package/README.md +2 -1
- package/bin/web-capture.js +3 -2
- package/examples/js/playwright_example.js +1 -1
- package/examples/python/capture_client.py +20 -0
- package/examples/python/html_download.py +2 -14
- package/examples/python/image_download.py +3 -7
- package/examples/python/markdown_download.py +2 -14
- package/examples/python/playwright_example.py +6 -6
- package/jest.config.mjs +1 -4
- package/package.json +11 -8
- package/src/archive.js +1 -1
- package/src/binary-download.js +14 -0
- package/src/docx.js +1 -1
- package/src/fetch.js +1 -1
- package/src/figures.js +2 -12
- package/src/lib.js +1 -1
- package/src/localize-images.js +2 -12
- package/tests/ci/detect-code-changes.test.js +36 -23
- package/tests/ci/workflow-policy.test.js +2 -2
- package/tests/helpers/screenshot.js +10 -0
- package/tests/integration/browser-engines.test.js +4 -52
- package/tests/integration/github-readme.test.js +2 -6
- package/tests/integration/wikipedia-download.test.js +3 -11
- package/tests/unit/publish-verification.test.js +13 -16
- package/tests/unit/stream.test.js +8 -10
- package/babel.config.cjs +0 -13
package/.changeset/config.json
CHANGED
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# @link-assistant/web-capture
|
|
2
2
|
|
|
3
|
+
## 2.0.0
|
|
4
|
+
|
|
5
|
+
### Major Changes
|
|
6
|
+
|
|
7
|
+
- b055a30: Require Node.js 26.10 or newer and update Playwright to 1.64. Use current Trixie browser images, native Jest ESM and reproducibly locked release tooling. Add locked Python examples and weekly dependency/security maintenance across all ecosystems.
|
|
8
|
+
|
|
3
9
|
## 1.12.0
|
|
4
10
|
|
|
5
11
|
### Minor Changes
|
package/Dockerfile
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
FROM node:
|
|
1
|
+
FROM node:26.10.0-trixie
|
|
2
2
|
|
|
3
3
|
WORKDIR /app
|
|
4
4
|
|
|
@@ -9,15 +9,15 @@ RUN apt-get update && \
|
|
|
9
9
|
apt-get install -y \
|
|
10
10
|
chromium \
|
|
11
11
|
fonts-liberation \
|
|
12
|
-
|
|
13
|
-
libatk-bridge2.0-
|
|
14
|
-
libatk1.0-
|
|
15
|
-
libatspi2.0-
|
|
16
|
-
|
|
12
|
+
libasound2t64 \
|
|
13
|
+
libatk-bridge2.0-0t64 \
|
|
14
|
+
libatk1.0-0t64 \
|
|
15
|
+
libatspi2.0-0t64 \
|
|
16
|
+
libcups2t64 \
|
|
17
17
|
libdbus-1-3 \
|
|
18
18
|
libdrm2 \
|
|
19
19
|
libgbm1 \
|
|
20
|
-
libgtk-3-
|
|
20
|
+
libgtk-3-0t64 \
|
|
21
21
|
libnspr4 \
|
|
22
22
|
libnss3 \
|
|
23
23
|
libxcomposite1 \
|
|
@@ -43,7 +43,7 @@ ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
|
|
|
43
43
|
ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium
|
|
44
44
|
|
|
45
45
|
COPY package.json package-lock.json ./
|
|
46
|
-
RUN npm ci
|
|
46
|
+
RUN npm install --global npm@12.2.0 && npm ci
|
|
47
47
|
|
|
48
48
|
COPY . .
|
|
49
49
|
|
package/README.md
CHANGED
|
@@ -19,7 +19,8 @@ A CLI and microservice to fetch URLs and render them as:
|
|
|
19
19
|
|
|
20
20
|
## Installation
|
|
21
21
|
|
|
22
|
-
Node.js
|
|
22
|
+
Node.js 26.10.0 or newer is required. CI uses 26.11.1 and the latest published
|
|
23
|
+
Node Docker image uses 26.10.0 on Debian Trixie. Use `npm ci`
|
|
23
24
|
for a locked repository install; Docker and CI share `package-lock.json` and its
|
|
24
25
|
security overrides.
|
|
25
26
|
|
package/bin/web-capture.js
CHANGED
|
@@ -1213,8 +1213,9 @@ async function captureUrl(url, options) {
|
|
|
1213
1213
|
baseDelay: 500,
|
|
1214
1214
|
});
|
|
1215
1215
|
if (resp.ok) {
|
|
1216
|
-
|
|
1217
|
-
|
|
1216
|
+
archive.append(Buffer.from(await resp.arrayBuffer()), {
|
|
1217
|
+
name: localPath,
|
|
1218
|
+
});
|
|
1218
1219
|
}
|
|
1219
1220
|
} catch {
|
|
1220
1221
|
/* skip failed image downloads */
|
|
@@ -16,7 +16,7 @@ console.log('Fetching screenshot using Playwright engine...');
|
|
|
16
16
|
const imageResponse = await fetch(
|
|
17
17
|
`${baseUrl}/image?url=${encodeURIComponent(targetUrl)}&engine=playwright`
|
|
18
18
|
);
|
|
19
|
-
const imageBuffer = await imageResponse.
|
|
19
|
+
const imageBuffer = Buffer.from(await imageResponse.arrayBuffer());
|
|
20
20
|
fs.writeFileSync('output/playwright_screenshot.png', imageBuffer);
|
|
21
21
|
console.log('Screenshot saved to output/playwright_screenshot.png');
|
|
22
22
|
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Shared request policy for the capture-service examples."""
|
|
2
|
+
import os
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import sys
|
|
5
|
+
import requests
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def capture(kind, url=None, **options):
|
|
9
|
+
if url is None:
|
|
10
|
+
url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
|
|
11
|
+
endpoint = os.environ.get('WEB_CAPTURE_URL', 'http://localhost:3000').rstrip('/')
|
|
12
|
+
response = requests.get(f'{endpoint}/{kind}', params={'url': url, **options}, timeout=30)
|
|
13
|
+
response.raise_for_status()
|
|
14
|
+
return response
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def download_text(kind, filename):
|
|
18
|
+
output = Path(__file__).with_name(filename)
|
|
19
|
+
output.write_text(capture(kind).text, encoding='utf-8')
|
|
20
|
+
print(f'{kind.title()} saved to {output}')
|
|
@@ -1,15 +1,3 @@
|
|
|
1
|
-
import
|
|
2
|
-
import requests
|
|
3
|
-
import os
|
|
1
|
+
from capture_client import download_text
|
|
4
2
|
|
|
5
|
-
|
|
6
|
-
endpoint = f'http://localhost:3000/html?url={url}'
|
|
7
|
-
|
|
8
|
-
response = requests.get(endpoint)
|
|
9
|
-
response.raise_for_status()
|
|
10
|
-
|
|
11
|
-
output_path = os.path.join(os.path.dirname(__file__), 'downloaded.html')
|
|
12
|
-
with open(output_path, 'w', encoding='utf-8') as f:
|
|
13
|
-
f.write(response.text)
|
|
14
|
-
|
|
15
|
-
print(f'HTML saved to {output_path}')
|
|
3
|
+
download_text('html', 'downloaded.html')
|
|
@@ -1,11 +1,7 @@
|
|
|
1
|
-
import
|
|
2
|
-
import requests
|
|
1
|
+
from capture_client import capture
|
|
3
2
|
import os
|
|
4
3
|
|
|
5
|
-
|
|
6
|
-
endpoint = f'http://localhost:3000/image?url={url}'
|
|
7
|
-
|
|
8
|
-
response = requests.get(endpoint)
|
|
4
|
+
response = capture('image')
|
|
9
5
|
print('Status:', response.status_code)
|
|
10
6
|
print('Content-Type:', response.headers.get('content-type'))
|
|
11
7
|
|
|
@@ -17,4 +13,4 @@ if response.content[:8] == png_signature:
|
|
|
17
13
|
f.write(response.content)
|
|
18
14
|
print(f'Image saved to {output_path} (valid PNG)')
|
|
19
15
|
else:
|
|
20
|
-
print('Response is not a valid PNG! First bytes:', response.content[:16])
|
|
16
|
+
print('Response is not a valid PNG! First bytes:', response.content[:16])
|
|
@@ -1,15 +1,3 @@
|
|
|
1
|
-
import
|
|
2
|
-
import requests
|
|
3
|
-
import os
|
|
1
|
+
from capture_client import download_text
|
|
4
2
|
|
|
5
|
-
|
|
6
|
-
endpoint = f'http://localhost:3000/markdown?url={url}'
|
|
7
|
-
|
|
8
|
-
response = requests.get(endpoint)
|
|
9
|
-
response.raise_for_status()
|
|
10
|
-
|
|
11
|
-
output_path = os.path.join(os.path.dirname(__file__), 'downloaded.md')
|
|
12
|
-
with open(output_path, 'w', encoding='utf-8') as f:
|
|
13
|
-
f.write(response.text)
|
|
14
|
-
|
|
15
|
-
print(f'Markdown saved to {output_path}')
|
|
3
|
+
download_text('markdown', 'downloaded.md')
|
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
import
|
|
1
|
+
import sys
|
|
2
|
+
from capture_client import capture
|
|
2
3
|
import os
|
|
3
4
|
|
|
4
|
-
|
|
5
|
-
target_url = 'https://example.com'
|
|
5
|
+
target_url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
|
|
6
6
|
|
|
7
7
|
# Create output directory if it doesn't exist
|
|
8
8
|
os.makedirs('output', exist_ok=True)
|
|
9
9
|
|
|
10
10
|
print('Fetching HTML using Playwright engine...')
|
|
11
|
-
html_response =
|
|
12
|
-
with open('output/playwright_html.html', 'w') as f:
|
|
11
|
+
html_response = capture('html', target_url, engine='playwright')
|
|
12
|
+
with open('output/playwright_html.html', 'w', encoding='utf-8') as f:
|
|
13
13
|
f.write(html_response.text)
|
|
14
14
|
print('HTML saved to output/playwright_html.html')
|
|
15
15
|
|
|
16
16
|
print('Fetching screenshot using Playwright engine...')
|
|
17
|
-
image_response =
|
|
17
|
+
image_response = capture('image', target_url, engine='playwright')
|
|
18
18
|
with open('output/playwright_screenshot.png', 'wb') as f:
|
|
19
19
|
f.write(image_response.content)
|
|
20
20
|
print('Screenshot saved to output/playwright_screenshot.png')
|
package/jest.config.mjs
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@link-assistant/web-capture",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
},
|
|
9
9
|
"scripts": {
|
|
10
10
|
"start": "docker compose up -d",
|
|
11
|
-
"dev": "
|
|
11
|
+
"dev": "node --watch bin/web-capture.js --serve",
|
|
12
12
|
"test": "node --experimental-vm-modules ./node_modules/.bin/jest",
|
|
13
13
|
"test:watch": "node --experimental-vm-modules ./node_modules/.bin/jest --watch",
|
|
14
14
|
"test:e2e:process": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e/process.test.js",
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"test:e2e": "npm run test:e2e:process && npm run test:e2e:docker",
|
|
17
17
|
"test:all": "npm run build && npm test && npm run test:e2e",
|
|
18
18
|
"build": "docker compose down && docker compose build && npm start",
|
|
19
|
-
"examples:python": "
|
|
19
|
+
"examples:python": "uv run --project .. --locked python examples/python/html_download.py && uv run --project .. --locked python examples/python/markdown_download.py && uv run --project .. --locked python examples/python/image_download.py",
|
|
20
20
|
"examples:javascript": "node examples/js/html_download.js && node examples/js/markdown_download.js && node examples/js/image_download.js",
|
|
21
21
|
"examples": "npm run build && npm run examples:javascript && npm run examples:python",
|
|
22
22
|
"lint": "eslint .",
|
|
@@ -43,18 +43,15 @@
|
|
|
43
43
|
"lino-arguments": "^0.3.0",
|
|
44
44
|
"log-lazy": "^1.0.4",
|
|
45
45
|
"node-fetch": "^3.3.2",
|
|
46
|
-
"playwright": "^1.
|
|
46
|
+
"playwright": "^1.64.0",
|
|
47
47
|
"puppeteer": "^25.12.0",
|
|
48
48
|
"puppeteer-core": "^25.12.0",
|
|
49
49
|
"turndown": "^7.2.4",
|
|
50
50
|
"turndown-plugin-gfm": "^1.0.2"
|
|
51
51
|
},
|
|
52
52
|
"devDependencies": {
|
|
53
|
-
"@babel/core": "^8.0.7",
|
|
54
|
-
"@babel/preset-env": "^8.0.7",
|
|
55
53
|
"@changesets/cli": "^3.0.3",
|
|
56
54
|
"@types/archiver": "^8.0.0",
|
|
57
|
-
"babel-jest": "^30.5.2",
|
|
58
55
|
"eslint": "^10.12.0",
|
|
59
56
|
"eslint-config-prettier": "^10.1.8",
|
|
60
57
|
"eslint-plugin-prettier": "^5.5.6",
|
|
@@ -83,7 +80,7 @@
|
|
|
83
80
|
"url": "https://github.com/link-assistant/web-capture"
|
|
84
81
|
},
|
|
85
82
|
"engines": {
|
|
86
|
-
"node": ">=
|
|
83
|
+
"node": ">=26.10.0"
|
|
87
84
|
},
|
|
88
85
|
"lint-staged": {
|
|
89
86
|
"*.{js,mjs,cjs}": [
|
|
@@ -106,5 +103,11 @@
|
|
|
106
103
|
"@istanbuljs/load-nyc-config": {
|
|
107
104
|
"js-yaml": "^4.1.1"
|
|
108
105
|
}
|
|
106
|
+
},
|
|
107
|
+
"packageManager": "npm@12.2.0",
|
|
108
|
+
"allowScripts": {
|
|
109
|
+
"@parcel/watcher": true,
|
|
110
|
+
"puppeteer": true,
|
|
111
|
+
"unrs-resolver": true
|
|
109
112
|
}
|
|
110
113
|
}
|
package/src/archive.js
CHANGED
|
@@ -200,7 +200,7 @@ export async function archiveHandler(req, res) {
|
|
|
200
200
|
baseDelay: 500,
|
|
201
201
|
});
|
|
202
202
|
if (imgResp.ok) {
|
|
203
|
-
const buffer = await imgResp.
|
|
203
|
+
const buffer = Buffer.from(await imgResp.arrayBuffer());
|
|
204
204
|
archive.append(buffer, { name: localPath });
|
|
205
205
|
}
|
|
206
206
|
} catch {
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import fetch from 'node-fetch';
|
|
2
|
+
import { retry } from './retry.js';
|
|
3
|
+
|
|
4
|
+
// Shared image-download policy; keep HTTP errors distinct from network retries.
|
|
5
|
+
export async function downloadBinary(url) {
|
|
6
|
+
const response = await retry(() => fetch(url), {
|
|
7
|
+
retries: 3,
|
|
8
|
+
baseDelay: 1000,
|
|
9
|
+
});
|
|
10
|
+
if (!response.ok) {
|
|
11
|
+
throw new Error(`HTTP ${response.status}`);
|
|
12
|
+
}
|
|
13
|
+
return Buffer.from(await response.arrayBuffer());
|
|
14
|
+
}
|
package/src/docx.js
CHANGED
|
@@ -75,7 +75,7 @@ export async function docxHandler(req, res) {
|
|
|
75
75
|
const imgUrl = new URL(src, absoluteUrl).href;
|
|
76
76
|
const imgResp = await fetch(imgUrl);
|
|
77
77
|
if (imgResp.ok) {
|
|
78
|
-
const buffer = await imgResp.
|
|
78
|
+
const buffer = Buffer.from(await imgResp.arrayBuffer());
|
|
79
79
|
const contentType = imgResp.headers.get('content-type') || '';
|
|
80
80
|
children.push(
|
|
81
81
|
new Paragraph({
|
package/src/fetch.js
CHANGED
|
@@ -14,7 +14,7 @@ export async function fetchHandler(req, res) {
|
|
|
14
14
|
copyProxyResponseHeaders(response.headers, res);
|
|
15
15
|
|
|
16
16
|
// Get the response body as buffer and send it
|
|
17
|
-
const buffer = await response.
|
|
17
|
+
const buffer = Buffer.from(await response.arrayBuffer());
|
|
18
18
|
res.send(buffer);
|
|
19
19
|
} catch (err) {
|
|
20
20
|
console.error('Fetch error:', err);
|
package/src/figures.js
CHANGED
|
@@ -10,10 +10,9 @@
|
|
|
10
10
|
* @module figures
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
|
-
import fetch from 'node-fetch';
|
|
14
13
|
import * as cheerio from 'cheerio';
|
|
15
14
|
import { URL } from 'url';
|
|
16
|
-
import {
|
|
15
|
+
import { downloadBinary } from './binary-download.js';
|
|
17
16
|
import { createBrowser, getBrowserEngine } from './browser.js';
|
|
18
17
|
import { dismissPopups, scrollToLoadContent } from './popups.js';
|
|
19
18
|
|
|
@@ -86,16 +85,7 @@ export async function downloadFigures(figures, options = {}) {
|
|
|
86
85
|
const filename = `figure-${figure.figureNum}.${ext}`;
|
|
87
86
|
|
|
88
87
|
try {
|
|
89
|
-
const
|
|
90
|
-
retries: 3,
|
|
91
|
-
baseDelay: 1000,
|
|
92
|
-
});
|
|
93
|
-
|
|
94
|
-
if (!resp.ok) {
|
|
95
|
-
throw new Error(`HTTP ${resp.status}`);
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
const buffer = await resp.buffer();
|
|
88
|
+
const buffer = await downloadBinary(figure.src);
|
|
99
89
|
|
|
100
90
|
if (options.onProgress) {
|
|
101
91
|
options.onProgress(figure.figureNum, 'downloaded');
|
package/src/lib.js
CHANGED
|
@@ -151,7 +151,7 @@ export async function fetchGoogleDriveImage(url) {
|
|
|
151
151
|
}
|
|
152
152
|
|
|
153
153
|
return {
|
|
154
|
-
buffer: await response.
|
|
154
|
+
buffer: Buffer.from(await response.arrayBuffer()),
|
|
155
155
|
contentType,
|
|
156
156
|
downloadUrl,
|
|
157
157
|
filename: `google-drive-${fileId}.${imageExtensionForContentType(contentType)}`,
|
package/src/localize-images.js
CHANGED
|
@@ -13,9 +13,8 @@
|
|
|
13
13
|
* @module localize-images
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
import fetch from 'node-fetch';
|
|
17
16
|
import { URL } from 'url';
|
|
18
|
-
import {
|
|
17
|
+
import { downloadBinary } from './binary-download.js';
|
|
19
18
|
|
|
20
19
|
/**
|
|
21
20
|
* Extract image references from markdown text.
|
|
@@ -165,16 +164,7 @@ export async function localizeImages(markdownText, options = {}) {
|
|
|
165
164
|
}
|
|
166
165
|
|
|
167
166
|
try {
|
|
168
|
-
const
|
|
169
|
-
retries: 3,
|
|
170
|
-
baseDelay: 1000,
|
|
171
|
-
});
|
|
172
|
-
|
|
173
|
-
if (!resp.ok) {
|
|
174
|
-
throw new Error(`HTTP ${resp.status}`);
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
const buffer = await resp.buffer();
|
|
167
|
+
const buffer = await downloadBinary(image.url);
|
|
178
168
|
downloadedCount++;
|
|
179
169
|
|
|
180
170
|
replacements.push({
|
|
@@ -39,7 +39,10 @@ function commitAll(repoPath, message) {
|
|
|
39
39
|
git(repoPath, ['commit', '-m', message]);
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
function createMergedFeatureRepo(
|
|
42
|
+
function createMergedFeatureRepo({
|
|
43
|
+
featureCode = true,
|
|
44
|
+
baseCode = false,
|
|
45
|
+
} = {}) {
|
|
43
46
|
const repoPath = mkdtempSync(join(tmpdir(), 'web-capture-detect-'));
|
|
44
47
|
|
|
45
48
|
initializeRepo(repoPath);
|
|
@@ -48,13 +51,19 @@ function createMergedFeatureRepo() {
|
|
|
48
51
|
commitAll(repoPath, 'initial commit');
|
|
49
52
|
|
|
50
53
|
git(repoPath, ['checkout', '-b', 'feature']);
|
|
51
|
-
|
|
52
|
-
|
|
54
|
+
if (featureCode) {
|
|
55
|
+
writeTrackedFile(repoPath, 'rust/src/lib.rs', 'pub fn capture() {}\n');
|
|
56
|
+
commitAll(repoPath, 'add rust code');
|
|
57
|
+
}
|
|
53
58
|
|
|
54
59
|
writeTrackedFile(repoPath, 'docs/notes.md', '# docs only tail commit\n');
|
|
55
60
|
commitAll(repoPath, 'document rust change');
|
|
56
61
|
|
|
57
62
|
git(repoPath, ['checkout', 'main']);
|
|
63
|
+
if (baseCode) {
|
|
64
|
+
writeTrackedFile(repoPath, 'rust/src/lib.rs', 'pub fn base_capture() {}\n');
|
|
65
|
+
commitAll(repoPath, 'base branch code change');
|
|
66
|
+
}
|
|
58
67
|
git(repoPath, ['merge', '--no-ff', 'feature', '-m', 'merge feature']);
|
|
59
68
|
|
|
60
69
|
return repoPath;
|
|
@@ -123,28 +132,32 @@ describe('detect-code-changes', () => {
|
|
|
123
132
|
}
|
|
124
133
|
});
|
|
125
134
|
|
|
126
|
-
test('
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
135
|
+
test.each(['push', 'pull_request'])(
|
|
136
|
+
'detects code before a docs-only final commit for %s',
|
|
137
|
+
(eventName) => {
|
|
138
|
+
repoPath = createMergedFeatureRepo();
|
|
139
|
+
|
|
140
|
+
expect(runDetector(repoPath, eventName)).toMatchObject({
|
|
141
|
+
'rust-code-changed': 'true',
|
|
142
|
+
'rust-changed': 'true',
|
|
143
|
+
'any-rust-code-changed': 'true',
|
|
144
|
+
'any-code-changed': 'true',
|
|
145
|
+
'docs-changed': 'true',
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
);
|
|
140
149
|
|
|
141
|
-
|
|
150
|
+
test.each([
|
|
151
|
+
['docs-only', { featureCode: false }],
|
|
152
|
+
['base branch code only', { featureCode: false, baseCode: true }],
|
|
153
|
+
])('keeps code checks disabled for %s PRs', (_name, options) => {
|
|
154
|
+
repoPath = createMergedFeatureRepo(options);
|
|
142
155
|
|
|
143
|
-
expect(
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
156
|
+
expect(runDetector(repoPath, 'pull_request')).toMatchObject({
|
|
157
|
+
'rust-code-changed': 'false',
|
|
158
|
+
'any-code-changed': 'false',
|
|
159
|
+
'docs-changed': 'true',
|
|
160
|
+
});
|
|
148
161
|
});
|
|
149
162
|
|
|
150
163
|
test('detects files when a push only has one commit', () => {
|
|
@@ -40,7 +40,7 @@ describe('workflow policy', () => {
|
|
|
40
40
|
(workflowFile) => {
|
|
41
41
|
const workflow = readWorkflow(workflowFile);
|
|
42
42
|
|
|
43
|
-
expect(workflow).toContain('uses: actions/checkout@
|
|
43
|
+
expect(workflow).toContain('uses: actions/checkout@v7.0.1');
|
|
44
44
|
expect(workflow).not.toContain('uses: actions/checkout@v4');
|
|
45
45
|
}
|
|
46
46
|
);
|
|
@@ -48,7 +48,7 @@ describe('workflow policy', () => {
|
|
|
48
48
|
test('Rust workflow uses the current cache action', () => {
|
|
49
49
|
const workflow = readWorkflow('.github/workflows/rust.yml');
|
|
50
50
|
|
|
51
|
-
expect(workflow).toContain('uses: actions/cache@
|
|
51
|
+
expect(workflow).toContain('uses: actions/cache@v6.1.0');
|
|
52
52
|
expect(workflow).not.toContain('uses: actions/cache@v4');
|
|
53
53
|
});
|
|
54
54
|
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { expect } from '@jest/globals';
|
|
2
|
+
|
|
3
|
+
// PNG magic number (89 50 4E 47 0D 0A 1A 0A).
|
|
4
|
+
const PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
|
|
5
|
+
|
|
6
|
+
export function expectPngScreenshot(screenshot, minimumLength = 1000) {
|
|
7
|
+
expect(screenshot).toBeInstanceOf(Uint8Array);
|
|
8
|
+
expect(screenshot.length).toBeGreaterThan(minimumLength);
|
|
9
|
+
expect(Buffer.from(screenshot).subarray(0, 8)).toEqual(PNG_SIGNATURE);
|
|
10
|
+
}
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { createBrowser } from '../../src/browser.js';
|
|
2
|
+
import { expectPngScreenshot } from '../helpers/screenshot.js';
|
|
2
3
|
|
|
3
4
|
describe('Browser Engine Integration Tests', () => {
|
|
4
|
-
describe('
|
|
5
|
+
describe.each(['puppeteer', 'playwright'])('%s Engine', (engine) => {
|
|
5
6
|
let browser;
|
|
6
7
|
|
|
7
8
|
beforeEach(async () => {
|
|
8
|
-
browser = await createBrowser(
|
|
9
|
+
browser = await createBrowser(engine);
|
|
9
10
|
});
|
|
10
11
|
|
|
11
12
|
afterEach(async () => {
|
|
@@ -32,56 +33,7 @@ describe('Browser Engine Integration Tests', () => {
|
|
|
32
33
|
timeout: 30000,
|
|
33
34
|
});
|
|
34
35
|
const screenshot = await page.screenshot({ type: 'png' });
|
|
35
|
-
|
|
36
|
-
expect(screenshot.length).toBeGreaterThan(100);
|
|
37
|
-
}, 60000);
|
|
38
|
-
|
|
39
|
-
it('can set custom headers and user agent', async () => {
|
|
40
|
-
const page = await browser.newPage();
|
|
41
|
-
await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-US' });
|
|
42
|
-
await page.setUserAgent('Test User Agent');
|
|
43
|
-
await page.goto('https://example.com', {
|
|
44
|
-
waitUntil: 'networkidle0',
|
|
45
|
-
timeout: 30000,
|
|
46
|
-
});
|
|
47
|
-
const content = await page.content();
|
|
48
|
-
expect(content).toBeTruthy();
|
|
49
|
-
}, 60000);
|
|
50
|
-
});
|
|
51
|
-
|
|
52
|
-
describe('Playwright Engine', () => {
|
|
53
|
-
let browser;
|
|
54
|
-
|
|
55
|
-
beforeEach(async () => {
|
|
56
|
-
browser = await createBrowser('playwright');
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
afterEach(async () => {
|
|
60
|
-
if (browser) {
|
|
61
|
-
await browser.close();
|
|
62
|
-
}
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
it('can navigate to a page and get content', async () => {
|
|
66
|
-
const page = await browser.newPage();
|
|
67
|
-
await page.goto('https://example.com', {
|
|
68
|
-
waitUntil: 'networkidle0',
|
|
69
|
-
timeout: 30000,
|
|
70
|
-
});
|
|
71
|
-
const content = await page.content();
|
|
72
|
-
expect(content).toContain('Example Domain');
|
|
73
|
-
}, 60000);
|
|
74
|
-
|
|
75
|
-
it('can take a screenshot', async () => {
|
|
76
|
-
const page = await browser.newPage();
|
|
77
|
-
await page.setViewport({ width: 1280, height: 800 });
|
|
78
|
-
await page.goto('https://example.com', {
|
|
79
|
-
waitUntil: 'networkidle0',
|
|
80
|
-
timeout: 30000,
|
|
81
|
-
});
|
|
82
|
-
const screenshot = await page.screenshot({ type: 'png' });
|
|
83
|
-
expect(screenshot).toBeInstanceOf(Buffer);
|
|
84
|
-
expect(screenshot.length).toBeGreaterThan(100);
|
|
36
|
+
expectPngScreenshot(screenshot, 100);
|
|
85
37
|
}, 60000);
|
|
86
38
|
|
|
87
39
|
it('can set custom headers and user agent', async () => {
|
|
@@ -12,6 +12,7 @@ import { jest } from '@jest/globals';
|
|
|
12
12
|
import { createBrowser } from '../../src/browser.js';
|
|
13
13
|
import { fetchHtml } from '../../src/lib.js';
|
|
14
14
|
import { retry } from '../../src/retry.js';
|
|
15
|
+
import { expectPngScreenshot } from '../helpers/screenshot.js';
|
|
15
16
|
import {
|
|
16
17
|
fetchGithubRepositorySnapshot,
|
|
17
18
|
formatGithubRepositoryMarkdown,
|
|
@@ -22,9 +23,6 @@ import {
|
|
|
22
23
|
const GITHUB_REPOSITORY_URL =
|
|
23
24
|
process.env.GITHUB_REPOSITORY_URL ||
|
|
24
25
|
'https://github.com/link-assistant/web-capture';
|
|
25
|
-
const PNG_SIGNATURE = Buffer.from([
|
|
26
|
-
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
|
|
27
|
-
]);
|
|
28
26
|
|
|
29
27
|
const SKIP_LIVE =
|
|
30
28
|
!process.env.GITHUB_REPOSITORY_INTEGRATION ||
|
|
@@ -128,9 +126,7 @@ describe.each(['puppeteer', 'playwright'])(
|
|
|
128
126
|
|
|
129
127
|
const screenshot = await page.screenshot({ type: 'png' });
|
|
130
128
|
|
|
131
|
-
|
|
132
|
-
expect(screenshot.length).toBeGreaterThan(1000);
|
|
133
|
-
expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
|
|
129
|
+
expectPngScreenshot(screenshot);
|
|
134
130
|
});
|
|
135
131
|
});
|
|
136
132
|
}
|
|
@@ -19,14 +19,10 @@ import { jest } from '@jest/globals';
|
|
|
19
19
|
import { createBrowser } from '../../src/browser.js';
|
|
20
20
|
import { convertHtmlToMarkdown } from '../../src/lib.js';
|
|
21
21
|
import { retry } from '../../src/retry.js';
|
|
22
|
+
import { expectPngScreenshot } from '../helpers/screenshot.js';
|
|
22
23
|
|
|
23
24
|
const WIKIPEDIA_URL = 'https://en.wikipedia.org/wiki/Wikipedia';
|
|
24
25
|
|
|
25
|
-
// PNG magic number (89 50 4E 47 0D 0A 1A 0A).
|
|
26
|
-
const PNG_SIGNATURE = Buffer.from([
|
|
27
|
-
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
|
|
28
|
-
]);
|
|
29
|
-
|
|
30
26
|
const SKIP_LIVE =
|
|
31
27
|
!process.env.WIKIPEDIA_INTEGRATION ||
|
|
32
28
|
process.env.WIKIPEDIA_INTEGRATION === 'false';
|
|
@@ -99,10 +95,7 @@ describe.each(['puppeteer', 'playwright'])(
|
|
|
99
95
|
it('downloads the Wikipedia page as an image (PNG screenshot)', async () => {
|
|
100
96
|
const screenshot = await page.screenshot({ type: 'png' });
|
|
101
97
|
|
|
102
|
-
|
|
103
|
-
expect(screenshot.length).toBeGreaterThan(1000);
|
|
104
|
-
// Verify the PNG signature so we know it is a real image.
|
|
105
|
-
expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
|
|
98
|
+
expectPngScreenshot(screenshot);
|
|
106
99
|
});
|
|
107
100
|
});
|
|
108
101
|
}
|
|
@@ -135,8 +128,7 @@ describeIfLive('Wikipedia Page Download (engine parity)', () => {
|
|
|
135
128
|
expect(markdown).toContain('Wikipedia');
|
|
136
129
|
expect(markdown.length).toBeGreaterThan(500);
|
|
137
130
|
// Image captured from both engines.
|
|
138
|
-
|
|
139
|
-
expect(screenshot.length).toBeGreaterThan(1000);
|
|
131
|
+
expectPngScreenshot(screenshot);
|
|
140
132
|
}
|
|
141
133
|
}, 180000);
|
|
142
134
|
});
|
|
@@ -1,23 +1,20 @@
|
|
|
1
1
|
import { jest } from '@jest/globals';
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
jest.unstable_mockModule('execa', () => ({
|
|
4
|
+
$: () => async () => ({ exitCode: 0, stdout: '', stderr: '' }),
|
|
5
|
+
}));
|
|
6
|
+
jest.unstable_mockModule('lino-arguments', () => ({
|
|
7
|
+
makeConfig: () => ({ shouldPull: false, jsRoot: 'js' }),
|
|
8
|
+
}));
|
|
4
9
|
|
|
5
10
|
beforeAll(() => {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
use: async (name) => {
|
|
10
|
-
if (name === 'command-stream') return { $: () => ({ run: async () => ({ code: 0, stdout: '', stderr: '' }) }) };
|
|
11
|
-
if (name === 'lino-arguments') return { makeConfig: () => ({ shouldPull: false, jsRoot: 'js' }) };
|
|
12
|
-
throw new Error('Unexpected dynamic import: ' + name);
|
|
13
|
-
}
|
|
14
|
-
})
|
|
15
|
-
`,
|
|
16
|
-
});
|
|
11
|
+
jest
|
|
12
|
+
.spyOn(global, 'fetch')
|
|
13
|
+
.mockRejectedValue(new Error('unexpected remote code download'));
|
|
17
14
|
});
|
|
18
15
|
|
|
19
16
|
afterAll(() => {
|
|
20
|
-
global.fetch
|
|
17
|
+
global.fetch.mockRestore();
|
|
21
18
|
});
|
|
22
19
|
|
|
23
20
|
describe('publish verification', () => {
|
|
@@ -28,17 +25,17 @@ describe('publish verification', () => {
|
|
|
28
25
|
const runVerify = jest
|
|
29
26
|
.fn()
|
|
30
27
|
.mockResolvedValueOnce({
|
|
31
|
-
|
|
28
|
+
exitCode: 1,
|
|
32
29
|
stdout: '',
|
|
33
30
|
stderr: 'npm error code E404',
|
|
34
31
|
})
|
|
35
32
|
.mockResolvedValueOnce({
|
|
36
|
-
|
|
33
|
+
exitCode: 1,
|
|
37
34
|
stdout: '',
|
|
38
35
|
stderr: 'npm error code E404',
|
|
39
36
|
})
|
|
40
37
|
.mockResolvedValueOnce({
|
|
41
|
-
|
|
38
|
+
exitCode: 0,
|
|
42
39
|
stdout: '1.7.1\n',
|
|
43
40
|
stderr: '',
|
|
44
41
|
});
|
|
@@ -17,10 +17,7 @@ describe('streamHandler', () => {
|
|
|
17
17
|
});
|
|
18
18
|
|
|
19
19
|
try {
|
|
20
|
-
const res = await
|
|
21
|
-
.get('/stream')
|
|
22
|
-
.query({ url })
|
|
23
|
-
.expect(200);
|
|
20
|
+
const res = await streamFrom(url);
|
|
24
21
|
|
|
25
22
|
expect(upstreamHeaders['accept-encoding']).toBe('identity');
|
|
26
23
|
expect(res.headers['content-type']).toBe('text/html; charset=utf-8');
|
|
@@ -39,15 +36,12 @@ describe('streamHandler', () => {
|
|
|
39
36
|
res.writeHead(200, {
|
|
40
37
|
'Content-Type': 'text/html; charset=utf-8',
|
|
41
38
|
});
|
|
42
|
-
|
|
43
|
-
res.socket.end();
|
|
39
|
+
// Wait until the body reaches the socket before deliberately truncating it.
|
|
40
|
+
res.write(body, () => res.socket.end());
|
|
44
41
|
});
|
|
45
42
|
|
|
46
43
|
try {
|
|
47
|
-
const res = await
|
|
48
|
-
.get('/stream')
|
|
49
|
-
.query({ url })
|
|
50
|
-
.expect(200);
|
|
44
|
+
const res = await streamFrom(url);
|
|
51
45
|
|
|
52
46
|
expect(res.text).toContain('Example Domain');
|
|
53
47
|
} finally {
|
|
@@ -83,3 +77,7 @@ async function closeServer(server) {
|
|
|
83
77
|
});
|
|
84
78
|
});
|
|
85
79
|
}
|
|
80
|
+
|
|
81
|
+
function streamFrom(url) {
|
|
82
|
+
return request(testApp()).get('/stream').query({ url }).expect(200);
|
|
83
|
+
}
|