@link-assistant/web-capture 1.12.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,5 @@
1
1
  {
2
- "$schema": "https://unpkg.com/@changesets/config@3.1.1/schema.json",
2
+ "$schema": "../node_modules/@changesets/config/schema.json",
3
3
  "changelog": "@changesets/cli/changelog",
4
4
  "commit": false,
5
5
  "fixed": [],
package/CHANGELOG.md CHANGED
@@ -1,5 +1,11 @@
1
1
  # @link-assistant/web-capture
2
2
 
3
+ ## 2.0.0
4
+
5
+ ### Major Changes
6
+
7
+ - b055a30: Require Node.js 26.10 or newer and update Playwright to 1.64. Use current Trixie browser images, native Jest ESM and reproducibly locked release tooling. Add locked Python examples and weekly dependency/security maintenance across all ecosystems.
8
+
3
9
  ## 1.12.0
4
10
 
5
11
  ### Minor Changes
package/Dockerfile CHANGED
@@ -1,4 +1,4 @@
1
- FROM node:24-bookworm
1
+ FROM node:26.10.0-trixie
2
2
 
3
3
  WORKDIR /app
4
4
 
@@ -9,15 +9,15 @@ RUN apt-get update && \
9
9
  apt-get install -y \
10
10
  chromium \
11
11
  fonts-liberation \
12
- libasound2 \
13
- libatk-bridge2.0-0 \
14
- libatk1.0-0 \
15
- libatspi2.0-0 \
16
- libcups2 \
12
+ libasound2t64 \
13
+ libatk-bridge2.0-0t64 \
14
+ libatk1.0-0t64 \
15
+ libatspi2.0-0t64 \
16
+ libcups2t64 \
17
17
  libdbus-1-3 \
18
18
  libdrm2 \
19
19
  libgbm1 \
20
- libgtk-3-0 \
20
+ libgtk-3-0t64 \
21
21
  libnspr4 \
22
22
  libnss3 \
23
23
  libxcomposite1 \
@@ -43,7 +43,7 @@ ENV PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1
43
43
  ENV PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=/usr/bin/chromium
44
44
 
45
45
  COPY package.json package-lock.json ./
46
- RUN npm ci
46
+ RUN npm install --global npm@12.2.0 && npm ci
47
47
 
48
48
  COPY . .
49
49
 
package/README.md CHANGED
@@ -19,7 +19,8 @@ A CLI and microservice to fetch URLs and render them as:
19
19
 
20
20
  ## Installation
21
21
 
22
- Node.js 22.22.1 or newer is required. CI and Docker use Node.js 24. Use `npm ci`
22
+ Node.js 26.10.0 or newer is required. CI uses 26.11.1 and the latest published
23
+ Node Docker image uses 26.10.0 on Debian Trixie. Use `npm ci`
23
24
  for a locked repository install; Docker and CI share `package-lock.json` and its
24
25
  security overrides.
25
26
 
@@ -1213,8 +1213,9 @@ async function captureUrl(url, options) {
1213
1213
  baseDelay: 500,
1214
1214
  });
1215
1215
  if (resp.ok) {
1216
- const buffer = await resp.buffer();
1217
- archive.append(buffer, { name: localPath });
1216
+ archive.append(Buffer.from(await resp.arrayBuffer()), {
1217
+ name: localPath,
1218
+ });
1218
1219
  }
1219
1220
  } catch {
1220
1221
  /* skip failed image downloads */
@@ -16,7 +16,7 @@ console.log('Fetching screenshot using Playwright engine...');
16
16
  const imageResponse = await fetch(
17
17
  `${baseUrl}/image?url=${encodeURIComponent(targetUrl)}&engine=playwright`
18
18
  );
19
- const imageBuffer = await imageResponse.buffer();
19
+ const imageBuffer = Buffer.from(await imageResponse.arrayBuffer());
20
20
  fs.writeFileSync('output/playwright_screenshot.png', imageBuffer);
21
21
  console.log('Screenshot saved to output/playwright_screenshot.png');
22
22
 
@@ -0,0 +1,20 @@
1
+ """Shared request policy for the capture-service examples."""
2
+ import os
3
+ from pathlib import Path
4
+ import sys
5
+ import requests
6
+
7
+
8
+ def capture(kind, url=None, **options):
9
+ if url is None:
10
+ url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
11
+ endpoint = os.environ.get('WEB_CAPTURE_URL', 'http://localhost:3000').rstrip('/')
12
+ response = requests.get(f'{endpoint}/{kind}', params={'url': url, **options}, timeout=30)
13
+ response.raise_for_status()
14
+ return response
15
+
16
+
17
+ def download_text(kind, filename):
18
+ output = Path(__file__).with_name(filename)
19
+ output.write_text(capture(kind).text, encoding='utf-8')
20
+ print(f'{kind.title()} saved to {output}')
@@ -1,15 +1,3 @@
1
- import sys
2
- import requests
3
- import os
1
+ from capture_client import download_text
4
2
 
5
- url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
6
- endpoint = f'http://localhost:3000/html?url={url}'
7
-
8
- response = requests.get(endpoint)
9
- response.raise_for_status()
10
-
11
- output_path = os.path.join(os.path.dirname(__file__), 'downloaded.html')
12
- with open(output_path, 'w', encoding='utf-8') as f:
13
- f.write(response.text)
14
-
15
- print(f'HTML saved to {output_path}')
3
+ download_text('html', 'downloaded.html')
@@ -1,11 +1,7 @@
1
- import sys
2
- import requests
1
+ from capture_client import capture
3
2
  import os
4
3
 
5
- url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
6
- endpoint = f'http://localhost:3000/image?url={url}'
7
-
8
- response = requests.get(endpoint)
4
+ response = capture('image')
9
5
  print('Status:', response.status_code)
10
6
  print('Content-Type:', response.headers.get('content-type'))
11
7
 
@@ -17,4 +13,4 @@ if response.content[:8] == png_signature:
17
13
  f.write(response.content)
18
14
  print(f'Image saved to {output_path} (valid PNG)')
19
15
  else:
20
- print('Response is not a valid PNG! First bytes:', response.content[:16])
16
+ print('Response is not a valid PNG! First bytes:', response.content[:16])
@@ -1,15 +1,3 @@
1
- import sys
2
- import requests
3
- import os
1
+ from capture_client import download_text
4
2
 
5
- url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
6
- endpoint = f'http://localhost:3000/markdown?url={url}'
7
-
8
- response = requests.get(endpoint)
9
- response.raise_for_status()
10
-
11
- output_path = os.path.join(os.path.dirname(__file__), 'downloaded.md')
12
- with open(output_path, 'w', encoding='utf-8') as f:
13
- f.write(response.text)
14
-
15
- print(f'Markdown saved to {output_path}')
3
+ download_text('markdown', 'downloaded.md')
@@ -1,20 +1,20 @@
1
- import requests
1
+ import sys
2
+ from capture_client import capture
2
3
  import os
3
4
 
4
- base_url = 'http://localhost:3000'
5
- target_url = 'https://example.com'
5
+ target_url = sys.argv[1] if len(sys.argv) > 1 else 'https://example.com'
6
6
 
7
7
  # Create output directory if it doesn't exist
8
8
  os.makedirs('output', exist_ok=True)
9
9
 
10
10
  print('Fetching HTML using Playwright engine...')
11
- html_response = requests.get(f'{base_url}/html', params={'url': target_url, 'engine': 'playwright'})
12
- with open('output/playwright_html.html', 'w') as f:
11
+ html_response = capture('html', target_url, engine='playwright')
12
+ with open('output/playwright_html.html', 'w', encoding='utf-8') as f:
13
13
  f.write(html_response.text)
14
14
  print('HTML saved to output/playwright_html.html')
15
15
 
16
16
  print('Fetching screenshot using Playwright engine...')
17
- image_response = requests.get(f'{base_url}/image', params={'url': target_url, 'engine': 'playwright'})
17
+ image_response = capture('image', target_url, engine='playwright')
18
18
  with open('output/playwright_screenshot.png', 'wb') as f:
19
19
  f.write(image_response.content)
20
20
  print('Screenshot saved to output/playwright_screenshot.png')
package/jest.config.mjs CHANGED
@@ -1,8 +1,5 @@
1
1
  export default {
2
- transform: {
3
- '^.+\\.js$': ['babel-jest', { configFile: './babel.config.cjs' }],
4
- },
5
- transformIgnorePatterns: ['node_modules/(?!(turndown)/)'],
2
+ transform: {},
6
3
  moduleNameMapper: {
7
4
  '^(\\.{1,2}/.*)\\.js$': '$1',
8
5
  },
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@link-assistant/web-capture",
3
- "version": "1.12.0",
3
+ "version": "2.0.0",
4
4
  "description": "CLI and microservice to render web pages as HTML, Markdown, or PNG",
5
5
  "main": "src/index.js",
6
6
  "bin": {
@@ -8,7 +8,7 @@
8
8
  },
9
9
  "scripts": {
10
10
  "start": "docker compose up -d",
11
- "dev": "nodemon bin/web-capture.js -- --serve",
11
+ "dev": "node --watch bin/web-capture.js --serve",
12
12
  "test": "node --experimental-vm-modules ./node_modules/.bin/jest",
13
13
  "test:watch": "node --experimental-vm-modules ./node_modules/.bin/jest --watch",
14
14
  "test:e2e:process": "node --experimental-vm-modules ./node_modules/.bin/jest tests/e2e/process.test.js",
@@ -16,7 +16,7 @@
16
16
  "test:e2e": "npm run test:e2e:process && npm run test:e2e:docker",
17
17
  "test:all": "npm run build && npm test && npm run test:e2e",
18
18
  "build": "docker compose down && docker compose build && npm start",
19
- "examples:python": "python3 examples/python/html_download.py && python3 examples/python/markdown_download.py && python3 examples/python/image_download.py",
19
+ "examples:python": "uv run --project .. --locked python examples/python/html_download.py && uv run --project .. --locked python examples/python/markdown_download.py && uv run --project .. --locked python examples/python/image_download.py",
20
20
  "examples:javascript": "node examples/js/html_download.js && node examples/js/markdown_download.js && node examples/js/image_download.js",
21
21
  "examples": "npm run build && npm run examples:javascript && npm run examples:python",
22
22
  "lint": "eslint .",
@@ -43,18 +43,15 @@
43
43
  "lino-arguments": "^0.3.0",
44
44
  "log-lazy": "^1.0.4",
45
45
  "node-fetch": "^3.3.2",
46
- "playwright": "^1.63.0",
46
+ "playwright": "^1.64.0",
47
47
  "puppeteer": "^25.12.0",
48
48
  "puppeteer-core": "^25.12.0",
49
49
  "turndown": "^7.2.4",
50
50
  "turndown-plugin-gfm": "^1.0.2"
51
51
  },
52
52
  "devDependencies": {
53
- "@babel/core": "^8.0.7",
54
- "@babel/preset-env": "^8.0.7",
55
53
  "@changesets/cli": "^3.0.3",
56
54
  "@types/archiver": "^8.0.0",
57
- "babel-jest": "^30.5.2",
58
55
  "eslint": "^10.12.0",
59
56
  "eslint-config-prettier": "^10.1.8",
60
57
  "eslint-plugin-prettier": "^5.5.6",
@@ -83,7 +80,7 @@
83
80
  "url": "https://github.com/link-assistant/web-capture"
84
81
  },
85
82
  "engines": {
86
- "node": ">=22.22.1"
83
+ "node": ">=26.10.0"
87
84
  },
88
85
  "lint-staged": {
89
86
  "*.{js,mjs,cjs}": [
@@ -106,5 +103,11 @@
106
103
  "@istanbuljs/load-nyc-config": {
107
104
  "js-yaml": "^4.1.1"
108
105
  }
106
+ },
107
+ "packageManager": "npm@12.2.0",
108
+ "allowScripts": {
109
+ "@parcel/watcher": true,
110
+ "puppeteer": true,
111
+ "unrs-resolver": true
109
112
  }
110
113
  }
package/src/archive.js CHANGED
@@ -200,7 +200,7 @@ export async function archiveHandler(req, res) {
200
200
  baseDelay: 500,
201
201
  });
202
202
  if (imgResp.ok) {
203
- const buffer = await imgResp.buffer();
203
+ const buffer = Buffer.from(await imgResp.arrayBuffer());
204
204
  archive.append(buffer, { name: localPath });
205
205
  }
206
206
  } catch {
@@ -0,0 +1,14 @@
1
+ import fetch from 'node-fetch';
2
+ import { retry } from './retry.js';
3
+
4
+ // Shared image-download policy; keep HTTP errors distinct from network retries.
5
+ export async function downloadBinary(url) {
6
+ const response = await retry(() => fetch(url), {
7
+ retries: 3,
8
+ baseDelay: 1000,
9
+ });
10
+ if (!response.ok) {
11
+ throw new Error(`HTTP ${response.status}`);
12
+ }
13
+ return Buffer.from(await response.arrayBuffer());
14
+ }
package/src/docx.js CHANGED
@@ -75,7 +75,7 @@ export async function docxHandler(req, res) {
75
75
  const imgUrl = new URL(src, absoluteUrl).href;
76
76
  const imgResp = await fetch(imgUrl);
77
77
  if (imgResp.ok) {
78
- const buffer = await imgResp.buffer();
78
+ const buffer = Buffer.from(await imgResp.arrayBuffer());
79
79
  const contentType = imgResp.headers.get('content-type') || '';
80
80
  children.push(
81
81
  new Paragraph({
package/src/fetch.js CHANGED
@@ -14,7 +14,7 @@ export async function fetchHandler(req, res) {
14
14
  copyProxyResponseHeaders(response.headers, res);
15
15
 
16
16
  // Get the response body as buffer and send it
17
- const buffer = await response.buffer();
17
+ const buffer = Buffer.from(await response.arrayBuffer());
18
18
  res.send(buffer);
19
19
  } catch (err) {
20
20
  console.error('Fetch error:', err);
package/src/figures.js CHANGED
@@ -10,10 +10,9 @@
10
10
  * @module figures
11
11
  */
12
12
 
13
- import fetch from 'node-fetch';
14
13
  import * as cheerio from 'cheerio';
15
14
  import { URL } from 'url';
16
- import { retry } from './retry.js';
15
+ import { downloadBinary } from './binary-download.js';
17
16
  import { createBrowser, getBrowserEngine } from './browser.js';
18
17
  import { dismissPopups, scrollToLoadContent } from './popups.js';
19
18
 
@@ -86,16 +85,7 @@ export async function downloadFigures(figures, options = {}) {
86
85
  const filename = `figure-${figure.figureNum}.${ext}`;
87
86
 
88
87
  try {
89
- const resp = await retry(() => fetch(figure.src), {
90
- retries: 3,
91
- baseDelay: 1000,
92
- });
93
-
94
- if (!resp.ok) {
95
- throw new Error(`HTTP ${resp.status}`);
96
- }
97
-
98
- const buffer = await resp.buffer();
88
+ const buffer = await downloadBinary(figure.src);
99
89
 
100
90
  if (options.onProgress) {
101
91
  options.onProgress(figure.figureNum, 'downloaded');
package/src/lib.js CHANGED
@@ -151,7 +151,7 @@ export async function fetchGoogleDriveImage(url) {
151
151
  }
152
152
 
153
153
  return {
154
- buffer: await response.buffer(),
154
+ buffer: Buffer.from(await response.arrayBuffer()),
155
155
  contentType,
156
156
  downloadUrl,
157
157
  filename: `google-drive-${fileId}.${imageExtensionForContentType(contentType)}`,
@@ -13,9 +13,8 @@
13
13
  * @module localize-images
14
14
  */
15
15
 
16
- import fetch from 'node-fetch';
17
16
  import { URL } from 'url';
18
- import { retry } from './retry.js';
17
+ import { downloadBinary } from './binary-download.js';
19
18
 
20
19
  /**
21
20
  * Extract image references from markdown text.
@@ -165,16 +164,7 @@ export async function localizeImages(markdownText, options = {}) {
165
164
  }
166
165
 
167
166
  try {
168
- const resp = await retry(() => fetch(image.url), {
169
- retries: 3,
170
- baseDelay: 1000,
171
- });
172
-
173
- if (!resp.ok) {
174
- throw new Error(`HTTP ${resp.status}`);
175
- }
176
-
177
- const buffer = await resp.buffer();
167
+ const buffer = await downloadBinary(image.url);
178
168
  downloadedCount++;
179
169
 
180
170
  replacements.push({
@@ -39,7 +39,10 @@ function commitAll(repoPath, message) {
39
39
  git(repoPath, ['commit', '-m', message]);
40
40
  }
41
41
 
42
- function createMergedFeatureRepo() {
42
+ function createMergedFeatureRepo({
43
+ featureCode = true,
44
+ baseCode = false,
45
+ } = {}) {
43
46
  const repoPath = mkdtempSync(join(tmpdir(), 'web-capture-detect-'));
44
47
 
45
48
  initializeRepo(repoPath);
@@ -48,13 +51,19 @@ function createMergedFeatureRepo() {
48
51
  commitAll(repoPath, 'initial commit');
49
52
 
50
53
  git(repoPath, ['checkout', '-b', 'feature']);
51
- writeTrackedFile(repoPath, 'rust/src/lib.rs', 'pub fn capture() {}\n');
52
- commitAll(repoPath, 'add rust code');
54
+ if (featureCode) {
55
+ writeTrackedFile(repoPath, 'rust/src/lib.rs', 'pub fn capture() {}\n');
56
+ commitAll(repoPath, 'add rust code');
57
+ }
53
58
 
54
59
  writeTrackedFile(repoPath, 'docs/notes.md', '# docs only tail commit\n');
55
60
  commitAll(repoPath, 'document rust change');
56
61
 
57
62
  git(repoPath, ['checkout', 'main']);
63
+ if (baseCode) {
64
+ writeTrackedFile(repoPath, 'rust/src/lib.rs', 'pub fn base_capture() {}\n');
65
+ commitAll(repoPath, 'base branch code change');
66
+ }
58
67
  git(repoPath, ['merge', '--no-ff', 'feature', '-m', 'merge feature']);
59
68
 
60
69
  return repoPath;
@@ -123,28 +132,32 @@ describe('detect-code-changes', () => {
123
132
  }
124
133
  });
125
134
 
126
- test('detects code introduced by a real merge commit pushed to main', () => {
127
- repoPath = createMergedFeatureRepo();
128
-
129
- const outputs = runDetector(repoPath, 'push');
130
-
131
- expect(outputs['rust-code-changed']).toBe('true');
132
- expect(outputs['rust-changed']).toBe('true');
133
- expect(outputs['any-rust-code-changed']).toBe('true');
134
- expect(outputs['any-code-changed']).toBe('true');
135
- expect(outputs['docs-changed']).toBe('true');
136
- });
137
-
138
- test('keeps pull request merge commits scoped to the PR head commit', () => {
139
- repoPath = createMergedFeatureRepo();
135
+ test.each(['push', 'pull_request'])(
136
+ 'detects code before a docs-only final commit for %s',
137
+ (eventName) => {
138
+ repoPath = createMergedFeatureRepo();
139
+
140
+ expect(runDetector(repoPath, eventName)).toMatchObject({
141
+ 'rust-code-changed': 'true',
142
+ 'rust-changed': 'true',
143
+ 'any-rust-code-changed': 'true',
144
+ 'any-code-changed': 'true',
145
+ 'docs-changed': 'true',
146
+ });
147
+ }
148
+ );
140
149
 
141
- const outputs = runDetector(repoPath, 'pull_request');
150
+ test.each([
151
+ ['docs-only', { featureCode: false }],
152
+ ['base branch code only', { featureCode: false, baseCode: true }],
153
+ ])('keeps code checks disabled for %s PRs', (_name, options) => {
154
+ repoPath = createMergedFeatureRepo(options);
142
155
 
143
- expect(outputs['rust-code-changed']).toBe('false');
144
- expect(outputs['rust-changed']).toBe('false');
145
- expect(outputs['any-rust-code-changed']).toBe('false');
146
- expect(outputs['any-code-changed']).toBe('false');
147
- expect(outputs['docs-changed']).toBe('true');
156
+ expect(runDetector(repoPath, 'pull_request')).toMatchObject({
157
+ 'rust-code-changed': 'false',
158
+ 'any-code-changed': 'false',
159
+ 'docs-changed': 'true',
160
+ });
148
161
  });
149
162
 
150
163
  test('detects files when a push only has one commit', () => {
@@ -40,7 +40,7 @@ describe('workflow policy', () => {
40
40
  (workflowFile) => {
41
41
  const workflow = readWorkflow(workflowFile);
42
42
 
43
- expect(workflow).toContain('uses: actions/checkout@v6');
43
+ expect(workflow).toContain('uses: actions/checkout@v7.0.1');
44
44
  expect(workflow).not.toContain('uses: actions/checkout@v4');
45
45
  }
46
46
  );
@@ -48,7 +48,7 @@ describe('workflow policy', () => {
48
48
  test('Rust workflow uses the current cache action', () => {
49
49
  const workflow = readWorkflow('.github/workflows/rust.yml');
50
50
 
51
- expect(workflow).toContain('uses: actions/cache@v5');
51
+ expect(workflow).toContain('uses: actions/cache@v6.1.0');
52
52
  expect(workflow).not.toContain('uses: actions/cache@v4');
53
53
  });
54
54
 
@@ -0,0 +1,10 @@
1
+ import { expect } from '@jest/globals';
2
+
3
+ // PNG magic number (89 50 4E 47 0D 0A 1A 0A).
4
+ const PNG_SIGNATURE = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]);
5
+
6
+ export function expectPngScreenshot(screenshot, minimumLength = 1000) {
7
+ expect(screenshot).toBeInstanceOf(Uint8Array);
8
+ expect(screenshot.length).toBeGreaterThan(minimumLength);
9
+ expect(Buffer.from(screenshot).subarray(0, 8)).toEqual(PNG_SIGNATURE);
10
+ }
@@ -1,11 +1,12 @@
1
1
  import { createBrowser } from '../../src/browser.js';
2
+ import { expectPngScreenshot } from '../helpers/screenshot.js';
2
3
 
3
4
  describe('Browser Engine Integration Tests', () => {
4
- describe('Puppeteer Engine', () => {
5
+ describe.each(['puppeteer', 'playwright'])('%s Engine', (engine) => {
5
6
  let browser;
6
7
 
7
8
  beforeEach(async () => {
8
- browser = await createBrowser('puppeteer');
9
+ browser = await createBrowser(engine);
9
10
  });
10
11
 
11
12
  afterEach(async () => {
@@ -32,56 +33,7 @@ describe('Browser Engine Integration Tests', () => {
32
33
  timeout: 30000,
33
34
  });
34
35
  const screenshot = await page.screenshot({ type: 'png' });
35
- expect(screenshot).toBeInstanceOf(Buffer);
36
- expect(screenshot.length).toBeGreaterThan(100);
37
- }, 60000);
38
-
39
- it('can set custom headers and user agent', async () => {
40
- const page = await browser.newPage();
41
- await page.setExtraHTTPHeaders({ 'Accept-Language': 'en-US' });
42
- await page.setUserAgent('Test User Agent');
43
- await page.goto('https://example.com', {
44
- waitUntil: 'networkidle0',
45
- timeout: 30000,
46
- });
47
- const content = await page.content();
48
- expect(content).toBeTruthy();
49
- }, 60000);
50
- });
51
-
52
- describe('Playwright Engine', () => {
53
- let browser;
54
-
55
- beforeEach(async () => {
56
- browser = await createBrowser('playwright');
57
- });
58
-
59
- afterEach(async () => {
60
- if (browser) {
61
- await browser.close();
62
- }
63
- });
64
-
65
- it('can navigate to a page and get content', async () => {
66
- const page = await browser.newPage();
67
- await page.goto('https://example.com', {
68
- waitUntil: 'networkidle0',
69
- timeout: 30000,
70
- });
71
- const content = await page.content();
72
- expect(content).toContain('Example Domain');
73
- }, 60000);
74
-
75
- it('can take a screenshot', async () => {
76
- const page = await browser.newPage();
77
- await page.setViewport({ width: 1280, height: 800 });
78
- await page.goto('https://example.com', {
79
- waitUntil: 'networkidle0',
80
- timeout: 30000,
81
- });
82
- const screenshot = await page.screenshot({ type: 'png' });
83
- expect(screenshot).toBeInstanceOf(Buffer);
84
- expect(screenshot.length).toBeGreaterThan(100);
36
+ expectPngScreenshot(screenshot, 100);
85
37
  }, 60000);
86
38
 
87
39
  it('can set custom headers and user agent', async () => {
@@ -12,6 +12,7 @@ import { jest } from '@jest/globals';
12
12
  import { createBrowser } from '../../src/browser.js';
13
13
  import { fetchHtml } from '../../src/lib.js';
14
14
  import { retry } from '../../src/retry.js';
15
+ import { expectPngScreenshot } from '../helpers/screenshot.js';
15
16
  import {
16
17
  fetchGithubRepositorySnapshot,
17
18
  formatGithubRepositoryMarkdown,
@@ -22,9 +23,6 @@ import {
22
23
  const GITHUB_REPOSITORY_URL =
23
24
  process.env.GITHUB_REPOSITORY_URL ||
24
25
  'https://github.com/link-assistant/web-capture';
25
- const PNG_SIGNATURE = Buffer.from([
26
- 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
27
- ]);
28
26
 
29
27
  const SKIP_LIVE =
30
28
  !process.env.GITHUB_REPOSITORY_INTEGRATION ||
@@ -128,9 +126,7 @@ describe.each(['puppeteer', 'playwright'])(
128
126
 
129
127
  const screenshot = await page.screenshot({ type: 'png' });
130
128
 
131
- expect(screenshot).toBeInstanceOf(Buffer);
132
- expect(screenshot.length).toBeGreaterThan(1000);
133
- expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
129
+ expectPngScreenshot(screenshot);
134
130
  });
135
131
  });
136
132
  }
@@ -19,14 +19,10 @@ import { jest } from '@jest/globals';
19
19
  import { createBrowser } from '../../src/browser.js';
20
20
  import { convertHtmlToMarkdown } from '../../src/lib.js';
21
21
  import { retry } from '../../src/retry.js';
22
+ import { expectPngScreenshot } from '../helpers/screenshot.js';
22
23
 
23
24
  const WIKIPEDIA_URL = 'https://en.wikipedia.org/wiki/Wikipedia';
24
25
 
25
- // PNG magic number (89 50 4E 47 0D 0A 1A 0A).
26
- const PNG_SIGNATURE = Buffer.from([
27
- 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
28
- ]);
29
-
30
26
  const SKIP_LIVE =
31
27
  !process.env.WIKIPEDIA_INTEGRATION ||
32
28
  process.env.WIKIPEDIA_INTEGRATION === 'false';
@@ -99,10 +95,7 @@ describe.each(['puppeteer', 'playwright'])(
99
95
  it('downloads the Wikipedia page as an image (PNG screenshot)', async () => {
100
96
  const screenshot = await page.screenshot({ type: 'png' });
101
97
 
102
- expect(screenshot).toBeInstanceOf(Buffer);
103
- expect(screenshot.length).toBeGreaterThan(1000);
104
- // Verify the PNG signature so we know it is a real image.
105
- expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
98
+ expectPngScreenshot(screenshot);
106
99
  });
107
100
  });
108
101
  }
@@ -135,8 +128,7 @@ describeIfLive('Wikipedia Page Download (engine parity)', () => {
135
128
  expect(markdown).toContain('Wikipedia');
136
129
  expect(markdown.length).toBeGreaterThan(500);
137
130
  // Image captured from both engines.
138
- expect(screenshot.slice(0, 8)).toEqual(PNG_SIGNATURE);
139
- expect(screenshot.length).toBeGreaterThan(1000);
131
+ expectPngScreenshot(screenshot);
140
132
  }
141
133
  }, 180000);
142
134
  });
@@ -1,23 +1,20 @@
1
1
  import { jest } from '@jest/globals';
2
2
 
3
- const originalFetch = global.fetch;
3
+ jest.unstable_mockModule('execa', () => ({
4
+ $: () => async () => ({ exitCode: 0, stdout: '', stderr: '' }),
5
+ }));
6
+ jest.unstable_mockModule('lino-arguments', () => ({
7
+ makeConfig: () => ({ shouldPull: false, jsRoot: 'js' }),
8
+ }));
4
9
 
5
10
  beforeAll(() => {
6
- global.fetch = jest.fn().mockResolvedValue({
7
- text: async () => `
8
- ({
9
- use: async (name) => {
10
- if (name === 'command-stream') return { $: () => ({ run: async () => ({ code: 0, stdout: '', stderr: '' }) }) };
11
- if (name === 'lino-arguments') return { makeConfig: () => ({ shouldPull: false, jsRoot: 'js' }) };
12
- throw new Error('Unexpected dynamic import: ' + name);
13
- }
14
- })
15
- `,
16
- });
11
+ jest
12
+ .spyOn(global, 'fetch')
13
+ .mockRejectedValue(new Error('unexpected remote code download'));
17
14
  });
18
15
 
19
16
  afterAll(() => {
20
- global.fetch = originalFetch;
17
+ global.fetch.mockRestore();
21
18
  });
22
19
 
23
20
  describe('publish verification', () => {
@@ -28,17 +25,17 @@ describe('publish verification', () => {
28
25
  const runVerify = jest
29
26
  .fn()
30
27
  .mockResolvedValueOnce({
31
- code: 1,
28
+ exitCode: 1,
32
29
  stdout: '',
33
30
  stderr: 'npm error code E404',
34
31
  })
35
32
  .mockResolvedValueOnce({
36
- code: 1,
33
+ exitCode: 1,
37
34
  stdout: '',
38
35
  stderr: 'npm error code E404',
39
36
  })
40
37
  .mockResolvedValueOnce({
41
- code: 0,
38
+ exitCode: 0,
42
39
  stdout: '1.7.1\n',
43
40
  stderr: '',
44
41
  });
@@ -17,10 +17,7 @@ describe('streamHandler', () => {
17
17
  });
18
18
 
19
19
  try {
20
- const res = await request(testApp())
21
- .get('/stream')
22
- .query({ url })
23
- .expect(200);
20
+ const res = await streamFrom(url);
24
21
 
25
22
  expect(upstreamHeaders['accept-encoding']).toBe('identity');
26
23
  expect(res.headers['content-type']).toBe('text/html; charset=utf-8');
@@ -39,15 +36,12 @@ describe('streamHandler', () => {
39
36
  res.writeHead(200, {
40
37
  'Content-Type': 'text/html; charset=utf-8',
41
38
  });
42
- res.write(body);
43
- res.socket.end();
39
+ // Wait until the body reaches the socket before deliberately truncating it.
40
+ res.write(body, () => res.socket.end());
44
41
  });
45
42
 
46
43
  try {
47
- const res = await request(testApp())
48
- .get('/stream')
49
- .query({ url })
50
- .expect(200);
44
+ const res = await streamFrom(url);
51
45
 
52
46
  expect(res.text).toContain('Example Domain');
53
47
  } finally {
@@ -83,3 +77,7 @@ async function closeServer(server) {
83
77
  });
84
78
  });
85
79
  }
80
+
81
+ function streamFrom(url) {
82
+ return request(testApp()).get('/stream').query({ url }).expect(200);
83
+ }
package/babel.config.cjs DELETED
@@ -1,13 +0,0 @@
1
- module.exports = {
2
- presets: [
3
- [
4
- '@babel/preset-env',
5
- {
6
- targets: {
7
- node: 'current',
8
- },
9
- modules: false,
10
- },
11
- ],
12
- ],
13
- };