browser-commander 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.changeset/config.json +1 -1
- package/.changeset/merged-loud-river.md +7 -0
- package/.jscpd-baseline.json +257 -0
- package/.jscpd.json +1 -3
- package/.prettierignore +1 -0
- package/CHANGELOG.md +22 -0
- package/eslint.config.js +23 -1
- package/package.json +13 -24
- package/scripts/changeset-version.mjs +2 -6
- package/scripts/create-github-release.mjs +7 -8
- package/scripts/create-manual-changeset.mjs +7 -8
- package/scripts/format-github-release.mjs +6 -7
- package/scripts/format-release-notes.mjs +7 -8
- package/scripts/instant-version-bump.mjs +7 -8
- package/scripts/publish-to-npm.mjs +7 -8
- package/scripts/setup-npm.mjs +2 -6
- package/scripts/version-and-commit.mjs +7 -8
- package/src/browser/browser-cookie-credentials.js +3 -1
- package/src/browser/browser-cookies.js +6 -2
- package/src/browser/browser-profiles.js +1 -1
- package/src/browser/launch-options.js +31 -8
- package/src/browser/launcher.js +16 -0
- package/src/exports.js +34 -0
- package/src/fingerprint/apply.js +116 -0
- package/src/fingerprint/automation-parity.js +224 -0
- package/src/fingerprint/cdp-overrides.js +176 -0
- package/src/fingerprint/derive.js +104 -0
- package/src/fingerprint/init-payload.js +181 -0
- package/src/fingerprint/init-script.js +104 -0
- package/src/fingerprint/limitations.js +107 -0
- package/src/fingerprint/limitations.json +96 -0
- package/src/fingerprint/presets.js +244 -0
- package/src/fingerprint/profile.js +428 -0
- package/src/tests/index.js +2 -1
- package/tests/e2e/fingerprint-parity.e2e.test.js +212 -0
- package/tests/helpers/mocks.js +4 -2
- package/tests/helpers/repo.js +74 -0
- package/tests/unit/browser/launch-options.test.js +9 -1
- package/tests/unit/experiments/fingerprint-parity-harness.test.js +97 -0
- package/tests/unit/fingerprint/apply.test.js +271 -0
- package/tests/unit/fingerprint/automation-parity.test.js +212 -0
- package/tests/unit/fingerprint/cdp-overrides.test.js +227 -0
- package/tests/unit/fingerprint/derive.test.js +105 -0
- package/tests/unit/fingerprint/init-script.test.js +357 -0
- package/tests/unit/fingerprint/limitations.test.js +184 -0
- package/tests/unit/fingerprint/presets.test.js +135 -0
- package/tests/unit/fingerprint/profile.test.js +282 -0
- package/tests/unit/scripts/check-pipeline-status.test.js +133 -0
- package/tests/unit/scripts/check-version-modification.test.js +70 -0
- package/tests/unit/scripts/check-web-archive.test.js +191 -0
- package/tests/unit/scripts/ci-timeout-budgets.test.js +311 -0
- package/tests/unit/scripts/ci-workflow-policy.test.js +217 -28
- package/tests/unit/scripts/debug-print.test.js +67 -0
- package/tests/unit/scripts/jscpd-config.test.js +141 -0
- package/tests/unit/scripts/pre-commit-config.test.js +209 -0
- package/tests/unit/scripts/read-manifest.test.js +194 -0
- package/tests/unit/scripts/run-with-budget-warning.test.js +115 -0
- package/tests/unit/scripts/use-module-adoption.test.js +74 -0
- package/tests/unit/scripts/use-module-integration.test.js +64 -0
- package/tests/unit/scripts/use-module.test.js +182 -0
- package/.husky/pre-commit +0 -1
- package/.prettierrc +0 -10
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The gate that makes a cancelled job visible.
|
|
3
|
+
*
|
|
4
|
+
* GitHub reports a job killed by its `timeout-minutes` as *cancelled*, not
|
|
5
|
+
* *failed*, and a run whose only casualty is a cancelled job is filed under
|
|
6
|
+
* `cancelled` as well - run 24045269874 of this repository is one such run on
|
|
7
|
+
* `main`. scripts/check-pipeline-status.sh is the only thing that looks at
|
|
8
|
+
* that, so these tests pin the four readings it has to get right: a failure is
|
|
9
|
+
* always an error, a cancellation off the default branch is a warning, a
|
|
10
|
+
* cancellation on the default branch is an error unless a newer run has already
|
|
11
|
+
* taken over the branch head.
|
|
12
|
+
*
|
|
13
|
+
* See docs/CI-TIMEOUT-BUDGETS.md.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import assert from 'node:assert/strict';
|
|
17
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
18
|
+
import { tmpdir } from 'node:os';
|
|
19
|
+
import path from 'node:path';
|
|
20
|
+
import { spawnSync } from 'node:child_process';
|
|
21
|
+
import { describe, it } from 'node:test';
|
|
22
|
+
|
|
23
|
+
import {
|
|
24
|
+
BASH_AVAILABLE,
|
|
25
|
+
REPO_ROOT,
|
|
26
|
+
repoPath,
|
|
27
|
+
runBashScript,
|
|
28
|
+
} from '../../helpers/repo.js';
|
|
29
|
+
|
|
30
|
+
const SCRIPT = repoPath('scripts/check-pipeline-status.sh');
|
|
31
|
+
|
|
32
|
+
function runGate(needs, env = {}, options = {}) {
|
|
33
|
+
return runBashScript(SCRIPT, [], {
|
|
34
|
+
cwd: options.cwd,
|
|
35
|
+
env: { NEEDS_JSON: JSON.stringify(needs), ...env },
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function results(map) {
|
|
40
|
+
return Object.fromEntries(
|
|
41
|
+
Object.entries(map).map(([job, result]) => [job, { result }])
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
describe('check-pipeline-status.sh', { skip: !BASH_AVAILABLE }, () => {
|
|
46
|
+
it('passes when every job succeeded or was legitimately skipped', () => {
|
|
47
|
+
const { status, output } = runGate(
|
|
48
|
+
results({ lint: 'success', release: 'skipped' }),
|
|
49
|
+
{ IS_MAIN: 'true', RUN_SHA: 'abc', BRANCH_HEAD_SHA: 'abc' }
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
assert.equal(status, 0);
|
|
53
|
+
assert.match(
|
|
54
|
+
output,
|
|
55
|
+
/All required jobs succeeded or were legitimately skipped/
|
|
56
|
+
);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('fails on a failed job and names it', () => {
|
|
60
|
+
const { status, output } = runGate(
|
|
61
|
+
results({ lint: 'failure', test: 'success' }),
|
|
62
|
+
{ IS_MAIN: 'true', RUN_SHA: 'abc', BRANCH_HEAD_SHA: 'abc' }
|
|
63
|
+
);
|
|
64
|
+
|
|
65
|
+
assert.equal(status, 1);
|
|
66
|
+
assert.match(output, /::error::Pipeline failed\. Failing jobs: lint/);
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it('only warns about a cancellation off the default branch', () => {
|
|
70
|
+
const { status, output } = runGate(results({ test: 'cancelled' }), {
|
|
71
|
+
IS_MAIN: 'false',
|
|
72
|
+
RUN_SHA: 'abc',
|
|
73
|
+
BRANCH_HEAD_SHA: 'abc',
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
assert.equal(status, 0);
|
|
77
|
+
assert.match(output, /::warning::Cancelled jobs: test/);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('fails on a cancellation on the default branch head', () => {
|
|
81
|
+
const { status, output } = runGate(results({ test: 'cancelled' }), {
|
|
82
|
+
IS_MAIN: 'true',
|
|
83
|
+
RUN_SHA: 'abc',
|
|
84
|
+
BRANCH_HEAD_SHA: 'abc',
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
assert.equal(status, 1);
|
|
88
|
+
assert.match(output, /::error::Pipeline has cancelled jobs on main: test/);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it('only warns when a newer commit already superseded the run', () => {
|
|
92
|
+
const { status, output } = runGate(results({ test: 'cancelled' }), {
|
|
93
|
+
IS_MAIN: 'true',
|
|
94
|
+
RUN_SHA: 'abc',
|
|
95
|
+
BRANCH_HEAD_SHA: 'def',
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
assert.equal(status, 0);
|
|
99
|
+
assert.match(output, /This run tests abc; main is at def/);
|
|
100
|
+
assert.match(output, /::warning::Cancelled jobs: test/);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it('fails rather than guesses when the branch head cannot be resolved', () => {
|
|
104
|
+
// A missed supersede costs one noisy warning; a missed overrun on main
|
|
105
|
+
// costs a silent failure, so the unresolvable case has to be the loud one.
|
|
106
|
+
const directory = mkdtempSync(path.join(tmpdir(), 'pipeline-status-'));
|
|
107
|
+
|
|
108
|
+
try {
|
|
109
|
+
const { status, output } = runGate(
|
|
110
|
+
results({ test: 'cancelled' }),
|
|
111
|
+
{ IS_MAIN: 'true', RUN_SHA: 'abc', BRANCH_HEAD_SHA: '' },
|
|
112
|
+
{ cwd: directory }
|
|
113
|
+
);
|
|
114
|
+
|
|
115
|
+
assert.equal(status, 1);
|
|
116
|
+
assert.match(output, /assuming it is current/);
|
|
117
|
+
assert.match(output, /::error::Pipeline has cancelled jobs on main/);
|
|
118
|
+
} finally {
|
|
119
|
+
rmSync(directory, { force: true, recursive: true });
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('refuses to run without the job results', () => {
|
|
124
|
+
const result = spawnSync('bash', [SCRIPT], {
|
|
125
|
+
cwd: REPO_ROOT,
|
|
126
|
+
encoding: 'utf8',
|
|
127
|
+
env: { ...process.env, NEEDS_JSON: '' },
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
assert.notEqual(result.status, 0);
|
|
131
|
+
assert.match(result.stderr, /NEEDS_JSON is required/);
|
|
132
|
+
});
|
|
133
|
+
});
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import assert from 'node:assert/strict';
|
|
2
|
+
import { describe, it } from 'node:test';
|
|
3
|
+
|
|
4
|
+
import { findManualVersionChanges } from '../../../../scripts/check-version-modification.mjs';
|
|
5
|
+
|
|
6
|
+
// The real implementation shells out to git; injecting the diff keeps these
|
|
7
|
+
// cases independent of the branch the suite happens to run on.
|
|
8
|
+
function stubDiff(diffsByPath) {
|
|
9
|
+
return (_baseRef, path) => diffsByPath[path] ?? '';
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
describe('check-version-modification', () => {
|
|
13
|
+
it('reports nothing when no manifest changed', () => {
|
|
14
|
+
assert.deepEqual(findManualVersionChanges('main', stubDiff({})), []);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it('detects a hand-edited package.json version', () => {
|
|
18
|
+
const diff = stubDiff({
|
|
19
|
+
'js/package.json': [
|
|
20
|
+
'--- a/js/package.json',
|
|
21
|
+
'+++ b/js/package.json',
|
|
22
|
+
'- "version": "0.16.0",',
|
|
23
|
+
'+ "version": "0.17.0",',
|
|
24
|
+
].join('\n'),
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
assert.deepEqual(findManualVersionChanges('main', diff), [
|
|
28
|
+
'js/package.json',
|
|
29
|
+
]);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
it('detects hand-edited pyproject and Cargo versions together', () => {
|
|
33
|
+
const diff = stubDiff({
|
|
34
|
+
'python/pyproject.toml': '+version = "0.5.4"',
|
|
35
|
+
'rust/Cargo.toml': '+version = "0.9.1"',
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
assert.deepEqual(findManualVersionChanges('main', diff), [
|
|
39
|
+
'python/pyproject.toml',
|
|
40
|
+
'rust/Cargo.toml',
|
|
41
|
+
]);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('ignores a non-numeric version key', () => {
|
|
45
|
+
// pyproject declares `version = "literal: pyproject.toml: project.version"`
|
|
46
|
+
// for its release tooling; that is not a version bump.
|
|
47
|
+
const diff = stubDiff({
|
|
48
|
+
'python/pyproject.toml':
|
|
49
|
+
'+version = "literal: pyproject.toml: project.version"',
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
assert.deepEqual(findManualVersionChanges('main', diff), []);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it('ignores an unrelated edit to a manifest', () => {
|
|
56
|
+
const diff = stubDiff({
|
|
57
|
+
'js/package.json': '+ "description": "A new description",',
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
assert.deepEqual(findManualVersionChanges('main', diff), []);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
it('ignores a removed version line', () => {
|
|
64
|
+
const diff = stubDiff({
|
|
65
|
+
'js/package.json': '- "version": "0.16.0",',
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
assert.deepEqual(findManualVersionChanges('main', diff), []);
|
|
69
|
+
});
|
|
70
|
+
});
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The link checker used to fail pull requests for links that were not broken.
|
|
3
|
+
*
|
|
4
|
+
* Run 33959793880 is the reference case: lychee reported exactly one error, a
|
|
5
|
+
* `[502]` on a GitHub issue that answered 200 minutes later, and listed a
|
|
6
|
+
* healthy `[302]` redirect in a separate section. Both were reported as broken
|
|
7
|
+
* links with no archived copy, and the job failed. The fixture below is that
|
|
8
|
+
* report; the assertions pin each step of the decision that now keeps such a
|
|
9
|
+
* run green while still failing on a link that is really gone.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import assert from 'node:assert/strict';
|
|
13
|
+
import { describe, it } from 'node:test';
|
|
14
|
+
|
|
15
|
+
import {
|
|
16
|
+
classifyStatus,
|
|
17
|
+
extractRejectedLinks,
|
|
18
|
+
judge,
|
|
19
|
+
} from '../../../../scripts/check-web-archive.mjs';
|
|
20
|
+
|
|
21
|
+
// Verbatim shape of lychee/out.md from run 33959793880.
|
|
22
|
+
const REPORT = `# Summary
|
|
23
|
+
|
|
24
|
+
| Status | Count |
|
|
25
|
+
|----------------|-------|
|
|
26
|
+
| 🔍 Total | 170 |
|
|
27
|
+
| 🔗 Unique | 123 |
|
|
28
|
+
| ✅ Successful | 166 |
|
|
29
|
+
| ⏳ Timeouts | 0 |
|
|
30
|
+
| 🔀 Redirected | 1 |
|
|
31
|
+
| 👻 Excluded | 3 |
|
|
32
|
+
| ❓ Unknown | 0 |
|
|
33
|
+
| 🚫 Errors | 1 |
|
|
34
|
+
| ⛔ Unsupported | 0 |
|
|
35
|
+
|
|
36
|
+
## Errors per input
|
|
37
|
+
|
|
38
|
+
### Errors in docs/case-studies/issue-53/README.md
|
|
39
|
+
|
|
40
|
+
* [502] <https://github.com/microsoft/playwright/issues/35743> (at 94:99) | Rejected status code: 502 Bad Gateway
|
|
41
|
+
|
|
42
|
+
## Redirects per input
|
|
43
|
+
|
|
44
|
+
### Redirects in docs/case-studies/issue-33/README.md
|
|
45
|
+
|
|
46
|
+
* https://docs.github.com/actions/security-guides/using-secrets-in-github-actions --[302]--> https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/use-secrets
|
|
47
|
+
`;
|
|
48
|
+
|
|
49
|
+
describe('reading lychee’s report', () => {
|
|
50
|
+
it('takes the rejected links from the errors section only', () => {
|
|
51
|
+
assert.deepEqual(extractRejectedLinks(REPORT), [
|
|
52
|
+
{
|
|
53
|
+
url: 'https://github.com/microsoft/playwright/issues/35743',
|
|
54
|
+
status: '502',
|
|
55
|
+
},
|
|
56
|
+
]);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('never reports a redirect as broken', () => {
|
|
60
|
+
const urls = extractRejectedLinks(REPORT).map((link) => link.url);
|
|
61
|
+
assert.ok(
|
|
62
|
+
!urls.some((url) => url.startsWith('https://docs.github.com/')),
|
|
63
|
+
'a 302 listed under "Redirects per input" is a working link'
|
|
64
|
+
);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('collects every input listed under the errors section', () => {
|
|
68
|
+
const report = [
|
|
69
|
+
'## Errors per input',
|
|
70
|
+
'',
|
|
71
|
+
'### Errors in a.md',
|
|
72
|
+
'',
|
|
73
|
+
'* [404] <https://example.com/gone> (at 1:1) | Not Found',
|
|
74
|
+
'',
|
|
75
|
+
'### Errors in b.md',
|
|
76
|
+
'',
|
|
77
|
+
'* [ERROR] <https://example.invalid/x> (at 2:2) | dns error',
|
|
78
|
+
'* [404] <https://example.com/gone> (at 3:3) | Not Found',
|
|
79
|
+
'',
|
|
80
|
+
'## Suggestions per input',
|
|
81
|
+
'',
|
|
82
|
+
'* [404] <https://example.com/not-an-error> (at 4:4)',
|
|
83
|
+
'',
|
|
84
|
+
].join('\n');
|
|
85
|
+
|
|
86
|
+
assert.deepEqual(extractRejectedLinks(report), [
|
|
87
|
+
{ url: 'https://example.com/gone', status: '404' },
|
|
88
|
+
{ url: 'https://example.invalid/x', status: 'ERROR' },
|
|
89
|
+
]);
|
|
90
|
+
});
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
describe('classifying a lychee status', () => {
|
|
94
|
+
it('treats a refusal to answer as saying nothing about the link', () => {
|
|
95
|
+
assert.equal(classifyStatus('429'), 'transient');
|
|
96
|
+
assert.equal(classifyStatus('500'), 'transient');
|
|
97
|
+
assert.equal(classifyStatus('502'), 'transient');
|
|
98
|
+
assert.equal(classifyStatus('503'), 'transient');
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it('treats a 4xx answer as the resource being gone', () => {
|
|
102
|
+
assert.equal(classifyStatus('404'), 'gone');
|
|
103
|
+
assert.equal(classifyStatus('410'), 'gone');
|
|
104
|
+
assert.equal(classifyStatus('403'), 'gone');
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
it('keeps network-level failures out of both buckets', () => {
|
|
108
|
+
assert.equal(classifyStatus('ERROR'), 'unknown');
|
|
109
|
+
assert.equal(classifyStatus('TIMEOUT'), 'unknown');
|
|
110
|
+
assert.equal(classifyStatus('UNKNOWN'), 'unknown');
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
// Stubs for the two questions `judge` asks. Naming them keeps each case below
|
|
115
|
+
// to the one thing it is about: what the second look said, and whether the
|
|
116
|
+
// Wayback Machine had a copy.
|
|
117
|
+
const answers = (status) => async () => ({ alive: status < 400, status });
|
|
118
|
+
const unreachable = () => async () => ({ alive: false, status: null });
|
|
119
|
+
|
|
120
|
+
const waybackNever = () => async () => {
|
|
121
|
+
throw new Error('the Wayback Machine must not be consulted here');
|
|
122
|
+
};
|
|
123
|
+
const waybackNothing = () => async () => ({
|
|
124
|
+
available: false,
|
|
125
|
+
archiveUrl: null,
|
|
126
|
+
timestamp: null,
|
|
127
|
+
});
|
|
128
|
+
const waybackHas = (timestamp, archiveUrl) => async () => ({
|
|
129
|
+
available: true,
|
|
130
|
+
archiveUrl,
|
|
131
|
+
timestamp,
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
const verdictFor = (link, recheckImpl, waybackImpl) =>
|
|
135
|
+
judge(link, { recheckImpl, waybackImpl });
|
|
136
|
+
|
|
137
|
+
describe('judging one rejected link', () => {
|
|
138
|
+
const playwright = {
|
|
139
|
+
url: 'https://github.com/microsoft/playwright/issues/35743',
|
|
140
|
+
status: '502',
|
|
141
|
+
};
|
|
142
|
+
const gone = { url: 'https://example.com/gone', status: '404' };
|
|
143
|
+
|
|
144
|
+
it('clears a link that answers on the second look', async () => {
|
|
145
|
+
const verdict = await verdictFor(playwright, answers(200), waybackNever());
|
|
146
|
+
assert.equal(verdict.verdict, 'alive');
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it('does not fail for a host that is down twice', async () => {
|
|
150
|
+
const verdict = await verdictFor(playwright, answers(503), waybackNever());
|
|
151
|
+
assert.equal(verdict.verdict, 'transient');
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
it('does not fail for a host that keeps rate-limiting the runner', async () => {
|
|
155
|
+
const verdict = await verdictFor(
|
|
156
|
+
{ url: 'https://example.com/x', status: '429' },
|
|
157
|
+
answers(429),
|
|
158
|
+
waybackNever()
|
|
159
|
+
);
|
|
160
|
+
assert.equal(verdict.verdict, 'transient');
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
it('suggests the snapshot when a confirmed 404 is archived', async () => {
|
|
164
|
+
const verdict = await verdictFor(
|
|
165
|
+
gone,
|
|
166
|
+
answers(404),
|
|
167
|
+
waybackHas(
|
|
168
|
+
'20231015143022',
|
|
169
|
+
'https://web.archive.org/web/20231015143022/https://example.com/gone'
|
|
170
|
+
)
|
|
171
|
+
);
|
|
172
|
+
assert.equal(verdict.verdict, 'archived');
|
|
173
|
+
assert.equal(verdict.date, '2023-10-15');
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it('fails on a confirmed 404 with no snapshot', async () => {
|
|
177
|
+
const verdict = await verdictFor(gone, answers(404), waybackNothing());
|
|
178
|
+
assert.equal(verdict.verdict, 'gone');
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
it('still fails on a domain that no longer resolves', async () => {
|
|
182
|
+
// A dead domain and a runner with no DNS look identical to lychee, so an
|
|
183
|
+
// unreachable host is not waved through the way a 5xx is.
|
|
184
|
+
const verdict = await verdictFor(
|
|
185
|
+
{ url: 'https://example.invalid/x', status: 'ERROR' },
|
|
186
|
+
unreachable(),
|
|
187
|
+
waybackNothing()
|
|
188
|
+
);
|
|
189
|
+
assert.equal(verdict.verdict, 'gone');
|
|
190
|
+
});
|
|
191
|
+
});
|
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `timeout-minutes` is a backstop, never a deadline.
|
|
3
|
+
*
|
|
4
|
+
* GitHub reports a job killed by `timeout-minutes` as *cancelled*, not
|
|
5
|
+
* *failed*, so an overrun used to leave no red check anywhere (see
|
|
6
|
+
* docs/CI-TIMEOUT-BUDGETS.md). Two things fix that: long steps own an explicit
|
|
7
|
+
* budget through scripts/run-with-budget-warning.sh, and every workflow ends in
|
|
8
|
+
* a `pipeline-status` job that reads the results of all the others.
|
|
9
|
+
*
|
|
10
|
+
* These assertions are what find the *next* occurrence instead of the one that
|
|
11
|
+
* already happened: a budget raised above its job's backstop, or a new job that
|
|
12
|
+
* the gate does not watch, fails here rather than in production.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import assert from 'node:assert/strict';
|
|
16
|
+
import { readFileSync, readdirSync } from 'node:fs';
|
|
17
|
+
import { describe, it } from 'node:test';
|
|
18
|
+
|
|
19
|
+
import { repoPath } from '../../helpers/repo.js';
|
|
20
|
+
|
|
21
|
+
const WORKFLOW_DIR = repoPath('.github/workflows');
|
|
22
|
+
|
|
23
|
+
// A budget must expire with room to spare before the backstop fires, because
|
|
24
|
+
// checkout, toolchain installation and dependency installs run on the same job
|
|
25
|
+
// clock and are not budgeted. The share matches BUDGET_WARN_PERCENT in
|
|
26
|
+
// scripts/run-with-budget-warning.sh.
|
|
27
|
+
const MAX_BUDGET_SHARE_PERCENT = 70;
|
|
28
|
+
|
|
29
|
+
const GATE_JOB = 'pipeline-status';
|
|
30
|
+
const GATE_SCRIPT = 'scripts/check-pipeline-status.sh';
|
|
31
|
+
|
|
32
|
+
function readWorkflow(fileName) {
|
|
33
|
+
return readFileSync(
|
|
34
|
+
repoPath('.github/workflows', fileName),
|
|
35
|
+
'utf8'
|
|
36
|
+
).replaceAll('\r\n', '\n');
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function listWorkflowFiles() {
|
|
40
|
+
return readdirSync(WORKFLOW_DIR)
|
|
41
|
+
.filter((fileName) => fileName.endsWith('.yml'))
|
|
42
|
+
.sort();
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Only the mapping under `jobs:` is scanned, so the two-space keys of `on:`
|
|
46
|
+
// (`push:`, `pull_request:`) are not mistaken for job names.
|
|
47
|
+
function listWorkflowJobs(workflow) {
|
|
48
|
+
const jobsStart = workflow.indexOf('\njobs:\n');
|
|
49
|
+
const jobsBody = jobsStart === -1 ? '' : workflow.slice(jobsStart);
|
|
50
|
+
|
|
51
|
+
return Array.from(
|
|
52
|
+
jobsBody.matchAll(/^ {2}([a-zA-Z0-9_-]+):\s*$/gm),
|
|
53
|
+
(match) => match[1]
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function getJobBlock(workflow, jobName) {
|
|
58
|
+
const lines = workflow.split('\n');
|
|
59
|
+
const start = lines.indexOf(` ${jobName}:`);
|
|
60
|
+
|
|
61
|
+
if (start === -1) {
|
|
62
|
+
return '';
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const end = lines.findIndex(
|
|
66
|
+
(line, index) => index > start && /^ {2}[a-zA-Z0-9_-]+:\s*$/.test(line)
|
|
67
|
+
);
|
|
68
|
+
|
|
69
|
+
return lines.slice(start, end === -1 ? lines.length : end).join('\n');
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function getTimeoutMinutes(workflow, jobName) {
|
|
73
|
+
const timeout = getJobBlock(workflow, jobName).match(
|
|
74
|
+
/^ {4}timeout-minutes:\s*(\d+)\s*$/m
|
|
75
|
+
);
|
|
76
|
+
|
|
77
|
+
return timeout ? Number(timeout[1]) : undefined;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function getStepBudgets(workflow, jobName) {
|
|
81
|
+
return Array.from(
|
|
82
|
+
getJobBlock(workflow, jobName).matchAll(
|
|
83
|
+
/run-with-budget-warning\.sh\s+(\d+)\s+"([^"]+)"/g
|
|
84
|
+
),
|
|
85
|
+
(match) => ({ label: match[2], seconds: Number(match[1]) })
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function getJobNeeds(workflow, jobName) {
|
|
90
|
+
const block = getJobBlock(workflow, jobName);
|
|
91
|
+
const inline = block.match(/^ {4}needs:\s*\[([^\]]*)\]\s*$/m);
|
|
92
|
+
|
|
93
|
+
if (inline) {
|
|
94
|
+
return inline[1]
|
|
95
|
+
.split(',')
|
|
96
|
+
.map((name) => name.trim())
|
|
97
|
+
.filter(Boolean);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const listStart = block.indexOf('\n needs:\n');
|
|
101
|
+
|
|
102
|
+
if (listStart === -1) {
|
|
103
|
+
return [];
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const lines = block
|
|
107
|
+
.slice(listStart + 1)
|
|
108
|
+
.split('\n')
|
|
109
|
+
.slice(1);
|
|
110
|
+
const needs = [];
|
|
111
|
+
|
|
112
|
+
for (const line of lines) {
|
|
113
|
+
const item = line.match(/^ {6}- ([a-zA-Z0-9_-]+)\s*$/);
|
|
114
|
+
|
|
115
|
+
if (!item) {
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
needs.push(item[1]);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
return needs;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const WORKFLOWS = listWorkflowFiles().map((fileName) => ({
|
|
126
|
+
fileName,
|
|
127
|
+
text: readWorkflow(fileName),
|
|
128
|
+
}));
|
|
129
|
+
|
|
130
|
+
describe('CI execution budgets', () => {
|
|
131
|
+
it('finds the workflows to check', () => {
|
|
132
|
+
assert.ok(
|
|
133
|
+
WORKFLOWS.length >= 9,
|
|
134
|
+
`expected the repository's workflows, found ${WORKFLOWS.length}`
|
|
135
|
+
);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
it('keeps every declared step budget under its job backstop', () => {
|
|
139
|
+
const violations = [];
|
|
140
|
+
let budgetedJobs = 0;
|
|
141
|
+
|
|
142
|
+
for (const { fileName, text } of WORKFLOWS) {
|
|
143
|
+
for (const jobName of listWorkflowJobs(text)) {
|
|
144
|
+
const budgets = getStepBudgets(text, jobName);
|
|
145
|
+
|
|
146
|
+
if (budgets.length === 0) {
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
budgetedJobs += 1;
|
|
151
|
+
|
|
152
|
+
const backstop = getTimeoutMinutes(text, jobName);
|
|
153
|
+
|
|
154
|
+
if (typeof backstop !== 'number') {
|
|
155
|
+
violations.push(`${fileName} ${jobName}: no timeout-minutes`);
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const allowedSeconds = Math.floor(
|
|
160
|
+
(backstop * 60 * MAX_BUDGET_SHARE_PERCENT) / 100
|
|
161
|
+
);
|
|
162
|
+
const totalSeconds = budgets.reduce(
|
|
163
|
+
(sum, budget) => sum + budget.seconds,
|
|
164
|
+
0
|
|
165
|
+
);
|
|
166
|
+
|
|
167
|
+
for (const budget of budgets) {
|
|
168
|
+
if (budget.seconds > allowedSeconds) {
|
|
169
|
+
violations.push(
|
|
170
|
+
`${fileName} ${jobName}: "${budget.label}" budget ${budget.seconds}s exceeds ${allowedSeconds}s (${MAX_BUDGET_SHARE_PERCENT}% of the ${backstop}min backstop)`
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
if (totalSeconds > allowedSeconds) {
|
|
176
|
+
violations.push(
|
|
177
|
+
`${fileName} ${jobName}: budgets total ${totalSeconds}s, exceeding ${allowedSeconds}s (${MAX_BUDGET_SHARE_PERCENT}% of the ${backstop}min backstop)`
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
assert.deepEqual(violations, []);
|
|
184
|
+
assert.ok(budgetedJobs > 0, 'no job declares a step budget');
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
it('wraps the long steps that used to be able to run past a backstop', () => {
|
|
188
|
+
const budgeted = WORKFLOWS.flatMap(({ fileName, text }) =>
|
|
189
|
+
listWorkflowJobs(text).flatMap((jobName) =>
|
|
190
|
+
getStepBudgets(text, jobName).map(
|
|
191
|
+
(budget) => `${fileName} ${jobName} ${budget.label}`
|
|
192
|
+
)
|
|
193
|
+
)
|
|
194
|
+
).sort();
|
|
195
|
+
|
|
196
|
+
assert.deepEqual(budgeted, [
|
|
197
|
+
'docs.yml build-docs Rust API docs',
|
|
198
|
+
'js.yml test Node.js test suite',
|
|
199
|
+
'parity.yml parity Fingerprint parity suite',
|
|
200
|
+
'python.yml test pytest suite',
|
|
201
|
+
'rust.yml coverage Rust code coverage',
|
|
202
|
+
'rust.yml test Rust doc tests',
|
|
203
|
+
'rust.yml test Rust test suite',
|
|
204
|
+
]);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it('warns at the same share of a budget that this invariant allows', () => {
|
|
208
|
+
const wrapper = readFileSync(
|
|
209
|
+
repoPath('scripts/run-with-budget-warning.sh'),
|
|
210
|
+
'utf8'
|
|
211
|
+
);
|
|
212
|
+
|
|
213
|
+
assert.match(
|
|
214
|
+
wrapper,
|
|
215
|
+
new RegExp(`BUDGET_WARN_PERCENT:-${MAX_BUDGET_SHARE_PERCENT}`)
|
|
216
|
+
);
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
it('runs the wrapper under bash, not the Windows default shell', () => {
|
|
220
|
+
for (const { fileName, text } of WORKFLOWS) {
|
|
221
|
+
const lines = text.split('\n');
|
|
222
|
+
|
|
223
|
+
lines.forEach((line, index) => {
|
|
224
|
+
if (!line.includes('run-with-budget-warning.sh')) {
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const step = lines
|
|
229
|
+
.slice(Math.max(0, index - 12), index + 1)
|
|
230
|
+
.join('\n')
|
|
231
|
+
.split(/^ {6}- /m)
|
|
232
|
+
.pop();
|
|
233
|
+
|
|
234
|
+
assert.ok(
|
|
235
|
+
/shell: bash/.test(step),
|
|
236
|
+
`${fileName}:${index + 1} wraps a command without "shell: bash"; the default shell on windows-latest is pwsh`
|
|
237
|
+
);
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
});
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
describe('pipeline status gate', () => {
|
|
244
|
+
it('is present in every workflow and needs every other job', () => {
|
|
245
|
+
for (const { fileName, text } of WORKFLOWS) {
|
|
246
|
+
const jobs = listWorkflowJobs(text);
|
|
247
|
+
|
|
248
|
+
assert.ok(
|
|
249
|
+
jobs.includes(GATE_JOB),
|
|
250
|
+
`${fileName} has no ${GATE_JOB} job, so a cancelled job there would go unreported`
|
|
251
|
+
);
|
|
252
|
+
|
|
253
|
+
const watched = getJobNeeds(text, GATE_JOB).sort();
|
|
254
|
+
const expected = jobs.filter((jobName) => jobName !== GATE_JOB).sort();
|
|
255
|
+
|
|
256
|
+
assert.deepEqual(
|
|
257
|
+
watched,
|
|
258
|
+
expected,
|
|
259
|
+
`${fileName}: ${GATE_JOB} does not watch every job`
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
it('runs the gate script after a failed dependency', () => {
|
|
265
|
+
for (const { fileName, text } of WORKFLOWS) {
|
|
266
|
+
const block = getJobBlock(text, GATE_JOB);
|
|
267
|
+
|
|
268
|
+
assert.ok(
|
|
269
|
+
block.includes(`run: bash ${GATE_SCRIPT}`),
|
|
270
|
+
`${fileName}: ${GATE_JOB} does not run ${GATE_SCRIPT}`
|
|
271
|
+
);
|
|
272
|
+
|
|
273
|
+
// Without a status function the gate inherits "every dependency
|
|
274
|
+
// succeeded" and is skipped in exactly the runs it exists to report.
|
|
275
|
+
assert.match(
|
|
276
|
+
block,
|
|
277
|
+
/if: >-\n\s*!cancelled\(\)/,
|
|
278
|
+
`${fileName}: ${GATE_JOB} must be conditioned on !cancelled()`
|
|
279
|
+
);
|
|
280
|
+
|
|
281
|
+
assert.match(
|
|
282
|
+
block,
|
|
283
|
+
/NEEDS_JSON: \$\{\{ toJSON\(needs\) \}\}/,
|
|
284
|
+
`${fileName}: ${GATE_JOB} does not pass the job results to the script`
|
|
285
|
+
);
|
|
286
|
+
|
|
287
|
+
assert.match(
|
|
288
|
+
block,
|
|
289
|
+
/RUN_SHA: \$\{\{ github\.sha \}\}/,
|
|
290
|
+
`${fileName}: ${GATE_JOB} cannot tell a supersede from an overrun without RUN_SHA`
|
|
291
|
+
);
|
|
292
|
+
}
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
it('reads the gate script from the repository root in scoped workflows', () => {
|
|
296
|
+
// js.yml, python.yml and rust.yml set defaults.run.working-directory to the
|
|
297
|
+
// language subdirectory, where scripts/check-pipeline-status.sh does not
|
|
298
|
+
// exist.
|
|
299
|
+
for (const { fileName, text } of WORKFLOWS) {
|
|
300
|
+
if (!/^defaults:\n {2}run:\n {4}working-directory: \S+$/m.test(text)) {
|
|
301
|
+
continue;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
assert.match(
|
|
305
|
+
getJobBlock(text, GATE_JOB),
|
|
306
|
+
/working-directory: \.$/m,
|
|
307
|
+
`${fileName}: ${GATE_JOB} runs in the language subdirectory, where ${GATE_SCRIPT} is not`
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
});
|
|
311
|
+
});
|