@avi2dg/checks 0.46.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +2 -1
- package/docs/gates/checks-advisories.md +2 -2
- package/docs/gates/checks-mutation-baseline.md +3 -3
- package/docs/gates/checks-mutation-compare.md +10 -8
- package/docs/gates/checks-mutation-scope.md +10 -10
- package/docs/gates/checks-mutation.md +4 -4
- package/docs/gates/checks-night-run.md +188 -0
- package/docs/gates/checks-release-notes.md +2 -1
- package/docs/gates/checks-release-pr.md +7 -2
- package/docs/gates/checks-test.md +5 -2
- package/package.json +4 -1
- package/src/delivery/github.ts +7 -8
- package/src/testing/mutation-compare.ts +1 -1
- package/src/testing/mutation-scope.ts +1 -1
- package/src/testing/night-run-findings.ts +74 -0
- package/src/testing/night-run.ts +253 -0
- package/src/testing/test.ts +24 -6
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
Every release of `@avi2dg/checks`, newest first, written by the release from its conventional commits.
|
|
4
4
|
|
|
5
|
+
## 0.47.0
|
|
6
|
+
|
|
7
|
+
Released 2026-10-19.
|
|
8
|
+
|
|
9
|
+
### Features
|
|
10
|
+
|
|
11
|
+
- **testing:** file night-run regressions as GitHub issues instead of gating pull requests [#174](https://github.com/avi2d/checks/pull/174)
|
|
12
|
+
|
|
5
13
|
## 0.46.0
|
|
6
14
|
|
|
7
15
|
Released 2026-10-19.
|
package/README.md
CHANGED
|
@@ -193,8 +193,9 @@ These bins run on their own:
|
|
|
193
193
|
- [`checks-mutation`](docs/gates/checks-mutation.md) runs Stryker for scoped checks and refuses a full run outside CI.
|
|
194
194
|
- [`checks-mutation-compare`](docs/gates/checks-mutation-compare.md) compares the mutants of two reports and lists each one that regresses.
|
|
195
195
|
- [`checks-mutation-baseline`](docs/gates/checks-mutation-baseline.md) restores the newest mutation baseline from `main`, saves it on a self-hosted runner's disk in place of an artifact, and deletes superseded baselines after a `main` run.
|
|
196
|
-
- [`checks-mutation-scope`](docs/gates/checks-mutation-scope.md) turns a
|
|
196
|
+
- [`checks-mutation-scope`](docs/gates/checks-mutation-scope.md) turns a branch's diff into the files its mutation runs mutate.
|
|
197
197
|
- [`checks-subsumed-tests`](docs/gates/checks-subsumed-tests.md) lists each test another test subsumes in a mutation run.
|
|
198
|
+
- [`checks-night-run`](docs/gates/checks-night-run.md) opens or comments on one GitHub issue per finding of a night run on `main`, each file whose mutation score fell and each failing test.
|
|
198
199
|
- [`checks-changelog`](docs/gates/checks-changelog.md) writes the pending release into `CHANGELOG.md` from the conventional commits since the last release.
|
|
199
200
|
- [`checks-release-notes`](docs/gates/checks-release-notes.md) writes one `CHANGELOG.md` section to a file for a GitHub release.
|
|
200
201
|
- [`checks-release-report`](docs/gates/checks-release-report.md) tells whether the history holds unreleased features or fixes since the last tag.
|
|
@@ -102,12 +102,12 @@ A range that leaves `bun.lock` unchanged runs in under a second, and one that ch
|
|
|
102
102
|
## Running it on a schedule
|
|
103
103
|
|
|
104
104
|
A range never fails on an advisory published after its package landed, so a scheduled `--all` run finds those.
|
|
105
|
-
A repository
|
|
105
|
+
A consuming repository pins advisories between 05:00 and 06:00 UTC at a minute chosen so repositories do not start together:
|
|
106
106
|
|
|
107
107
|
```yaml
|
|
108
108
|
on:
|
|
109
109
|
schedule:
|
|
110
|
-
- cron: "
|
|
110
|
+
- cron: "15 5 * * *"
|
|
111
111
|
workflow_dispatch:
|
|
112
112
|
jobs:
|
|
113
113
|
advisories:
|
|
@@ -166,7 +166,7 @@ The saved baseline uses none of the account's Actions storage, which every priva
|
|
|
166
166
|
That storage accrues by the hour over the month, so an artifact deleted early still counts until the month ends.
|
|
167
167
|
A baseline job on a GitHub-hosted runner uploads the same two files as a `mutation-baseline` artifact with `retention-days: 7` instead, since the restore reads only the newest `main` run.
|
|
168
168
|
|
|
169
|
-
A full run uploads its report as the `mutation-baseline-full` artifact, which a
|
|
169
|
+
A full run uploads its report as the `mutation-baseline-full` artifact, which a branch comparison's scope step reads on any runner.
|
|
170
170
|
It slims the report first, so the artifact holds about 0.5 MB in place of about 100 MB:
|
|
171
171
|
|
|
172
172
|
```yaml
|
|
@@ -179,9 +179,9 @@ It slims the report first, so the artifact holds about 0.5 MB in place of about
|
|
|
179
179
|
```
|
|
180
180
|
|
|
181
181
|
The full upload keeps 30 days, so the newest full baseline outlives the gap between full runs.
|
|
182
|
-
A
|
|
182
|
+
A branch comparison's scope step loses its coverage when no full baseline outlives the gap.
|
|
183
183
|
|
|
184
|
-
A
|
|
184
|
+
A branch comparison's scope step restores the full report with `--full`, and [checks-mutation-scope](checks-mutation-scope.md) reads it:
|
|
185
185
|
|
|
186
186
|
```yaml
|
|
187
187
|
- name: Select mutation scope
|
|
@@ -68,22 +68,23 @@ mutation-compare: REGRESSION (1 mutant(s))
|
|
|
68
68
|
## When it runs
|
|
69
69
|
|
|
70
70
|
Only a CI step the repository writes runs it.
|
|
71
|
-
A
|
|
71
|
+
A person starts that step by hand on a branch, and no pull request triggers it or waits for it.
|
|
72
|
+
[checks-night-run](checks-night-run.md) judges the mutation score of `main` each night and opens an issue for each file whose score fell.
|
|
73
|
+
A repository runs it with `--advisory`, so the comparison stays advisory.
|
|
72
74
|
Its lines land only in the CI log.
|
|
73
|
-
A repository that wants a reviewer to answer each regression line puts the lines in front of that reviewer with its own CI step.
|
|
74
|
-
That step can post them as a pull request comment.
|
|
75
75
|
A full sweep runs in CI and never on a laptop.
|
|
76
76
|
Start a baseline with `gh workflow run mutation` and keep its report as an artifact.
|
|
77
77
|
The shared preset refuses a full `stryker run` outside CI and names that workflow command instead, as [checks-mutation](checks-mutation.md) says.
|
|
78
78
|
|
|
79
79
|
## Running it in CI
|
|
80
80
|
|
|
81
|
-
|
|
81
|
+
A person starts a workflow named `mutation-compare` on a branch with `gh workflow run mutation-compare --ref <branch>`.
|
|
82
|
+
It compares the head report against a report built at the merge-base with `main` over the same scope:
|
|
82
83
|
|
|
83
84
|
```yaml
|
|
84
85
|
name: mutation-compare
|
|
85
86
|
on:
|
|
86
|
-
|
|
87
|
+
workflow_dispatch:
|
|
87
88
|
jobs:
|
|
88
89
|
mutation-compare:
|
|
89
90
|
runs-on: ubuntu-latest
|
|
@@ -102,7 +103,7 @@ jobs:
|
|
|
102
103
|
env:
|
|
103
104
|
GH_TOKEN: ${{ github.token }}
|
|
104
105
|
run: |
|
|
105
|
-
base="$(git merge-base HEAD
|
|
106
|
+
base="$(git merge-base HEAD origin/main)"
|
|
106
107
|
echo "BASE_SHA=$base" >> "$GITHUB_ENV"
|
|
107
108
|
bunx checks-mutation-baseline --full "$RUNNER_TEMP/baseline-full/mutation.json"
|
|
108
109
|
bunx checks-mutation-scope "$base" "$RUNNER_TEMP/baseline-full/mutation.json" >> "$GITHUB_ENV"
|
|
@@ -125,7 +126,7 @@ jobs:
|
|
|
125
126
|
git worktree prune
|
|
126
127
|
```
|
|
127
128
|
|
|
128
|
-
The scope step picks the files the
|
|
129
|
+
The scope step picks the files the branch changes and the sources its changed unit tests cover or kill, as [checks-mutation-scope](checks-mutation-scope.md) says.
|
|
129
130
|
Neither run mutates a file outside that scope, so the job's time follows the size of the change and not the size of the repository.
|
|
130
131
|
Both runs take the same `--mutate "$SCOPE"`, so the two reports hold the same files.
|
|
131
132
|
A scope file the merge base lacks matches nothing there, and Stryker still writes a report.
|
|
@@ -135,7 +136,7 @@ A changed unit test that `main` edited after that baseline widens the scope to e
|
|
|
135
136
|
Start a full baseline with `gh workflow run mutation`, as [checks-mutation](checks-mutation.md#running-it-in-ci) shows.
|
|
136
137
|
A run with `--mutate` is scoped, so the preset lets it through without `CI=true`.
|
|
137
138
|
The base worktree's path carries the run's id and attempt, and the last step removes it even when a run fails, so a runner kept between jobs starts each job clean.
|
|
138
|
-
A public repository keeps `runs-on: ubuntu-latest`,
|
|
139
|
+
A public repository keeps `runs-on: ubuntu-latest`, as [checks-ci-wiring](checks-ci-wiring.md#runners) allows a public repository on every job.
|
|
139
140
|
A private repository sets `runs-on: [self-hosted, Linux, X64, winbox]` instead, which is the fleet's self-hosted Linux runner where it sends its mutation runs.
|
|
140
141
|
The job never reads `CI_RUNS_ON`, so an override that moves a repository's other jobs during a runner outage leaves the comparison waiting for `winbox`.
|
|
141
142
|
[checks-ci-wiring](checks-ci-wiring.md#runners) refuses a mutation job that names it in `runs-on`.
|
|
@@ -144,4 +145,5 @@ The job never reads `CI_RUNS_ON`, so an override that moves a repository's other
|
|
|
144
145
|
|
|
145
146
|
- [checks-mutation](checks-mutation.md)
|
|
146
147
|
- [checks-mutation-scope](checks-mutation-scope.md)
|
|
148
|
+
- [checks-night-run](checks-night-run.md)
|
|
147
149
|
- [checks-test-layout](checks-test-layout.md)
|
|
@@ -4,7 +4,7 @@ audience: consumers
|
|
|
4
4
|
---
|
|
5
5
|
# checks-mutation-scope
|
|
6
6
|
|
|
7
|
-
`checks-mutation-scope` turns a
|
|
7
|
+
`checks-mutation-scope` turns a branch's diff into the files its mutation runs mutate, and prints them as `NAME=a,b` lines a CI step appends to its environment file.
|
|
8
8
|
|
|
9
9
|
## What it checks
|
|
10
10
|
|
|
@@ -17,23 +17,23 @@ A `mutate` entry with a mutation range, such as `src/a.ts:1-10`, fails the run,
|
|
|
17
17
|
A renamed source enters under both names, and a deleted source does not enter.
|
|
18
18
|
|
|
19
19
|
A unit test is a `.test.ts` or `.test.tsx` file under `tests/unit/`, the files the preset's `unitTestFiles` lists.
|
|
20
|
-
A unit test the
|
|
20
|
+
A unit test the branch adds, edits, deletes or renames is changed.
|
|
21
21
|
A unit test that reads a changed file under `tests/` is changed too.
|
|
22
22
|
It reads a file when it imports it, directly or through another file under `tests/`.
|
|
23
23
|
An import resolves as Bun resolves it, to the file it names, then with an extension added, and only then to the directory's `index`.
|
|
24
24
|
A unit test also reads a changed fixture under `tests/fixtures/` when it names the fixture's path or a directory above it.
|
|
25
25
|
Bun loads each `[test] preload` in `bunfig.toml` before every test file.
|
|
26
26
|
So every unit test is changed when the preload list differs between the merge base and `HEAD`.
|
|
27
|
-
The same holds when the
|
|
27
|
+
The same holds when the branch changes a preload or a file a preload imports, wherever that file lives.
|
|
28
28
|
|
|
29
29
|
Each source whose mutants a changed unit test covers or kills in the full baseline report then enters the scope, when it still exists at `HEAD`.
|
|
30
|
-
So a kill lost through a test change alone still shows in the comparison, in a source the
|
|
30
|
+
So a kill lost through a test change alone still shows in the comparison, in a source the branch leaves alone.
|
|
31
31
|
A changed unit test with no baseline report fails the run rather than reading incomplete coverage.
|
|
32
32
|
Stryker records each test file's source in the report's `testFiles`.
|
|
33
33
|
A changed unit test the merge base holds must have that recorded source match its content at the merge base.
|
|
34
34
|
Otherwise the report ran an older test, so its coverage cannot say what the test covers now.
|
|
35
35
|
The scope then widens to every file at `HEAD` the `mutate` globs match.
|
|
36
|
-
A unit test the
|
|
36
|
+
A unit test the branch adds is new to every report, so it needs no entry.
|
|
37
37
|
Stryker writes no source without mutants into the report, so a mutated source the report lacks may be newer than it.
|
|
38
38
|
When a unit test changed, each such source that exists at `HEAD` enters the scope.
|
|
39
39
|
A change under `tests/e2e/` changes no unit test, so it needs no baseline report.
|
|
@@ -61,7 +61,7 @@ The report must come from a run that restored no state and passed `--disableBail
|
|
|
61
61
|
checks-mutation-scope <base> [<full-baseline-report>]
|
|
62
62
|
```
|
|
63
63
|
|
|
64
|
-
`<base>` is the branch
|
|
64
|
+
`<base>` is the target branch, such as `origin/main`, or the merge base itself.
|
|
65
65
|
`<full-baseline-report>` is the path the full baseline was restored to, and a path with no file reads as no report.
|
|
66
66
|
|
|
67
67
|
## Output
|
|
@@ -93,7 +93,7 @@ It also names each stale test that widens the scope.
|
|
|
93
93
|
|
|
94
94
|
## Sample output
|
|
95
95
|
|
|
96
|
-
A helper change that reaches a source the
|
|
96
|
+
A helper change that reaches a source the branch leaves alone:
|
|
97
97
|
|
|
98
98
|
```
|
|
99
99
|
mutation-scope: tests/unit/dashboard/snapshot.test.ts reads tests/lib/session-fixture.ts
|
|
@@ -116,20 +116,20 @@ mutation-scope: the baseline report did not run tests/unit/dashboard/snapshot.te
|
|
|
116
116
|
|
|
117
117
|
## When it runs
|
|
118
118
|
|
|
119
|
-
Only a CI step the repository writes runs it, in
|
|
119
|
+
Only a CI step the repository writes runs it, in a branch's mutation comparison, before either Stryker run:
|
|
120
120
|
|
|
121
121
|
```yaml
|
|
122
122
|
- name: Select mutation scope
|
|
123
123
|
env:
|
|
124
124
|
GH_TOKEN: ${{ github.token }}
|
|
125
125
|
run: |
|
|
126
|
-
base="$(git merge-base HEAD
|
|
126
|
+
base="$(git merge-base HEAD origin/main)"
|
|
127
127
|
echo "BASE_SHA=$base" >> "$GITHUB_ENV"
|
|
128
128
|
bunx checks-mutation-baseline --full "$RUNNER_TEMP/baseline-full/mutation.json"
|
|
129
129
|
bunx checks-mutation-scope "$base" "$RUNNER_TEMP/baseline-full/mutation.json" >> "$GITHUB_ENV"
|
|
130
130
|
```
|
|
131
131
|
|
|
132
|
-
Each later step reads
|
|
132
|
+
Each later step reads `SCOPE` from the environment, and a failed scope step adds nothing to it.
|
|
133
133
|
[checks-mutation-compare](checks-mutation-compare.md#running-it-in-ci) shows the whole job.
|
|
134
134
|
The full baseline comes from the `mutation` workflow, as [checks-mutation](checks-mutation.md#running-it-in-ci) shows.
|
|
135
135
|
|
|
@@ -12,7 +12,7 @@ It refuses a full mutation run when `CI` is not `true`.
|
|
|
12
12
|
A full run is one with no `--mutate <glob>` and no `--incremental` flag.
|
|
13
13
|
A `--mutate` without a glob, `--incrementalFile` alone or `--incremental` with `--force` is still a full run.
|
|
14
14
|
Its refusal names `gh workflow run mutation` as the command that starts the same run in CI.
|
|
15
|
-
A run scoped with `--mutate <glob>`, `--mutate=<glob>` or `-m <glob>` stays allowed locally, because
|
|
15
|
+
A run scoped with `--mutate <glob>`, `--mutate=<glob>` or `-m <glob>` stays allowed locally, because a branch comparison scopes to named files.
|
|
16
16
|
An incremental run stays allowed locally only when its incremental report exists, because it reuses the results of mutants that did not change.
|
|
17
17
|
That report is the `incrementalFile` Stryker resolves from the command line and the config file, else `reports/stryker-incremental.json`.
|
|
18
18
|
With no report there, the refusal says to pass `--mutate <glob>`, or to start the full baseline in CI with `gh workflow run mutation`.
|
|
@@ -75,7 +75,7 @@ An `--incremental` run with no report stops before instrumenting with the missin
|
|
|
75
75
|
## When it runs
|
|
76
76
|
|
|
77
77
|
A repository runs full baselines from the mutation workflow on `workflow_dispatch`.
|
|
78
|
-
A repository that picks a
|
|
78
|
+
A repository that picks a branch scope from a baseline starts that workflow weekly with `gh workflow run mutation`, because a GitHub schedule can start hours late.
|
|
79
79
|
Run scoped checks locally during development.
|
|
80
80
|
Each baseline costs a full Stryker run.
|
|
81
81
|
|
|
@@ -106,9 +106,9 @@ jobs:
|
|
|
106
106
|
```
|
|
107
107
|
|
|
108
108
|
The run restores no state and passes `--disableBail`, so its report names every test that covers or kills each mutant.
|
|
109
|
-
The upload is the `mutation-baseline-full` artifact that a
|
|
109
|
+
The upload is the `mutation-baseline-full` artifact that a branch comparison's scope step restores with `checks-mutation-baseline --full`.
|
|
110
110
|
[checks-mutation-scope](checks-mutation-scope.md) reads it to bring in the sources a changed unit test covers or kills.
|
|
111
|
-
The restore reads only a successful run of `mutation.yml` on `main`, so a sweep that fails publishes nothing a
|
|
111
|
+
The restore reads only a successful run of `mutation.yml` on `main`, so a sweep that fails publishes nothing a scope step reads.
|
|
112
112
|
The artifact keeps 30 days, so the newest full baseline outlives the week between full runs.
|
|
113
113
|
The upload holds the report without each mutant's `statusReason`, as [checks-mutation-baseline](checks-mutation-baseline.md#the-saved-baselines) says, since the full report of a large repository passes 100 MB as an artifact.
|
|
114
114
|
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
---
|
|
2
|
+
kind: reference
|
|
3
|
+
audience: consumers
|
|
4
|
+
---
|
|
5
|
+
# checks-night-run
|
|
6
|
+
|
|
7
|
+
`checks-night-run` turns a night run on `main` into GitHub issues, one for each file whose mutation score fell and one for each failing test.
|
|
8
|
+
|
|
9
|
+
## What it checks
|
|
10
|
+
|
|
11
|
+
It scores each file of tonight's Stryker report as Stryker does, the `Killed` and `Timeout` mutants over every mutant but the `Ignored`, `CompileError`, `RuntimeError` and `Pending` ones.
|
|
12
|
+
It compares each file's score with the same file's score from the previous night.
|
|
13
|
+
A file whose score is lower than the previous night's is a finding.
|
|
14
|
+
A file only one night holds is no finding, since it lacks one of the two scores.
|
|
15
|
+
A file with no scored mutant has no score, so it is no finding either.
|
|
16
|
+
An equal score is no finding, even when it counts other mutants.
|
|
17
|
+
Each failing test in a JUnit report is a finding, whatever the previous night holds.
|
|
18
|
+
A file whose test worker crashed is one finding, since Bun reports none of its tests.
|
|
19
|
+
With no previous night it compares no score and still files each failing test.
|
|
20
|
+
With no mutation report it compares and records no score and still files each failing test.
|
|
21
|
+
|
|
22
|
+
Each finding has a key that stays the same from night to night:
|
|
23
|
+
|
|
24
|
+
| Finding | Key |
|
|
25
|
+
| --- | --- |
|
|
26
|
+
| a file whose mutation score fell | `mutation:<file>` |
|
|
27
|
+
| a failing test | `test:<file> > <describe> > <name>` |
|
|
28
|
+
| a file whose test worker crashed | `test:<file>` |
|
|
29
|
+
|
|
30
|
+
## How it files a finding
|
|
31
|
+
|
|
32
|
+
It creates the `night-run` label when the repository lacks it.
|
|
33
|
+
It reads every open issue labelled `night-run`, and reads the key each one carries in a hidden marker in its body.
|
|
34
|
+
A finding whose key an open issue carries gets one comment on that issue, so a regression that lasts two nights stays one issue.
|
|
35
|
+
Any other finding opens one issue labelled `night-run`.
|
|
36
|
+
The issue and the comment name tonight's run, the file and either the test or the score of both nights.
|
|
37
|
+
The body holds the file, the full test name and the finding key.
|
|
38
|
+
A title longer than GitHub's 256 characters is cut to at most 256 and ends in `…`.
|
|
39
|
+
A night with no finding makes no call to GitHub's issues or labels at all.
|
|
40
|
+
Closing an issue ends it, and the same finding on a later night opens a new one.
|
|
41
|
+
Agents and people fix each issue as a task of its own, so no pull request waits for a night run.
|
|
42
|
+
A run on any ref but `refs/heads/main` files no issue and records no score, and lists each finding in its log only.
|
|
43
|
+
|
|
44
|
+
## What it reads
|
|
45
|
+
|
|
46
|
+
It reads tonight's Stryker `mutation.json` report, when the mutation run left one, and any number of JUnit reports, such as the ones `checks-test --junit=<file>` keeps, as [checks-test](checks-test.md#arguments) says.
|
|
47
|
+
It reads the previous night from the `night-run` artifact of the newest successful run of the same workflow on `main`.
|
|
48
|
+
That artifact holds `night-run.json`, the scores the run recorded, and a newest run without it counts as no previous night.
|
|
49
|
+
A run on another branch never becomes the previous night.
|
|
50
|
+
On `main` with a mutation report it writes tonight's scores to `reports/night-run.json`, and the workflow uploads that file as the `night-run` artifact.
|
|
51
|
+
It reads the run from `GITHUB_SERVER_URL`, `GITHUB_REPOSITORY`, `GITHUB_RUN_ID`, `GITHUB_WORKFLOW_REF` and `GITHUB_REF`, so it runs only inside a GitHub Actions job.
|
|
52
|
+
It calls `gh` with the job's `GH_TOKEN`, which needs `actions: read` for the previous night and `issues: write` for the issues.
|
|
53
|
+
|
|
54
|
+
## Arguments
|
|
55
|
+
|
|
56
|
+
```sh
|
|
57
|
+
checks-night-run [--mutation <mutation-report>] [--tests <junit-report>]...
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
`--mutation` names tonight's Stryker report and is given at most once.
|
|
61
|
+
`--tests` names one JUnit report and is given once per report.
|
|
62
|
+
At least one report is given.
|
|
63
|
+
|
|
64
|
+
## Exit codes
|
|
65
|
+
|
|
66
|
+
| Code | When |
|
|
67
|
+
| --- | --- |
|
|
68
|
+
| 0 | it filed every finding and recorded tonight's scores, it found nothing to file, or the run is off `main` and it listed each finding |
|
|
69
|
+
| 2 | an argument does not parse, a report does not decode, a GitHub Actions variable is unset or a `gh` call fails |
|
|
70
|
+
|
|
71
|
+
It never exits 1, because a finding becomes an issue rather than a failed run.
|
|
72
|
+
|
|
73
|
+
## Sample output
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
checks-night-run: the night run https://github.com/acme/widget/actions/runs/12 compared with the night run https://github.com/acme/widget/actions/runs/11; 2 finding(s)
|
|
77
|
+
opened #31 for mutation:src/cart.ts
|
|
78
|
+
commented on #27 for test:tests/unit/cart.test.ts > totals > rounds half to even
|
|
79
|
+
checks-night-run: recorded 42 file score(s) in reports/night-run.json
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## When it runs
|
|
83
|
+
|
|
84
|
+
Only a night workflow the repository writes runs it, after that workflow's full mutation run and its long test tiers.
|
|
85
|
+
|
|
86
|
+
## Running it at night
|
|
87
|
+
|
|
88
|
+
A repository runs it from a workflow named `night-run` on a schedule, and starts the same workflow by hand with `gh workflow run night-run`:
|
|
89
|
+
|
|
90
|
+
```yaml
|
|
91
|
+
name: night-run
|
|
92
|
+
on:
|
|
93
|
+
schedule:
|
|
94
|
+
- cron: "7 22 * * *"
|
|
95
|
+
workflow_dispatch:
|
|
96
|
+
permissions:
|
|
97
|
+
contents: read
|
|
98
|
+
concurrency:
|
|
99
|
+
group: night-run
|
|
100
|
+
cancel-in-progress: false
|
|
101
|
+
jobs:
|
|
102
|
+
mutation:
|
|
103
|
+
runs-on: ubuntu-latest
|
|
104
|
+
timeout-minutes: 60
|
|
105
|
+
steps:
|
|
106
|
+
- uses: actions/checkout@v5
|
|
107
|
+
- uses: oven-sh/setup-bun@v2
|
|
108
|
+
with:
|
|
109
|
+
bun-version-file: .bun-version
|
|
110
|
+
- run: bun install --frozen-lockfile
|
|
111
|
+
- run: bunx stryker run
|
|
112
|
+
- uses: actions/upload-artifact@v4
|
|
113
|
+
with:
|
|
114
|
+
name: night-run-mutation
|
|
115
|
+
path: reports/mutation/mutation.json
|
|
116
|
+
retention-days: 1
|
|
117
|
+
tests:
|
|
118
|
+
runs-on: ${{ vars.CI_RUNS_ON || 'ubuntu-latest' }}
|
|
119
|
+
steps:
|
|
120
|
+
- uses: actions/checkout@v5
|
|
121
|
+
- uses: oven-sh/setup-bun@v2
|
|
122
|
+
with:
|
|
123
|
+
bun-version-file: .bun-version
|
|
124
|
+
- run: bun install --frozen-lockfile
|
|
125
|
+
- run: |
|
|
126
|
+
bunx checks-test --tier=live --junit=reports/night-run-tests/live.xml || true
|
|
127
|
+
bunx checks-test --tier=pixel --junit=reports/night-run-tests/pixel.xml || true
|
|
128
|
+
- uses: actions/upload-artifact@v4
|
|
129
|
+
with:
|
|
130
|
+
name: night-run-tests
|
|
131
|
+
path: reports/night-run-tests/
|
|
132
|
+
retention-days: 1
|
|
133
|
+
report:
|
|
134
|
+
needs: [mutation, tests]
|
|
135
|
+
if: ${{ !cancelled() && needs.tests.result == 'success' }}
|
|
136
|
+
runs-on: ${{ vars.CI_RUNS_ON || 'ubuntu-latest' }}
|
|
137
|
+
permissions:
|
|
138
|
+
contents: read
|
|
139
|
+
actions: read
|
|
140
|
+
issues: write
|
|
141
|
+
steps:
|
|
142
|
+
- uses: actions/checkout@v5
|
|
143
|
+
- uses: oven-sh/setup-bun@v2
|
|
144
|
+
with:
|
|
145
|
+
bun-version-file: .bun-version
|
|
146
|
+
- run: bun install --frozen-lockfile
|
|
147
|
+
- if: needs.mutation.result == 'success'
|
|
148
|
+
uses: actions/download-artifact@v4
|
|
149
|
+
with:
|
|
150
|
+
name: night-run-mutation
|
|
151
|
+
path: reports/mutation
|
|
152
|
+
- uses: actions/download-artifact@v4
|
|
153
|
+
with:
|
|
154
|
+
name: night-run-tests
|
|
155
|
+
path: reports/night-run-tests
|
|
156
|
+
- env:
|
|
157
|
+
GH_TOKEN: ${{ github.token }}
|
|
158
|
+
run: bunx checks-night-run ${{ needs.mutation.result == 'success' && '--mutation reports/mutation/mutation.json' || '' }} --tests reports/night-run-tests/live.xml --tests reports/night-run-tests/pixel.xml
|
|
159
|
+
- uses: actions/upload-artifact@v4
|
|
160
|
+
with:
|
|
161
|
+
name: night-run
|
|
162
|
+
path: reports/night-run.json
|
|
163
|
+
if-no-files-found: ignore
|
|
164
|
+
retention-days: 90
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
The cron starts the run at 22:07 UTC, inside the kit's own night window from 22:00 to 22:30 UTC.
|
|
168
|
+
A repository takes minute 7 of its own tier's 30 minute night window, so repositories do not start together.
|
|
169
|
+
The tests job runs each long tier the repository holds, `live` and `pixel` as [checks-test](checks-test.md#test-tiers) names them, and keeps each JUnit report.
|
|
170
|
+
Each `|| true` lets the job go on to the upload after a failing test, since that test becomes an issue.
|
|
171
|
+
A tier that writes no report fails the report job, because `checks-night-run` cannot read it.
|
|
172
|
+
The report job runs whenever the tests job succeeds, so a failing test that also stops Stryker still becomes an issue.
|
|
173
|
+
It passes `--mutation` only when the mutation job succeeded, and the failed mutation job still fails the run.
|
|
174
|
+
The concurrency group queues a run started by hand behind the scheduled one, so two runs never file the same finding twice.
|
|
175
|
+
A dispatch on a branch runs both jobs and lists its findings in the report job's log, and `if-no-files-found: ignore` lets that run upload no record.
|
|
176
|
+
`issues: write` on the report job is the one permission beyond reading the repository and its runs.
|
|
177
|
+
A public repository keeps `runs-on: ubuntu-latest` on the mutation job.
|
|
178
|
+
A private repository sets `runs-on: [self-hosted, Linux, X64, winbox]` on the mutation job, and [checks-ci-wiring](checks-ci-wiring.md#runners) refuses a mutation job that names `CI_RUNS_ON`.
|
|
179
|
+
The report job reads `CI_RUNS_ON`, because it runs no mutant.
|
|
180
|
+
A successful run uploads `night-run`, so the next night compares with it, and a failed run uploads nothing a later night reads.
|
|
181
|
+
The artifact keeps 90 days, so a gap of nights still leaves a previous night to compare with.
|
|
182
|
+
|
|
183
|
+
## Related topics
|
|
184
|
+
|
|
185
|
+
- [checks-test](checks-test.md)
|
|
186
|
+
- [checks-mutation](checks-mutation.md)
|
|
187
|
+
- [checks-mutation-compare](checks-mutation-compare.md)
|
|
188
|
+
- [checks-ci-wiring](checks-ci-wiring.md)
|
|
@@ -140,7 +140,8 @@ jobs:
|
|
|
140
140
|
```
|
|
141
141
|
|
|
142
142
|
A private repository sets `runs-on: ${{ vars.CI_RUNS_ON || 'ubuntu-latest' }}` on each job in either workflow, as [checks-ci-wiring](checks-ci-wiring.md#runners) requires.
|
|
143
|
-
A release is a pull request that holds only the version bump and the built changelog
|
|
143
|
+
A release is a pull request that holds only the version bump and the built changelog.
|
|
144
|
+
[checks-release-pr](checks-release-pr.md) opens it on the daily release's schedule, or on every push to `main` in a package repository.
|
|
144
145
|
When it lands, [checks-release-tag](checks-release-tag.md) tags the merge commit and dispatches this workflow on the tag.
|
|
145
146
|
A tag the workflow token pushes starts no `push` run, so the workflow triggers on `workflow_dispatch` as well, and refuses a dispatch on a ref that is not a tag.
|
|
146
147
|
A tag a person pushes starts the same workflow through its `push` trigger.
|
|
@@ -75,19 +75,23 @@ release-pr: https://github.com/acme/widget/pull/12 releases 0.4.0 from 3f2a9c81d
|
|
|
75
75
|
## When it runs
|
|
76
76
|
|
|
77
77
|
The daily release workflow below runs it once a day on the default branch.
|
|
78
|
+
A package repository runs it on every push to `main` other than a release commit.
|
|
78
79
|
Run the workflow by hand with `gh workflow run daily-release` to refresh the release pull request sooner.
|
|
79
80
|
|
|
80
81
|
## Running it in CI
|
|
81
82
|
|
|
82
83
|
A repository takes the daily release as `.github/workflows/daily-release.yml`.
|
|
83
|
-
A repository
|
|
84
|
+
A consuming repository pins daily-release between 03:00 and 04:00 UTC at a minute chosen so repositories do not start together.
|
|
85
|
+
A package repository released on push drops the `schedule` trigger, so every push to `main` but a release commit opens or refreshes the release pull request.
|
|
86
|
+
Its `pull-request` job's `if` reads `"github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'chore: release ')"`.
|
|
87
|
+
The `pull-request` job checks out its branch's tip when it starts, so a run that starts late never rebuilds the release on an older commit.
|
|
84
88
|
The `pull-request` job runs on the schedule, and the `tag` job runs when a release commit lands on `main`, as [checks-release-tag](checks-release-tag.md) says:
|
|
85
89
|
|
|
86
90
|
```yaml
|
|
87
91
|
name: daily-release
|
|
88
92
|
on:
|
|
89
93
|
schedule:
|
|
90
|
-
- cron: "7
|
|
94
|
+
- cron: "7 3 * * *"
|
|
91
95
|
workflow_dispatch:
|
|
92
96
|
push:
|
|
93
97
|
branches: [main]
|
|
@@ -108,6 +112,7 @@ jobs:
|
|
|
108
112
|
steps:
|
|
109
113
|
- uses: actions/checkout@v5
|
|
110
114
|
with:
|
|
115
|
+
ref: ${{ github.ref }}
|
|
111
116
|
fetch-depth: 0
|
|
112
117
|
- uses: oven-sh/setup-bun@v2
|
|
113
118
|
with:
|
|
@@ -47,7 +47,7 @@ A skipped test or a todo without `skipReason` fails in every environment.
|
|
|
47
47
|
It does not read a skip list from `package.json`.
|
|
48
48
|
It counts a run as `ci` when `CI` is true and as `local` otherwise.
|
|
49
49
|
|
|
50
|
-
The command takes no arguments for the default suite, except the worker count of [Parallel runs](#parallel-runs).
|
|
50
|
+
The command takes no arguments for the default suite, except the worker count of [Parallel runs](#parallel-runs) and the file `--junit` keeps the report at.
|
|
51
51
|
Use `checks-test --tier=live` or `checks-test --tier=pixel` for a named test tier.
|
|
52
52
|
Use `checks-test --tier=unit` or `checks-test --tier=e2e` to run only one level of the default suite.
|
|
53
53
|
A live or pixel run clears Bun's ignored paths, runs only `./tests/live` or `./tests/pixel`, and still checks each skip.
|
|
@@ -102,8 +102,10 @@ A timings option waits for Bun to honour timings under random order.
|
|
|
102
102
|
|
|
103
103
|
## Arguments
|
|
104
104
|
|
|
105
|
-
The default command takes no arguments except `--parallel=<workers>`, as [Parallel runs](#parallel-runs) says
|
|
105
|
+
The default command takes no arguments except `--parallel=<workers>`, as [Parallel runs](#parallel-runs) says, and `--junit=<file>`.
|
|
106
106
|
Its named tier options are `--tier=live`, `--tier=pixel`, `--tier=unit` and `--tier=e2e`, and a tier run takes `--parallel=<workers>` too.
|
|
107
|
+
`--junit=<file>` keeps the JUnit report Bun wrote at `<file>`, whether the tests passed or failed, so a night run hands it to [checks-night-run](checks-night-run.md).
|
|
108
|
+
A `package.json` script never takes `--junit`, as [checks-test-layout](checks-test-layout.md) says of `scripts.test`.
|
|
107
109
|
Each option appears at most once.
|
|
108
110
|
It refuses test filters because every test excluded by a filter would look skipped.
|
|
109
111
|
|
|
@@ -138,3 +140,4 @@ A repository that tracks no TypeScript source does not need it, as [checks-test-
|
|
|
138
140
|
- [checks-test-layout](checks-test-layout.md)
|
|
139
141
|
- [checks-flake](checks-flake.md)
|
|
140
142
|
- [checks-quarantine-clock](checks-quarantine-clock.md)
|
|
143
|
+
- [checks-night-run](checks-night-run.md)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@avi2dg/checks",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.47.0",
|
|
4
4
|
"description": "Deterministic checks shared across a set of TypeScript repositories",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -51,6 +51,8 @@
|
|
|
51
51
|
"src/testing/mutation-baseline.ts",
|
|
52
52
|
"src/testing/mutation-baseline-saved.ts",
|
|
53
53
|
"src/testing/subsumed-tests.ts",
|
|
54
|
+
"src/testing/night-run.ts",
|
|
55
|
+
"src/testing/night-run-findings.ts",
|
|
54
56
|
"src/delivery/ci-wiring.ts",
|
|
55
57
|
"src/delivery/shell-command.ts",
|
|
56
58
|
"src/delivery/workflow-references.ts",
|
|
@@ -156,6 +158,7 @@
|
|
|
156
158
|
"checks-mutation-baseline": "src/testing/mutation-baseline.ts",
|
|
157
159
|
"checks-mutation-scope": "src/testing/mutation-scope.ts",
|
|
158
160
|
"checks-subsumed-tests": "src/testing/subsumed-tests.ts",
|
|
161
|
+
"checks-night-run": "src/testing/night-run.ts",
|
|
159
162
|
"checks-ci-wiring": "src/delivery/ci-wiring.ts",
|
|
160
163
|
"checks-comment-gate": "src/quality/comment-gate.ts",
|
|
161
164
|
"checks-css-size": "src/quality/css-size.ts",
|
package/src/delivery/github.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Effect, Schema } from "effect";
|
|
2
2
|
import { ChildProcessSpawner } from "effect/process";
|
|
3
|
-
import { collect } from "../core/git.ts";
|
|
3
|
+
import { collect, type Feed } from "../core/git.ts";
|
|
4
4
|
|
|
5
5
|
export class GitHubFailure extends Schema.TaggedError<GitHubFailure>()("GitHubFailure", {
|
|
6
6
|
message: Schema.String,
|
|
@@ -13,17 +13,16 @@ type Decode<A> = (text: string) => Effect.Effect<A, Schema.SchemaError>;
|
|
|
13
13
|
// gh fills {owner} and {repo} from the checkout's remote, so a workflow names no repository.
|
|
14
14
|
export const REPOSITORY = "repos/{owner}/{repo}";
|
|
15
15
|
|
|
16
|
-
const
|
|
17
|
-
const failed = (reason: string): GitHubFailure => new GitHubFailure({ message:
|
|
18
|
-
const
|
|
19
|
-
const feed = body === undefined ? {} : { input: JSON.stringify(body) };
|
|
20
|
-
const { stdout, stderr, exitCode } = yield* collect("gh", ["api", "--method", method, path, ...input], undefined, feed).pipe(
|
|
21
|
-
Effect.mapError((cause) => failed(cause.message)),
|
|
22
|
-
);
|
|
16
|
+
export const gh = Effect.fn("gh")(function* (args: readonly string[], call = `gh ${args.join(" ")}`, feed: Feed = {}) {
|
|
17
|
+
const failed = (reason: string): GitHubFailure => new GitHubFailure({ message: `${call}: ${reason.trim()}` });
|
|
18
|
+
const { stdout, stderr, exitCode } = yield* collect("gh", args, undefined, feed).pipe(Effect.mapError((cause) => failed(cause.message)));
|
|
23
19
|
if (exitCode !== ChildProcessSpawner.ExitCode(0)) return yield* failed(stderr);
|
|
24
20
|
return stdout;
|
|
25
21
|
});
|
|
26
22
|
|
|
23
|
+
const gitHub = (method: Method, path: string, body?: unknown) =>
|
|
24
|
+
gh(["api", "--method", method, path, ...(body === undefined ? [] : ["--input", "-"])], `gh api ${method} ${path}`, body === undefined ? {} : { input: JSON.stringify(body) });
|
|
25
|
+
|
|
27
26
|
export function gitHubJson<A>(decode: Decode<A>, method: Method, path: string, body?: unknown): Effect.Effect<A, GitHubFailure, ChildProcessSpawner.ChildProcessSpawner> {
|
|
28
27
|
return gitHub(method, path, body).pipe(
|
|
29
28
|
Effect.flatMap((text) =>
|
|
@@ -186,7 +186,7 @@ export function readersOf(sources: ReadonlyMap<string, string>, changed: readonl
|
|
|
186
186
|
return new Map([...reached].filter(([file, via]) => file !== via));
|
|
187
187
|
}
|
|
188
188
|
|
|
189
|
-
// Names what every unit test loads that the
|
|
189
|
+
// Names what every unit test loads that the branch changes: the preload list, a preload, or a file a preload imports.
|
|
190
190
|
export function preloadChange(
|
|
191
191
|
changed: ReadonlySet<string>,
|
|
192
192
|
preloads: { readonly atBase: readonly string[]; readonly atHead: readonly string[] },
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { Option, Schema } from "effect";
|
|
2
|
+
import { OUTCOME, type ReportFile } from "./mutation-compare.ts";
|
|
3
|
+
import { NAME_SEPARATOR, type TestResult } from "./test-report.ts";
|
|
4
|
+
|
|
5
|
+
const ScoreSchema = Schema.Struct({ detected: Schema.Int, valid: Schema.Int });
|
|
6
|
+
|
|
7
|
+
export type Score = typeof ScoreSchema.Type;
|
|
8
|
+
|
|
9
|
+
export const NightRecordSchema = Schema.Struct({ scores: Schema.Record(Schema.String, ScoreSchema) });
|
|
10
|
+
|
|
11
|
+
export type NightRecord = typeof NightRecordSchema.Type;
|
|
12
|
+
|
|
13
|
+
export type TestReport = { readonly results: readonly TestResult[]; readonly crashedFiles: readonly string[] };
|
|
14
|
+
|
|
15
|
+
export type Tonight = { readonly scores: NightRecord["scores"]; readonly tests: readonly TestReport[] };
|
|
16
|
+
|
|
17
|
+
export type PreviousNight = { readonly url: string; readonly record: NightRecord };
|
|
18
|
+
|
|
19
|
+
export type Finding =
|
|
20
|
+
| { readonly kind: "mutation"; readonly file: string; readonly before: Score; readonly after: Score; readonly since: string }
|
|
21
|
+
| { readonly kind: "test"; readonly file: string; readonly name: string }
|
|
22
|
+
| { readonly kind: "crash"; readonly file: string };
|
|
23
|
+
|
|
24
|
+
function scoreOf(file: ReportFile): Score {
|
|
25
|
+
let detected = 0;
|
|
26
|
+
let valid = 0;
|
|
27
|
+
for (const mutant of file.mutants) {
|
|
28
|
+
const outcome = OUTCOME[mutant.status];
|
|
29
|
+
if (outcome === "leaves-score") continue;
|
|
30
|
+
valid += 1;
|
|
31
|
+
if (outcome === "detected") detected += 1;
|
|
32
|
+
}
|
|
33
|
+
return { detected, valid };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function scoresOf(files: ReadonlyMap<string, ReportFile>): NightRecord["scores"] {
|
|
37
|
+
return Object.fromEntries([...files].map(([path, file]) => [path, scoreOf(file)] as const).sort(([one], [other]) => one.localeCompare(other)));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Cross-multiplied, so two equal scores never differ by how their quotients round.
|
|
41
|
+
function fell(before: Score, after: Score): boolean {
|
|
42
|
+
return before.valid > 0 && after.valid > 0 && after.detected * before.valid < before.detected * after.valid;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function keyOf(finding: Finding): string {
|
|
46
|
+
if (finding.kind === "mutation") return `mutation:${finding.file}`;
|
|
47
|
+
if (finding.kind === "test") return `test:${finding.file}${NAME_SEPARATOR}${finding.name}`;
|
|
48
|
+
return `test:${finding.file}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function scoreFindings(previous: Option.Option<PreviousNight>, scores: NightRecord["scores"]): readonly Finding[] {
|
|
52
|
+
if (Option.isNone(previous)) return [];
|
|
53
|
+
const { url, record } = previous.value;
|
|
54
|
+
return Object.entries(scores).flatMap(([file, after]): readonly Finding[] => {
|
|
55
|
+
const before = record.scores[file];
|
|
56
|
+
return before !== undefined && fell(before, after) ? [{ kind: "mutation", file, before, after, since: url }] : [];
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function testFindings(reports: readonly TestReport[]): readonly Finding[] {
|
|
61
|
+
const failing = reports.flatMap(({ results, crashedFiles }) => [
|
|
62
|
+
...results.filter((result) => result.outcome === "failed").map((result): Finding => ({ kind: "test", file: result.file, name: result.name })),
|
|
63
|
+
...crashedFiles.map((file): Finding => ({ kind: "crash", file })),
|
|
64
|
+
]);
|
|
65
|
+
return [...new Map(failing.map((finding) => [keyOf(finding), finding])).values()];
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function findingsOf(previous: Option.Option<PreviousNight>, tonight: Tonight): readonly Finding[] {
|
|
69
|
+
return [...scoreFindings(previous, tonight.scores), ...testFindings(tonight.tests)].sort((one, other) => keyOf(one).localeCompare(keyOf(other)));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function percent(score: Score): string {
|
|
73
|
+
return `${((score.detected / score.valid) * 100).toFixed(2)}%`;
|
|
74
|
+
}
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
import { Config, Console, Context, Effect, FileSystem, Layer, Option, Path, Schema } from "effect";
|
|
3
|
+
import { ChildProcessSpawner } from "effect/process";
|
|
4
|
+
import { DEFAULT_BRANCH } from "../core/gates.ts";
|
|
5
|
+
import { runMain, Usage } from "../core/main.ts";
|
|
6
|
+
import { gh, type GitHubFailure, gitHubJson, REPOSITORY } from "../delivery/github.ts";
|
|
7
|
+
import { parseReport as parseMutationReport } from "./mutation-compare.ts";
|
|
8
|
+
import { type Finding, findingsOf, keyOf, NightRecordSchema, type NightRecord, percent, type PreviousNight, scoresOf, type TestReport } from "./night-run-findings.ts";
|
|
9
|
+
import { parseReport as parseTestReport } from "./test-report.ts";
|
|
10
|
+
|
|
11
|
+
const NAME = "checks-night-run";
|
|
12
|
+
const USAGE = `usage: ${NAME} [--mutation <mutation-report>] [--tests <junit-report>]...`;
|
|
13
|
+
export const LABEL = "night-run";
|
|
14
|
+
const LABEL_COLOR = "5319e7";
|
|
15
|
+
const LABEL_DESCRIPTION = "A regression a night run on main found";
|
|
16
|
+
const ARTIFACT = "night-run";
|
|
17
|
+
const RECORD_NAME = "night-run.json";
|
|
18
|
+
export const RECORD = `reports/${RECORD_NAME}`;
|
|
19
|
+
const PAGE_SIZE = 100;
|
|
20
|
+
const GITHUB_TITLE_LIMIT = 256;
|
|
21
|
+
|
|
22
|
+
class NightRunFailure extends Schema.TaggedError<NightRunFailure>()("NightRunFailure", {
|
|
23
|
+
message: Schema.String,
|
|
24
|
+
}) {}
|
|
25
|
+
|
|
26
|
+
export type Options = { readonly mutation: Option.Option<string>; readonly tests: readonly string[] };
|
|
27
|
+
|
|
28
|
+
export const parseArgs = Effect.fnUntraced(function* (argv: readonly string[]): Effect.fn.Return<Options, Usage> {
|
|
29
|
+
const mutation: string[] = [];
|
|
30
|
+
const tests: string[] = [];
|
|
31
|
+
for (let at = 0; at < argv.length; at += 2) {
|
|
32
|
+
const [flag, value] = [argv[at], argv[at + 1]];
|
|
33
|
+
const into = flag === "--mutation" ? mutation : flag === "--tests" ? tests : undefined;
|
|
34
|
+
if (into === undefined || value === undefined || value.startsWith("--")) return yield* new Usage({ message: USAGE });
|
|
35
|
+
into.push(value);
|
|
36
|
+
}
|
|
37
|
+
if (mutation.length > 1 || mutation.length + tests.length === 0) return yield* new Usage({ message: USAGE });
|
|
38
|
+
return { mutation: Option.fromNullishOr(mutation[0]), tests };
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
export type OpenIssue = { readonly number: number; readonly key: string };
|
|
42
|
+
|
|
43
|
+
export type Filed = { readonly key: string; readonly issue: number; readonly action: "opened" | "commented on" };
|
|
44
|
+
|
|
45
|
+
export class Issues extends Context.Service<
|
|
46
|
+
Issues,
|
|
47
|
+
{
|
|
48
|
+
readonly ensureLabel: Effect.Effect<void, GitHubFailure>;
|
|
49
|
+
readonly open: Effect.Effect<readonly OpenIssue[], GitHubFailure>;
|
|
50
|
+
readonly create: (title: string, body: string) => Effect.Effect<number, GitHubFailure>;
|
|
51
|
+
readonly comment: (issue: number, body: string) => Effect.Effect<void, GitHubFailure>;
|
|
52
|
+
}
|
|
53
|
+
>()("@avi2dg/checks/testing/Issues") {
|
|
54
|
+
static readonly gh = Layer.effect(
|
|
55
|
+
Issues,
|
|
56
|
+
Effect.gen(function* () {
|
|
57
|
+
const services = yield* Effect.context<ChildProcessSpawner.ChildProcessSpawner>();
|
|
58
|
+
const provided = <A>(effect: Effect.Effect<A, GitHubFailure, ChildProcessSpawner.ChildProcessSpawner>) => effect.pipe(Effect.provideContext(services));
|
|
59
|
+
return Issues.of({
|
|
60
|
+
ensureLabel: provided(ensureLabel),
|
|
61
|
+
open: provided(openIssues),
|
|
62
|
+
create: (title, body) => provided(gitHubJson(decodeNumbered, "POST", `${REPOSITORY}/issues`, { title, body, labels: [LABEL] })).pipe(Effect.map(({ number }) => number)),
|
|
63
|
+
comment: (issue, body) => provided(gitHubJson(decodeAny, "POST", `${REPOSITORY}/issues/${issue}/comments`, { body })).pipe(Effect.asVoid),
|
|
64
|
+
});
|
|
65
|
+
}),
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const decodeNumbered = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Struct({ number: Schema.Int })));
|
|
70
|
+
const decodeAny = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
|
|
71
|
+
const decodeIssues = Schema.decodeUnknownEffect(
|
|
72
|
+
Schema.fromJsonString(Schema.Array(Schema.Struct({ number: Schema.Int, body: Schema.NullOr(Schema.String), pull_request: Schema.optionalKey(Schema.Unknown) }))),
|
|
73
|
+
);
|
|
74
|
+
const decodeKey = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.String));
|
|
75
|
+
|
|
76
|
+
const MARKER = /<!-- checks-night-run ("(?:[^"\\]|\\.)*") -->/;
|
|
77
|
+
|
|
78
|
+
// The key is JSON with every > escaped, so no key can close the HTML comment early.
|
|
79
|
+
export function marker(key: string): string {
|
|
80
|
+
return `<!-- checks-night-run ${JSON.stringify(key).replaceAll(">", "\\u003e")} -->`;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function markedKey(body: string | null): Option.Option<string> {
|
|
84
|
+
const quoted = body === null ? undefined : MARKER.exec(body)?.[1];
|
|
85
|
+
return quoted === undefined ? Option.none() : decodeKey(quoted);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const ensureLabel = gh(["label", "create", LABEL, "--color", LABEL_COLOR, "--description", LABEL_DESCRIPTION, "--force"]).pipe(Effect.asVoid);
|
|
89
|
+
|
|
90
|
+
const openIssues = Effect.gen(function* () {
|
|
91
|
+
const found: OpenIssue[] = [];
|
|
92
|
+
for (let page = 1; ; page += 1) {
|
|
93
|
+
const batch = yield* gitHubJson(decodeIssues, "GET", `${REPOSITORY}/issues?labels=${LABEL}&state=open&per_page=${PAGE_SIZE}&page=${page}`);
|
|
94
|
+
for (const issue of batch) {
|
|
95
|
+
const key = markedKey(issue.body);
|
|
96
|
+
if (issue.pull_request === undefined && Option.isSome(key)) found.push({ number: issue.number, key: key.value });
|
|
97
|
+
}
|
|
98
|
+
if (batch.length < PAGE_SIZE) return found;
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
function fullTitleOf(finding: Finding): string {
|
|
103
|
+
if (finding.kind === "mutation") return `Night run: mutation score fell in ${finding.file}`;
|
|
104
|
+
if (finding.kind === "test") return `Night run: ${finding.file} > ${finding.name} fails`;
|
|
105
|
+
return `Night run: ${finding.file} crashed its test worker`;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export function titleOf(finding: Finding): string {
|
|
109
|
+
const title = fullTitleOf(finding);
|
|
110
|
+
if (title.length <= GITHUB_TITLE_LIMIT) return title;
|
|
111
|
+
const end = GITHUB_TITLE_LIMIT - 1;
|
|
112
|
+
const splitsPair = /[\uD800-\uDBFF]/.test(title.charAt(end - 1));
|
|
113
|
+
return `${title.slice(0, splitsPair ? end - 1 : end)}…`;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function findingText(finding: Finding, run: string): string {
|
|
117
|
+
if (finding.kind === "mutation") {
|
|
118
|
+
const { file, before, after, since } = finding;
|
|
119
|
+
return [
|
|
120
|
+
`The night run ${run} scored \`${file}\` lower than the night run ${since}.`,
|
|
121
|
+
"",
|
|
122
|
+
"| File | Previous night | Tonight |",
|
|
123
|
+
"| --- | --- | --- |",
|
|
124
|
+
`| \`${file}\` | ${percent(before)}, ${before.detected} of ${before.valid} mutants detected | ${percent(after)}, ${after.detected} of ${after.valid} mutants detected |`,
|
|
125
|
+
].join("\n");
|
|
126
|
+
}
|
|
127
|
+
if (finding.kind === "test") return `The night run ${run} failed \`${finding.name}\` in \`${finding.file}\`.`;
|
|
128
|
+
return `The night run ${run} lost every result of \`${finding.file}\`, because the worker running it crashed.`;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export function issueBody(finding: Finding, run: string): string {
|
|
132
|
+
return `${findingText(finding, run)}\n\nFinding key: \`${keyOf(finding)}\`\n\n${marker(keyOf(finding))}\n`;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Nothing to file means no call at all, so a quiet night never touches the label or the issues.
|
|
136
|
+
export const fileFindings = Effect.fn("fileFindings")(function* (findings: readonly Finding[], run: string) {
|
|
137
|
+
if (findings.length === 0) return [];
|
|
138
|
+
const issues = yield* Issues;
|
|
139
|
+
yield* issues.ensureLabel;
|
|
140
|
+
const open = new Map((yield* issues.open).map((issue) => [issue.key, issue.number]));
|
|
141
|
+
const filed: Filed[] = [];
|
|
142
|
+
for (const finding of findings) {
|
|
143
|
+
const key = keyOf(finding);
|
|
144
|
+
const known = open.get(key);
|
|
145
|
+
if (known === undefined) {
|
|
146
|
+
const issue = yield* issues.create(titleOf(finding), issueBody(finding, run));
|
|
147
|
+
open.set(key, issue);
|
|
148
|
+
filed.push({ key, issue, action: "opened" });
|
|
149
|
+
} else {
|
|
150
|
+
yield* issues.comment(known, findingText(finding, run));
|
|
151
|
+
filed.push({ key, issue: known, action: "commented on" });
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return filed;
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
const required = (name: string) =>
|
|
158
|
+
Config.String(name).pipe(Effect.mapError(() => new NightRunFailure({ message: `${name} is unset, so ${NAME} runs only inside a GitHub Actions job` })));
|
|
159
|
+
|
|
160
|
+
const WORKFLOW_FILE = /\/\.github\/workflows\/([^/@]+)@/;
|
|
161
|
+
|
|
162
|
+
type Actions = { readonly runs: string; readonly run: string; readonly workflow: string; readonly ref: string };
|
|
163
|
+
|
|
164
|
+
const actionsRun = Effect.gen(function* () {
|
|
165
|
+
const runs = `${yield* required("GITHUB_SERVER_URL")}/${yield* required("GITHUB_REPOSITORY")}/actions/runs`;
|
|
166
|
+
const reference = yield* required("GITHUB_WORKFLOW_REF");
|
|
167
|
+
const workflow = WORKFLOW_FILE.exec(reference)?.[1];
|
|
168
|
+
if (workflow === undefined) return yield* new NightRunFailure({ message: `GITHUB_WORKFLOW_REF ${reference} names no workflow file` });
|
|
169
|
+
return { runs, run: `${runs}/${yield* required("GITHUB_RUN_ID")}`, workflow, ref: yield* required("GITHUB_REF") } satisfies Actions;
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
const decodeRuns = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(Schema.Struct({ databaseId: Schema.Int }))));
|
|
173
|
+
const decodeArtifacts = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Struct({ artifacts: Schema.Array(Schema.Struct({ name: Schema.String, expired: Schema.Boolean })) })));
|
|
174
|
+
const decodeRecord = Schema.decodeUnknownEffect(Schema.fromJsonString(NightRecordSchema));
|
|
175
|
+
|
|
176
|
+
const unreadable = (source: string) => (cause: { readonly message: string }) => new NightRunFailure({ message: `cannot read ${source}: ${cause.message}` });
|
|
177
|
+
|
|
178
|
+
// Only the newest successful run on main counts, so a branch dispatch never becomes the night a later run compares with.
|
|
179
|
+
const previousNight = Effect.fn("previousNight")(
|
|
180
|
+
function* ({ runs, workflow }: Actions) {
|
|
181
|
+
const listed = ["run", "list", "--workflow", workflow, "--branch", DEFAULT_BRANCH, "--status", "success", "--limit", "1", "--json", "databaseId,event,createdAt"];
|
|
182
|
+
const [newest] = yield* gh(listed).pipe(Effect.flatMap(decodeRuns), Effect.mapError(unreadable(`the runs of ${workflow}`)));
|
|
183
|
+
if (newest === undefined) return Option.none<PreviousNight>();
|
|
184
|
+
const run = newest.databaseId;
|
|
185
|
+
const { artifacts } = yield* gh(["api", `${REPOSITORY}/actions/runs/${run}/artifacts?name=${ARTIFACT}`]).pipe(
|
|
186
|
+
Effect.flatMap(decodeArtifacts),
|
|
187
|
+
Effect.mapError(unreadable(`the artifacts of run ${run}`)),
|
|
188
|
+
);
|
|
189
|
+
if (!artifacts.some((artifact) => artifact.name === ARTIFACT && !artifact.expired)) return Option.none<PreviousNight>();
|
|
190
|
+
const fs = yield* FileSystem.FileSystem;
|
|
191
|
+
const dir = yield* fs.makeTempDirectoryScoped({ prefix: "checks-night-run-" });
|
|
192
|
+
yield* gh(["run", "download", String(run), "--name", ARTIFACT, "--dir", dir]);
|
|
193
|
+
const file = (yield* Path.Path).join(dir, RECORD_NAME);
|
|
194
|
+
const record = yield* fs.readFileString(file).pipe(Effect.flatMap(decodeRecord), Effect.mapError(unreadable(`the ${ARTIFACT} artifact of run ${run}`)));
|
|
195
|
+
return Option.some<PreviousNight>({ url: `${runs}/${run}`, record });
|
|
196
|
+
},
|
|
197
|
+
Effect.scoped,
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
const readText = Effect.fn("readText")(function* (file: string) {
|
|
201
|
+
return yield* (yield* FileSystem.FileSystem).readFileString(file).pipe(Effect.mapError(unreadable(file)));
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
const testReport = Effect.fn("testReport")(function* (file: string) {
|
|
205
|
+
return yield* parseTestReport(yield* readText(file)).pipe(Effect.mapError(unreadable(file)));
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
const writeRecord = Effect.fn("writeRecord")(function* (record: NightRecord) {
|
|
209
|
+
const fs = yield* FileSystem.FileSystem;
|
|
210
|
+
yield* fs.makeDirectory((yield* Path.Path).dirname(RECORD), { recursive: true });
|
|
211
|
+
yield* fs.writeFileString(RECORD, `${JSON.stringify(record, null, 2)}\n`);
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
const tonightScores = Effect.fn("tonightScores")(function* (file: string) {
|
|
215
|
+
return scoresOf(yield* parseMutationReport(file, yield* readText(file)));
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
type Filing = { readonly onMain: true; readonly filed: readonly Filed[] } | { readonly onMain: false; readonly ref: string };
|
|
219
|
+
|
|
220
|
+
export function report(run: string, scored: Option.Option<number>, previous: Option.Option<PreviousNight>, findings: readonly Finding[], filing: Filing): string {
|
|
221
|
+
const against = Option.isNone(scored)
|
|
222
|
+
? "had no mutation report, so it compared no score"
|
|
223
|
+
: Option.isSome(previous)
|
|
224
|
+
? `compared with the night run ${previous.value.url}`
|
|
225
|
+
: `found no earlier successful run on ${DEFAULT_BRANCH} holding a ${ARTIFACT} artifact, so it compared no score`;
|
|
226
|
+
const verdict = findings.length === 0 ? "no finding" : `${findings.length} finding(s)`;
|
|
227
|
+
const listed = filing.onMain ? filing.filed.map((one) => ` ${one.action} #${one.issue} for ${one.key}`) : findings.map((finding) => ` found ${keyOf(finding)}`);
|
|
228
|
+
const recorded = !filing.onMain
|
|
229
|
+
? `${NAME}: the run is on ${filing.ref}, not ${DEFAULT_BRANCH}, so it filed no issue and recorded no score`
|
|
230
|
+
: Option.isSome(scored)
|
|
231
|
+
? `${NAME}: recorded ${scored.value} file score(s) in ${RECORD}`
|
|
232
|
+
: `${NAME}: recorded no score, since it had no mutation report`;
|
|
233
|
+
return [`${NAME}: the night run ${run} ${against}; ${verdict}`, ...listed, recorded].join("\n");
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const nightRun = Effect.gen(function* () {
|
|
237
|
+
const options = yield* parseArgs(process.argv.slice(2));
|
|
238
|
+
const actions = yield* actionsRun;
|
|
239
|
+
const scores = Option.isSome(options.mutation) ? Option.some(yield* tonightScores(options.mutation.value)) : Option.none<NightRecord["scores"]>();
|
|
240
|
+
const tests: TestReport[] = yield* Effect.forEach(options.tests, testReport);
|
|
241
|
+
const previous = Option.isSome(scores) ? yield* previousNight(actions) : Option.none<PreviousNight>();
|
|
242
|
+
const findings = findingsOf(previous, { scores: Option.getOrElse(scores, () => ({})), tests });
|
|
243
|
+
const filing: Filing = actions.ref === `refs/heads/${DEFAULT_BRANCH}` ? { onMain: true, filed: yield* fileFindings(findings, actions.run) } : { onMain: false, ref: actions.ref };
|
|
244
|
+
if (filing.onMain && Option.isSome(scores)) yield* writeRecord({ scores: scores.value });
|
|
245
|
+
yield* Console.log(report(actions.run, Option.map(scores, (tonight) => Object.keys(tonight).length), previous, findings, filing));
|
|
246
|
+
return true;
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
export function main(issues: Layer.Layer<Issues, never, ChildProcessSpawner.ChildProcessSpawner>): void {
|
|
250
|
+
runMain(NAME, nightRun.pipe(Effect.provide(issues)));
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
if (import.meta.main) main(Issues.gh);
|
package/src/testing/test.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
|
-
import { Config, Console, Effect, FileSystem, Path, Schema } from "effect";
|
|
2
|
+
import { Config, Console, Effect, FileSystem, Option, Path, Schema } from "effect";
|
|
3
3
|
import { ChildProcess, ChildProcessSpawner } from "effect/process";
|
|
4
4
|
import { TEST_ENTRY_POINT } from "../core/gates.ts";
|
|
5
5
|
import { runMain, Usage } from "../core/main.ts";
|
|
@@ -26,7 +26,7 @@ export type TestOptions = {
|
|
|
26
26
|
};
|
|
27
27
|
|
|
28
28
|
const NAME = TEST_ENTRY_POINT.bin;
|
|
29
|
-
const USAGE = `usage: ${NAME} [--tier=live | --tier=pixel | --tier=unit | --tier=e2e] [--parallel=<workers>], and it refuses test filters`;
|
|
29
|
+
const USAGE = `usage: ${NAME} [--tier=live | --tier=pixel | --tier=unit | --tier=e2e] [--parallel=<workers>] [--junit=<file>], and it refuses test filters`;
|
|
30
30
|
const TIERS = new Map<string, TestTier>([
|
|
31
31
|
["--tier=live", "live"],
|
|
32
32
|
["--tier=pixel", "pixel"],
|
|
@@ -34,6 +34,7 @@ const TIERS = new Map<string, TestTier>([
|
|
|
34
34
|
["--tier=e2e", "e2e"],
|
|
35
35
|
]);
|
|
36
36
|
const PARALLEL = /^--parallel=([1-9][0-9]*)$/;
|
|
37
|
+
const JUNIT = /^--junit=(.+)$/;
|
|
37
38
|
|
|
38
39
|
function optionOf(arg: string): TestOptions | undefined {
|
|
39
40
|
const tier = TIERS.get(arg);
|
|
@@ -52,6 +53,16 @@ export function testOptions(args: readonly string[]): TestOptions | undefined {
|
|
|
52
53
|
return options;
|
|
53
54
|
}
|
|
54
55
|
|
|
56
|
+
export type EntryOptions = { readonly options: TestOptions; readonly junit: Option.Option<string> };
|
|
57
|
+
|
|
58
|
+
// Kept out of testOptions, so checks-test-layout still holds a package.json test script to --parallel alone.
|
|
59
|
+
export function entryOptions(args: readonly string[]): EntryOptions | undefined {
|
|
60
|
+
const junit = args.flatMap((arg) => JUNIT.exec(arg)?.[1] ?? []);
|
|
61
|
+
const options = testOptions(args.filter((arg) => !JUNIT.test(arg)));
|
|
62
|
+
if (options === undefined || junit.length > 1) return undefined;
|
|
63
|
+
return { options, junit: Option.fromNullishOr(junit[0]) };
|
|
64
|
+
}
|
|
65
|
+
|
|
55
66
|
const TEST_SCRIPTS: readonly string[] = [NAME, `bun src/${TEST_ENTRY_POINT.script}`];
|
|
56
67
|
|
|
57
68
|
export function scriptOptions(script: unknown): TestOptions | undefined {
|
|
@@ -169,15 +180,22 @@ const runSuite = Effect.fn("runSuite")(function* (outfile: string, { tier, worke
|
|
|
169
180
|
});
|
|
170
181
|
|
|
171
182
|
const testEntry = Effect.gen(function* () {
|
|
172
|
-
const
|
|
173
|
-
if (
|
|
183
|
+
const entry = entryOptions(process.argv.slice(2));
|
|
184
|
+
if (entry === undefined) return yield* new Usage({ message: USAGE });
|
|
185
|
+
const { options, junit } = entry;
|
|
174
186
|
const fs = yield* FileSystem.FileSystem;
|
|
187
|
+
const path = yield* Path.Path;
|
|
175
188
|
const declarations = yield* readSkipDeclarations(process.cwd(), options.tier);
|
|
176
189
|
const environment = yield* environmentOf;
|
|
177
190
|
|
|
178
|
-
const outfile =
|
|
191
|
+
const outfile = path.join(yield* fs.makeTempDirectoryScoped({ prefix: "checks-test-" }), "junit.xml");
|
|
179
192
|
const suitePassed = (yield* runSuite(outfile, options)) === ChildProcessSpawner.ExitCode(0);
|
|
180
|
-
|
|
193
|
+
const written = yield* fs.exists(outfile);
|
|
194
|
+
if (written && Option.isSome(junit)) {
|
|
195
|
+
yield* fs.makeDirectory(path.dirname(junit.value), { recursive: true });
|
|
196
|
+
yield* fs.copyFile(outfile, junit.value);
|
|
197
|
+
}
|
|
198
|
+
if (!written) {
|
|
181
199
|
if (!suitePassed) {
|
|
182
200
|
yield* Console.error(`${NAME}: bun test failed before it wrote a report, so no skip was judged`);
|
|
183
201
|
return false;
|