@avi2dg/checks 0.70.0 → 0.71.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/docs/bins.md +6 -6
- package/docs/configs/native-settings.md +1 -1
- package/docs/design.md +2 -2
- package/docs/gates/checks-browser.md +2 -2
- package/docs/gates/checks-ci-wiring.md +3 -22
- package/docs/gates/checks-frontend-syntax.md +2 -2
- package/docs/gates/checks-lint-coverage.md +1 -1
- package/docs/gates/checks-lint.md +5 -5
- package/docs/gates/checks-mutation-compare.md +2 -2
- package/docs/gates/checks-quarantine-clock.md +2 -2
- package/docs/gates/checks-repetition.md +3 -3
- package/docs/gates/checks-setup.md +1 -4
- package/docs/gates/checks-suppressions-ratchet.md +3 -3
- package/docs/gates/checks-test.md +3 -3
- package/docs/gates/checks-whole-tree-run.md +281 -0
- package/docs/how-to/set-up-a-repository.md +3 -3
- package/docs/how-to/upgrade-the-kit.md +5 -2
- package/package.json +4 -5
- package/src/core/gates.ts +7 -7
- package/src/core/lint.ts +12 -13
- package/src/delivery/ci-wiring.ts +2 -7
- package/src/delivery/setup-workflow.ts +0 -25
- package/src/delivery/setup.ts +1 -2
- package/src/delivery/shell-command.ts +0 -4
- package/src/testing/{night-run-findings.ts → whole-tree-run-findings.ts} +10 -10
- package/src/testing/{night-run.ts → whole-tree-run.ts} +44 -44
- package/docs/gates/checks-night-run.md +0 -257
- package/src/delivery/night-step.ts +0 -79
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
---
|
|
2
|
+
kind: reference
|
|
3
|
+
audience: consumers
|
|
4
|
+
---
|
|
5
|
+
# checks-whole-tree-run
|
|
6
|
+
|
|
7
|
+
`checks-whole-tree-run` turns each finding of a whole-tree run on `main` into a GitHub issue.
|
|
8
|
+
A finding is a file whose mutation score fell, a failing test, a failed whole-tree gate or a missing mutation report.
|
|
9
|
+
It closes the issue of a test, gate or missing mutation report that a later run no longer reports, and it comments on a mutation issue it no longer reports without closing it, but only when a previous run exists and every requested mutation report is present.
|
|
10
|
+
|
|
11
|
+
## What it checks
|
|
12
|
+
|
|
13
|
+
It scores each file of this run's Stryker reports as Stryker does, the `Killed` and `Timeout` mutants over every mutant but the `Ignored`, `CompileError`, `RuntimeError` and `Pending` ones.
|
|
14
|
+
It compares each file's score with the same file's score from the previous run.
|
|
15
|
+
A file whose score is lower than the previous run's is a finding.
|
|
16
|
+
A file only one run holds is no finding, since it lacks one of the two scores.
|
|
17
|
+
A file with no scored mutant has no score, so it is no finding either.
|
|
18
|
+
An equal score is no finding, even when it counts other mutants.
|
|
19
|
+
Each failing test in a JUnit report is a finding, whatever the previous run holds.
|
|
20
|
+
Each whole-tree gate the lint log names as failed is a finding.
|
|
21
|
+
A log where every named gate is undecided files `checks-lint` itself, with no per-gate issue.
|
|
22
|
+
A log with no passing run and no named gate files `checks-lint` itself, so a gate run that never finished still reaches an issue.
|
|
23
|
+
A file whose test worker crashed is one finding, since Bun reports none of its tests.
|
|
24
|
+
With no previous run it compares no score and still files each failing test.
|
|
25
|
+
Each mutation report `--mutation` names that does not exist is a finding, since the mutation job that writes it failed, timed out or was cancelled.
|
|
26
|
+
With no mutation report it compares and records no score and still files each failing test.
|
|
27
|
+
With some mutation reports missing it compares the files the others hold and records no score.
|
|
28
|
+
The next run then compares every file with the last run that scored them all.
|
|
29
|
+
|
|
30
|
+
Each finding has a key that stays the same from run to run:
|
|
31
|
+
|
|
32
|
+
| Finding | Key |
|
|
33
|
+
| --- | --- |
|
|
34
|
+
| a file whose mutation score fell | `mutation:<file>` |
|
|
35
|
+
| a failing test | `test:<file> > <describe> > <name>` |
|
|
36
|
+
| a file whose test worker crashed | `test:<file>` |
|
|
37
|
+
| a whole-tree gate that failed | `gates:<gate>` |
|
|
38
|
+
| a missing mutation report | `mutation-report:<path>` |
|
|
39
|
+
|
|
40
|
+
## How it files a finding
|
|
41
|
+
|
|
42
|
+
It creates the `whole-tree-run` label when the repository lacks it.
|
|
43
|
+
It reads every open issue labelled `whole-tree-run`, and reads the key each one carries in a hidden marker in its body.
|
|
44
|
+
A finding whose key an open issue carries gets one comment on that issue, so a regression that lasts two runs stays one issue.
|
|
45
|
+
Any other finding opens one issue labelled `whole-tree-run`.
|
|
46
|
+
The issue and the comment name this run, the file and the test, the score of both runs or the failed gate.
|
|
47
|
+
The body holds the file, the full test name or the failed gate, and the finding key.
|
|
48
|
+
A title longer than GitHub's 256 characters is cut to at most 256 and ends in `…`.
|
|
49
|
+
A run with no finding creates no label and writes no issue, and only reads the open issues to close any test, gate or mutation-report issue whose finding is gone and, when a previous run exists and every requested mutation report is present, to comment on any mutation issue whose finding is gone.
|
|
50
|
+
Closing an issue ends it, and the same finding on a later run opens a new one.
|
|
51
|
+
|
|
52
|
+
## How it closes an issue
|
|
53
|
+
|
|
54
|
+
An open test, gate or mutation-report issue whose key this run's findings lack gets one comment naming this run and the key.
|
|
55
|
+
It then closes as completed.
|
|
56
|
+
When a previous run exists and every requested mutation report is present, an open mutation issue whose key this run's findings lack gets one comment that names this run and the key, and stays open.
|
|
57
|
+
A closed issue is no longer read, so a later run without the finding comments on it no more.
|
|
58
|
+
A comment that reports the finding carries a reported marker, and a note that a mutation finding is gone carries a noted marker.
|
|
59
|
+
A mutation issue whose latest marked comment already notes the finding gone gets no further note.
|
|
60
|
+
So a mutation finding gone for many runs is noted once, and noted again only after a later run reports it again.
|
|
61
|
+
A person's comment carries no marker, so it never makes the run note an issue twice.
|
|
62
|
+
It judges a key only from a report it read this run:
|
|
63
|
+
|
|
64
|
+
| Key | Closes when |
|
|
65
|
+
| --- | --- |
|
|
66
|
+
| `test:<file> > <describe> > <name>` | this run read a JUnit report, the test is not failing, and `<file>`'s worker did not crash |
|
|
67
|
+
| `test:<file>` | this run read a JUnit report and `<file>`'s worker did not crash |
|
|
68
|
+
| `gates:<gate>` | this run read a lint log that passes or names its failed gates, and `<gate>` is not among them |
|
|
69
|
+
| `mutation-report:<path>` | this run read the mutation report at `<path>` |
|
|
70
|
+
| `mutation:<file>` | never |
|
|
71
|
+
|
|
72
|
+
A crashed worker loses every result of its file, so the issue of a test in that file stays open.
|
|
73
|
+
A lint log that is undecided or never finished says nothing of any one gate, so every gate issue stays open.
|
|
74
|
+
A mutation finding is a fall from one run to the next.
|
|
75
|
+
A later run that does not fall again still holds the lower score.
|
|
76
|
+
Only a run that compared scores with every requested mutation report present comments on the mutation issue without closing it, since a score can recover and drop again.
|
|
77
|
+
So a person closes a mutation issue once the score is back.
|
|
78
|
+
An issue with no key in its body is never read, so a person's issue under the `whole-tree-run` label is never commented on or closed.
|
|
79
|
+
A run off `main` comments on and closes no issue.
|
|
80
|
+
Agents and people fix each issue as a task of its own, so no pull request waits for a whole-tree run.
|
|
81
|
+
A run on any ref but `refs/heads/main` files no issue and records no score, and lists each finding in its log only.
|
|
82
|
+
|
|
83
|
+
## What it reads
|
|
84
|
+
|
|
85
|
+
It reads this run's Stryker `mutation.json` reports and any number of JUnit reports, such as the ones `checks-test --junit=<file>` keeps, as [checks-test](checks-test.md#arguments) says.
|
|
86
|
+
It reads the lint log `--gates` names for the `gate(s) failed:` line `checks-lint --whole-tree` prints.
|
|
87
|
+
It scans the newest 30 successful runs of the same workflow on `main` from newest to oldest.
|
|
88
|
+
It takes the first run whose `whole-tree-run` artifact still exists as the previous run.
|
|
89
|
+
That artifact holds `whole-tree-run.json`, the scores the run recorded.
|
|
90
|
+
When no scanned run holds the artifact it compares no score.
|
|
91
|
+
A run on another branch never becomes the previous run.
|
|
92
|
+
On `main` with every mutation report present it writes this run's scores to `reports/whole-tree-run.json`, and the workflow uploads that file as the `whole-tree-run` artifact.
|
|
93
|
+
It reads the run from `GITHUB_SERVER_URL`, `GITHUB_REPOSITORY`, `GITHUB_RUN_ID`, `GITHUB_WORKFLOW_REF` and `GITHUB_REF`, so it runs only inside a GitHub Actions job.
|
|
94
|
+
It calls `gh` with the job's `GH_TOKEN`, which needs `actions: read` for the previous run and `issues: write` for the issues.
|
|
95
|
+
|
|
96
|
+
## Arguments
|
|
97
|
+
|
|
98
|
+
```sh
|
|
99
|
+
checks-whole-tree-run [--mutation <mutation-report>]... [--tests <junit-report>]... [--gates <lint-log>]
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
`--mutation` names one of this run's Stryker reports and is given once per report the workflow's mutation jobs write, whether or not each job wrote it.
|
|
103
|
+
`--tests` names one JUnit report and is given once per report.
|
|
104
|
+
`--gates` names this run's `checks-lint --whole-tree` log and is given at most once.
|
|
105
|
+
At least one report is given.
|
|
106
|
+
|
|
107
|
+
## Exit codes
|
|
108
|
+
|
|
109
|
+
| Code | When |
|
|
110
|
+
| --- | --- |
|
|
111
|
+
| 0 | it filed every finding, closed each issue whose finding is gone and recorded this run's scores, it found nothing to file, or the run is off `main` and it listed each finding |
|
|
112
|
+
| 2 | an argument does not parse, a report or the lint log does not decode, a GitHub Actions variable is unset or a `gh` call fails |
|
|
113
|
+
|
|
114
|
+
It never exits 1, because a finding becomes an issue rather than a failed run.
|
|
115
|
+
|
|
116
|
+
## Sample output
|
|
117
|
+
|
|
118
|
+
```
|
|
119
|
+
checks-whole-tree-run: the whole-tree run https://github.com/acme/widget/actions/runs/12 compared with the whole-tree run https://github.com/acme/widget/actions/runs/11; 2 finding(s)
|
|
120
|
+
opened #31 for mutation:src/cart.ts
|
|
121
|
+
commented on #27 for test:tests/unit/cart.test.ts > totals > rounds half to even
|
|
122
|
+
closed #24 for gates:checks-repetition
|
|
123
|
+
checks-whole-tree-run: recorded 42 file score(s) in reports/whole-tree-run.json
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
## When it runs
|
|
127
|
+
|
|
128
|
+
Only a whole-tree workflow the repository writes runs it, after that workflow's full mutation run, its long test tiers and its whole-tree lint gates.
|
|
129
|
+
|
|
130
|
+
## Coverage that stops running
|
|
131
|
+
|
|
132
|
+
Neither release nor CI runs a full mutation run, the four whole-tree gates or the issue filing in any repository below.
|
|
133
|
+
Each answer comes from the workflows under `.github/workflows` of that repository.
|
|
134
|
+
|
|
135
|
+
- checks keeps none of the three on a workflow.
|
|
136
|
+
Mutation testing runs only by hand through the mutation workflow baseline.
|
|
137
|
+
The four whole-tree gates run only by hand through `checks-lint --whole-tree`.
|
|
138
|
+
Nothing files issues until someone adds a `whole-tree-run` workflow.
|
|
139
|
+
- skills runs all three by hand through its `night-run` workflow.
|
|
140
|
+
That workflow runs a full Stryker mutation run, the four gates and the unit and e2e tiers with issue filing.
|
|
141
|
+
- dotfiles runs mutation testing on pull requests as an advisory scoped compare, and the full baseline by hand, both through `mutation.yml`.
|
|
142
|
+
The four gates run only by hand through its `night-run` workflow.
|
|
143
|
+
Nothing files issues.
|
|
144
|
+
- career runs the four gates only by hand through its `night-run` workflow.
|
|
145
|
+
No mutation testing and no issue filing run anywhere.
|
|
146
|
+
- home-vpn runs the four gates only by hand through its `night-run` workflow.
|
|
147
|
+
No mutation testing and no issue filing run anywhere.
|
|
148
|
+
- orc runs mutation testing only by hand through its mutation workflow baseline.
|
|
149
|
+
The four gates run only by hand through its `night-run` workflow.
|
|
150
|
+
Nothing files issues.
|
|
151
|
+
- shopping runs the four gates only by hand through its `night-run` workflow.
|
|
152
|
+
No mutation testing and no issue filing run anywhere.
|
|
153
|
+
|
|
154
|
+
## Running it by hand
|
|
155
|
+
|
|
156
|
+
A repository runs it from a workflow named `whole-tree-run`, which someone starts with `gh workflow run whole-tree-run`.
|
|
157
|
+
The workflow holds no schedule, since [checks-ci-wiring](checks-ci-wiring.md#schedules) refuses one:
|
|
158
|
+
|
|
159
|
+
```yaml
|
|
160
|
+
name: whole-tree-run
|
|
161
|
+
on:
|
|
162
|
+
workflow_dispatch:
|
|
163
|
+
permissions:
|
|
164
|
+
contents: read
|
|
165
|
+
concurrency:
|
|
166
|
+
group: whole-tree-run
|
|
167
|
+
cancel-in-progress: false
|
|
168
|
+
jobs:
|
|
169
|
+
mutation:
|
|
170
|
+
runs-on: ubuntu-latest
|
|
171
|
+
timeout-minutes: 60
|
|
172
|
+
steps:
|
|
173
|
+
- uses: actions/checkout@v5
|
|
174
|
+
- uses: oven-sh/setup-bun@v2
|
|
175
|
+
with:
|
|
176
|
+
bun-version-file: .bun-version
|
|
177
|
+
- run: bun install --frozen-lockfile
|
|
178
|
+
- run: bunx stryker run
|
|
179
|
+
- uses: actions/upload-artifact@v4
|
|
180
|
+
with:
|
|
181
|
+
name: whole-tree-run-mutation
|
|
182
|
+
path: reports/mutation/mutation.json
|
|
183
|
+
retention-days: 1
|
|
184
|
+
tests:
|
|
185
|
+
runs-on: ubuntu-latest
|
|
186
|
+
timeout-minutes: 20
|
|
187
|
+
steps:
|
|
188
|
+
- uses: actions/checkout@v5
|
|
189
|
+
- uses: oven-sh/setup-bun@v2
|
|
190
|
+
with:
|
|
191
|
+
bun-version-file: .bun-version
|
|
192
|
+
- run: bun install --frozen-lockfile
|
|
193
|
+
- run: |
|
|
194
|
+
bunx checks-test --tier=live --junit=reports/whole-tree-run-tests/live.xml || true
|
|
195
|
+
bunx checks-test --tier=pixel --junit=reports/whole-tree-run-tests/pixel.xml || true
|
|
196
|
+
- uses: actions/upload-artifact@v4
|
|
197
|
+
with:
|
|
198
|
+
name: whole-tree-run-tests
|
|
199
|
+
path: reports/whole-tree-run-tests/
|
|
200
|
+
retention-days: 1
|
|
201
|
+
gates:
|
|
202
|
+
runs-on: ubuntu-latest
|
|
203
|
+
timeout-minutes: 20
|
|
204
|
+
steps:
|
|
205
|
+
- uses: actions/checkout@v5
|
|
206
|
+
with:
|
|
207
|
+
fetch-depth: 0
|
|
208
|
+
- uses: oven-sh/setup-bun@v2
|
|
209
|
+
with:
|
|
210
|
+
bun-version-file: .bun-version
|
|
211
|
+
- run: bun install --frozen-lockfile
|
|
212
|
+
- run: mkdir -p reports/whole-tree-run-gates && bunx checks-lint --whole-tree > reports/whole-tree-run-gates/gates.log 2>&1 || true
|
|
213
|
+
- uses: actions/upload-artifact@v4
|
|
214
|
+
with:
|
|
215
|
+
name: whole-tree-run-gates
|
|
216
|
+
path: reports/whole-tree-run-gates/
|
|
217
|
+
retention-days: 1
|
|
218
|
+
report:
|
|
219
|
+
needs: [mutation, gates, tests]
|
|
220
|
+
if: ${{ !cancelled() && needs.tests.result == 'success' }}
|
|
221
|
+
runs-on: ubuntu-latest
|
|
222
|
+
timeout-minutes: 15
|
|
223
|
+
permissions:
|
|
224
|
+
contents: read
|
|
225
|
+
actions: read
|
|
226
|
+
issues: write
|
|
227
|
+
steps:
|
|
228
|
+
- uses: actions/checkout@v5
|
|
229
|
+
- uses: oven-sh/setup-bun@v2
|
|
230
|
+
with:
|
|
231
|
+
bun-version-file: .bun-version
|
|
232
|
+
- run: bun install --frozen-lockfile
|
|
233
|
+
- uses: actions/download-artifact@v4
|
|
234
|
+
with:
|
|
235
|
+
pattern: whole-tree-run-mutation
|
|
236
|
+
path: reports/mutation
|
|
237
|
+
- if: needs.gates.result == 'success'
|
|
238
|
+
uses: actions/download-artifact@v4
|
|
239
|
+
with:
|
|
240
|
+
name: whole-tree-run-gates
|
|
241
|
+
path: reports/whole-tree-run-gates
|
|
242
|
+
- uses: actions/download-artifact@v4
|
|
243
|
+
with:
|
|
244
|
+
name: whole-tree-run-tests
|
|
245
|
+
path: reports/whole-tree-run-tests
|
|
246
|
+
- env:
|
|
247
|
+
GH_TOKEN: ${{ github.token }}
|
|
248
|
+
run: bunx checks-whole-tree-run --mutation reports/mutation/whole-tree-run-mutation/mutation.json --tests reports/whole-tree-run-tests/live.xml --tests reports/whole-tree-run-tests/pixel.xml ${{ needs.gates.result == 'success' && '--gates reports/whole-tree-run-gates/gates.log' || '' }}
|
|
249
|
+
- uses: actions/upload-artifact@v4
|
|
250
|
+
with:
|
|
251
|
+
name: whole-tree-run
|
|
252
|
+
path: reports/whole-tree-run.json
|
|
253
|
+
if-no-files-found: ignore
|
|
254
|
+
retention-days: 90
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
The tests job runs each long tier the repository holds, `live` and `pixel` as [checks-test](checks-test.md#test-tiers) names them, and keeps each JUnit report.
|
|
258
|
+
Each `|| true` lets the job go on to the upload after a failing test, since that test becomes an issue.
|
|
259
|
+
A tier that writes no report fails the report job, because `checks-whole-tree-run` cannot read it.
|
|
260
|
+
The gates job runs the whole-tree lint gates and keeps the log, since a failing gate becomes an issue.
|
|
261
|
+
The report job runs whenever the tests job succeeds, so a failing test that also stops Stryker still becomes an issue.
|
|
262
|
+
It always passes `--mutation`, so a mutation job that failed or timed out leaves a missing report that becomes an issue.
|
|
263
|
+
The download names the artifact by `pattern`, which downloads nothing rather than failing when the artifact is missing.
|
|
264
|
+
A full run too long for one job splits into a matrix of shards.
|
|
265
|
+
Each shard uploads its own artifact under the one pattern, and each report gets its own `--mutation`.
|
|
266
|
+
It passes `--gates` only when the gates job succeeded, and the failed gates job still fails the run.
|
|
267
|
+
The concurrency group queues a run behind one already going, so two runs never file the same finding twice.
|
|
268
|
+
A dispatch on a branch runs both jobs and lists its findings in the report job's log, and `if-no-files-found: ignore` lets that run upload no record.
|
|
269
|
+
`issues: write` on the report job is the one permission beyond reading the repository and its runs.
|
|
270
|
+
A public repository keeps `runs-on: ubuntu-latest` on every job.
|
|
271
|
+
A private repository sets `runs-on` on the mutation job to the labels declared under `checks.mutationRunner` in `package.json`, and [checks-ci-wiring](checks-ci-wiring.md#runners) refuses a mutation job that names `CI_RUNS_ON`.
|
|
272
|
+
A private repository sets `runs-on: ${{ vars.CI_RUNS_ON || 'ubuntu-latest' }}` on the tests, gates, and report jobs.
|
|
273
|
+
A successful run with every mutation report present uploads `whole-tree-run`, so the next run compares with it, and a run that failed or missed a report uploads nothing a later run reads.
|
|
274
|
+
The artifact keeps 90 days, so a gap of runs still leaves a previous run to compare with.
|
|
275
|
+
|
|
276
|
+
## Related topics
|
|
277
|
+
|
|
278
|
+
- [checks-test](checks-test.md)
|
|
279
|
+
- [checks-mutation](checks-mutation.md)
|
|
280
|
+
- [checks-mutation-compare](checks-mutation-compare.md)
|
|
281
|
+
- [checks-ci-wiring](checks-ci-wiring.md)
|
|
@@ -147,10 +147,10 @@ The same job reports the title step's outcome under that name on every run, whic
|
|
|
147
147
|
```
|
|
148
148
|
|
|
149
149
|
A private repository sets `runs-on: ${{ vars.CI_RUNS_ON || 'ubuntu-latest' }}` on each job, as [checks-ci-wiring](../gates/checks-ci-wiring.md#runners) requires.
|
|
150
|
-
|
|
150
|
+
To run the four whole-tree gates on demand, a repository adds a whole-tree-run workflow:
|
|
151
151
|
|
|
152
152
|
```yaml
|
|
153
|
-
name:
|
|
153
|
+
name: whole-tree-run
|
|
154
154
|
on:
|
|
155
155
|
workflow_dispatch:
|
|
156
156
|
jobs:
|
|
@@ -165,7 +165,7 @@ Run the four nightly gates from a night-run workflow once the repository tracks
|
|
|
165
165
|
with:
|
|
166
166
|
bun-version-file: .bun-version
|
|
167
167
|
- run: bun install --frozen-lockfile
|
|
168
|
-
- run: bunx checks-lint --
|
|
168
|
+
- run: bunx checks-lint --whole-tree
|
|
169
169
|
```
|
|
170
170
|
|
|
171
171
|
`checks-effect-scope` writes the Effect paths of `oxlint.config.ts` and the kit's severities into `tsconfig.json`, as [The Effect rules](../configs/effect-rules.md#language-service) says.
|
|
@@ -31,8 +31,11 @@ To move a repository to a newer release of the kit:
|
|
|
31
31
|
1. Run `bun run build`, whose `checks-effect-scope` rewrites the Effect override and the kit's severities in `tsconfig.json` when a release changes them.
|
|
32
32
|
A severity key the release drops stays in `tsconfig.json` until the repository removes it.
|
|
33
33
|
1. Copy `node_modules/@avi2dg/checks/bunfig.toml` over `bunfig.toml` again, since `checks-test-layout` compares the copy with the installed preset.
|
|
34
|
-
1. When the repository
|
|
35
|
-
|
|
34
|
+
1. When the repository still holds the workflow the previous kit wrote for the whole-tree gates, delete that workflow.
|
|
35
|
+
Run the gates with `checks-lint --whole-tree` instead.
|
|
36
|
+
[Set up a repository](set-up-a-repository.md) and [checks-whole-tree-run](../gates/checks-whole-tree-run.md#running-it-by-hand) show the current names.
|
|
37
|
+
1. A stale workflow left behind still passes `checks-ci-wiring`, but dispatching it fails, so delete it before dispatching it.
|
|
38
|
+
1. The release notes hold the old-to-new name mapping for the workflow, label, artifacts and report file.
|
|
36
39
|
1. Run `bun run lint`, `bun run typecheck` and `bun run test`.
|
|
37
40
|
|
|
38
41
|
The repository's lockfile pins the kit, so a repository moves only when it runs these steps.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@avi2dg/checks",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.71.0",
|
|
4
4
|
"description": "Deterministic checks shared across a set of TypeScript repositories",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "avi2d",
|
|
@@ -71,10 +71,9 @@
|
|
|
71
71
|
"src/testing/mutation-baseline-full.ts",
|
|
72
72
|
"src/testing/mutation-baseline-saved.ts",
|
|
73
73
|
"src/testing/subsumed-tests.ts",
|
|
74
|
-
"src/testing/
|
|
75
|
-
"src/testing/
|
|
74
|
+
"src/testing/whole-tree-run.ts",
|
|
75
|
+
"src/testing/whole-tree-run-findings.ts",
|
|
76
76
|
"src/delivery/ci-wiring.ts",
|
|
77
|
-
"src/delivery/night-step.ts",
|
|
78
77
|
"src/delivery/shell-command.ts",
|
|
79
78
|
"src/delivery/workflow-references.ts",
|
|
80
79
|
"src/delivery/bun-pin.ts",
|
|
@@ -195,7 +194,7 @@
|
|
|
195
194
|
"checks-mutation-baseline": "src/testing/mutation-baseline.ts",
|
|
196
195
|
"checks-mutation-scope": "src/testing/mutation-scope.ts",
|
|
197
196
|
"checks-subsumed-tests": "src/testing/subsumed-tests.ts",
|
|
198
|
-
"checks-
|
|
197
|
+
"checks-whole-tree-run": "src/testing/whole-tree-run.ts",
|
|
199
198
|
"checks-ci-wiring": "src/delivery/ci-wiring.ts",
|
|
200
199
|
"checks-comment-gate": "src/quality/comment-gate.ts",
|
|
201
200
|
"checks-css-size": "src/quality/css-size.ts",
|
package/src/core/gates.ts
CHANGED
|
@@ -26,7 +26,7 @@ export type KitGate = {
|
|
|
26
26
|
readonly args?: readonly string[];
|
|
27
27
|
readonly env?: Readonly<Record<string, string>>;
|
|
28
28
|
readonly appliesTo: typeof EVERY_REPOSITORY | TrackedContent;
|
|
29
|
-
readonly
|
|
29
|
+
readonly wholeTree?: true;
|
|
30
30
|
};
|
|
31
31
|
|
|
32
32
|
export const ENTRY_POINT: Program = { bin: "checks-lint", script: "core/lint.ts" };
|
|
@@ -52,7 +52,7 @@ const COPY_DECLARATION: TrackedContent = { pathspecs: ["copy.json"], content: "a
|
|
|
52
52
|
export const WHOLE_TREE = "--whole-tree";
|
|
53
53
|
|
|
54
54
|
export const KIT_GATES = [
|
|
55
|
-
{ bin: "checks-lint-coverage", vector: "quality", file: "lint-coverage.sh", reads: "tree", appliesTo: LINTED_SOURCE,
|
|
55
|
+
{ bin: "checks-lint-coverage", vector: "quality", file: "lint-coverage.sh", reads: "tree", appliesTo: LINTED_SOURCE, wholeTree: true },
|
|
56
56
|
{ bin: "checks-test-layout", vector: "testing", file: "test-layout.ts", reads: "tree", appliesTo: TYPESCRIPT_SOURCE },
|
|
57
57
|
{ bin: "checks-commit-identity", vector: "delivery", file: "commit-identity.ts", reads: "range", appliesTo: EVERY_REPOSITORY },
|
|
58
58
|
{ bin: "checks-comment-gate", vector: "quality", file: "comment-gate.ts", reads: "range", appliesTo: EVERY_REPOSITORY },
|
|
@@ -60,14 +60,14 @@ export const KIT_GATES = [
|
|
|
60
60
|
{ bin: "checks-frontend-syntax", vector: "quality", file: "frontend-syntax.ts", reads: "tree", appliesTo: FRONTEND_SYNTAX_DECLARATION },
|
|
61
61
|
{ bin: "checks-css-size", vector: "quality", file: "css-size.ts", reads: "tree", appliesTo: STYLESHEET },
|
|
62
62
|
{ bin: "checks-no-plain-js", vector: "quality", file: "no-plain-js.ts", reads: "tree", appliesTo: TYPESCRIPT_SOURCE },
|
|
63
|
-
{ bin: "checks-suppressions-ratchet", vector: "complexity", file: "suppressions-ratchet.ts", reads: "range", appliesTo: EVERY_REPOSITORY,
|
|
63
|
+
{ bin: "checks-suppressions-ratchet", vector: "complexity", file: "suppressions-ratchet.ts", reads: "range", appliesTo: EVERY_REPOSITORY, wholeTree: true },
|
|
64
64
|
{ bin: "checks-ci-wiring", vector: "delivery", file: "ci-wiring.ts", reads: "tree", appliesTo: EVERY_REPOSITORY },
|
|
65
65
|
{ bin: "checks-secrets", vector: "delivery", file: "secrets.ts", reads: "range", appliesTo: EVERY_REPOSITORY },
|
|
66
66
|
{ bin: "checks-docs", vector: "docs", file: "docs.ts", reads: "range", alsoReads: "every agent file at the head commit", appliesTo: EVERY_REPOSITORY },
|
|
67
67
|
{ bin: "checks-copy", vector: "docs", file: "copy.ts", reads: "tree", appliesTo: COPY_DECLARATION },
|
|
68
|
-
{ bin: "checks-repetition", vector: "complexity", file: "repetition.ts", reads: "range", appliesTo: TYPESCRIPT_SOURCE,
|
|
68
|
+
{ bin: "checks-repetition", vector: "complexity", file: "repetition.ts", reads: "range", appliesTo: TYPESCRIPT_SOURCE, wholeTree: true },
|
|
69
69
|
{ bin: "checks-graphwalk", vector: "complexity", file: "graphwalk.ts", reads: "range", appliesTo: LINTED_SOURCE },
|
|
70
|
-
{ bin: "checks-quarantine-clock", vector: "testing", file: "quarantine-clock.ts", reads: "range", appliesTo: EVERY_REPOSITORY,
|
|
70
|
+
{ bin: "checks-quarantine-clock", vector: "testing", file: "quarantine-clock.ts", reads: "range", appliesTo: EVERY_REPOSITORY, wholeTree: true },
|
|
71
71
|
{
|
|
72
72
|
bin: "checks-advisories",
|
|
73
73
|
vector: "dependencies",
|
|
@@ -78,6 +78,6 @@ export const KIT_GATES = [
|
|
|
78
78
|
}
|
|
79
79
|
] as const satisfies readonly KitGate[];
|
|
80
80
|
|
|
81
|
-
export const
|
|
81
|
+
export const WHOLE_TREE_GATES: readonly KitGate[] = KIT_GATES.filter((gate) => "wholeTree" in gate);
|
|
82
82
|
|
|
83
|
-
export const PR_GATES: readonly KitGate[] = KIT_GATES.filter((gate) => !("
|
|
83
|
+
export const PR_GATES: readonly KitGate[] = KIT_GATES.filter((gate) => !("wholeTree" in gate));
|
package/src/core/lint.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import { Config, Console, Effect, FileSystem, Option, Path, Schema } from "effect";
|
|
3
3
|
import { ChildProcess, ChildProcessSpawner } from "effect/process";
|
|
4
|
-
import { EVERY_REPOSITORY,
|
|
4
|
+
import { EVERY_REPOSITORY, PR_GATES, WHOLE_TREE, WHOLE_TREE_GATES, type KitGate } from "./gates.ts";
|
|
5
5
|
import { changedPaths, defaultBranch, git, pathsOf } from "./git.ts";
|
|
6
6
|
import { runMain, Usage } from "./main.ts";
|
|
7
7
|
import { FAST_TREE_GATES, selectScope } from "./scope.ts";
|
|
@@ -22,8 +22,7 @@ class GatesUndecided extends Schema.TaggedError<GatesUndecided>()("GatesUndecide
|
|
|
22
22
|
}) {}
|
|
23
23
|
|
|
24
24
|
const NAME = "checks-lint";
|
|
25
|
-
const USAGE = "usage: lint.ts [--full | --desk] [<base-ref> <head-ref>] | --
|
|
26
|
-
const NIGHT = "--night";
|
|
25
|
+
const USAGE = "usage: lint.ts [--full | --desk] [<base-ref> <head-ref>] | --whole-tree";
|
|
27
26
|
const PULL_REQUEST_EVENT = "pull_request";
|
|
28
27
|
const SHALLOW_HINT = "a CI checkout needs actions/checkout fetch-depth: 0";
|
|
29
28
|
|
|
@@ -220,34 +219,34 @@ const pullRequestSelection = Effect.fn("pullRequestSelection")(function* (
|
|
|
220
219
|
return yield* scopedGates(range, root);
|
|
221
220
|
});
|
|
222
221
|
|
|
223
|
-
function selectionDetail(
|
|
224
|
-
if (
|
|
222
|
+
function selectionDetail(wholeTree: boolean, mode: LintMode, scopeDetail: string, count: number): string | undefined {
|
|
223
|
+
if (wholeTree) return undefined;
|
|
225
224
|
if (mode === "scoped" && scopeDetail === "") return `${count} gate(s) for this change`;
|
|
226
225
|
return scopeDetail === "" ? undefined : scopeDetail;
|
|
227
226
|
}
|
|
228
227
|
|
|
229
228
|
const lint = Effect.gen(function* () {
|
|
230
229
|
const raw = process.argv.slice(2);
|
|
231
|
-
const
|
|
232
|
-
if (
|
|
233
|
-
const split =
|
|
230
|
+
const wholeTree = raw.includes(WHOLE_TREE);
|
|
231
|
+
if (wholeTree && raw.length > 1) return yield* new Usage({ message: USAGE });
|
|
232
|
+
const split = wholeTree ? { mode: "full" as const, rest: [] as readonly string[] } : splitMode(raw);
|
|
234
233
|
if (split === undefined) return yield* new Usage({ message: USAGE });
|
|
235
234
|
const root = (yield* git(["rev-parse", "--show-toplevel"])).trim();
|
|
236
|
-
const range =
|
|
235
|
+
const range = wholeTree ? yield* wholeTreeRange() : yield* resolveRange(split.rest);
|
|
237
236
|
yield* Console.log(`${NAME}: ${describe(range)} from ${range.source}`);
|
|
238
|
-
const applicable = yield* Effect.filter(
|
|
237
|
+
const applicable = yield* Effect.filter(wholeTree ? WHOLE_TREE_GATES : PR_GATES, (gate) =>
|
|
239
238
|
gate.appliesTo === EVERY_REPOSITORY
|
|
240
239
|
? Effect.succeed(true)
|
|
241
240
|
: git(["ls-files", "--", ...gate.appliesTo.pathspecs], root).pipe(Effect.map((files) => files.trim() !== "")),
|
|
242
241
|
);
|
|
243
|
-
const scoped =
|
|
242
|
+
const scoped = wholeTree
|
|
244
243
|
? { bins: applicable.map((gate) => gate.bin) as readonly string[] | undefined, detail: "" }
|
|
245
244
|
: yield* pullRequestSelection(split, range, root, applicable);
|
|
246
245
|
const bins = scoped.bins;
|
|
247
246
|
const gates = bins === undefined ? applicable : applicable.filter((gate) => bins.includes(gate.bin));
|
|
248
|
-
const detail = selectionDetail(
|
|
247
|
+
const detail = selectionDetail(wholeTree, split.mode, scoped.detail, gates.length);
|
|
249
248
|
if (detail !== undefined) yield* Console.log(`${NAME}: ${detail}`);
|
|
250
|
-
const verdicts = yield* Effect.forEach(gates, (gate) => runGate(gate, range,
|
|
249
|
+
const verdicts = yield* Effect.forEach(gates, (gate) => runGate(gate, range, wholeTree));
|
|
251
250
|
const failed = verdicts.filter((verdict) => verdict.outcome !== "passed");
|
|
252
251
|
if (failed.length === 0) {
|
|
253
252
|
yield* Console.log(`${NAME}: ${gates.length} gate(s) pass`);
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { Config, Console, Effect, FileSystem, Path, Schema } from "effect";
|
|
3
3
|
import { defaultBranch, git } from "../core/git.ts";
|
|
4
4
|
import { runMain } from "../core/main.ts";
|
|
5
|
-
import { ENTRY_POINT, KIT_GATES,
|
|
5
|
+
import { ENTRY_POINT, KIT_GATES, type KitGate } from "../core/gates.ts";
|
|
6
6
|
import { findBunPinFaults, formatBunPinReport, readBunPinPolicy } from "./bun-pin.ts";
|
|
7
7
|
import { findLongJobs, formatLongJobReport } from "./job-timeout.ts";
|
|
8
8
|
import { readScheduleFiles, readShellFiles } from "./mutation-job.ts";
|
|
@@ -10,7 +10,6 @@ import { findScheduledReleases, formatScheduleReport } from "./release-schedule.
|
|
|
10
10
|
import { findRunnerFaults, formatRunnerReport, jobRuns, readRunnerConfig } from "./runner-policy.ts";
|
|
11
11
|
import { findScheduledRuns, formatScheduledRunReport } from "./schedule-policy.ts";
|
|
12
12
|
import { invokes, mentions, plainCommand, type Command } from "./shell-command.ts";
|
|
13
|
-
import { findNightStepFaults, formatNightStepReport } from "./night-step.ts";
|
|
14
13
|
import { findMissingWorkflows, formatMissingReport, isRecord, localUses, names, skippedBy, switchedOff, workflowJobs, type Workflow } from "./workflow-references.ts";
|
|
15
14
|
|
|
16
15
|
export { BunPinError, findBunPinFaults, formatBunPinReport } from "./bun-pin.ts";
|
|
@@ -311,11 +310,7 @@ const wiring = Effect.gen(function* () {
|
|
|
311
310
|
if (scheduledRuns.length > 0) yield* Console.error(formatScheduledRunReport(scheduledRuns));
|
|
312
311
|
const longJobs = findLongJobs(workflows);
|
|
313
312
|
if (longJobs.length > 0) yield* Console.error(formatLongJobReport(longJobs));
|
|
314
|
-
|
|
315
|
-
(yield* git(["ls-files", "--", ...TYPESCRIPT_SOURCE.pathspecs], root)).trim() !== "";
|
|
316
|
-
const nightFaults = findNightStepFaults(tracksTypeScript, workflows, { scripts, files: scheduleFiles });
|
|
317
|
-
if (nightFaults.length > 0) yield* Console.error(formatNightStepReport(nightFaults));
|
|
318
|
-
return [gaps, faults, missing, bunFaults, scheduled, scheduledRuns, longJobs, nightFaults].every((found) => found.length === 0);
|
|
313
|
+
return [gaps, faults, missing, bunFaults, scheduled, scheduledRuns, longJobs].every((found) => found.length === 0);
|
|
319
314
|
});
|
|
320
315
|
|
|
321
316
|
if (import.meta.main) runMain("ci-wiring", wiring);
|
|
@@ -1,31 +1,6 @@
|
|
|
1
1
|
const HOSTED_FALLBACK = "ubuntu-latest";
|
|
2
2
|
const HOSTED_DEFAULT = "${{ vars.CI_RUNS_ON || 'ubuntu-latest' }}";
|
|
3
3
|
|
|
4
|
-
export function nightWorkflow(isPrivate: boolean): string {
|
|
5
|
-
const lines = [
|
|
6
|
-
"name: night-run",
|
|
7
|
-
"",
|
|
8
|
-
"on:",
|
|
9
|
-
" workflow_dispatch:",
|
|
10
|
-
"",
|
|
11
|
-
"jobs:",
|
|
12
|
-
" gates:",
|
|
13
|
-
" runs-on: " + (isPrivate ? HOSTED_DEFAULT : HOSTED_FALLBACK),
|
|
14
|
-
" timeout-minutes: 20",
|
|
15
|
-
" steps:",
|
|
16
|
-
" - uses: actions/checkout@v5",
|
|
17
|
-
" with:",
|
|
18
|
-
" fetch-depth: 0",
|
|
19
|
-
" - uses: oven-sh/setup-bun@v2",
|
|
20
|
-
" with:",
|
|
21
|
-
" bun-version-file: .bun-version",
|
|
22
|
-
" - run: bun install --frozen-lockfile",
|
|
23
|
-
" - run: bunx checks-lint --night",
|
|
24
|
-
"",
|
|
25
|
-
];
|
|
26
|
-
return lines.join("\n");
|
|
27
|
-
}
|
|
28
|
-
|
|
29
4
|
export function ciWorkflow(isPrivate: boolean, hasBuild: boolean): string {
|
|
30
5
|
const lines = [
|
|
31
6
|
"name: ci",
|
package/src/delivery/setup.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import { Console, Effect, FileSystem, Path, Schema } from "effect";
|
|
3
3
|
import { runMain, Usage } from "../core/main.ts";
|
|
4
|
-
import { ciWorkflow
|
|
4
|
+
import { ciWorkflow } from "./setup-workflow.ts";
|
|
5
5
|
import { TEMPLATES } from "../docs/doc-templates.ts";
|
|
6
6
|
import { scriptOptions } from "../testing/test.ts";
|
|
7
7
|
import { withServiceOverrides, type ServiceOverride } from "../quality/effect-scope.ts";
|
|
@@ -349,7 +349,6 @@ const check = Effect.gen(function* () {
|
|
|
349
349
|
yield* setupTsconfig(root, oxlintText);
|
|
350
350
|
yield* setupFile(root, "knip.config.ts", KNIP_CONFIG);
|
|
351
351
|
yield* setupFile(root, ".github/workflows/ci.yml", ciWorkflow(isPrivate, outcome.hasBuild));
|
|
352
|
-
yield* setupFile(root, ".github/workflows/night-run.yml", nightWorkflow(isPrivate));
|
|
353
352
|
for (const kind of Object.keys(TEMPLATES)) {
|
|
354
353
|
yield* setupFile(root, "templates/" + kind + ".md", yield* kitText("dist/templates/" + kind + ".md"));
|
|
355
354
|
}
|
|
@@ -289,10 +289,6 @@ export function shellWords(script: string): readonly (readonly Word[])[] {
|
|
|
289
289
|
return parse.commands.filter((command) => command.length > 0);
|
|
290
290
|
}
|
|
291
291
|
|
|
292
|
-
export function shellCommands(script: string): readonly Command[] {
|
|
293
|
-
return shellWords(script).map(texts);
|
|
294
|
-
}
|
|
295
|
-
|
|
296
292
|
export function withoutOptions(words: Command, operands: ReadonlySet<string>): Command {
|
|
297
293
|
const [first = "", ...rest] = words;
|
|
298
294
|
if (first === "--") return rest;
|
|
@@ -6,15 +6,15 @@ const ScoreSchema = Schema.Struct({ detected: Schema.Int, valid: Schema.Int });
|
|
|
6
6
|
|
|
7
7
|
export type Score = typeof ScoreSchema.Type;
|
|
8
8
|
|
|
9
|
-
export const
|
|
9
|
+
export const WholeTreeRecordSchema = Schema.Struct({ scores: Schema.Record(Schema.String, ScoreSchema) });
|
|
10
10
|
|
|
11
|
-
export type
|
|
11
|
+
export type WholeTreeRecord = typeof WholeTreeRecordSchema.Type;
|
|
12
12
|
|
|
13
13
|
export type TestReport = { readonly results: readonly TestResult[]; readonly crashedFiles: readonly string[] };
|
|
14
14
|
|
|
15
|
-
export type
|
|
15
|
+
export type CurrentRun = { readonly scores: WholeTreeRecord["scores"]; readonly tests: readonly TestReport[] };
|
|
16
16
|
|
|
17
|
-
export type
|
|
17
|
+
export type PreviousRun = { readonly url: string; readonly record: WholeTreeRecord };
|
|
18
18
|
|
|
19
19
|
export type Finding =
|
|
20
20
|
| { readonly kind: "mutation"; readonly file: string; readonly before: Score; readonly after: Score; readonly since: string }
|
|
@@ -35,7 +35,7 @@ function scoreOf(file: ReportFile): Score {
|
|
|
35
35
|
return { detected, valid };
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
export function scoresOf(files: ReadonlyMap<string, ReportFile>):
|
|
38
|
+
export function scoresOf(files: ReadonlyMap<string, ReportFile>): WholeTreeRecord["scores"] {
|
|
39
39
|
return Object.fromEntries([...files].map(([path, file]) => [path, scoreOf(file)] as const).sort(([one], [other]) => one.localeCompare(other)));
|
|
40
40
|
}
|
|
41
41
|
|
|
@@ -44,7 +44,7 @@ function fell(before: Score, after: Score): boolean {
|
|
|
44
44
|
return before.valid > 0 && after.valid > 0 && after.detected * before.valid < before.detected * after.valid;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
// A
|
|
47
|
+
// A run judges a key only from a report it read, since a report it lacks leaves every finding in it absent.
|
|
48
48
|
export type Observed = { readonly tests: boolean; readonly gates: boolean; readonly mutation: boolean; readonly mutationReports: ReadonlySet<string> };
|
|
49
49
|
|
|
50
50
|
export function keyOf(finding: Finding): string {
|
|
@@ -55,7 +55,7 @@ export function keyOf(finding: Finding): string {
|
|
|
55
55
|
return `test:${finding.file}`;
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
function scoreFindings(previous: Option.Option<
|
|
58
|
+
function scoreFindings(previous: Option.Option<PreviousRun>, scores: WholeTreeRecord["scores"]): readonly Finding[] {
|
|
59
59
|
if (Option.isNone(previous)) return [];
|
|
60
60
|
const { url, record } = previous.value;
|
|
61
61
|
return Object.entries(scores).flatMap(([file, after]): readonly Finding[] => {
|
|
@@ -72,8 +72,8 @@ function testFindings(reports: readonly TestReport[]): readonly Finding[] {
|
|
|
72
72
|
return [...new Map(failing.map((finding) => [keyOf(finding), finding])).values()];
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
-
export function findingsOf(previous: Option.Option<
|
|
76
|
-
return [...scoreFindings(previous,
|
|
75
|
+
export function findingsOf(previous: Option.Option<PreviousRun>, current: CurrentRun, others: readonly Finding[] = []): readonly Finding[] {
|
|
76
|
+
return [...scoreFindings(previous, current.scores), ...testFindings(current.tests), ...others].sort((one, other) =>
|
|
77
77
|
keyOf(one).localeCompare(keyOf(other)),
|
|
78
78
|
);
|
|
79
79
|
}
|
|
@@ -95,7 +95,7 @@ export function gatesDecided(log: string): boolean {
|
|
|
95
95
|
return FAILED_GATES.test(log) || PASSED_GATES.test(log);
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
-
// A mutation finding is a fall between two
|
|
98
|
+
// A mutation finding is a fall between two runs, so a run without it shows no score recovered.
|
|
99
99
|
export function cleared(key: string, present: ReadonlySet<string>, observed: Observed): boolean {
|
|
100
100
|
if (present.has(key)) return false;
|
|
101
101
|
if (key.startsWith("gates:")) return observed.gates;
|