@avi2dg/checks 0.39.0 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +31 -3
- package/docs/configs/commit-messages.md +13 -7
- package/docs/gates/checks-ci-wiring.md +2 -1
- package/docs/gates/checks-commit-title.md +57 -0
- package/docs/gates/checks-docs-only.md +50 -0
- package/docs/gates/checks-flake.md +7 -2
- package/docs/gates/checks-mutation-baseline.md +66 -16
- package/docs/gates/checks-mutation.md +6 -1
- package/docs/gates/checks-release-pr.md +4 -5
- package/docs/gates/checks-release-tag.md +1 -1
- package/package.json +6 -1
- package/src/delivery/ci-wiring.ts +11 -2
- package/src/delivery/commit-title.ts +82 -0
- package/src/delivery/docs-only.ts +38 -0
- package/src/testing/mutation-baseline-saved.ts +105 -0
- package/src/testing/mutation-baseline.ts +95 -29
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,22 @@
|
|
|
2
2
|
|
|
3
3
|
Every release of `@avi2dg/checks`, newest first, written by the release from its conventional commits.
|
|
4
4
|
|
|
5
|
+
## 0.41.0
|
|
6
|
+
|
|
7
|
+
Released 2026-10-12.
|
|
8
|
+
|
|
9
|
+
### Features
|
|
10
|
+
|
|
11
|
+
- **testing:** keep mutation baselines and flake reports out of Actions artifact storage [#150](https://github.com/avi2d/checks/pull/150)
|
|
12
|
+
|
|
13
|
+
## 0.40.0
|
|
14
|
+
|
|
15
|
+
Released 2026-10-11.
|
|
16
|
+
|
|
17
|
+
### Features
|
|
18
|
+
|
|
19
|
+
- **delivery:** fold title lint into the ci job and trim docs-only runs [#146](https://github.com/avi2d/checks/pull/146)
|
|
20
|
+
|
|
5
21
|
## 0.39.0
|
|
6
22
|
|
|
7
23
|
Released 2026-10-11.
|
package/README.md
CHANGED
|
@@ -117,7 +117,33 @@ To consume the kit from a repository:
|
|
|
117
117
|
- run: bun run test
|
|
118
118
|
```
|
|
119
119
|
|
|
120
|
-
|
|
120
|
+
Lint the pull request title in the same job with one step:
|
|
121
|
+
|
|
122
|
+
```yaml
|
|
123
|
+
- id: title
|
|
124
|
+
name: lint the title
|
|
125
|
+
env:
|
|
126
|
+
PR_TITLE: ${{ github.event.pull_request.title }}
|
|
127
|
+
GH_TOKEN: ${{ github.token }}
|
|
128
|
+
run: bunx checks-commit-title
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
A repository whose branch protection requires `commitlint` keeps that required status name while `commitlint.yml` goes away.
|
|
132
|
+
The same job reports the title step's outcome under that name on every run, which needs `statuses: write`:
|
|
133
|
+
|
|
134
|
+
```yaml
|
|
135
|
+
- name: report the title as commitlint
|
|
136
|
+
if: always()
|
|
137
|
+
env:
|
|
138
|
+
GH_TOKEN: ${{ github.token }}
|
|
139
|
+
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
|
140
|
+
OUTCOME: ${{ steps.title.outcome }}
|
|
141
|
+
run: |
|
|
142
|
+
state=failure
|
|
143
|
+
if [ "$OUTCOME" = success ]; then state=success; fi
|
|
144
|
+
gh api "repos/$GITHUB_REPOSITORY/statuses/${HEAD_SHA:-$GITHUB_SHA}" -f state="$state" -f context=commitlint -f target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" --silent
|
|
145
|
+
```
|
|
146
|
+
|
|
121
147
|
A private repository sets `runs-on: ${{ vars.CI_RUNS_ON || 'ubuntu-latest' }}` on each job, as [checks-ci-wiring](docs/gates/checks-ci-wiring.md#runners) requires.
|
|
122
148
|
`checks-effect-scope` writes the Effect paths of `oxlint.config.ts` and the kit's severities into `tsconfig.json`, as [The Effect rules](docs/configs/effect-rules.md#language-service) says.
|
|
123
149
|
`bun run lint` then ends with `checks-lint: <count> gate(s) pass`.
|
|
@@ -157,7 +183,7 @@ These bins run on their own:
|
|
|
157
183
|
- [`checks-flake`](docs/gates/checks-flake.md) runs the suite on a schedule and records the seeds a flaky test fails with.
|
|
158
184
|
- [`checks-mutation`](docs/gates/checks-mutation.md) runs Stryker for scoped checks and refuses a full run outside CI.
|
|
159
185
|
- [`checks-mutation-compare`](docs/gates/checks-mutation-compare.md) compares the mutants of two reports and lists each one that regresses.
|
|
160
|
-
- [`checks-mutation-baseline`](docs/gates/checks-mutation-baseline.md) restores the newest mutation baseline from `main`,
|
|
186
|
+
- [`checks-mutation-baseline`](docs/gates/checks-mutation-baseline.md) restores the newest mutation baseline from `main`, saves it on a self-hosted runner's disk in place of an artifact, and deletes superseded baselines after a `main` run.
|
|
161
187
|
- [`checks-subsumed-tests`](docs/gates/checks-subsumed-tests.md) lists each test another test subsumes in a mutation run.
|
|
162
188
|
- [`checks-changelog`](docs/gates/checks-changelog.md) writes the pending release into `CHANGELOG.md` from the conventional commits since the last release.
|
|
163
189
|
- [`checks-release-notes`](docs/gates/checks-release-notes.md) writes one `CHANGELOG.md` section to a file for a GitHub release.
|
|
@@ -166,9 +192,11 @@ These bins run on their own:
|
|
|
166
192
|
- [`checks-release-tag`](docs/gates/checks-release-tag.md) tags a landed release commit with its version and dispatches the release workflow on the tag.
|
|
167
193
|
- [`checks-vendor`](docs/gates/checks-vendor.md) pins each library its `prepare` arguments name to a shared read-only clone and links it under `repos/`.
|
|
168
194
|
- [`checks-browser`](docs/gates/checks-browser.md) opens a product's built pages in Chrome and fails on the layout, keyboard, motion, accessibility, nesting and asset checks its `browser-checks.json` declares.
|
|
195
|
+
- [`checks-commit-title`](docs/gates/checks-commit-title.md) lints the pull request title in one step of the `ci` job.
|
|
196
|
+
- [`checks-docs-only`](docs/gates/checks-docs-only.md) tells whether a change touches only top-level Markdown and Markdown under `docs/`, so a docs-only change skips the full run.
|
|
169
197
|
|
|
170
198
|
`checks-lint` has [its own page](docs/gates/checks-lint.md), which says which range it resolves.
|
|
171
|
-
oxlint
|
|
199
|
+
oxlint runs through its own tool, as the pages under Related topics say.
|
|
172
200
|
|
|
173
201
|
## Upgrade
|
|
174
202
|
|
|
@@ -14,20 +14,26 @@ It arrives with the kit, since `@commitlint/cli` and `@commitlint/config-convent
|
|
|
14
14
|
## Workflow
|
|
15
15
|
|
|
16
16
|
The lint runs in CI on pull requests, because `jj` never fires a git hook.
|
|
17
|
-
Each repository
|
|
17
|
+
Each repository lints the title in one step of its `ci` job with `bunx checks-commit-title`.
|
|
18
18
|
The workflow lints with the installed kit's `commitlint.config.js`, so every repository holds titles to the same rules.
|
|
19
|
+
The `ci` job reports the title step's outcome as a `commitlint` status on every run, so a required `commitlint` status stays while `commitlint.yml` goes away.
|
|
20
|
+
That post needs the workflow's `statuses: write` permission.
|
|
21
|
+
The title lint runs before the docs-only filter, so a change to top-level Markdown and `docs/` Markdown alone still lints its title.
|
|
19
22
|
|
|
20
23
|
## What it lints
|
|
21
24
|
|
|
22
25
|
It lints the pull request title and nothing else.
|
|
23
26
|
The title is the enforced subject because a squash merge uses it as the main commit subject, and per-commit messages are not linted.
|
|
24
|
-
GitHub appends
|
|
25
|
-
The
|
|
26
|
-
The
|
|
27
|
-
|
|
27
|
+
GitHub appends the pull request number to the squashed subject, so the bin lints the title with ` (#0000)` attached.
|
|
28
|
+
The header length limit applies to the landed subject, not the bare title.
|
|
29
|
+
The bin moves the git comment character off `#`, so a title starting with `#` is linted like any other.
|
|
30
|
+
The `ci` job also triggers on `workflow_dispatch`, as [checks-release-pr](../gates/checks-release-pr.md) says.
|
|
31
|
+
GitHub holds the `pull_request` runs of a release pull request the workflow token opens until a maintainer approves them.
|
|
32
|
+
A dispatched run carries no pull request, so the bin reads the title of the one open pull request its branch heads.
|
|
33
|
+
It fails when there is none.
|
|
28
34
|
That lookup needs the workflow's `pull-requests: read` permission.
|
|
29
|
-
|
|
30
|
-
The kit's own `.github/workflows/
|
|
35
|
+
The `commitlint` status covers a dispatched run too, since GitHub keeps a dispatched run's check off the pull request.
|
|
36
|
+
The kit's own `.github/workflows/ci.yml` shows the step.
|
|
31
37
|
|
|
32
38
|
It never sees a commit's author or committer fields, nor the `Co-authored-by` trailer GitHub writes from a foreign author when it squashes, so it cannot enforce who a commit belongs to.
|
|
33
39
|
[checks-commit-identity](../gates/checks-commit-identity.md) is that enforcement.
|
|
@@ -8,7 +8,8 @@ audience: consumers
|
|
|
8
8
|
|
|
9
9
|
## What it checks
|
|
10
10
|
|
|
11
|
-
The kit requires
|
|
11
|
+
The kit requires `bunx checks-commit-title` on every pull request.
|
|
12
|
+
A step that calls `bun src/delivery/commit-title.ts` or `./node_modules/.bin/commitlint` directly still satisfies the check.
|
|
12
13
|
It requires `bun run lint`, `bun run build`, `bun run typecheck` and `bun run test` for each of those scripts that `package.json` defines.
|
|
13
14
|
A `build` script also requires `git diff --exit-code`.
|
|
14
15
|
The target branch is the pull request base in CI, else the branch `refs/remotes/origin/HEAD` names, else the default branch of the repository in GitHub's event, else `main`.
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
---
|
|
2
|
+
kind: reference
|
|
3
|
+
audience: consumers
|
|
4
|
+
---
|
|
5
|
+
# checks-commit-title
|
|
6
|
+
|
|
7
|
+
`checks-commit-title` lints the pull request title against the shared commitlint config in one step of the `ci` job.
|
|
8
|
+
|
|
9
|
+
## What it checks
|
|
10
|
+
|
|
11
|
+
It lints the pull request title and nothing else.
|
|
12
|
+
The title comes from `PR_TITLE` when the event carries one.
|
|
13
|
+
A dispatched run carries no pull request, so the bin reads the title of the one open pull request its branch heads.
|
|
14
|
+
It fails when there is none.
|
|
15
|
+
A run with no title to lint passes and says so.
|
|
16
|
+
The bin lints the title with ` (#0000)` attached, since GitHub appends the pull request number to the squashed subject.
|
|
17
|
+
It moves the git comment character off `#`, so a title starting with `#` is linted like any other.
|
|
18
|
+
|
|
19
|
+
## What it reads
|
|
20
|
+
|
|
21
|
+
It reads `PR_TITLE` and `GITHUB_EVENT_NAME` from the environment.
|
|
22
|
+
A dispatched run also reads `GITHUB_REPOSITORY`, `GITHUB_REPOSITORY_OWNER` and `GITHUB_REF_NAME`, and calls `gh api` with `GH_TOKEN` for the open pull request.
|
|
23
|
+
It lints with the installed kit's commitlint config through the installed commitlint cli, as [The commit message lint](../configs/commit-messages.md) says.
|
|
24
|
+
|
|
25
|
+
## Arguments
|
|
26
|
+
|
|
27
|
+
The bin takes no arguments.
|
|
28
|
+
|
|
29
|
+
## Exit codes
|
|
30
|
+
|
|
31
|
+
| Code | Result |
|
|
32
|
+
| --- | --- |
|
|
33
|
+
| 0 | The title lints clean, or there is no title to lint. |
|
|
34
|
+
| 1 | The title breaks the rules, or no one open pull request heads the branch. |
|
|
35
|
+
| 2 | The kit config or binary cannot be resolved, or a dispatched run misses its environment. |
|
|
36
|
+
|
|
37
|
+
## Sample output
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
⧗ --- input ---
|
|
41
|
+
# feat(lint): add x (#0000)
|
|
42
|
+
✖ subject may not be empty [subject-empty]
|
|
43
|
+
✖ type may not be empty [type-empty]
|
|
44
|
+
|
|
45
|
+
✖ found 2 problems, 0 warnings
|
|
46
|
+
ⓘ Get help: https://github.com/conventional-changelog/commitlint/#what-is-commitlint
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## When it runs
|
|
50
|
+
|
|
51
|
+
The `ci` job runs it on `pull_request` and `workflow_dispatch`, as [checks-ci-wiring](checks-ci-wiring.md) requires.
|
|
52
|
+
The job then posts its outcome as a `commitlint` status, as [The commit message lint](../configs/commit-messages.md) says.
|
|
53
|
+
|
|
54
|
+
## Related topics
|
|
55
|
+
|
|
56
|
+
- [The commit message lint](../configs/commit-messages.md)
|
|
57
|
+
- [checks-ci-wiring](checks-ci-wiring.md)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
---
|
|
2
|
+
kind: reference
|
|
3
|
+
audience: consumers
|
|
4
|
+
---
|
|
5
|
+
# checks-docs-only
|
|
6
|
+
|
|
7
|
+
`checks-docs-only` tells whether a change touches only top-level Markdown and Markdown under `docs/`, so a docs-only change skips the full `ci` run.
|
|
8
|
+
|
|
9
|
+
## What it checks
|
|
10
|
+
|
|
11
|
+
A change is docs-only when every file it touches is a `.md` file at the top level or under `docs/`, and it touches at least one file.
|
|
12
|
+
Markdown anywhere else, such as a test fixture under `tests/` or a template under `dist/`, keeps the full run.
|
|
13
|
+
A rename counts both sides.
|
|
14
|
+
An empty change and a change with no range keep the full run.
|
|
15
|
+
|
|
16
|
+
## What it reads
|
|
17
|
+
|
|
18
|
+
It reads the base and head refs from its arguments.
|
|
19
|
+
It compares their merge base with the head through `git merge-base` and `git diff`.
|
|
20
|
+
|
|
21
|
+
## Arguments
|
|
22
|
+
|
|
23
|
+
```sh
|
|
24
|
+
checks-docs-only <base-ref> <head-ref>
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
With no arguments, the bin reports `docs-only=false`.
|
|
28
|
+
|
|
29
|
+
## Exit codes
|
|
30
|
+
|
|
31
|
+
| Code | Result |
|
|
32
|
+
| --- | --- |
|
|
33
|
+
| 0 | The bin classified the change, and the verdict is on stdout. |
|
|
34
|
+
| 2 | The range cannot be decoded. |
|
|
35
|
+
|
|
36
|
+
## Sample output
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
docs-only=true
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## When it runs
|
|
43
|
+
|
|
44
|
+
The `ci` job has a filter step that appends the verdict to `GITHUB_OUTPUT`, and the typecheck and test steps skip when the verdict reads `true`.
|
|
45
|
+
The build, its `git diff --exit-code` and `bun run lint` still run, so `checks-docs` and the generated blocks hold on a docs-only change.
|
|
46
|
+
The job itself always runs and reports, so every required check stays green on a docs-only change.
|
|
47
|
+
|
|
48
|
+
## Related topics
|
|
49
|
+
|
|
50
|
+
- [checks-ci-wiring](checks-ci-wiring.md)
|
|
@@ -54,7 +54,7 @@ Only a schedule the repository writes runs it.
|
|
|
54
54
|
|
|
55
55
|
## Running it on a schedule
|
|
56
56
|
|
|
57
|
-
A repository runs it on a schedule and keeps the record as an artifact:
|
|
57
|
+
A repository runs it on a schedule and keeps the record of a failed run as an artifact:
|
|
58
58
|
|
|
59
59
|
```yaml
|
|
60
60
|
on:
|
|
@@ -70,12 +70,17 @@ jobs:
|
|
|
70
70
|
- run: bun install --frozen-lockfile
|
|
71
71
|
- run: bunx checks-flake --runs 10 --report flake-report.json
|
|
72
72
|
- uses: actions/upload-artifact@v4
|
|
73
|
-
if:
|
|
73
|
+
if: failure()
|
|
74
74
|
with:
|
|
75
75
|
name: flake-report
|
|
76
76
|
path: flake-report.json
|
|
77
|
+
retention-days: 7
|
|
77
78
|
```
|
|
78
79
|
|
|
80
|
+
A passing run's job summary holds all its seeds, so a passing run uploads nothing.
|
|
81
|
+
In a private repository every upload counts against the account's Actions storage, and GitHub refuses each one once that storage is used up.
|
|
82
|
+
A run that only uploads on failure stays green through that refusal, and a failing run is red for its tests either way.
|
|
83
|
+
|
|
79
84
|
A private repository sets `runs-on: ${{ vars.CI_RUNS_ON || 'ubuntu-latest' }}` on the job, as [checks-ci-wiring](checks-ci-wiring.md#runners) requires.
|
|
80
85
|
|
|
81
86
|
## Related topics
|
|
@@ -4,14 +4,16 @@ audience: consumers
|
|
|
4
4
|
---
|
|
5
5
|
# checks-mutation-baseline
|
|
6
6
|
|
|
7
|
-
`checks-mutation-baseline` restores the newest mutation baseline that `main` published, and deletes superseded baselines after a `main`
|
|
7
|
+
`checks-mutation-baseline` restores the newest mutation baseline that `main` published, and deletes superseded baselines after a `main` run.
|
|
8
8
|
On a self-hosted runner it keeps each baseline it downloads, so the next job copies it instead.
|
|
9
|
+
There it also saves a run's baseline on the runner's disk, so the baseline never counts against the account's Actions storage.
|
|
9
10
|
|
|
10
11
|
## What it checks
|
|
11
12
|
|
|
12
13
|
It reads the successful runs of `mutation.yml` on `main`.
|
|
13
14
|
Without `--full` it takes the newest of the 20 newest of those runs that a pull request did not start.
|
|
14
|
-
It restores that run
|
|
15
|
+
It restores the baseline that run saved on the runner's disk, as [The saved baselines](#the-saved-baselines) says.
|
|
16
|
+
Without one there, it restores that run's `mutation-baseline` artifact, and takes the run whether or not it holds one.
|
|
15
17
|
It tries no older run, so a newest run without the artifact restores nothing.
|
|
16
18
|
An artifact without `stryker-incremental.json` counts as no baseline.
|
|
17
19
|
It copies `stryker-incremental.json` to the first destination, and `mutation/mutation.json` to the second destination when the artifact holds it.
|
|
@@ -25,6 +27,7 @@ It creates the directories each destination needs.
|
|
|
25
27
|
|
|
26
28
|
With `--prune` it deletes the repository's superseded baselines instead of restoring one.
|
|
27
29
|
It keeps the newest unexpired `mutation-baseline` among the runs the restore reads, and the newest unexpired `mutation-baseline-full` the restore takes first.
|
|
30
|
+
A run whose baseline the runner's disk holds stops that search, and every older `mutation-baseline` goes, since the restore reads that run's saved baseline first.
|
|
28
31
|
It also keeps the unexpired baselines of the run it runs in, which `GITHUB_RUN_ID` names.
|
|
29
32
|
The restore reads only successful runs, so it takes them once that run succeeds.
|
|
30
33
|
A failed artifact lookup while it picks what to keep stops it before it deletes anything.
|
|
@@ -39,7 +42,8 @@ It deletes through `gh`, which needs a token that deletes the repository's Actio
|
|
|
39
42
|
It reads the runs and their artifacts through `gh`, which needs a token that reads the repository's Actions, such as `GH_TOKEN: ${{ github.token }}` with `actions: read`.
|
|
40
43
|
The `mutation-baseline` artifact holds `stryker-incremental.json` and `mutation/mutation.json`, which an upload of `reports/stryker-incremental.json` and `reports/mutation/mutation.json` writes.
|
|
41
44
|
The `mutation-baseline-full` artifact holds `mutation.json` at its root, which an upload of `reports/mutation/mutation.json` alone writes.
|
|
42
|
-
It reads `RUNNER_ENVIRONMENT` and `HOME` to place the runner cache, and
|
|
45
|
+
It reads `RUNNER_ENVIRONMENT` and `HOME` to place the runner cache, and `GITHUB_REPOSITORY_ID` to find the saved baselines.
|
|
46
|
+
`--save` and `--prune` read `GITHUB_RUN_ID`.
|
|
43
47
|
|
|
44
48
|
## The runner cache
|
|
45
49
|
|
|
@@ -58,24 +62,53 @@ A 100 MB baseline zip can unpack to about 500 MB, so a repository restoring both
|
|
|
58
62
|
When `RUNNER_ENVIRONMENT` is `github-hosted`, it downloads into a temporary directory and writes no cache, since a hosted runner starts every job on a fresh machine.
|
|
59
63
|
When `HOME` is unset it works the same way, and prints to stderr that the runner cache is disabled.
|
|
60
64
|
|
|
65
|
+
## The saved baselines
|
|
66
|
+
|
|
67
|
+
`--save` writes the run's baseline to `$HOME/.cache/avi2dg-checks/mutation-baseline/<repository id>/saved/mutation-baseline/<run id>.json`.
|
|
68
|
+
The entry is one JSON file that holds `stryker-incremental.json` and, when given, `mutation/mutation.json`, keyed by those names.
|
|
69
|
+
It writes both files without each mutant's `statusReason`.
|
|
70
|
+
Stryker puts the test runner's failure message there, which no reader of a baseline uses, and which makes up most of a large report.
|
|
71
|
+
A report of about 280 MB comes out at about 5 MB.
|
|
72
|
+
The entry lands through a rename beside it, so a job sharing the runner sees it whole or not at all.
|
|
73
|
+
A re-run attempt keeps its run's id, so its entry replaces the one an earlier attempt saved.
|
|
74
|
+
|
|
75
|
+
The restore reads only the successful runs, so a run that saves and then fails leaves an entry no restore reads.
|
|
76
|
+
A re-run attempt that succeeds replaces it first.
|
|
77
|
+
A restore reads the entry in one open, so a job that removes or replaces it afterwards changes nothing the restore copies.
|
|
78
|
+
A restore from the disk removes the entries of runs older than the one it read.
|
|
79
|
+
It keeps the entries of newer runs, which may still be running.
|
|
80
|
+
A job that picked one of the removed entries and had not opened it yet finds none, and falls back to the artifact.
|
|
81
|
+
|
|
82
|
+
Every job that restores the baseline must run on the machine that saved it, as the same user.
|
|
83
|
+
A job on another machine finds no entry and falls back to the artifact, which the prune deletes once a saved baseline supersedes it.
|
|
84
|
+
Losing the disk costs the next `main` run a fresh baseline, since an incremental run with no state tests every mutant.
|
|
85
|
+
|
|
86
|
+
`--save` refuses on a GitHub-hosted runner, since its disk ends with the job.
|
|
87
|
+
There a repository uploads the baseline as an artifact.
|
|
88
|
+
It also refuses when `HOME`, `GITHUB_REPOSITORY_ID` or `GITHUB_RUN_ID` is unset.
|
|
89
|
+
|
|
61
90
|
## Arguments
|
|
62
91
|
|
|
63
92
|
```sh
|
|
64
93
|
checks-mutation-baseline <incremental-dest> [mutation-json-dest]
|
|
65
94
|
checks-mutation-baseline --full <mutation-json-dest>
|
|
95
|
+
checks-mutation-baseline --save <incremental-src> [mutation-json-src]
|
|
96
|
+
checks-mutation-baseline --slim <mutation-json-src> <dest>
|
|
66
97
|
checks-mutation-baseline --prune [--dry-run]
|
|
67
98
|
```
|
|
68
99
|
|
|
69
100
|
`--full` restores only the report of a run that started with no state.
|
|
70
|
-
`--
|
|
101
|
+
`--save` keeps this run's baseline on the runner's disk in place of an upload.
|
|
102
|
+
`--slim` writes a report without `statusReason` to `<dest>`, for a report that still goes up as an artifact.
|
|
103
|
+
`--prune` deletes superseded baselines after a save or an upload.
|
|
71
104
|
`--dry-run` lists what `--prune` would delete, and deletes nothing.
|
|
72
105
|
|
|
73
106
|
## Exit codes
|
|
74
107
|
|
|
75
108
|
| Code | When |
|
|
76
109
|
| --- | --- |
|
|
77
|
-
| 0 | it restored a baseline,
|
|
78
|
-
| 2 | the arguments do not parse, `gh` cannot list the runs, the cache cannot be written,
|
|
110
|
+
| 0 | it restored a baseline, found none to restore, saved the baseline or wrote the slim report |
|
|
111
|
+
| 2 | the arguments do not parse, `gh` cannot list the runs, the cache cannot be written, another job removed the entry before it was copied, a file to save or slim does not parse, or `--save` runs where it refuses |
|
|
79
112
|
|
|
80
113
|
A failed artifact lookup or download prints the `gh` error and moves on, as a missing artifact does.
|
|
81
114
|
With `--prune`, `0` means it deleted the superseded artifacts, found none, or could not delete some.
|
|
@@ -92,6 +125,12 @@ mutation-baseline: restored it from the runner's cache, mutation-baseline artifa
|
|
|
92
125
|
|
|
93
126
|
A miss prints `downloaded it into the runner's cache` instead, and a hosted runner prints `downloaded it`.
|
|
94
127
|
|
|
128
|
+
```
|
|
129
|
+
mutation-baseline: saved the mutation-baseline of run 38058558974 on the runner's disk
|
|
130
|
+
mutation-baseline: restored it from the runner's disk, mutation-baseline saved by run 38058558974
|
|
131
|
+
mutation-baseline: wrote full/mutation.json without statusReason, 4788924 bytes from 272879455
|
|
132
|
+
```
|
|
133
|
+
|
|
95
134
|
```
|
|
96
135
|
mutation-baseline: deleted artifacts 11542133901, 11553708473, 11632727585; kept mutation-baseline artifact 11670749656 from run 38055231501
|
|
97
136
|
```
|
|
@@ -110,16 +149,12 @@ A baseline job restores the state the previous `main` run left before an increme
|
|
|
110
149
|
run: bunx checks-mutation-baseline reports/stryker-incremental.json
|
|
111
150
|
```
|
|
112
151
|
|
|
113
|
-
A baseline job
|
|
152
|
+
A baseline job on a self-hosted runner saves the state the incremental run wrote, then on `main` prunes the superseded baselines:
|
|
114
153
|
|
|
115
154
|
```yaml
|
|
116
|
-
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
path: |
|
|
120
|
-
reports/mutation/mutation.json
|
|
121
|
-
reports/stryker-incremental.json
|
|
122
|
-
retention-days: 7
|
|
155
|
+
- name: Save baseline state
|
|
156
|
+
if: github.ref == 'refs/heads/main'
|
|
157
|
+
run: bunx checks-mutation-baseline --save reports/stryker-incremental.json reports/mutation/mutation.json
|
|
123
158
|
- name: Prune superseded baselines
|
|
124
159
|
if: github.ref == 'refs/heads/main'
|
|
125
160
|
env:
|
|
@@ -127,8 +162,23 @@ A baseline job uploads the state the incremental run wrote, then on `main` prune
|
|
|
127
162
|
run: bunx checks-mutation-baseline --prune
|
|
128
163
|
```
|
|
129
164
|
|
|
130
|
-
The
|
|
131
|
-
|
|
165
|
+
The saved baseline uses none of the account's Actions storage, which every private repository's artifacts share.
|
|
166
|
+
That storage accrues by the hour over the month, so an artifact deleted early still counts until the month ends.
|
|
167
|
+
A baseline job on a GitHub-hosted runner uploads the same two files as a `mutation-baseline` artifact with `retention-days: 7` instead, since the restore reads only the newest `main` run.
|
|
168
|
+
|
|
169
|
+
A full run uploads its report as the `mutation-baseline-full` artifact, which a pull request's scope step reads on any runner.
|
|
170
|
+
It slims the report first, so the artifact holds about 0.5 MB in place of about 100 MB:
|
|
171
|
+
|
|
172
|
+
```yaml
|
|
173
|
+
- run: bunx checks-mutation-baseline --slim reports/mutation/mutation.json "$RUNNER_TEMP/full/mutation.json"
|
|
174
|
+
- uses: actions/upload-artifact@v4
|
|
175
|
+
with:
|
|
176
|
+
name: mutation-baseline-full
|
|
177
|
+
path: ${{ runner.temp }}/full/mutation.json
|
|
178
|
+
retention-days: 30
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
The full upload keeps 30 days, so the newest full baseline outlives the gap between full runs.
|
|
132
182
|
A pull request's scope step loses its coverage when no full baseline outlives the gap.
|
|
133
183
|
|
|
134
184
|
A pull request's scope step restores the full report with `--full`:
|
|
@@ -97,13 +97,18 @@ jobs:
|
|
|
97
97
|
bun-version-file: .bun-version
|
|
98
98
|
- run: bun install --frozen-lockfile
|
|
99
99
|
- run: bunx stryker run
|
|
100
|
+
- if: always()
|
|
101
|
+
run: bunx checks-mutation-baseline --slim reports/mutation/mutation.json "$RUNNER_TEMP/report/mutation.json"
|
|
100
102
|
- uses: actions/upload-artifact@v4
|
|
101
103
|
if: always()
|
|
102
104
|
with:
|
|
103
105
|
name: mutation-report
|
|
104
|
-
path:
|
|
106
|
+
path: ${{ runner.temp }}/report/mutation.json
|
|
107
|
+
retention-days: 7
|
|
105
108
|
```
|
|
106
109
|
|
|
110
|
+
The upload holds the report without each mutant's `statusReason`, as [checks-mutation-baseline](checks-mutation-baseline.md#the-saved-baselines) says, since the full report of a large repository passes 100 MB as an artifact.
|
|
111
|
+
|
|
107
112
|
The job runs on the fleet's self-hosted Linux runner labelled `winbox`, which is where a repository sends its full sweeps.
|
|
108
113
|
It never reads `CI_RUNS_ON`, so an override that moves a repository's other jobs during a runner outage leaves the sweep waiting for `winbox`.
|
|
109
114
|
[checks-ci-wiring](checks-ci-wiring.md#runners) refuses a mutation job that names it in `runs-on`.
|
|
@@ -47,7 +47,7 @@ The repository needs **Allow GitHub Actions to create and approve pull requests*
|
|
|
47
47
|
checks-release-pr <workflow>...
|
|
48
48
|
```
|
|
49
49
|
|
|
50
|
-
Each argument names a workflow file under `.github/workflows/` whose jobs report the checks the default branch requires, such as `ci.yml
|
|
50
|
+
Each argument names a workflow file under `.github/workflows/` whose jobs report the checks the default branch requires, such as `ci.yml`.
|
|
51
51
|
Each named workflow triggers on `workflow_dispatch`.
|
|
52
52
|
Name no workflow that runs something else on dispatch, such as a mutation baseline.
|
|
53
53
|
|
|
@@ -63,7 +63,7 @@ Name no workflow that runs something else on dispatch, such as a mutation baseli
|
|
|
63
63
|
A run that opens the pull request prints the build's own output, then one line:
|
|
64
64
|
|
|
65
65
|
```
|
|
66
|
-
release-pr: opened https://github.com/acme/widget/pull/12 to release 0.4.0, and dispatched ci.yml
|
|
66
|
+
release-pr: opened https://github.com/acme/widget/pull/12 to release 0.4.0, and dispatched ci.yml on release/main
|
|
67
67
|
```
|
|
68
68
|
|
|
69
69
|
A run on the same `HEAD` the next day prints one line:
|
|
@@ -115,7 +115,7 @@ jobs:
|
|
|
115
115
|
- name: open or refresh the release pull request
|
|
116
116
|
env:
|
|
117
117
|
GH_TOKEN: ${{ github.token }}
|
|
118
|
-
run: ./node_modules/.bin/checks-release-pr ci.yml
|
|
118
|
+
run: ./node_modules/.bin/checks-release-pr ci.yml
|
|
119
119
|
tag:
|
|
120
120
|
if: "github.event_name == 'push' && startsWith(github.event.head_commit.message, 'chore: release ')"
|
|
121
121
|
runs-on: ubuntu-latest
|
|
@@ -150,8 +150,7 @@ Nothing requires them, and approving them runs the same checks again.
|
|
|
150
150
|
|
|
151
151
|
The daily release needs the repository's other workflows to accept the dispatch:
|
|
152
152
|
|
|
153
|
-
- `.github/workflows/ci.yml`
|
|
154
|
-
`commitlint.yml` also grants `pull-requests: read`, which the title lookup needs.
|
|
153
|
+
- `.github/workflows/ci.yml` triggers on `workflow_dispatch`, grants `statuses: write` and `pull-requests: read`, and ends each required job with the step below.
|
|
155
154
|
- The title lint reads the title of the one open pull request its branch heads when the event carries none, as [Commit messages](../configs/commit-messages.md) says.
|
|
156
155
|
- `.github/workflows/release.yml` triggers on `workflow_dispatch` and refuses a ref that is not a tag, as [checks-release-notes](checks-release-notes.md) shows.
|
|
157
156
|
- **Allow GitHub Actions to create and approve pull requests** is on, which the call after the step sets.
|
|
@@ -8,7 +8,7 @@ audience: consumers
|
|
|
8
8
|
|
|
9
9
|
## What it checks
|
|
10
10
|
|
|
11
|
-
It reads the subject of `HEAD`, and does nothing unless the subject is `chore: release <version>`, with or without the
|
|
11
|
+
It reads the subject of `HEAD`, and does nothing unless the subject is `chore: release <version>`, with or without the pull request number a squash merge appends.
|
|
12
12
|
It refuses a release commit whose `package.json` holds another version.
|
|
13
13
|
It tags `HEAD` as `v<version>` and pushes the tag to `origin`.
|
|
14
14
|
A tag the workflow token pushes starts no `push` workflow, so it then dispatches the workflow its argument names on the tag.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@avi2dg/checks",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.41.0",
|
|
4
4
|
"description": "Deterministic checks shared across a set of TypeScript repositories",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -40,11 +40,14 @@
|
|
|
40
40
|
"src/testing/test-report.ts",
|
|
41
41
|
"src/testing/flake.ts",
|
|
42
42
|
"src/delivery/commit-identity.ts",
|
|
43
|
+
"src/delivery/commit-title.ts",
|
|
44
|
+
"src/delivery/docs-only.ts",
|
|
43
45
|
"src/testing/mutation.ts",
|
|
44
46
|
"src/testing/mutation-scope.js",
|
|
45
47
|
"src/testing/mutation-guard-plugin.js",
|
|
46
48
|
"src/testing/mutation-compare.ts",
|
|
47
49
|
"src/testing/mutation-baseline.ts",
|
|
50
|
+
"src/testing/mutation-baseline-saved.ts",
|
|
48
51
|
"src/testing/subsumed-tests.ts",
|
|
49
52
|
"src/delivery/ci-wiring.ts",
|
|
50
53
|
"src/delivery/shell-command.ts",
|
|
@@ -132,6 +135,8 @@
|
|
|
132
135
|
"checks-test": "src/testing/test.ts",
|
|
133
136
|
"checks-flake": "src/testing/flake.ts",
|
|
134
137
|
"checks-commit-identity": "src/delivery/commit-identity.ts",
|
|
138
|
+
"checks-commit-title": "src/delivery/commit-title.ts",
|
|
139
|
+
"checks-docs-only": "src/delivery/docs-only.ts",
|
|
135
140
|
"checks-mutation": "src/testing/mutation.ts",
|
|
136
141
|
"checks-mutation-compare": "src/testing/mutation-compare.ts",
|
|
137
142
|
"checks-mutation-baseline": "src/testing/mutation-baseline.ts",
|
|
@@ -11,6 +11,7 @@ export type { Command };
|
|
|
11
11
|
export type Gate = {
|
|
12
12
|
readonly command: string;
|
|
13
13
|
readonly words: Command;
|
|
14
|
+
readonly alternatives: readonly Command[];
|
|
14
15
|
};
|
|
15
16
|
|
|
16
17
|
export type Declaration = {
|
|
@@ -63,7 +64,10 @@ export class WiringError extends Schema.TaggedError<WiringError>()("WiringError"
|
|
|
63
64
|
const WORKFLOWS = ".github/workflows";
|
|
64
65
|
const SCRIPTED = ["lint", "build", "typecheck", "test"];
|
|
65
66
|
const BUILT_TREE = "git diff --exit-code";
|
|
66
|
-
const TITLE_LINT = "
|
|
67
|
+
const TITLE_LINT = "bunx checks-commit-title";
|
|
68
|
+
// The gate holds until every consumer migrates, so the direct commitlint call still counts.
|
|
69
|
+
// The kit calls its own file through bun, since bunx never resolves the working tree.
|
|
70
|
+
const TITLE_CALLS = ["bun src/delivery/commit-title.ts", "./node_modules/.bin/commitlint"];
|
|
67
71
|
const Manifest = Schema.Struct({ scripts: Schema.optionalKey(Schema.Record(Schema.String, Schema.String)) });
|
|
68
72
|
// Without these a pull_request workflow never sees the commits a pull request pushes.
|
|
69
73
|
const GATING_TYPES = ["opened", "synchronize"];
|
|
@@ -207,6 +211,7 @@ function gapsAmong(gates: readonly Gate[], steps: readonly RunStep[], lintGates:
|
|
|
207
211
|
const entryPoint = entryPointCommand(gate.words, lintGates);
|
|
208
212
|
const commands = [
|
|
209
213
|
{ command: gate.command, words: gate.words },
|
|
214
|
+
...gate.alternatives.map((words) => ({ command: gate.command, words })),
|
|
210
215
|
...(entryPoint === undefined ? [] : [{ command: entryPoint.join(" "), words: entryPoint }]),
|
|
211
216
|
];
|
|
212
217
|
const invoking = steps.flatMap(({ location, blocker, script }) => {
|
|
@@ -258,7 +263,11 @@ export function requiredCommands(scripts: readonly string[]): readonly string[]
|
|
|
258
263
|
|
|
259
264
|
export function declarationFor(scripts: readonly string[], branch: string): Declaration {
|
|
260
265
|
return {
|
|
261
|
-
gates: requiredCommands(scripts).map((command) => ({
|
|
266
|
+
gates: requiredCommands(scripts).map((command) => ({
|
|
267
|
+
command,
|
|
268
|
+
words: command.split(" "),
|
|
269
|
+
alternatives: command === TITLE_LINT ? TITLE_CALLS.map((call) => call.split(" ")) : [],
|
|
270
|
+
})),
|
|
262
271
|
defaultBranch: branch,
|
|
263
272
|
lintGates: KIT_GATES,
|
|
264
273
|
};
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
import { Config, Console, Effect, FileSystem, Path, Schema } from "effect";
|
|
3
|
+
import { collect } from "../core/git.ts";
|
|
4
|
+
import { runMain } from "../core/main.ts";
|
|
5
|
+
|
|
6
|
+
const NAME = "commit-title";
|
|
7
|
+
|
|
8
|
+
class TitleSetup extends Schema.TaggedError<TitleSetup>()("TitleSetup", {
|
|
9
|
+
message: Schema.String,
|
|
10
|
+
}) {}
|
|
11
|
+
|
|
12
|
+
const setup = (message: string): TitleSetup => new TitleSetup({ message });
|
|
13
|
+
|
|
14
|
+
// The cli entry stays inside its own package, so this resolves from the install cache as well as node_modules.
|
|
15
|
+
const cliOf = Effect.fn("cliOf")(function* () {
|
|
16
|
+
const entry = yield* Effect.try({
|
|
17
|
+
try: () => new URL("./cli.js", import.meta.resolve("@commitlint/cli")),
|
|
18
|
+
catch: () => setup("cannot resolve @commitlint/cli from the installed kit"),
|
|
19
|
+
});
|
|
20
|
+
return yield* (yield* Path.Path).fromFileUrl(entry);
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
// The config ships with the kit, so it sits two levels above this file wherever the kit is installed.
|
|
24
|
+
const configOf = Effect.fn("configOf")(function* () {
|
|
25
|
+
const file = yield* Effect.try({
|
|
26
|
+
try: () => new URL("../../commitlint.config.js", import.meta.url),
|
|
27
|
+
catch: () => setup("cannot resolve commitlint.config.js from the installed kit"),
|
|
28
|
+
});
|
|
29
|
+
return yield* (yield* Path.Path).fromFileUrl(file);
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
const dispatchedTitle = Effect.fn("dispatchedTitle")(function* () {
|
|
33
|
+
const repository = yield* Config.String("GITHUB_REPOSITORY").pipe(Config.withDefault(""));
|
|
34
|
+
const owner = yield* Config.String("GITHUB_REPOSITORY_OWNER").pipe(Config.withDefault(""));
|
|
35
|
+
const ref = yield* Config.String("GITHUB_REF_NAME").pipe(Config.withDefault(""));
|
|
36
|
+
if (repository === "" || owner === "" || ref === "") {
|
|
37
|
+
return yield* setup("a dispatched run lints the open pull request its branch heads, so it needs GITHUB_REPOSITORY, GITHUB_REPOSITORY_OWNER and GITHUB_REF_NAME");
|
|
38
|
+
}
|
|
39
|
+
// A dispatched run carries no pull request, so the title comes from the one open pull request the branch heads.
|
|
40
|
+
const found = yield* collect("gh", [
|
|
41
|
+
"api",
|
|
42
|
+
`repos/${repository}/pulls?head=${owner}:${ref}&state=open`,
|
|
43
|
+
"--jq",
|
|
44
|
+
`if length == 1 then .[0].title else error("no one open pull request heads the branch") end`,
|
|
45
|
+
]);
|
|
46
|
+
const title = found.stdout.trim();
|
|
47
|
+
if (found.exitCode !== 0 || title === "" || title === "null") {
|
|
48
|
+
const report = found.stderr.trim() === "" ? found.stdout.trim() : found.stderr.trim();
|
|
49
|
+
if (report !== "") yield* Console.error(report);
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
return title;
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
const lintTitle = Effect.fn("lintTitle")(function* (title: string) {
|
|
56
|
+
const system = yield* FileSystem.FileSystem;
|
|
57
|
+
const path = yield* Path.Path;
|
|
58
|
+
const file = path.join(yield* system.makeTempDirectoryScoped({ prefix: "checks-commit-title-" }), "pr-title");
|
|
59
|
+
yield* system.writeFileString(file, `${title} (#0000)`);
|
|
60
|
+
// A title opening with the git comment character lints as empty without this: commentChar moves off the hash.
|
|
61
|
+
const linted = yield* collect(process.execPath, [yield* cliOf(), "--config", yield* configOf(), "--edit", file], undefined, {
|
|
62
|
+
env: { GIT_CONFIG_COUNT: "1", GIT_CONFIG_KEY_0: "core.commentChar", GIT_CONFIG_VALUE_0: "\x01" },
|
|
63
|
+
});
|
|
64
|
+
if (linted.stdout.trim() !== "") yield* Console.log(linted.stdout.trim());
|
|
65
|
+
if (linted.stderr.trim() !== "") yield* Console.error(linted.stderr.trim());
|
|
66
|
+
return linted.exitCode === 0;
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
const check = Effect.scoped(Effect.gen(function* () {
|
|
70
|
+
const direct = yield* Config.String("PR_TITLE").pipe(Config.withDefault(""));
|
|
71
|
+
if (direct !== "") return yield* lintTitle(direct);
|
|
72
|
+
const event = yield* Config.String("GITHUB_EVENT_NAME").pipe(Config.withDefault(""));
|
|
73
|
+
if (event === "workflow_dispatch") {
|
|
74
|
+
const title = yield* dispatchedTitle();
|
|
75
|
+
if (title === undefined) return false;
|
|
76
|
+
return yield* lintTitle(title);
|
|
77
|
+
}
|
|
78
|
+
yield* Console.log(`${NAME}: no pull request title is available, so there is nothing to lint`);
|
|
79
|
+
return true;
|
|
80
|
+
}));
|
|
81
|
+
|
|
82
|
+
if (import.meta.main) runMain(NAME, check);
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
import { Console, Effect } from "effect";
|
|
3
|
+
import { changedPaths, git } from "../core/git.ts";
|
|
4
|
+
import { runMain } from "../core/main.ts";
|
|
5
|
+
|
|
6
|
+
const NAME = "docs-only";
|
|
7
|
+
|
|
8
|
+
export function isDocsPath(path: string): boolean {
|
|
9
|
+
const lower = path.toLowerCase();
|
|
10
|
+
return lower.endsWith(".md") && (!lower.includes("/") || lower.startsWith("docs/"));
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// The verdict goes to stdout as one line, so a step appends it to GITHUB_OUTPUT; the detail goes to stderr.
|
|
14
|
+
const check = Effect.gen(function* () {
|
|
15
|
+
const [base, head] = process.argv.slice(2);
|
|
16
|
+
if (base === undefined || head === undefined) {
|
|
17
|
+
yield* Console.error(`${NAME}: no range is given, so the full run stays`);
|
|
18
|
+
yield* Console.log("docs-only=false");
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
const mergeBase = (yield* git(["merge-base", base, head])).trim();
|
|
22
|
+
const changes = yield* changedPaths(mergeBase, head, []);
|
|
23
|
+
const paths = changes.flatMap((change) => (change.kind === "renamed" ? [change.from, change.path] : [change.path]));
|
|
24
|
+
const full = paths.filter((path) => !isDocsPath(path));
|
|
25
|
+
if (paths.length === 0) {
|
|
26
|
+
yield* Console.error(`${NAME}: the range changes no file, so the full run stays`);
|
|
27
|
+
yield* Console.log("docs-only=false");
|
|
28
|
+
} else if (full.length === 0) {
|
|
29
|
+
yield* Console.error(`${NAME}: ${paths.length} changed file(s) are all top-level or docs/ Markdown, so the full run can skip`);
|
|
30
|
+
yield* Console.log("docs-only=true");
|
|
31
|
+
} else {
|
|
32
|
+
yield* Console.error(`${NAME}: ${full.length} changed file(s) need the full run`);
|
|
33
|
+
yield* Console.log("docs-only=false");
|
|
34
|
+
}
|
|
35
|
+
return true;
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
if (import.meta.main) runMain(NAME, check);
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { Config, Console, Effect, FileSystem, Option, Path, Schema } from "effect";
|
|
2
|
+
import { cacheRoot } from "../dependencies/cache-root.ts";
|
|
3
|
+
|
|
4
|
+
export const LATEST_ARTIFACT = "mutation-baseline";
|
|
5
|
+
export const INCREMENTAL = "stryker-incremental.json";
|
|
6
|
+
export const REPORT = "mutation/mutation.json";
|
|
7
|
+
const SAVED = "saved";
|
|
8
|
+
|
|
9
|
+
class SaveFailure extends Schema.TaggedError<SaveFailure>()("SaveFailure", {
|
|
10
|
+
message: Schema.String,
|
|
11
|
+
}) {}
|
|
12
|
+
|
|
13
|
+
const decodeReport = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
|
|
14
|
+
|
|
15
|
+
const SavedFiles = Schema.Struct({ [INCREMENTAL]: Schema.Unknown, [REPORT]: Schema.optionalKey(Schema.Unknown) });
|
|
16
|
+
export type SavedFiles = typeof SavedFiles.Type;
|
|
17
|
+
const decodeSavedFiles = Schema.decodeUnknownEffect(Schema.fromJsonString(SavedFiles));
|
|
18
|
+
|
|
19
|
+
const isRecord = (value: unknown): value is Readonly<Record<string, unknown>> => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
20
|
+
|
|
21
|
+
const withoutReason = (mutant: unknown): unknown => (isRecord(mutant) ? Object.fromEntries(Object.entries(mutant).filter(([key]) => key !== "statusReason")) : mutant);
|
|
22
|
+
|
|
23
|
+
function slimFile(file: unknown): unknown {
|
|
24
|
+
if (!isRecord(file)) return file;
|
|
25
|
+
const mutants = file["mutants"];
|
|
26
|
+
return Array.isArray(mutants) ? { ...file, mutants: mutants.map((mutant: unknown) => withoutReason(mutant)) } : file;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// statusReason holds the test runner's failure message for each mutant, which no reader of a baseline uses and which fills most of a large report.
|
|
30
|
+
export function withoutReasons(report: unknown): unknown {
|
|
31
|
+
if (!isRecord(report) || !isRecord(report["files"])) return report;
|
|
32
|
+
return { ...report, files: Object.fromEntries(Object.entries(report["files"]).map(([name, file]) => [name, slimFile(file)])) };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const slimOf = Effect.fn("slimOf")(function* (from: string) {
|
|
36
|
+
const report = yield* (yield* FileSystem.FileSystem).readFileString(from).pipe(
|
|
37
|
+
Effect.flatMap(decodeReport),
|
|
38
|
+
Effect.mapError((cause) => new SaveFailure({ message: `cannot read ${from}: ${cause.message}` })),
|
|
39
|
+
);
|
|
40
|
+
return withoutReasons(report);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
export const slimmed = Effect.fn("slimmed")(function* (from: string, to: string) {
|
|
44
|
+
const fs = yield* FileSystem.FileSystem;
|
|
45
|
+
const report = yield* slimOf(from);
|
|
46
|
+
yield* fs.makeDirectory((yield* Path.Path).dirname(to), { recursive: true });
|
|
47
|
+
yield* fs.writeFileString(to, JSON.stringify(report));
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
const shelfOf = Effect.fn("shelfOf")(function* (root: string, repository: number) {
|
|
51
|
+
return (yield* Path.Path).join(root, "mutation-baseline", String(repository), SAVED, LATEST_ARTIFACT);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
// GitHub sets GITHUB_REPOSITORY_ID on every job, so a run outside Actions has no saved baselines.
|
|
55
|
+
export const savedShelf = Effect.fn("savedShelf")(function* (root: Option.Option<string>) {
|
|
56
|
+
const repository = yield* Config.Int("GITHUB_REPOSITORY_ID").pipe(Config.option);
|
|
57
|
+
if (Option.isNone(root) || Option.isNone(repository)) return Option.none<string>();
|
|
58
|
+
return Option.some(yield* shelfOf(root.value, repository.value));
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
const savedEntry = Effect.fn("savedEntry")(function* (shelf: string, run: number) {
|
|
62
|
+
return (yield* Path.Path).join(shelf, `${run}.json`);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
// One read opens the entry once, so a job that removes or replaces it afterwards leaves what this job read whole.
|
|
66
|
+
export const savedFiles = Effect.fn("savedFiles")(function* (shelf: string, run: number) {
|
|
67
|
+
const entry = yield* savedEntry(shelf, run);
|
|
68
|
+
const text = yield* (yield* FileSystem.FileSystem).readFileString(entry).pipe(
|
|
69
|
+
Effect.asSome,
|
|
70
|
+
Effect.catchReason("PlatformError", "NotFound", () => Effect.succeedNone),
|
|
71
|
+
);
|
|
72
|
+
if (Option.isNone(text)) return Option.none<SavedFiles>();
|
|
73
|
+
return Option.some(yield* decodeSavedFiles(text.value).pipe(Effect.mapError((cause) => new SaveFailure({ message: `cannot read ${entry}: ${cause.message}` }))));
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// GitHub numbers runs in the order it creates them, so a lower id is an older run the restore never reads again.
|
|
77
|
+
export function outlived(entries: readonly string[], restoring: number): readonly string[] {
|
|
78
|
+
return entries.filter((entry) => {
|
|
79
|
+
const run = /^(\d+)\.json$/.exec(entry)?.[1];
|
|
80
|
+
return run !== undefined && Number(run) < restoring;
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const required = (name: string) =>
|
|
85
|
+
Config.Int(name).pipe(Effect.mapError(() => new SaveFailure({ message: `${name} is unset, so --save runs only inside a GitHub Actions job` })));
|
|
86
|
+
|
|
87
|
+
// The entry is one file renamed into place, so a job sharing the runner reads one attempt's entry whole even while a re-run replaces it.
|
|
88
|
+
export const saveBaseline = Effect.fn("saveBaseline")(function* (incremental: string, report: Option.Option<string>) {
|
|
89
|
+
const fs = yield* FileSystem.FileSystem;
|
|
90
|
+
const path = yield* Path.Path;
|
|
91
|
+
const environment = yield* Config.String("RUNNER_ENVIRONMENT").pipe(Config.withDefault(""));
|
|
92
|
+
if (environment === "github-hosted") {
|
|
93
|
+
return yield* new SaveFailure({ message: "a GitHub-hosted runner discards its disk when the job ends, so upload the baseline as an artifact there" });
|
|
94
|
+
}
|
|
95
|
+
const root = yield* cacheRoot().pipe(Effect.mapError((cause) => new SaveFailure({ message: cause.message })));
|
|
96
|
+
const shelf = yield* shelfOf(root, yield* required("GITHUB_REPOSITORY_ID"));
|
|
97
|
+
const run = yield* required("GITHUB_RUN_ID");
|
|
98
|
+
const files: SavedFiles = { [INCREMENTAL]: yield* slimOf(incremental), ...(Option.isSome(report) ? { [REPORT]: yield* slimOf(report.value) } : {}) };
|
|
99
|
+
yield* fs.makeDirectory(shelf, { recursive: true });
|
|
100
|
+
const staged = path.join(yield* fs.makeTempDirectoryScoped({ directory: shelf, prefix: ".staging-" }), "entry.json");
|
|
101
|
+
yield* fs.writeFileString(staged, JSON.stringify(files));
|
|
102
|
+
yield* fs.rename(staged, yield* savedEntry(shelf, run));
|
|
103
|
+
yield* Console.log(`mutation-baseline: saved the ${LATEST_ARTIFACT} of run ${run} on the runner's disk`);
|
|
104
|
+
return true;
|
|
105
|
+
}, Effect.scoped);
|
|
@@ -4,20 +4,19 @@ import { ChildProcessSpawner } from "effect/process";
|
|
|
4
4
|
import { collect } from "../core/git.ts";
|
|
5
5
|
import { runMain, Usage } from "../core/main.ts";
|
|
6
6
|
import { cacheRoot } from "../dependencies/cache-root.ts";
|
|
7
|
+
import { INCREMENTAL, LATEST_ARTIFACT, outlived, REPORT, saveBaseline, savedFiles, savedShelf, slimmed } from "./mutation-baseline-saved.ts";
|
|
7
8
|
|
|
8
9
|
class BaselineFailure extends Schema.TaggedError<BaselineFailure>()("BaselineFailure", {
|
|
9
10
|
message: Schema.String,
|
|
10
11
|
}) {}
|
|
11
12
|
|
|
12
13
|
const NAME = "checks-mutation-baseline";
|
|
13
|
-
const USAGE =
|
|
14
|
+
const USAGE =
|
|
15
|
+
"usage: checks-mutation-baseline <incremental-dest> [mutation-json-dest] | --full <mutation-json-dest> | --save <incremental-src> [mutation-json-src] | --slim <mutation-json-src> <dest> | --prune [--dry-run]";
|
|
14
16
|
const WORKFLOW = "mutation.yml";
|
|
15
17
|
const BRANCH = "main";
|
|
16
|
-
const INCREMENTAL = "stryker-incremental.json";
|
|
17
|
-
const REPORT = "mutation/mutation.json";
|
|
18
18
|
const REPORT_ALONE = "mutation.json";
|
|
19
19
|
const KEPT_PER_ARTIFACT = 2;
|
|
20
|
-
const LATEST_ARTIFACT = "mutation-baseline";
|
|
21
20
|
const FULL_ARTIFACT = "mutation-baseline-full";
|
|
22
21
|
const LATEST_LIMIT = 20;
|
|
23
22
|
const FULL_LIMIT = 50;
|
|
@@ -63,12 +62,12 @@ export function newestFirst(runs: readonly Run[]): readonly number[] {
|
|
|
63
62
|
return [...runs].sort((one, other) => other.createdAt.localeCompare(one.createdAt)).map((run) => run.databaseId);
|
|
64
63
|
}
|
|
65
64
|
|
|
65
|
+
// All non-pull-request runs newest first: a newer full run holds no baseline, and the newest baseline must survive it.
|
|
66
66
|
const latestCandidates = Effect.gen(function* () {
|
|
67
67
|
const newest = (yield* listed(undefined, LATEST_LIMIT)).find((run) => run.event !== "pull_request");
|
|
68
68
|
return newest === undefined ? [] : [newest.databaseId];
|
|
69
69
|
});
|
|
70
70
|
|
|
71
|
-
// All non-pull-request runs newest first: a newer full run holds no baseline, and the newest baseline must survive it.
|
|
72
71
|
const latestRuns = listed(undefined, LATEST_LIMIT).pipe(Effect.map((runs) => newestFirst(runs.filter((run) => run.event !== "pull_request"))));
|
|
73
72
|
|
|
74
73
|
// Each full event is listed on its own, because one shared window lets push runs crowd out every full run.
|
|
@@ -108,12 +107,22 @@ const missedDeletion = Effect.fn("missedDeletion")(function* (artifact: Artifact
|
|
|
108
107
|
);
|
|
109
108
|
});
|
|
110
109
|
|
|
110
|
+
// A run whose baseline the runner's disk holds supersedes every older artifact, since the restore reads that run's entry first.
|
|
111
|
+
const keptLatest = Effect.fn("keptLatest")(function* (runs: readonly number[], shelf: Option.Option<string>) {
|
|
112
|
+
for (const run of runs) {
|
|
113
|
+
if (Option.isSome(shelf) && Option.isSome(yield* savedFiles(shelf.value, run))) return { saved: Option.some(run), kept: Option.none<Kept>() };
|
|
114
|
+
const artifact = yield* artifactOf(run, LATEST_ARTIFACT);
|
|
115
|
+
if (Option.isSome(artifact)) return { saved: Option.none<number>(), kept: Option.some<Kept>({ run, id: artifact.value.id, name: LATEST_ARTIFACT }) };
|
|
116
|
+
}
|
|
117
|
+
return { saved: Option.none<number>(), kept: Option.none<Kept>() };
|
|
118
|
+
});
|
|
119
|
+
|
|
111
120
|
const pruneBaselines = Effect.fn("pruneBaselines")(function* (dryRun: boolean) {
|
|
112
121
|
const running = yield* Config.Int("GITHUB_RUN_ID").pipe(Config.option);
|
|
113
|
-
const latest = yield*
|
|
122
|
+
const latest = yield* keptLatest(yield* latestRuns, yield* savedShelf(yield* runnerCache));
|
|
114
123
|
const full = yield* keptArtifact(yield* fullCandidates, FULL_ARTIFACT);
|
|
115
124
|
const artifacts = yield* everyArtifact();
|
|
116
|
-
const keeping = [...Option.toArray(latest), ...Option.toArray(full), ...uploadedBy(artifacts, running)];
|
|
125
|
+
const keeping = [...Option.toArray(latest.kept), ...Option.toArray(full), ...uploadedBy(artifacts, running)];
|
|
117
126
|
const victims = superseded(artifacts, new Set(keeping.map((keep) => keep.id)));
|
|
118
127
|
const removed: number[] = [];
|
|
119
128
|
const missed: Missed[] = [];
|
|
@@ -124,7 +133,7 @@ const pruneBaselines = Effect.fn("pruneBaselines")(function* (dryRun: boolean) {
|
|
|
124
133
|
else removed.push(victim.id);
|
|
125
134
|
}
|
|
126
135
|
}
|
|
127
|
-
yield* Console.log(pruneReport(dryRun, dryRun ? victims.map((victim) => victim.id) : removed, missed, keeping));
|
|
136
|
+
yield* Console.log(pruneReport(dryRun, dryRun ? victims.map((victim) => victim.id) : removed, missed, keeping, latest.saved));
|
|
128
137
|
return true;
|
|
129
138
|
});
|
|
130
139
|
|
|
@@ -175,8 +184,12 @@ export function superseded(artifacts: readonly Artifact[], kept: ReadonlySet<num
|
|
|
175
184
|
// A workflow command ends at a newline, so the error's own newlines and percent signs are escaped.
|
|
176
185
|
const annotated = (text: string): string => text.replaceAll("%", "%25").replaceAll("\r", "%0D").replaceAll("\n", "%0A");
|
|
177
186
|
|
|
178
|
-
export function pruneReport(dryRun: boolean, removed: readonly number[], missed: readonly Missed[], keeping: readonly Kept[]): string {
|
|
179
|
-
const
|
|
187
|
+
export function pruneReport(dryRun: boolean, removed: readonly number[], missed: readonly Missed[], keeping: readonly Kept[], saved: Option.Option<number>): string {
|
|
188
|
+
const holdings = [
|
|
189
|
+
...keeping.map((keep) => `${keep.name} artifact ${keep.id} from run ${keep.run}`),
|
|
190
|
+
...Option.toArray(saved).map((run) => `the ${LATEST_ARTIFACT} run ${run} saved on the runner's disk`),
|
|
191
|
+
];
|
|
192
|
+
const kept = holdings.length === 0 ? "no baseline artifacts" : holdings.join(", ");
|
|
180
193
|
const deleted = removed.length === 0 ? "nothing to delete" : `deleted artifacts ${removed.join(", ")}`;
|
|
181
194
|
if (dryRun) return `mutation-baseline: dry run, ${removed.length === 0 ? "nothing to delete" : `would delete artifacts ${removed.join(", ")}`}; would keep ${kept}`;
|
|
182
195
|
if (missed.length === 0) return `mutation-baseline: ${deleted}; kept ${kept}`;
|
|
@@ -265,36 +278,80 @@ const runnerCache = Effect.gen(function* () {
|
|
|
265
278
|
);
|
|
266
279
|
});
|
|
267
280
|
|
|
281
|
+
const restoredFromSaved = Effect.fn("restoredFromSaved")(function* (shelf: string, run: number, request: Request) {
|
|
282
|
+
const fs = yield* FileSystem.FileSystem;
|
|
283
|
+
const path = yield* Path.Path;
|
|
284
|
+
const files = yield* savedFiles(shelf, run);
|
|
285
|
+
if (Option.isNone(files)) return false;
|
|
286
|
+
const stored = new Map(Object.entries(files.value));
|
|
287
|
+
for (const copy of [request.needed, ...request.optional]) {
|
|
288
|
+
const name = copy.from.find((from) => stored.has(from));
|
|
289
|
+
if (name === undefined) continue;
|
|
290
|
+
yield* fs.makeDirectory(path.dirname(copy.to), { recursive: true });
|
|
291
|
+
yield* fs.writeFileString(copy.to, JSON.stringify(stored.get(name)));
|
|
292
|
+
}
|
|
293
|
+
for (const older of outlived(yield* fs.readDirectory(shelf), run)) {
|
|
294
|
+
yield* fs.remove(path.join(shelf, older), { force: true });
|
|
295
|
+
}
|
|
296
|
+
yield* Console.log(`mutation-baseline: restored it from the runner's disk, ${LATEST_ARTIFACT} saved by run ${run}`);
|
|
297
|
+
return true;
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
const slimReport = Effect.fn("slimReport")(function* (from: string, to: string) {
|
|
301
|
+
const fs = yield* FileSystem.FileSystem;
|
|
302
|
+
const before = (yield* fs.stat(from)).size;
|
|
303
|
+
yield* slimmed(from, to);
|
|
304
|
+
const after = (yield* fs.stat(to)).size;
|
|
305
|
+
yield* Console.log(`mutation-baseline: wrote ${to} without statusReason, ${after} bytes from ${before}`);
|
|
306
|
+
return true;
|
|
307
|
+
});
|
|
308
|
+
|
|
268
309
|
// An artifact that lacks `needed` counts as no baseline, and `optional` copies only what the artifact holds.
|
|
269
|
-
type Request = { readonly full: boolean; readonly needed: Copy; readonly optional: readonly Copy[] };
|
|
310
|
+
type Request = { readonly kind: "restore"; readonly full: boolean; readonly needed: Copy; readonly optional: readonly Copy[] };
|
|
270
311
|
|
|
271
|
-
type
|
|
312
|
+
type Command =
|
|
313
|
+
| Request
|
|
314
|
+
| { readonly kind: "prune"; readonly dryRun: boolean }
|
|
315
|
+
| { readonly kind: "save"; readonly incremental: string; readonly report: Option.Option<string> }
|
|
316
|
+
| { readonly kind: "slim"; readonly from: string; readonly to: string };
|
|
272
317
|
|
|
273
318
|
const isDestination = (arg: string | undefined): arg is string => arg !== undefined && !arg.startsWith("-");
|
|
274
319
|
|
|
275
|
-
function
|
|
276
|
-
if (args[0] === "--prune") {
|
|
277
|
-
const [flag, ...extra] = args.slice(1);
|
|
278
|
-
if ((flag === undefined || flag === "--dry-run") && extra.length === 0) return { prune: true, dryRun: flag !== undefined };
|
|
279
|
-
return undefined;
|
|
280
|
-
}
|
|
281
|
-
if (args[0] === "--full") {
|
|
282
|
-
const [report, ...extra] = args.slice(1);
|
|
283
|
-
if (!isDestination(report) || extra.length > 0) return undefined;
|
|
284
|
-
return { full: true, needed: { from: [REPORT_ALONE, REPORT], to: report }, optional: [] };
|
|
285
|
-
}
|
|
286
|
-
const [incremental, report, ...extra] = args;
|
|
320
|
+
function restoreRequest(incremental: string | undefined, report: string | undefined, extra: readonly string[]): Command | undefined {
|
|
287
321
|
if (!isDestination(incremental) || extra.length > 0) return undefined;
|
|
288
|
-
return { full: false, needed: { from: [INCREMENTAL], to: incremental }, optional: report === undefined ? [] : [{ from: [REPORT], to: report }] };
|
|
322
|
+
return { kind: "restore", full: false, needed: { from: [INCREMENTAL], to: incremental }, optional: report === undefined ? [] : [{ from: [REPORT], to: report }] };
|
|
289
323
|
}
|
|
290
324
|
|
|
291
|
-
const
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
325
|
+
const FLAGS = new Map<string, (rest: readonly string[]) => Command | undefined>([
|
|
326
|
+
["--prune", ([flag, ...extra]) => ((flag === undefined || flag === "--dry-run") && extra.length === 0 ? { kind: "prune", dryRun: flag !== undefined } : undefined)],
|
|
327
|
+
[
|
|
328
|
+
"--save",
|
|
329
|
+
([incremental, report, ...extra]) =>
|
|
330
|
+
isDestination(incremental) && (report === undefined || isDestination(report)) && extra.length === 0
|
|
331
|
+
? { kind: "save", incremental, report: Option.fromNullishOr(report) }
|
|
332
|
+
: undefined,
|
|
333
|
+
],
|
|
334
|
+
["--slim", ([from, to, ...extra]) => (isDestination(from) && isDestination(to) && extra.length === 0 ? { kind: "slim", from, to } : undefined)],
|
|
335
|
+
[
|
|
336
|
+
"--full",
|
|
337
|
+
([report, ...extra]) =>
|
|
338
|
+
isDestination(report) && extra.length === 0 ? { kind: "restore", full: true, needed: { from: [REPORT_ALONE, REPORT], to: report }, optional: [] } : undefined,
|
|
339
|
+
],
|
|
340
|
+
]);
|
|
341
|
+
|
|
342
|
+
function parsed([first, ...rest]: readonly string[]): Command | undefined {
|
|
343
|
+
const flag = first === undefined ? undefined : FLAGS.get(first);
|
|
344
|
+
if (flag !== undefined) return flag(rest);
|
|
345
|
+
const [report, ...extra] = rest;
|
|
346
|
+
return restoreRequest(first, report, extra);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
const restoreBaseline = Effect.fn("restoreBaseline")(function* (request: Request) {
|
|
295
350
|
const name = request.full ? FULL_ARTIFACT : LATEST_ARTIFACT;
|
|
296
351
|
const root = yield* runnerCache;
|
|
352
|
+
const shelf = request.full ? Option.none<string>() : yield* savedShelf(root);
|
|
297
353
|
for (const run of yield* request.full ? fullCandidates : latestCandidates) {
|
|
354
|
+
if (Option.isSome(shelf) && (yield* restoredFromSaved(shelf.value, run, request))) return true;
|
|
298
355
|
const artifact = yield* artifactOf(run, name).pipe(Effect.catch(warned(Option.none<Artifact>())));
|
|
299
356
|
if (Option.isSome(artifact) && (yield* restoredFrom(run, artifact.value, root, request))) return true;
|
|
300
357
|
}
|
|
@@ -302,4 +359,13 @@ const mutationBaseline = Effect.gen(function* () {
|
|
|
302
359
|
return true;
|
|
303
360
|
});
|
|
304
361
|
|
|
362
|
+
const mutationBaseline = Effect.gen(function* () {
|
|
363
|
+
const command = parsed(process.argv.slice(2));
|
|
364
|
+
if (command === undefined) return yield* new Usage({ message: USAGE });
|
|
365
|
+
if (command.kind === "prune") return yield* pruneBaselines(command.dryRun);
|
|
366
|
+
if (command.kind === "save") return yield* saveBaseline(command.incremental, command.report);
|
|
367
|
+
if (command.kind === "slim") return yield* slimReport(command.from, command.to);
|
|
368
|
+
return yield* restoreBaseline(command);
|
|
369
|
+
});
|
|
370
|
+
|
|
305
371
|
if (import.meta.main) runMain(NAME, mutationBaseline);
|