@avi2dg/checks 0.45.1 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +3 -1
- package/dist/presets/stylelint.js +8 -0
- package/docs/design.md +6 -2
- package/docs/gates/checks-advisories.md +3 -2
- package/docs/gates/checks-css-size.md +53 -0
- package/docs/gates/checks-flake.md +3 -2
- package/docs/gates/checks-frontend-syntax.md +5 -0
- package/docs/gates/checks-lint.md +2 -1
- package/docs/gates/checks-mutation-baseline.md +3 -3
- package/docs/gates/checks-mutation-compare.md +10 -8
- package/docs/gates/checks-mutation-scope.md +10 -10
- package/docs/gates/checks-mutation.md +4 -4
- package/docs/gates/checks-night-run.md +188 -0
- package/docs/gates/checks-release-notes.md +2 -1
- package/docs/gates/checks-release-pr.md +7 -1
- package/docs/gates/checks-test.md +5 -2
- package/package.json +6 -1
- package/src/core/gates.ts +3 -0
- package/src/core/main.ts +14 -0
- package/src/delivery/github.ts +7 -8
- package/src/quality/css-size.ts +69 -0
- package/src/quality/no-plain-js.ts +3 -10
- package/src/quality/presets/stylelint.ts +12 -0
- package/src/testing/mutation-compare.ts +1 -1
- package/src/testing/mutation-scope.ts +1 -1
- package/src/testing/night-run-findings.ts +74 -0
- package/src/testing/night-run.ts +253 -0
- package/src/testing/test.ts +24 -6
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,22 @@
|
|
|
2
2
|
|
|
3
3
|
Every release of `@avi2dg/checks`, newest first, written by the release from its conventional commits.
|
|
4
4
|
|
|
5
|
+
## 0.47.0
|
|
6
|
+
|
|
7
|
+
Released 2026-10-19.
|
|
8
|
+
|
|
9
|
+
### Features
|
|
10
|
+
|
|
11
|
+
- **testing:** file night-run regressions as GitHub issues instead of gating pull requests [#174](https://github.com/avi2d/checks/pull/174)
|
|
12
|
+
|
|
13
|
+
## 0.46.0
|
|
14
|
+
|
|
15
|
+
Released 2026-10-19.
|
|
16
|
+
|
|
17
|
+
### Features
|
|
18
|
+
|
|
19
|
+
- **quality:** refuse CSS comments, non-ASCII selectors and stylesheets over 400 lines [#172](https://github.com/avi2d/checks/pull/172)
|
|
20
|
+
|
|
5
21
|
## 0.45.1
|
|
6
22
|
|
|
7
23
|
Released 2026-10-19.
|
package/README.md
CHANGED
|
@@ -172,6 +172,7 @@ The table groups the gates by vector, the part of a repository each one judges.
|
|
|
172
172
|
| quality | [`checks-comment-gate`](docs/gates/checks-comment-gate.md) | the range | every repository |
|
|
173
173
|
| quality | [`checks-effect-scope`](docs/gates/checks-effect-scope.md) | the working tree | a repository tracking `oxlint.config.ts` |
|
|
174
174
|
| quality | [`checks-frontend-syntax`](docs/gates/checks-frontend-syntax.md) | the working tree | a repository tracking `frontend-syntax.json` |
|
|
175
|
+
| quality | [`checks-css-size`](docs/gates/checks-css-size.md) | the working tree | a repository tracking `*.css` |
|
|
175
176
|
| quality | [`checks-no-plain-js`](docs/gates/checks-no-plain-js.md) | the working tree | a repository tracking `*.ts` or `*.tsx` |
|
|
176
177
|
| testing | [`checks-test-layout`](docs/gates/checks-test-layout.md) | the working tree | a repository tracking `*.ts` or `*.tsx` |
|
|
177
178
|
| testing | [`checks-quarantine-clock`](docs/gates/checks-quarantine-clock.md) | the range | every repository |
|
|
@@ -192,8 +193,9 @@ These bins run on their own:
|
|
|
192
193
|
- [`checks-mutation`](docs/gates/checks-mutation.md) runs Stryker for scoped checks and refuses a full run outside CI.
|
|
193
194
|
- [`checks-mutation-compare`](docs/gates/checks-mutation-compare.md) compares the mutants of two reports and lists each one that regresses.
|
|
194
195
|
- [`checks-mutation-baseline`](docs/gates/checks-mutation-baseline.md) restores the newest mutation baseline from `main`, saves it on a self-hosted runner's disk in place of an artifact, and deletes superseded baselines after a `main` run.
|
|
195
|
-
- [`checks-mutation-scope`](docs/gates/checks-mutation-scope.md) turns a
|
|
196
|
+
- [`checks-mutation-scope`](docs/gates/checks-mutation-scope.md) turns a branch's diff into the files its mutation runs mutate.
|
|
196
197
|
- [`checks-subsumed-tests`](docs/gates/checks-subsumed-tests.md) lists each test another test subsumes in a mutation run.
|
|
198
|
+
- [`checks-night-run`](docs/gates/checks-night-run.md) opens or comments on one GitHub issue per finding of a night run on `main`, each file whose mutation score fell and each failing test.
|
|
197
199
|
- [`checks-changelog`](docs/gates/checks-changelog.md) writes the pending release into `CHANGELOG.md` from the conventional commits since the last release.
|
|
198
200
|
- [`checks-release-notes`](docs/gates/checks-release-notes.md) writes one `CHANGELOG.md` section to a file for a GitHub release.
|
|
199
201
|
- [`checks-release-report`](docs/gates/checks-release-report.md) tells whether the history holds unreleased features or fixes since the last tag.
|
|
@@ -13,6 +13,12 @@ var SELECTOR_COMPLEXITY = {
|
|
|
13
13
|
"selector-max-specificity": "0,3,2",
|
|
14
14
|
"selector-max-compound-selectors": 3
|
|
15
15
|
};
|
|
16
|
+
var COMMENT_REFUSAL = {
|
|
17
|
+
"comment-word-disallowed-list": ["/^[\\s\\S]*$/"]
|
|
18
|
+
};
|
|
19
|
+
var LANGUAGE_FREE_SELECTORS = {
|
|
20
|
+
"selector-disallowed-list": ["/[^\\x00-\\x7F]/"]
|
|
21
|
+
};
|
|
16
22
|
var COLOUR_FUNCTIONS = ["rgb", "rgba", "hsl", "hsla", "hwb", "lab", "lch", "oklab", "oklch", "color"];
|
|
17
23
|
var DESIGN_TOKENS = {
|
|
18
24
|
"color-no-hex": true,
|
|
@@ -61,6 +67,8 @@ function defineConfig({ baseline, tokens, rules, overrides = [], ...rest }) {
|
|
|
61
67
|
const merged = {
|
|
62
68
|
...CONVENTION_CHANGES,
|
|
63
69
|
...SELECTOR_COMPLEXITY,
|
|
70
|
+
...COMMENT_REFUSAL,
|
|
71
|
+
...LANGUAGE_FREE_SELECTORS,
|
|
64
72
|
[BASELINE_RULE]: [true, { available }],
|
|
65
73
|
...tokenRules.rules,
|
|
66
74
|
...rules
|
package/docs/design.md
CHANGED
|
@@ -145,14 +145,18 @@ The violation it catches is a deleted workflow step, so it has to fail in the lo
|
|
|
145
145
|
It parses workflows with `Bun.YAML`, which the `bun` shebang already provides, so it adds no dependency.
|
|
146
146
|
It reads `on` as a string key, not as the YAML 1.1 boolean.
|
|
147
147
|
|
|
148
|
-
##
|
|
148
|
+
## TypeScript size and repetition only fail on growth
|
|
149
149
|
|
|
150
|
-
The size limits are oxlint's own rules at `error`, not a script of the kit's.
|
|
150
|
+
The TypeScript size limits are oxlint's own rules at `error`, not a script of the kit's.
|
|
151
151
|
A script of the kit's own would restate how oxlint reads its config and compares sites, and each restatement adds cases oxlint itself does not have.
|
|
152
152
|
A repository records its existing violations with `oxlint --suppress-all`, and `checks-suppressions-ratchet` refuses any count that rises.
|
|
153
153
|
The cost is that a file or function already over its limit can grow without adding a site the count sees.
|
|
154
154
|
Each repository works its counts down to zero.
|
|
155
155
|
|
|
156
|
+
`checks-css-size` holds each stylesheet to the same 400 lines as an absolute budget, with no count to ratchet.
|
|
157
|
+
oxlint does not read CSS, and no maintained stylelint rule or plugin limits lines per file, so the budget is a kit script.
|
|
158
|
+
A stylesheet already over 400 lines fails until it is split, whatever the suppression counts hold.
|
|
159
|
+
|
|
156
160
|
`checks-repetition` runs jscpd at both ends of the range with the head's `.jscpd.json`, so the same `path` and `ignore` globs pick the files at both ends.
|
|
157
161
|
It compares each file's count of repeated lines, and does not use jscpd's `--baseline-from-ref`.
|
|
158
162
|
That flag reports a repeated block as new once its text changes, so a change that shortens an existing repeated block would fail.
|
|
@@ -101,12 +101,13 @@ A range that leaves `bun.lock` unchanged runs in under a second, and one that ch
|
|
|
101
101
|
|
|
102
102
|
## Running it on a schedule
|
|
103
103
|
|
|
104
|
-
A range never fails on an advisory published after its package landed, so a scheduled `--all` run finds those
|
|
104
|
+
A range never fails on an advisory published after its package landed, so a scheduled `--all` run finds those.
|
|
105
|
+
A consuming repository pins advisories between 05:00 and 06:00 UTC at a minute chosen so repositories do not start together:
|
|
105
106
|
|
|
106
107
|
```yaml
|
|
107
108
|
on:
|
|
108
109
|
schedule:
|
|
109
|
-
- cron: "
|
|
110
|
+
- cron: "15 5 * * *"
|
|
110
111
|
workflow_dispatch:
|
|
111
112
|
jobs:
|
|
112
113
|
advisories:
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
---
|
|
2
|
+
kind: reference
|
|
3
|
+
audience: consumers
|
|
4
|
+
---
|
|
5
|
+
# checks-css-size
|
|
6
|
+
|
|
7
|
+
`checks-css-size` is the gate that holds each tracked stylesheet to 400 lines, the kit's own `max-lines` for code files.
|
|
8
|
+
|
|
9
|
+
## What it checks
|
|
10
|
+
|
|
11
|
+
It fails on every `*.css` file in the working tree that holds more than 400 lines.
|
|
12
|
+
It counts every line, blank lines and comments included, the way `max-lines` counts code with blank lines and comments kept.
|
|
13
|
+
Each line names the file and its line count, and tells its writer to split the stylesheet into smaller files.
|
|
14
|
+
|
|
15
|
+
## What it reads
|
|
16
|
+
|
|
17
|
+
It reads the working tree.
|
|
18
|
+
It scans the tracked and untracked files that `git ls-files --cached --others --exclude-standard` reports.
|
|
19
|
+
Ignore rules leave out untracked files only, so an untracked `node_modules/` is out of reach while a tracked stylesheet under an ignored directory is still measured.
|
|
20
|
+
A local run agrees with CI before `git add`.
|
|
21
|
+
|
|
22
|
+
## Arguments
|
|
23
|
+
|
|
24
|
+
```sh
|
|
25
|
+
checks-css-size [<directory>]
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
It checks the directory it runs in, or the directory it is given.
|
|
29
|
+
|
|
30
|
+
## Exit codes
|
|
31
|
+
|
|
32
|
+
| Code | When |
|
|
33
|
+
| --- | --- |
|
|
34
|
+
| 0 | every stylesheet holds 400 lines or fewer |
|
|
35
|
+
| 1 | a stylesheet holds more than 400 lines |
|
|
36
|
+
| 2 | git does not list the working tree, or a listed stylesheet cannot be read |
|
|
37
|
+
|
|
38
|
+
## Sample output
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
css-size: 1 violation(s)
|
|
42
|
+
src/styles.css: 1901 lines, over the 400-line budget; split the stylesheet into smaller files
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## When it runs
|
|
46
|
+
|
|
47
|
+
`checks-lint` runs it when the repository tracks a `*.css` file.
|
|
48
|
+
|
|
49
|
+
## Related topics
|
|
50
|
+
|
|
51
|
+
- [checks-frontend-syntax](checks-frontend-syntax.md)
|
|
52
|
+
- [checks-lint](checks-lint.md)
|
|
53
|
+
- [checks-repetition](checks-repetition.md)
|
|
@@ -101,12 +101,13 @@ Only a schedule the repository writes runs it.
|
|
|
101
101
|
|
|
102
102
|
## Running it on a schedule
|
|
103
103
|
|
|
104
|
-
A repository runs it on a schedule and keeps the record of a failed run as an artifact
|
|
104
|
+
A repository runs it on a schedule and keeps the record of a failed run as an artifact.
|
|
105
|
+
A repository schedules it at minute 27 of its tier's 30 minute night window, so repositories do not start together:
|
|
105
106
|
|
|
106
107
|
```yaml
|
|
107
108
|
on:
|
|
108
109
|
schedule:
|
|
109
|
-
- cron: "
|
|
110
|
+
- cron: "27 22 * * *"
|
|
110
111
|
workflow_dispatch:
|
|
111
112
|
jobs:
|
|
112
113
|
flake:
|
|
@@ -39,6 +39,8 @@ A control that needs selection off sets it on its own selector, such as `.drag-h
|
|
|
39
39
|
| G3 selector complexity | `selector-max-specificity` at `0,3,2` and `selector-max-compound-selectors` at 3 | on |
|
|
40
40
|
| G4 design tokens | `color-no-hex`, `color-named: "never"`, `function-disallowed-list` over `rgb`, `rgba`, `hsl`, `hsla`, `hwb`, `lab`, `lch`, `oklab`, `oklch` and `color`, and `font-size` only as one `var(--…)` with no fallback, or `inherit` | off until `tokens` names the token files, and off in those files |
|
|
41
41
|
| G5 Baseline target | `plugin/use-baseline` from `stylelint-plugin-use-baseline`, which judges each property, value, function, at-rule, selector and unit against the Baseline data, and passes a property inside an `@supports` block. It passes `@property` at every target, since a browser without `@property` treats the declaration as an unregistered custom property | on, at the target `baseline` sets |
|
|
42
|
+
| G6 comments refused | `comment-word-disallowed-list` over `/^[\s\S]*$/`, which refuses every comment, including an empty `/* */` | on |
|
|
43
|
+
| G7 language-free selectors | `selector-disallowed-list` over `/[^\x00-\x7F]/`, which refuses any selector holding a non-ASCII character, in attribute values, class names and ids alike | on |
|
|
42
44
|
|
|
43
45
|
The builder changes three of the shared config's rules:
|
|
44
46
|
|
|
@@ -47,7 +49,9 @@ The builder changes three of the shared config's rules:
|
|
|
47
49
|
- `property-no-vendor-prefix` keeps reporting without a fix, and passes `-webkit-user-select`, since its fix deletes the prefix Safari still needs.
|
|
48
50
|
- In an `.html` or `.astro` file, `declaration-block-single-line-max-declarations` is off, since a `style` attribute is one line.
|
|
49
51
|
|
|
52
|
+
The comment rule refuses every comment but a sourcemap `/*# */` one, which stylelint passes to keep the map.
|
|
50
53
|
CSS repetition is not a family of this gate: a repository adds `"css"` to `format` in `.jscpd.json`, and [checks-repetition](checks-repetition.md) then fails when a stylesheet's repeated lines grow.
|
|
54
|
+
A stylesheet's length is not a family of this gate either, and [checks-css-size](checks-css-size.md) fails one that holds more than 400 lines.
|
|
51
55
|
|
|
52
56
|
### What no comment turns off
|
|
53
57
|
|
|
@@ -169,6 +173,7 @@ A repository without that file never runs it and needs neither `stylelint.config
|
|
|
169
173
|
- [checks-browser](checks-browser.md)
|
|
170
174
|
- [checks-lint](checks-lint.md)
|
|
171
175
|
- [checks-comment-gate](checks-comment-gate.md)
|
|
176
|
+
- [checks-css-size](checks-css-size.md)
|
|
172
177
|
- [checks-suppressions-ratchet](checks-suppressions-ratchet.md)
|
|
173
178
|
- [checks-repetition](checks-repetition.md)
|
|
174
179
|
- [Native settings](../configs/native-settings.md)
|
|
@@ -14,6 +14,7 @@ Gates requiring tracked TypeScript files begin running when the repository track
|
|
|
14
14
|
`checks-advisories` begins running when the repository tracks `bun.lock`.
|
|
15
15
|
`checks-frontend-syntax` begins running when the repository tracks `frontend-syntax.json` at its root.
|
|
16
16
|
`checks-copy` begins running when the repository tracks `copy.json` at its root.
|
|
17
|
+
`checks-css-size` begins running when the repository tracks a `*.css` file.
|
|
17
18
|
All other gates run for every repository.
|
|
18
19
|
|
|
19
20
|
## What it reads
|
|
@@ -48,7 +49,7 @@ With two arguments, the base and head override range discovery.
|
|
|
48
49
|
|
|
49
50
|
```
|
|
50
51
|
checks-lint: range 2504acf098d120e73a8ece3c96f22b934f35c6a8..10ba7d8935b73ed72624120a1542e51bd21ca7c7 from HEAD against origin/main
|
|
51
|
-
checks-lint: 1 of
|
|
52
|
+
checks-lint: 1 of 19 gate(s) failed: checks-comment-gate
|
|
52
53
|
```
|
|
53
54
|
|
|
54
55
|
<!-- end generated lint-sample -->
|
|
@@ -166,7 +166,7 @@ The saved baseline uses none of the account's Actions storage, which every priva
|
|
|
166
166
|
That storage accrues by the hour over the month, so an artifact deleted early still counts until the month ends.
|
|
167
167
|
A baseline job on a GitHub-hosted runner uploads the same two files as a `mutation-baseline` artifact with `retention-days: 7` instead, since the restore reads only the newest `main` run.
|
|
168
168
|
|
|
169
|
-
A full run uploads its report as the `mutation-baseline-full` artifact, which a
|
|
169
|
+
A full run uploads its report as the `mutation-baseline-full` artifact, which a branch comparison's scope step reads on any runner.
|
|
170
170
|
It slims the report first, so the artifact holds about 0.5 MB in place of about 100 MB:
|
|
171
171
|
|
|
172
172
|
```yaml
|
|
@@ -179,9 +179,9 @@ It slims the report first, so the artifact holds about 0.5 MB in place of about
|
|
|
179
179
|
```
|
|
180
180
|
|
|
181
181
|
The full upload keeps 30 days, so the newest full baseline outlives the gap between full runs.
|
|
182
|
-
A
|
|
182
|
+
A branch comparison's scope step loses its coverage when no full baseline outlives the gap.
|
|
183
183
|
|
|
184
|
-
A
|
|
184
|
+
A branch comparison's scope step restores the full report with `--full`, and [checks-mutation-scope](checks-mutation-scope.md) reads it:
|
|
185
185
|
|
|
186
186
|
```yaml
|
|
187
187
|
- name: Select mutation scope
|
|
@@ -68,22 +68,23 @@ mutation-compare: REGRESSION (1 mutant(s))
|
|
|
68
68
|
## When it runs
|
|
69
69
|
|
|
70
70
|
Only a CI step the repository writes runs it.
|
|
71
|
-
A
|
|
71
|
+
A person starts that step by hand on a branch, and no pull request triggers it or waits for it.
|
|
72
|
+
[checks-night-run](checks-night-run.md) judges the mutation score of `main` each night and opens an issue for each file whose score fell.
|
|
73
|
+
A repository runs it with `--advisory`, so the comparison stays advisory.
|
|
72
74
|
Its lines land only in the CI log.
|
|
73
|
-
A repository that wants a reviewer to answer each regression line puts the lines in front of that reviewer with its own CI step.
|
|
74
|
-
That step can post them as a pull request comment.
|
|
75
75
|
A full sweep runs in CI and never on a laptop.
|
|
76
76
|
Start a baseline with `gh workflow run mutation` and keep its report as an artifact.
|
|
77
77
|
The shared preset refuses a full `stryker run` outside CI and names that workflow command instead, as [checks-mutation](checks-mutation.md) says.
|
|
78
78
|
|
|
79
79
|
## Running it in CI
|
|
80
80
|
|
|
81
|
-
|
|
81
|
+
A person starts a workflow named `mutation-compare` on a branch with `gh workflow run mutation-compare --ref <branch>`.
|
|
82
|
+
It compares the head report against a report built at the merge-base with `main` over the same scope:
|
|
82
83
|
|
|
83
84
|
```yaml
|
|
84
85
|
name: mutation-compare
|
|
85
86
|
on:
|
|
86
|
-
|
|
87
|
+
workflow_dispatch:
|
|
87
88
|
jobs:
|
|
88
89
|
mutation-compare:
|
|
89
90
|
runs-on: ubuntu-latest
|
|
@@ -102,7 +103,7 @@ jobs:
|
|
|
102
103
|
env:
|
|
103
104
|
GH_TOKEN: ${{ github.token }}
|
|
104
105
|
run: |
|
|
105
|
-
base="$(git merge-base HEAD
|
|
106
|
+
base="$(git merge-base HEAD origin/main)"
|
|
106
107
|
echo "BASE_SHA=$base" >> "$GITHUB_ENV"
|
|
107
108
|
bunx checks-mutation-baseline --full "$RUNNER_TEMP/baseline-full/mutation.json"
|
|
108
109
|
bunx checks-mutation-scope "$base" "$RUNNER_TEMP/baseline-full/mutation.json" >> "$GITHUB_ENV"
|
|
@@ -125,7 +126,7 @@ jobs:
|
|
|
125
126
|
git worktree prune
|
|
126
127
|
```
|
|
127
128
|
|
|
128
|
-
The scope step picks the files the
|
|
129
|
+
The scope step picks the files the branch changes and the sources its changed unit tests cover or kill, as [checks-mutation-scope](checks-mutation-scope.md) says.
|
|
129
130
|
Neither run mutates a file outside that scope, so the job's time follows the size of the change and not the size of the repository.
|
|
130
131
|
Both runs take the same `--mutate "$SCOPE"`, so the two reports hold the same files.
|
|
131
132
|
A scope file the merge base lacks matches nothing there, and Stryker still writes a report.
|
|
@@ -135,7 +136,7 @@ A changed unit test that `main` edited after that baseline widens the scope to e
|
|
|
135
136
|
Start a full baseline with `gh workflow run mutation`, as [checks-mutation](checks-mutation.md#running-it-in-ci) shows.
|
|
136
137
|
A run with `--mutate` is scoped, so the preset lets it through without `CI=true`.
|
|
137
138
|
The base worktree's path carries the run's id and attempt, and the last step removes it even when a run fails, so a runner kept between jobs starts each job clean.
|
|
138
|
-
A public repository keeps `runs-on: ubuntu-latest`,
|
|
139
|
+
A public repository keeps `runs-on: ubuntu-latest`, as [checks-ci-wiring](checks-ci-wiring.md#runners) allows a public repository on every job.
|
|
139
140
|
A private repository sets `runs-on: [self-hosted, Linux, X64, winbox]` instead, which is the fleet's self-hosted Linux runner where it sends its mutation runs.
|
|
140
141
|
The job never reads `CI_RUNS_ON`, so an override that moves a repository's other jobs during a runner outage leaves the comparison waiting for `winbox`.
|
|
141
142
|
[checks-ci-wiring](checks-ci-wiring.md#runners) refuses a mutation job that names it in `runs-on`.
|
|
@@ -144,4 +145,5 @@ The job never reads `CI_RUNS_ON`, so an override that moves a repository's other
|
|
|
144
145
|
|
|
145
146
|
- [checks-mutation](checks-mutation.md)
|
|
146
147
|
- [checks-mutation-scope](checks-mutation-scope.md)
|
|
148
|
+
- [checks-night-run](checks-night-run.md)
|
|
147
149
|
- [checks-test-layout](checks-test-layout.md)
|
|
@@ -4,7 +4,7 @@ audience: consumers
|
|
|
4
4
|
---
|
|
5
5
|
# checks-mutation-scope
|
|
6
6
|
|
|
7
|
-
`checks-mutation-scope` turns a
|
|
7
|
+
`checks-mutation-scope` turns a branch's diff into the files its mutation runs mutate, and prints them as `NAME=a,b` lines a CI step appends to its environment file.
|
|
8
8
|
|
|
9
9
|
## What it checks
|
|
10
10
|
|
|
@@ -17,23 +17,23 @@ A `mutate` entry with a mutation range, such as `src/a.ts:1-10`, fails the run,
|
|
|
17
17
|
A renamed source enters under both names, and a deleted source does not enter.
|
|
18
18
|
|
|
19
19
|
A unit test is a `.test.ts` or `.test.tsx` file under `tests/unit/`, the files the preset's `unitTestFiles` lists.
|
|
20
|
-
A unit test the
|
|
20
|
+
A unit test the branch adds, edits, deletes or renames is changed.
|
|
21
21
|
A unit test that reads a changed file under `tests/` is changed too.
|
|
22
22
|
It reads a file when it imports it, directly or through another file under `tests/`.
|
|
23
23
|
An import resolves as Bun resolves it, to the file it names, then with an extension added, and only then to the directory's `index`.
|
|
24
24
|
A unit test also reads a changed fixture under `tests/fixtures/` when it names the fixture's path or a directory above it.
|
|
25
25
|
Bun loads each `[test] preload` in `bunfig.toml` before every test file.
|
|
26
26
|
So every unit test is changed when the preload list differs between the merge base and `HEAD`.
|
|
27
|
-
The same holds when the
|
|
27
|
+
The same holds when the branch changes a preload or a file a preload imports, wherever that file lives.
|
|
28
28
|
|
|
29
29
|
Each source whose mutants a changed unit test covers or kills in the full baseline report then enters the scope, when it still exists at `HEAD`.
|
|
30
|
-
So a kill lost through a test change alone still shows in the comparison, in a source the
|
|
30
|
+
So a kill lost through a test change alone still shows in the comparison, in a source the branch leaves alone.
|
|
31
31
|
A changed unit test with no baseline report fails the run rather than reading incomplete coverage.
|
|
32
32
|
Stryker records each test file's source in the report's `testFiles`.
|
|
33
33
|
A changed unit test the merge base holds must have that recorded source match its content at the merge base.
|
|
34
34
|
Otherwise the report ran an older test, so its coverage cannot say what the test covers now.
|
|
35
35
|
The scope then widens to every file at `HEAD` the `mutate` globs match.
|
|
36
|
-
A unit test the
|
|
36
|
+
A unit test the branch adds is new to every report, so it needs no entry.
|
|
37
37
|
Stryker writes no source without mutants into the report, so a mutated source the report lacks may be newer than it.
|
|
38
38
|
When a unit test changed, each such source that exists at `HEAD` enters the scope.
|
|
39
39
|
A change under `tests/e2e/` changes no unit test, so it needs no baseline report.
|
|
@@ -61,7 +61,7 @@ The report must come from a run that restored no state and passed `--disableBail
|
|
|
61
61
|
checks-mutation-scope <base> [<full-baseline-report>]
|
|
62
62
|
```
|
|
63
63
|
|
|
64
|
-
`<base>` is the branch
|
|
64
|
+
`<base>` is the target branch, such as `origin/main`, or the merge base itself.
|
|
65
65
|
`<full-baseline-report>` is the path the full baseline was restored to, and a path with no file reads as no report.
|
|
66
66
|
|
|
67
67
|
## Output
|
|
@@ -93,7 +93,7 @@ It also names each stale test that widens the scope.
|
|
|
93
93
|
|
|
94
94
|
## Sample output
|
|
95
95
|
|
|
96
|
-
A helper change that reaches a source the
|
|
96
|
+
A helper change that reaches a source the branch leaves alone:
|
|
97
97
|
|
|
98
98
|
```
|
|
99
99
|
mutation-scope: tests/unit/dashboard/snapshot.test.ts reads tests/lib/session-fixture.ts
|
|
@@ -116,20 +116,20 @@ mutation-scope: the baseline report did not run tests/unit/dashboard/snapshot.te
|
|
|
116
116
|
|
|
117
117
|
## When it runs
|
|
118
118
|
|
|
119
|
-
Only a CI step the repository writes runs it, in
|
|
119
|
+
Only a CI step the repository writes runs it, in a branch's mutation comparison, before either Stryker run:
|
|
120
120
|
|
|
121
121
|
```yaml
|
|
122
122
|
- name: Select mutation scope
|
|
123
123
|
env:
|
|
124
124
|
GH_TOKEN: ${{ github.token }}
|
|
125
125
|
run: |
|
|
126
|
-
base="$(git merge-base HEAD
|
|
126
|
+
base="$(git merge-base HEAD origin/main)"
|
|
127
127
|
echo "BASE_SHA=$base" >> "$GITHUB_ENV"
|
|
128
128
|
bunx checks-mutation-baseline --full "$RUNNER_TEMP/baseline-full/mutation.json"
|
|
129
129
|
bunx checks-mutation-scope "$base" "$RUNNER_TEMP/baseline-full/mutation.json" >> "$GITHUB_ENV"
|
|
130
130
|
```
|
|
131
131
|
|
|
132
|
-
Each later step reads
|
|
132
|
+
Each later step reads `SCOPE` from the environment, and a failed scope step adds nothing to it.
|
|
133
133
|
[checks-mutation-compare](checks-mutation-compare.md#running-it-in-ci) shows the whole job.
|
|
134
134
|
The full baseline comes from the `mutation` workflow, as [checks-mutation](checks-mutation.md#running-it-in-ci) shows.
|
|
135
135
|
|
|
@@ -12,7 +12,7 @@ It refuses a full mutation run when `CI` is not `true`.
|
|
|
12
12
|
A full run is one with no `--mutate <glob>` and no `--incremental` flag.
|
|
13
13
|
A `--mutate` without a glob, `--incrementalFile` alone or `--incremental` with `--force` is still a full run.
|
|
14
14
|
Its refusal names `gh workflow run mutation` as the command that starts the same run in CI.
|
|
15
|
-
A run scoped with `--mutate <glob>`, `--mutate=<glob>` or `-m <glob>` stays allowed locally, because
|
|
15
|
+
A run scoped with `--mutate <glob>`, `--mutate=<glob>` or `-m <glob>` stays allowed locally, because a branch comparison scopes to named files.
|
|
16
16
|
An incremental run stays allowed locally only when its incremental report exists, because it reuses the results of mutants that did not change.
|
|
17
17
|
That report is the `incrementalFile` Stryker resolves from the command line and the config file, else `reports/stryker-incremental.json`.
|
|
18
18
|
With no report there, the refusal says to pass `--mutate <glob>`, or to start the full baseline in CI with `gh workflow run mutation`.
|
|
@@ -75,7 +75,7 @@ An `--incremental` run with no report stops before instrumenting with the missin
|
|
|
75
75
|
## When it runs
|
|
76
76
|
|
|
77
77
|
A repository runs full baselines from the mutation workflow on `workflow_dispatch`.
|
|
78
|
-
A repository that picks a
|
|
78
|
+
A repository that picks a branch scope from a baseline starts that workflow weekly with `gh workflow run mutation`, because a GitHub schedule can start hours late.
|
|
79
79
|
Run scoped checks locally during development.
|
|
80
80
|
Each baseline costs a full Stryker run.
|
|
81
81
|
|
|
@@ -106,9 +106,9 @@ jobs:
|
|
|
106
106
|
```
|
|
107
107
|
|
|
108
108
|
The run restores no state and passes `--disableBail`, so its report names every test that covers or kills each mutant.
|
|
109
|
-
The upload is the `mutation-baseline-full` artifact that a
|
|
109
|
+
The upload is the `mutation-baseline-full` artifact that a branch comparison's scope step restores with `checks-mutation-baseline --full`.
|
|
110
110
|
[checks-mutation-scope](checks-mutation-scope.md) reads it to bring in the sources a changed unit test covers or kills.
|
|
111
|
-
The restore reads only a successful run of `mutation.yml` on `main`, so a sweep that fails publishes nothing a
|
|
111
|
+
The restore reads only a successful run of `mutation.yml` on `main`, so a sweep that fails publishes nothing a scope step reads.
|
|
112
112
|
The artifact keeps 30 days, so the newest full baseline outlives the week between full runs.
|
|
113
113
|
The upload holds the report without each mutant's `statusReason`, as [checks-mutation-baseline](checks-mutation-baseline.md#the-saved-baselines) says, since the full report of a large repository passes 100 MB as an artifact.
|
|
114
114
|
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
---
|
|
2
|
+
kind: reference
|
|
3
|
+
audience: consumers
|
|
4
|
+
---
|
|
5
|
+
# checks-night-run
|
|
6
|
+
|
|
7
|
+
`checks-night-run` turns a night run on `main` into GitHub issues, one for each file whose mutation score fell and one for each failing test.
|
|
8
|
+
|
|
9
|
+
## What it checks
|
|
10
|
+
|
|
11
|
+
It scores each file of tonight's Stryker report as Stryker does, the `Killed` and `Timeout` mutants over every mutant but the `Ignored`, `CompileError`, `RuntimeError` and `Pending` ones.
|
|
12
|
+
It compares each file's score with the same file's score from the previous night.
|
|
13
|
+
A file whose score is lower than the previous night's is a finding.
|
|
14
|
+
A file only one night holds is no finding, since it lacks one of the two scores.
|
|
15
|
+
A file with no scored mutant has no score, so it is no finding either.
|
|
16
|
+
An equal score is no finding, even when it counts other mutants.
|
|
17
|
+
Each failing test in a JUnit report is a finding, whatever the previous night holds.
|
|
18
|
+
A file whose test worker crashed is one finding, since Bun reports none of its tests.
|
|
19
|
+
With no previous night it compares no score and still files each failing test.
|
|
20
|
+
With no mutation report it compares and records no score and still files each failing test.
|
|
21
|
+
|
|
22
|
+
Each finding has a key that stays the same from night to night:
|
|
23
|
+
|
|
24
|
+
| Finding | Key |
|
|
25
|
+
| --- | --- |
|
|
26
|
+
| a file whose mutation score fell | `mutation:<file>` |
|
|
27
|
+
| a failing test | `test:<file> > <describe> > <name>` |
|
|
28
|
+
| a file whose test worker crashed | `test:<file>` |
|
|
29
|
+
|
|
30
|
+
## How it files a finding
|
|
31
|
+
|
|
32
|
+
It creates the `night-run` label when the repository lacks it.
|
|
33
|
+
It reads every open issue labelled `night-run`, and reads the key each one carries in a hidden marker in its body.
|
|
34
|
+
A finding whose key an open issue carries gets one comment on that issue, so a regression that lasts two nights stays one issue.
|
|
35
|
+
Any other finding opens one issue labelled `night-run`.
|
|
36
|
+
The issue and the comment name tonight's run, the file and either the test or the score of both nights.
|
|
37
|
+
The body holds the file, the full test name and the finding key.
|
|
38
|
+
A title longer than GitHub's 256 characters is cut to at most 256 and ends in `…`.
|
|
39
|
+
A night with no finding makes no call to GitHub's issues or labels at all.
|
|
40
|
+
Closing an issue ends it, and the same finding on a later night opens a new one.
|
|
41
|
+
Agents and people fix each issue as a task of its own, so no pull request waits for a night run.
|
|
42
|
+
A run on any ref but `refs/heads/main` files no issue and records no score, and lists each finding in its log only.
|
|
43
|
+
|
|
44
|
+
## What it reads
|
|
45
|
+
|
|
46
|
+
It reads tonight's Stryker `mutation.json` report, when the mutation run left one, and any number of JUnit reports, such as the ones `checks-test --junit=<file>` keeps, as [checks-test](checks-test.md#arguments) says.
|
|
47
|
+
It reads the previous night from the `night-run` artifact of the newest successful run of the same workflow on `main`.
|
|
48
|
+
That artifact holds `night-run.json`, the scores the run recorded, and a newest run without it counts as no previous night.
|
|
49
|
+
A run on another branch never becomes the previous night.
|
|
50
|
+
On `main` with a mutation report it writes tonight's scores to `reports/night-run.json`, and the workflow uploads that file as the `night-run` artifact.
|
|
51
|
+
It reads the run from `GITHUB_SERVER_URL`, `GITHUB_REPOSITORY`, `GITHUB_RUN_ID`, `GITHUB_WORKFLOW_REF` and `GITHUB_REF`, so it runs only inside a GitHub Actions job.
|
|
52
|
+
It calls `gh` with the job's `GH_TOKEN`, which needs `actions: read` for the previous night and `issues: write` for the issues.
|
|
53
|
+
|
|
54
|
+
## Arguments
|
|
55
|
+
|
|
56
|
+
```sh
|
|
57
|
+
checks-night-run [--mutation <mutation-report>] [--tests <junit-report>]...
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
`--mutation` names tonight's Stryker report and is given at most once.
|
|
61
|
+
`--tests` names one JUnit report and is given once per report.
|
|
62
|
+
At least one report is given.
|
|
63
|
+
|
|
64
|
+
## Exit codes
|
|
65
|
+
|
|
66
|
+
| Code | When |
|
|
67
|
+
| --- | --- |
|
|
68
|
+
| 0 | it filed every finding and recorded tonight's scores, it found nothing to file, or the run is off `main` and it listed each finding |
|
|
69
|
+
| 2 | an argument does not parse, a report does not decode, a GitHub Actions variable is unset or a `gh` call fails |
|
|
70
|
+
|
|
71
|
+
It never exits 1, because a finding becomes an issue rather than a failed run.
|
|
72
|
+
|
|
73
|
+
## Sample output
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
checks-night-run: the night run https://github.com/acme/widget/actions/runs/12 compared with the night run https://github.com/acme/widget/actions/runs/11; 2 finding(s)
|
|
77
|
+
opened #31 for mutation:src/cart.ts
|
|
78
|
+
commented on #27 for test:tests/unit/cart.test.ts > totals > rounds half to even
|
|
79
|
+
checks-night-run: recorded 42 file score(s) in reports/night-run.json
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
## When it runs
|
|
83
|
+
|
|
84
|
+
Only a night workflow the repository writes runs it, after that workflow's full mutation run and its long test tiers.
|
|
85
|
+
|
|
86
|
+
## Running it at night
|
|
87
|
+
|
|
88
|
+
A repository runs it from a workflow named `night-run` on a schedule, and starts the same workflow by hand with `gh workflow run night-run`:
|
|
89
|
+
|
|
90
|
+
```yaml
|
|
91
|
+
name: night-run
|
|
92
|
+
on:
|
|
93
|
+
schedule:
|
|
94
|
+
- cron: "7 22 * * *"
|
|
95
|
+
workflow_dispatch:
|
|
96
|
+
permissions:
|
|
97
|
+
contents: read
|
|
98
|
+
concurrency:
|
|
99
|
+
group: night-run
|
|
100
|
+
cancel-in-progress: false
|
|
101
|
+
jobs:
|
|
102
|
+
mutation:
|
|
103
|
+
runs-on: ubuntu-latest
|
|
104
|
+
timeout-minutes: 60
|
|
105
|
+
steps:
|
|
106
|
+
- uses: actions/checkout@v5
|
|
107
|
+
- uses: oven-sh/setup-bun@v2
|
|
108
|
+
with:
|
|
109
|
+
bun-version-file: .bun-version
|
|
110
|
+
- run: bun install --frozen-lockfile
|
|
111
|
+
- run: bunx stryker run
|
|
112
|
+
- uses: actions/upload-artifact@v4
|
|
113
|
+
with:
|
|
114
|
+
name: night-run-mutation
|
|
115
|
+
path: reports/mutation/mutation.json
|
|
116
|
+
retention-days: 1
|
|
117
|
+
tests:
|
|
118
|
+
runs-on: ${{ vars.CI_RUNS_ON || 'ubuntu-latest' }}
|
|
119
|
+
steps:
|
|
120
|
+
- uses: actions/checkout@v5
|
|
121
|
+
- uses: oven-sh/setup-bun@v2
|
|
122
|
+
with:
|
|
123
|
+
bun-version-file: .bun-version
|
|
124
|
+
- run: bun install --frozen-lockfile
|
|
125
|
+
- run: |
|
|
126
|
+
bunx checks-test --tier=live --junit=reports/night-run-tests/live.xml || true
|
|
127
|
+
bunx checks-test --tier=pixel --junit=reports/night-run-tests/pixel.xml || true
|
|
128
|
+
- uses: actions/upload-artifact@v4
|
|
129
|
+
with:
|
|
130
|
+
name: night-run-tests
|
|
131
|
+
path: reports/night-run-tests/
|
|
132
|
+
retention-days: 1
|
|
133
|
+
report:
|
|
134
|
+
needs: [mutation, tests]
|
|
135
|
+
if: ${{ !cancelled() && needs.tests.result == 'success' }}
|
|
136
|
+
runs-on: ${{ vars.CI_RUNS_ON || 'ubuntu-latest' }}
|
|
137
|
+
permissions:
|
|
138
|
+
contents: read
|
|
139
|
+
actions: read
|
|
140
|
+
issues: write
|
|
141
|
+
steps:
|
|
142
|
+
- uses: actions/checkout@v5
|
|
143
|
+
- uses: oven-sh/setup-bun@v2
|
|
144
|
+
with:
|
|
145
|
+
bun-version-file: .bun-version
|
|
146
|
+
- run: bun install --frozen-lockfile
|
|
147
|
+
- if: needs.mutation.result == 'success'
|
|
148
|
+
uses: actions/download-artifact@v4
|
|
149
|
+
with:
|
|
150
|
+
name: night-run-mutation
|
|
151
|
+
path: reports/mutation
|
|
152
|
+
- uses: actions/download-artifact@v4
|
|
153
|
+
with:
|
|
154
|
+
name: night-run-tests
|
|
155
|
+
path: reports/night-run-tests
|
|
156
|
+
- env:
|
|
157
|
+
GH_TOKEN: ${{ github.token }}
|
|
158
|
+
run: bunx checks-night-run ${{ needs.mutation.result == 'success' && '--mutation reports/mutation/mutation.json' || '' }} --tests reports/night-run-tests/live.xml --tests reports/night-run-tests/pixel.xml
|
|
159
|
+
- uses: actions/upload-artifact@v4
|
|
160
|
+
with:
|
|
161
|
+
name: night-run
|
|
162
|
+
path: reports/night-run.json
|
|
163
|
+
if-no-files-found: ignore
|
|
164
|
+
retention-days: 90
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
The cron starts the run at 22:07 UTC, inside the kit's own night window from 22:00 to 22:30 UTC.
|
|
168
|
+
A repository takes minute 7 of its own tier's 30 minute night window, so repositories do not start together.
|
|
169
|
+
The tests job runs each long tier the repository holds, `live` and `pixel` as [checks-test](checks-test.md#test-tiers) names them, and keeps each JUnit report.
|
|
170
|
+
Each `|| true` lets the job go on to the upload after a failing test, since that test becomes an issue.
|
|
171
|
+
A tier that writes no report fails the report job, because `checks-night-run` cannot read it.
|
|
172
|
+
The report job runs whenever the tests job succeeds, so a failing test that also stops Stryker still becomes an issue.
|
|
173
|
+
It passes `--mutation` only when the mutation job succeeded, and the failed mutation job still fails the run.
|
|
174
|
+
The concurrency group queues a run started by hand behind the scheduled one, so two runs never file the same finding twice.
|
|
175
|
+
A dispatch on a branch runs both jobs and lists its findings in the report job's log, and `if-no-files-found: ignore` lets that run upload no record.
|
|
176
|
+
`issues: write` on the report job is the one permission beyond reading the repository and its runs.
|
|
177
|
+
A public repository keeps `runs-on: ubuntu-latest` on the mutation job.
|
|
178
|
+
A private repository sets `runs-on: [self-hosted, Linux, X64, winbox]` on the mutation job, and [checks-ci-wiring](checks-ci-wiring.md#runners) refuses a mutation job that names `CI_RUNS_ON`.
|
|
179
|
+
The report job reads `CI_RUNS_ON`, because it runs no mutant.
|
|
180
|
+
A successful run uploads `night-run`, so the next night compares with it, and a failed run uploads nothing a later night reads.
|
|
181
|
+
The artifact keeps 90 days, so a gap of nights still leaves a previous night to compare with.
|
|
182
|
+
|
|
183
|
+
## Related topics
|
|
184
|
+
|
|
185
|
+
- [checks-test](checks-test.md)
|
|
186
|
+
- [checks-mutation](checks-mutation.md)
|
|
187
|
+
- [checks-mutation-compare](checks-mutation-compare.md)
|
|
188
|
+
- [checks-ci-wiring](checks-ci-wiring.md)
|
|
@@ -140,7 +140,8 @@ jobs:
|
|
|
140
140
|
```
|
|
141
141
|
|
|
142
142
|
A private repository sets `runs-on: ${{ vars.CI_RUNS_ON || 'ubuntu-latest' }}` on each job in either workflow, as [checks-ci-wiring](checks-ci-wiring.md#runners) requires.
|
|
143
|
-
A release is a pull request that holds only the version bump and the built changelog
|
|
143
|
+
A release is a pull request that holds only the version bump and the built changelog.
|
|
144
|
+
[checks-release-pr](checks-release-pr.md) opens it on the daily release's schedule, or on every push to `main` in a package repository.
|
|
144
145
|
When it lands, [checks-release-tag](checks-release-tag.md) tags the merge commit and dispatches this workflow on the tag.
|
|
145
146
|
A tag the workflow token pushes starts no `push` run, so the workflow triggers on `workflow_dispatch` as well, and refuses a dispatch on a ref that is not a tag.
|
|
146
147
|
A tag a person pushes starts the same workflow through its `push` trigger.
|