@avi2dg/checks 0.19.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +1 -0
- package/dist/feature-rules.js +5 -0
- package/docs/configs/commit-messages.md +1 -0
- package/docs/configs/quality-file.md +2 -0
- package/docs/gates/checks-quality.md +3 -1
- package/docs/gates/checks-subsumed-tests.md +68 -0
- package/package.json +4 -1
- package/quality.schema.json +18 -0
- package/scripts/mutation-compare.ts +51 -17
- package/scripts/quality-file.ts +6 -0
- package/scripts/quality.ts +44 -8
- package/scripts/subsumed-tests.ts +208 -0
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
Every release of `@avi2dg/checks`, newest first, written by the release from its conventional commits.
|
|
4
4
|
|
|
5
|
+
## 0.21.0
|
|
6
|
+
|
|
7
|
+
Released 2026-09-26.
|
|
8
|
+
|
|
9
|
+
### Features
|
|
10
|
+
|
|
11
|
+
- **scripts:** let quality.json set the runs-on of generated workflows (#66)
|
|
12
|
+
|
|
13
|
+
## 0.20.0
|
|
14
|
+
|
|
15
|
+
Released 2026-09-26.
|
|
16
|
+
|
|
17
|
+
### Features
|
|
18
|
+
|
|
19
|
+
- **scripts:** pin node from .node-version in the generated CI workflow (#64)
|
|
20
|
+
- **scripts:** add checks-subsumed-tests to report tests another test subsumes in a (#62)
|
|
21
|
+
|
|
22
|
+
### Fixes
|
|
23
|
+
|
|
24
|
+
- **scripts:** lint a PR title that starts with # in the generated commitlint workflow (#63)
|
|
25
|
+
|
|
5
26
|
## 0.19.0
|
|
6
27
|
|
|
7
28
|
Released 2026-09-26.
|
package/README.md
CHANGED
|
@@ -132,6 +132,7 @@ These bins run on their own:
|
|
|
132
132
|
- [`checks-test`](docs/gates/checks-test.md) runs the suite as `scripts.test` and refuses a skip the repository has not declared.
|
|
133
133
|
- [`checks-flake`](docs/gates/checks-flake.md) runs the suite on a schedule and records the seeds a flaky test fails with.
|
|
134
134
|
- [`checks-mutation-compare`](docs/gates/checks-mutation-compare.md) holds every mutant in a pull request to no regression.
|
|
135
|
+
- [`checks-subsumed-tests`](docs/gates/checks-subsumed-tests.md) lists each test another test subsumes in a mutation run.
|
|
135
136
|
- [`checks-backtest`](docs/gates/checks-backtest.md) reports what the comment check would have refused in recent history.
|
|
136
137
|
- [`checks-vendor`](docs/gates/checks-vendor.md) pins each library `quality.json` declares to a shared read-only clone and links it under `repos/`.
|
|
137
138
|
|
package/dist/feature-rules.js
CHANGED
|
@@ -153,6 +153,10 @@ var Gates = Schema3.Struct({
|
|
|
153
153
|
scheduled: Schema3.optionalKey(Schema3.Array(Command).annotate({ description: "The commands a cron-scheduled workflow runs" })),
|
|
154
154
|
lint: Schema3.optionalKey(LintGates)
|
|
155
155
|
});
|
|
156
|
+
var RunsOn = Schema3.NonEmptyArray(Schema3.NonEmptyString).annotate({
|
|
157
|
+
identifier: "RunsOn",
|
|
158
|
+
description: "The runner labels every job the kit generates runs on; ubuntu-latest when absent"
|
|
159
|
+
});
|
|
156
160
|
var EffectSources = Schema3.Struct({
|
|
157
161
|
paths: Schema3.NonEmptyArray(PathGlob).annotate({
|
|
158
162
|
description: "Where source is written in Effect, held to the Effect rules of oxlint and the language service"
|
|
@@ -233,6 +237,7 @@ var Quality = Schema3.Struct({
|
|
|
233
237
|
$schema: Schema3.optionalKey(Schema3.String),
|
|
234
238
|
defaultBranch: Schema3.optionalKey(Schema3.NonEmptyString.annotate({ description: "The branch pull requests merge into; main when absent" })),
|
|
235
239
|
gates: Schema3.optionalKey(Gates),
|
|
240
|
+
runsOn: Schema3.optionalKey(RunsOn),
|
|
236
241
|
commitIdentity: Schema3.optionalKey(CommitIdentity),
|
|
237
242
|
sources: Schema3.optionalKey(Sources),
|
|
238
243
|
size: Schema3.optionalKey(Size),
|
|
@@ -18,6 +18,7 @@ The workflow lints with the installed kit's `commitlint.config.js`, so every rep
|
|
|
18
18
|
It lints the pull request title and nothing else.
|
|
19
19
|
The title is the enforced subject because a squash merge uses it as the main commit subject, and per-commit messages are not linted.
|
|
20
20
|
GitHub appends ` (#N)` to the squashed subject, so the workflow lints the title with that suffix attached, and the header length limit applies to the landed subject, not the bare title.
|
|
21
|
+
The workflow moves git's comment character off `#`, so a title starting with `#` is linted like any other.
|
|
21
22
|
|
|
22
23
|
It never sees a commit's author or committer fields, nor the `Co-authored-by` trailer GitHub writes from a foreign author when it squashes, so it cannot enforce who a commit belongs to.
|
|
23
24
|
[checks-commit-identity](../gates/checks-commit-identity.md) is that enforcement.
|
|
@@ -14,6 +14,7 @@ The kit's bins find the file at the git root and read it there:
|
|
|
14
14
|
"ci": ["bun run lint", "bun run typecheck", "bun run test"],
|
|
15
15
|
"scheduled": ["bunx checks-flake --runs 10 --report flake-report.json"]
|
|
16
16
|
},
|
|
17
|
+
"runsOn": ["self-hosted", "Linux", "X64", "winbox"],
|
|
17
18
|
"commitIdentity": { "authors": [{ "name": "avi2d", "email": "avi2dg@gmail.com" }] },
|
|
18
19
|
"sources": {
|
|
19
20
|
"production": ["src/**/*.ts"],
|
|
@@ -43,6 +44,7 @@ The kit's bins find the file at the git root and read it there:
|
|
|
43
44
|
| `gates.ci` | `checks-ci-wiring`, `checks-quality` | the commands CI runs on every pull request, as [checks-ci-wiring](../gates/checks-ci-wiring.md) says |
|
|
44
45
|
| `gates.scheduled` | `checks-ci-wiring` | the commands a schedule runs |
|
|
45
46
|
| `gates.lint` | `checks-lint`, `checks-ci-wiring` | the gates `checks-lint` runs when not all apply, as [Gate selection](../gates/checks-lint.md#gate-selection) says |
|
|
47
|
+
| `runsOn` | `checks-quality` | the runner labels every job the ci and commitlint workflows run on, `ubuntu-latest` when absent |
|
|
46
48
|
| `commitIdentity.authors` | `checks-commit-identity` | the identities allowed to author and commit, as [checks-commit-identity](../gates/checks-commit-identity.md) says |
|
|
47
49
|
| `sources.production` | `checks-size-budget`, `checks-repetition`, `checks-quality` | the source the repository ships, as [checks-size-budget](../gates/checks-size-budget.md) and [checks-repetition](../gates/checks-repetition.md) say |
|
|
48
50
|
| `sources.effect` | `checks-quality` | the paths held to the Effect rules, and the files under them that are not, as [The Effect rules](effect-rules.md) says |
|
|
@@ -32,6 +32,8 @@ A rule only this repository needs stays in its own `.oxlintrc.json`, whose overr
|
|
|
32
32
|
GitHub Actions reads its own YAML and nothing else, so `checks-quality generate` also writes the kit recipe workflows whole.
|
|
33
33
|
`.github/workflows/ci.yml` runs every `gates.ci` command but the title lint as its own step after a frozen install.
|
|
34
34
|
`.github/workflows/commitlint.yml` lints the pull request title with the installed kit config.
|
|
35
|
+
Both jobs run on the runner labels `quality.json` `runsOn` lists, or on `ubuntu-latest` when the key is absent.
|
|
36
|
+
A self-hosted runner in a public repository runs the code of any pull request from a fork.
|
|
35
37
|
A step one repository alone needs lives in another workflow file, never in the recipe.
|
|
36
38
|
All generated workflows are committed.
|
|
37
39
|
|
|
@@ -59,7 +61,7 @@ Two details of the fragments are easy to get wrong, so the kit's tests pin both:
|
|
|
59
61
|
It reads the working tree: `quality.json`, the presets of the installed kit, `.oxlintrc.json`, `tsconfig.json`, the two fragments and the generated workflows.
|
|
60
62
|
It reads the root `package.json` name, since only the kit's own tree lints titles with its root `commitlint.config.js`.
|
|
61
63
|
The name also decides which kit configs `extends` must list, since the kit's own tree extends its root `oxlintrc.json` and `tsconfig.effect.json`.
|
|
62
|
-
It looks for `.bun-version`, and the suite pins its bun to
|
|
64
|
+
It looks for `.bun-version` and `.node-version`, and the suite pins its bun and its node to whichever of the two files exists.
|
|
63
65
|
It lists the tracked and untracked files to see what each declared glob matches.
|
|
64
66
|
|
|
65
67
|
## Arguments
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# checks-subsumed-tests
|
|
2
|
+
|
|
3
|
+
`checks-subsumed-tests` is the report that lists each test another test subsumes in a Stryker mutation run, and a reader looks it up to judge whether the suite carries tests it no longer needs.
|
|
4
|
+
|
|
5
|
+
## What it checks
|
|
6
|
+
|
|
7
|
+
It checks nothing and fails nothing.
|
|
8
|
+
It reads one Stryker `mutation.json` report, gives each test the set of mutants it kills, and prints each test whose kill set sits inside the kill set of one other test beside that test, with both kill counts.
|
|
9
|
+
It then prints the tests whose kill sets are identical, then the size of a greedy cover that keeps every kill out of every test the report lists.
|
|
10
|
+
A test is subsumed when its kill set sits inside the kill set of one other test, so dropping every subsumed test loses no kill in this run.
|
|
11
|
+
The report informs a person and decides nothing, so keep a subsumed test unless reading the pair shows the same scenario.
|
|
12
|
+
A subsumed verdict trusts the mutants the run covers, so read the files the report names before judging a test redundant.
|
|
13
|
+
A test judged against a module that is not its subject looks redundant until its own subject is mutated.
|
|
14
|
+
A test whose subject no mutant can touch, such as frontmatter or links, looks redundant because mutation cannot see what it checks.
|
|
15
|
+
|
|
16
|
+
## What it reads
|
|
17
|
+
|
|
18
|
+
It reads one Stryker `mutation.json` report built with bail off, which the shared Stryker preset's `json` reporter writes to `reports/mutation/mutation.json`.
|
|
19
|
+
With `disableBail` Stryker runs every covering test for each mutant and records every test that fails as a killer, so each test gets a kill set.
|
|
20
|
+
A report built with bail on records one killer per mutant, which makes every test look unique.
|
|
21
|
+
Stryker writes its options into the report's `config`, so `checks-subsumed-tests` refuses a report whose `config.disableBail` is not `true`.
|
|
22
|
+
A report with no `config` shows nothing about bail, so `checks-subsumed-tests` prints one warning and reads it anyway.
|
|
23
|
+
Build a bail-off report with this command:
|
|
24
|
+
|
|
25
|
+
```sh
|
|
26
|
+
bunx stryker run --disableBail
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
The report records each killer as a test index, so it names each test by its file and its name from the report's `testFiles` table.
|
|
30
|
+
|
|
31
|
+
## Arguments
|
|
32
|
+
|
|
33
|
+
```sh
|
|
34
|
+
checks-subsumed-tests <mutation-report>
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Exit codes
|
|
38
|
+
|
|
39
|
+
| Code | When |
|
|
40
|
+
| --- | --- |
|
|
41
|
+
| 0 | the report printed |
|
|
42
|
+
| 2 | the report is not a Stryker mutation report, it was built with bail on, or the arguments do not parse |
|
|
43
|
+
|
|
44
|
+
## Sample output
|
|
45
|
+
|
|
46
|
+
It prints the files the run mutated, then each subsumed test beside the test that subsumes it, then the tests whose kill sets are identical, then the greedy cover:
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
subsumed-tests: 2 file(s) mutated
|
|
50
|
+
mutated src/add.ts
|
|
51
|
+
mutated src/mul.ts
|
|
52
|
+
subsumed tests (1):
|
|
53
|
+
"tests/add.test.ts > add sums two numbers" (1 kill) subsumed by "tests/add.test.ts > add covers every operator" (3 kills)
|
|
54
|
+
identical kill sets (1 group(s)):
|
|
55
|
+
"tests/mul.test.ts > mul multiplies" (2 kills) = "tests/mul.test.ts > mul multiplies in either order" (2 kills)
|
|
56
|
+
greedy cover: 3 of 6 test(s) keep all 6 kill(s)
|
|
57
|
+
cover "tests/add.test.ts > add covers every operator"
|
|
58
|
+
cover "tests/mul.test.ts > mul multiplies"
|
|
59
|
+
cover "tests/mul.test.ts > mul checks its guard"
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Opting out
|
|
63
|
+
|
|
64
|
+
Nothing runs it but a person who wants the figures.
|
|
65
|
+
|
|
66
|
+
## Related topics
|
|
67
|
+
|
|
68
|
+
- [checks-mutation-compare](checks-mutation-compare.md)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@avi2dg/checks",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.21.0",
|
|
4
4
|
"description": "Deterministic checks shared across the captain's TypeScript repos",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -31,6 +31,7 @@
|
|
|
31
31
|
"scripts/flake.ts",
|
|
32
32
|
"scripts/commit-identity.ts",
|
|
33
33
|
"scripts/mutation-compare.ts",
|
|
34
|
+
"scripts/subsumed-tests.ts",
|
|
34
35
|
"scripts/ci-wiring.ts",
|
|
35
36
|
"scripts/shell-command.ts",
|
|
36
37
|
"scripts/comment-matchers.ts",
|
|
@@ -75,6 +76,7 @@
|
|
|
75
76
|
"./scripts/flake.ts": "./scripts/flake.ts",
|
|
76
77
|
"./scripts/commit-identity.ts": "./scripts/commit-identity.ts",
|
|
77
78
|
"./scripts/mutation-compare.ts": "./scripts/mutation-compare.ts",
|
|
79
|
+
"./scripts/subsumed-tests.ts": "./scripts/subsumed-tests.ts",
|
|
78
80
|
"./scripts/ci-wiring.ts": "./scripts/ci-wiring.ts",
|
|
79
81
|
"./scripts/comment-matchers.ts": "./scripts/comment-matchers.ts",
|
|
80
82
|
"./scripts/prose-matchers.ts": "./scripts/prose-matchers.ts",
|
|
@@ -116,6 +118,7 @@
|
|
|
116
118
|
"checks-flake": "scripts/flake.ts",
|
|
117
119
|
"checks-commit-identity": "scripts/commit-identity.ts",
|
|
118
120
|
"checks-mutation-compare": "scripts/mutation-compare.ts",
|
|
121
|
+
"checks-subsumed-tests": "scripts/subsumed-tests.ts",
|
|
119
122
|
"checks-ci-wiring": "scripts/ci-wiring.ts",
|
|
120
123
|
"checks-comment-gate": "scripts/comment-gate.ts",
|
|
121
124
|
"checks-suppressions-ratchet": "scripts/suppressions-ratchet.ts",
|
package/quality.schema.json
CHANGED
|
@@ -88,6 +88,9 @@
|
|
|
88
88
|
},
|
|
89
89
|
"additionalProperties": false
|
|
90
90
|
},
|
|
91
|
+
"runsOn": {
|
|
92
|
+
"$ref": "#/$defs/RunsOn"
|
|
93
|
+
},
|
|
91
94
|
"commitIdentity": {
|
|
92
95
|
"type": "object",
|
|
93
96
|
"properties": {
|
|
@@ -406,6 +409,21 @@
|
|
|
406
409
|
"type": "string",
|
|
407
410
|
"minLength": 1
|
|
408
411
|
},
|
|
412
|
+
"RunsOn": {
|
|
413
|
+
"type": "array",
|
|
414
|
+
"prefixItems": [
|
|
415
|
+
{
|
|
416
|
+
"type": "string",
|
|
417
|
+
"minLength": 1
|
|
418
|
+
}
|
|
419
|
+
],
|
|
420
|
+
"minItems": 1,
|
|
421
|
+
"items": {
|
|
422
|
+
"type": "string",
|
|
423
|
+
"minLength": 1
|
|
424
|
+
},
|
|
425
|
+
"description": "The runner labels every job the kit generates runs on; ubuntu-latest when absent"
|
|
426
|
+
},
|
|
409
427
|
"Identity": {
|
|
410
428
|
"type": "object",
|
|
411
429
|
"properties": {
|
|
@@ -11,6 +11,7 @@ export type Mutant = {
|
|
|
11
11
|
readonly status: string;
|
|
12
12
|
readonly mutatorName: string;
|
|
13
13
|
readonly replacement: string;
|
|
14
|
+
readonly killedBy?: readonly string[];
|
|
14
15
|
readonly location: Location;
|
|
15
16
|
};
|
|
16
17
|
|
|
@@ -19,6 +20,16 @@ export type ReportFile = {
|
|
|
19
20
|
readonly mutants: readonly Mutant[];
|
|
20
21
|
};
|
|
21
22
|
|
|
23
|
+
export type TestFile = {
|
|
24
|
+
readonly tests: readonly { readonly id: string; readonly name: string }[];
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
export type KillRun = {
|
|
28
|
+
readonly files: ReadonlyMap<string, ReportFile>;
|
|
29
|
+
readonly testFiles: ReadonlyMap<string, TestFile>;
|
|
30
|
+
readonly bail: "off" | "on" | "unrecorded";
|
|
31
|
+
};
|
|
32
|
+
|
|
22
33
|
export type MutantChange = {
|
|
23
34
|
readonly path: string;
|
|
24
35
|
readonly location: Location;
|
|
@@ -59,33 +70,56 @@ const UNDETECTED = new Set(["Survived", "NoCoverage"]);
|
|
|
59
70
|
const LEAVES_SCORE = new Set(["CompileError", "RuntimeError", "Ignored", "Pending"]);
|
|
60
71
|
const USAGE = "usage: mutation-compare.ts [--advisory] <base-report> <head-report>";
|
|
61
72
|
|
|
62
|
-
const
|
|
73
|
+
const Files = Schema.Record(
|
|
74
|
+
Schema.String,
|
|
63
75
|
Schema.Struct({
|
|
64
|
-
|
|
65
|
-
|
|
76
|
+
source: Schema.String,
|
|
77
|
+
mutants: Schema.Array(
|
|
66
78
|
Schema.Struct({
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
end: Schema.Struct({ line: Schema.Finite, column: Schema.Finite }),
|
|
76
|
-
}),
|
|
77
|
-
}),
|
|
78
|
-
),
|
|
79
|
+
status: Schema.String,
|
|
80
|
+
mutatorName: Schema.String,
|
|
81
|
+
replacement: Schema.String,
|
|
82
|
+
killedBy: Schema.optionalKey(Schema.Array(Schema.String)),
|
|
83
|
+
location: Schema.Struct({
|
|
84
|
+
start: Schema.Struct({ line: Schema.Finite, column: Schema.Finite }),
|
|
85
|
+
end: Schema.Struct({ line: Schema.Finite, column: Schema.Finite }),
|
|
86
|
+
}),
|
|
79
87
|
}),
|
|
80
88
|
),
|
|
81
89
|
}),
|
|
82
90
|
);
|
|
83
|
-
const decodeReport = Schema.decodeUnknownEffect(
|
|
91
|
+
const decodeReport = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Struct({ files: Files })));
|
|
92
|
+
const decodeKillRun = Schema.decodeUnknownEffect(
|
|
93
|
+
Schema.fromJsonString(
|
|
94
|
+
Schema.Struct({
|
|
95
|
+
files: Files,
|
|
96
|
+
testFiles: Schema.Record(Schema.String, Schema.Struct({ tests: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.String })) })),
|
|
97
|
+
config: Schema.optionalKey(Schema.Struct({ disableBail: Schema.optionalKey(Schema.Boolean) })),
|
|
98
|
+
}),
|
|
99
|
+
),
|
|
100
|
+
);
|
|
101
|
+
|
|
102
|
+
const notAReport = (source: string) => (cause: { readonly message: string }) => new ReportError({ message: `${source} is not a Stryker mutation report: ${cause.message}` });
|
|
84
103
|
|
|
85
104
|
export const parseReport = (source: string, text: string): Effect.Effect<Map<string, ReportFile>, ReportError> =>
|
|
86
105
|
decodeReport(text).pipe(
|
|
87
106
|
Effect.map(({ files }) => new Map(Object.entries(files))),
|
|
88
|
-
Effect.mapError((
|
|
107
|
+
Effect.mapError(notAReport(source)),
|
|
108
|
+
);
|
|
109
|
+
|
|
110
|
+
function bailOf(config: { readonly disableBail?: boolean } | undefined): KillRun["bail"] {
|
|
111
|
+
if (config === undefined) return "unrecorded";
|
|
112
|
+
return config.disableBail === true ? "off" : "on";
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export const parseKillRun = (source: string, text: string): Effect.Effect<KillRun, ReportError> =>
|
|
116
|
+
decodeKillRun(text).pipe(
|
|
117
|
+
Effect.map(({ files, testFiles, config }) => ({
|
|
118
|
+
files: new Map(Object.entries(files)),
|
|
119
|
+
testFiles: new Map(Object.entries(testFiles)),
|
|
120
|
+
bail: bailOf(config),
|
|
121
|
+
})),
|
|
122
|
+
Effect.mapError(notAReport(source)),
|
|
89
123
|
);
|
|
90
124
|
|
|
91
125
|
function byPosition(a: { readonly path: string; readonly location: Location }, b: { readonly path: string; readonly location: Location }): number {
|
package/scripts/quality-file.ts
CHANGED
|
@@ -82,6 +82,11 @@ const Gates = Schema.Struct({
|
|
|
82
82
|
lint: Schema.optionalKey(LintGates),
|
|
83
83
|
});
|
|
84
84
|
|
|
85
|
+
const RunsOn = Schema.NonEmptyArray(Schema.NonEmptyString).annotate({
|
|
86
|
+
identifier: "RunsOn",
|
|
87
|
+
description: "The runner labels every job the kit generates runs on; ubuntu-latest when absent",
|
|
88
|
+
});
|
|
89
|
+
|
|
85
90
|
const EffectSources = Schema.Struct({
|
|
86
91
|
paths: Schema.NonEmptyArray(PathGlob).annotate({
|
|
87
92
|
description: "Where source is written in Effect, held to the Effect rules of oxlint and the language service",
|
|
@@ -205,6 +210,7 @@ export const Quality = Schema.Struct({
|
|
|
205
210
|
Schema.NonEmptyString.annotate({ description: "The branch pull requests merge into; main when absent" }),
|
|
206
211
|
),
|
|
207
212
|
gates: Schema.optionalKey(Gates),
|
|
213
|
+
runsOn: Schema.optionalKey(RunsOn),
|
|
208
214
|
commitIdentity: Schema.optionalKey(CommitIdentity),
|
|
209
215
|
sources: Schema.optionalKey(Sources),
|
|
210
216
|
size: Schema.optionalKey(Size),
|
package/scripts/quality.ts
CHANGED
|
@@ -38,6 +38,7 @@ export type GeneratedWorkflow = {
|
|
|
38
38
|
export type WorkflowRecipe = {
|
|
39
39
|
readonly commitlintConfig: string;
|
|
40
40
|
readonly bunVersionFile: boolean;
|
|
41
|
+
readonly nodeVersionFile: boolean;
|
|
41
42
|
};
|
|
42
43
|
|
|
43
44
|
class NativeConfigUnreadable extends Schema.TaggedError<NativeConfigUnreadable>()("NativeConfigUnreadable", {
|
|
@@ -104,17 +105,34 @@ function runsInTitleLint(gate: string, commitlintConfig: string): boolean {
|
|
|
104
105
|
return words !== undefined && invokes(plainCommand(titleLint(commitlintConfig)), words);
|
|
105
106
|
}
|
|
106
107
|
|
|
108
|
+
// YAML 1.2's core schema resolves these plain scalars to a null, a boolean or a number, and YAML 1.1 parsers also read yes, no, on and off as booleans.
|
|
109
|
+
const NON_STRING_SCALAR =
|
|
110
|
+
/^(?:~|null|Null|NULL|true|True|TRUE|false|False|FALSE|y|Y|yes|Yes|YES|n|N|no|No|NO|on|On|ON|off|Off|OFF|[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?|0o[0-7]+|0x[0-9a-fA-F]+|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$/;
|
|
111
|
+
|
|
107
112
|
function runScalar(command: string): string {
|
|
108
|
-
return /(^[\s#&*!|>@`'"%{[-])|:\s|\s#|:$|[\n\\]/.test(command)
|
|
113
|
+
return /(^[\s#&*!|>@`'"%{[-])|:\s|\s#|:$|[\n\\]/.test(command) || NON_STRING_SCALAR.test(command)
|
|
114
|
+
? JSON.stringify(command)
|
|
115
|
+
: command;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function flowScalar(label: string): string {
|
|
119
|
+
return /[,[\]{}]/.test(label) ? JSON.stringify(label) : runScalar(label);
|
|
109
120
|
}
|
|
110
121
|
|
|
111
|
-
|
|
112
|
-
return
|
|
122
|
+
function runsOnLine(runsOn: Quality["runsOn"]): string {
|
|
123
|
+
if (runsOn === undefined) return "ubuntu-latest";
|
|
124
|
+
return `[${runsOn.map(flowScalar).join(", ")}]`;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function commitlintWorkflow(commitlintConfig: string, runsOn: Quality["runsOn"]): string {
|
|
128
|
+
return `permissions:
|
|
129
|
+
contents: read
|
|
130
|
+
on:
|
|
113
131
|
pull_request:
|
|
114
132
|
types: [opened, edited, synchronize, reopened]
|
|
115
133
|
jobs:
|
|
116
134
|
commitlint:
|
|
117
|
-
runs-on:
|
|
135
|
+
runs-on: ${runsOnLine(runsOn)}
|
|
118
136
|
steps:
|
|
119
137
|
- uses: actions/checkout@v5
|
|
120
138
|
- uses: oven-sh/setup-bun@v2
|
|
@@ -123,27 +141,44 @@ jobs:
|
|
|
123
141
|
- run: printf '%s' "$PR_TITLE (#0000)" > "$RUNNER_TEMP/pr-title"
|
|
124
142
|
env:
|
|
125
143
|
PR_TITLE: \${{ github.event.pull_request.title }}
|
|
144
|
+
# A title starting with git's comment character lints as empty without this: commentChar moves off '#'.
|
|
126
145
|
- run: ${titleLint(commitlintConfig)}
|
|
146
|
+
env:
|
|
147
|
+
GIT_CONFIG_COUNT: "1"
|
|
148
|
+
GIT_CONFIG_KEY_0: core.commentChar
|
|
149
|
+
GIT_CONFIG_VALUE_0: "\\x01"
|
|
127
150
|
`;
|
|
128
151
|
}
|
|
129
152
|
|
|
130
|
-
export function suiteWorkflow(
|
|
153
|
+
export function suiteWorkflow(
|
|
154
|
+
defaultBranch: string,
|
|
155
|
+
gates: readonly string[],
|
|
156
|
+
bunVersionFile: boolean,
|
|
157
|
+
nodeVersionFile: boolean,
|
|
158
|
+
runsOn: Quality["runsOn"],
|
|
159
|
+
): string {
|
|
160
|
+
const node = nodeVersionFile
|
|
161
|
+
? " - uses: actions/setup-node@v5\n with:\n node-version-file: .node-version\n"
|
|
162
|
+
: "";
|
|
131
163
|
const setup = bunVersionFile
|
|
132
164
|
? " - uses: oven-sh/setup-bun@v2\n with:\n bun-version-file: .bun-version\n"
|
|
133
165
|
: " - uses: oven-sh/setup-bun@v2\n";
|
|
134
166
|
const steps = gates.map((gate) => ` - run: ${runScalar(gate)}\n`).join("");
|
|
135
|
-
return `name: ci\non:\n push:\n branches: [${defaultBranch}]\n pull_request:\n types: [opened, edited, synchronize, reopened]\njobs:\n checks:\n runs-on:
|
|
167
|
+
return `name: ci\non:\n push:\n branches: [${defaultBranch}]\n pull_request:\n types: [opened, edited, synchronize, reopened]\njobs:\n checks:\n runs-on: ${runsOnLine(runsOn)}\n steps:\n # The head, not GitHub's merge ref, so the tree the gates read is the commit the range ends at.\n # The whole history, since the range starts where the head branched from the base branch.\n - uses: actions/checkout@v5\n with:\n ref: \${{ github.event.pull_request.head.sha || github.sha }}\n fetch-depth: 0\n${node}${setup} - run: bun install --frozen-lockfile\n${steps}`;
|
|
136
168
|
}
|
|
137
169
|
|
|
138
170
|
export function workflowsFor(quality: Quality, recipe: WorkflowRecipe): readonly GeneratedWorkflow[] {
|
|
139
171
|
const commitlint: GeneratedWorkflow = {
|
|
140
172
|
file: COMMITLINT_WORKFLOW,
|
|
141
|
-
content: commitlintWorkflow(recipe.commitlintConfig),
|
|
173
|
+
content: commitlintWorkflow(recipe.commitlintConfig, quality.runsOn),
|
|
142
174
|
};
|
|
143
175
|
if (quality.gates?.ci === undefined) return [commitlint];
|
|
144
176
|
const suite = quality.gates.ci.filter((gate) => !runsInTitleLint(gate, recipe.commitlintConfig));
|
|
145
177
|
return [
|
|
146
|
-
{
|
|
178
|
+
{
|
|
179
|
+
file: SUITE_WORKFLOW,
|
|
180
|
+
content: suiteWorkflow(quality.defaultBranch ?? DEFAULT_BRANCH, suite, recipe.bunVersionFile, recipe.nodeVersionFile, quality.runsOn),
|
|
181
|
+
},
|
|
147
182
|
commitlint,
|
|
148
183
|
];
|
|
149
184
|
}
|
|
@@ -247,6 +282,7 @@ const recipeOf = Effect.fn("recipeOf")(function* (root: string) {
|
|
|
247
282
|
// The kit never installs itself, so its own tree lints with its root config.
|
|
248
283
|
commitlintConfig: (yield* isKit(root)) ? OWN_COMMITLINT_CONFIG : KIT_COMMITLINT_CONFIG,
|
|
249
284
|
bunVersionFile: yield* fs.exists(path.join(root, ".bun-version")),
|
|
285
|
+
nodeVersionFile: yield* fs.exists(path.join(root, ".node-version")),
|
|
250
286
|
} satisfies WorkflowRecipe;
|
|
251
287
|
});
|
|
252
288
|
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
import { Console, Effect, FileSystem } from "effect";
|
|
3
|
+
import { runMain, Usage } from "./main.ts";
|
|
4
|
+
import { type KillRun, parseKillRun, ReportError } from "./mutation-compare.ts";
|
|
5
|
+
|
|
6
|
+
export type KillSets = ReadonlyMap<string, ReadonlySet<string>>;
|
|
7
|
+
|
|
8
|
+
export type Subsumed = {
|
|
9
|
+
readonly test: string;
|
|
10
|
+
readonly kills: number;
|
|
11
|
+
readonly subsumedBy: string;
|
|
12
|
+
readonly subsumerKills: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export type IdenticalGroup = {
|
|
16
|
+
readonly tests: readonly string[];
|
|
17
|
+
readonly kills: number;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
export type Cover = {
|
|
21
|
+
readonly members: readonly string[];
|
|
22
|
+
readonly tests: number;
|
|
23
|
+
readonly kills: number;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export type Report = {
|
|
27
|
+
readonly files: readonly string[];
|
|
28
|
+
readonly subsumed: readonly Subsumed[];
|
|
29
|
+
readonly identical: readonly IdenticalGroup[];
|
|
30
|
+
readonly cover: Cover;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export type Options = {
|
|
34
|
+
readonly reportPath: string;
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const USAGE = "usage: subsumed-tests.ts <mutation-report>";
|
|
38
|
+
|
|
39
|
+
export function killSetsOf(run: KillRun): KillSets {
|
|
40
|
+
const kills = new Map<string, Set<string>>();
|
|
41
|
+
for (const file of run.testFiles.values()) for (const test of file.tests) kills.set(test.id, new Set<string>());
|
|
42
|
+
for (const [path, file] of run.files) {
|
|
43
|
+
file.mutants.forEach((mutant, index) => {
|
|
44
|
+
for (const test of mutant.killedBy ?? []) {
|
|
45
|
+
let set = kills.get(test);
|
|
46
|
+
if (set === undefined) {
|
|
47
|
+
set = new Set<string>();
|
|
48
|
+
kills.set(test, set);
|
|
49
|
+
}
|
|
50
|
+
set.add(`${path}#${index}`);
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
return kills;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function isSuperset(candidate: ReadonlySet<string>, other: ReadonlySet<string>): boolean {
|
|
58
|
+
if (candidate.size <= other.size) return false;
|
|
59
|
+
for (const kill of other) if (!candidate.has(kill)) return false;
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function subsumerOf(test: string, kills: ReadonlySet<string>, sets: KillSets): { readonly name: string; readonly size: number } | undefined {
|
|
64
|
+
let best: { readonly name: string; readonly size: number } | undefined;
|
|
65
|
+
for (const [other, otherKills] of sets) {
|
|
66
|
+
if (other !== test && isSuperset(otherKills, kills) && (best === undefined || otherKills.size > best.size || (otherKills.size === best.size && other < best.name))) best = { name: other, size: otherKills.size };
|
|
67
|
+
}
|
|
68
|
+
return best;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function findSubsumed(sets: KillSets): readonly Subsumed[] {
|
|
72
|
+
const subsumed: Subsumed[] = [];
|
|
73
|
+
for (const [test, kills] of sets) {
|
|
74
|
+
if (kills.size === 0) continue;
|
|
75
|
+
const subsumer = subsumerOf(test, kills, sets);
|
|
76
|
+
if (subsumer !== undefined) subsumed.push({ test, kills: kills.size, subsumedBy: subsumer.name, subsumerKills: subsumer.size });
|
|
77
|
+
}
|
|
78
|
+
return subsumed;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function findIdentical(sets: KillSets): readonly IdenticalGroup[] {
|
|
82
|
+
const groups = new Map<string, string[]>();
|
|
83
|
+
for (const [test, kills] of sets) {
|
|
84
|
+
if (kills.size === 0) continue;
|
|
85
|
+
const key = [...kills].toSorted().join("\n");
|
|
86
|
+
const group = groups.get(key);
|
|
87
|
+
if (group === undefined) groups.set(key, [test]);
|
|
88
|
+
else group.push(test);
|
|
89
|
+
}
|
|
90
|
+
return [...groups.values()].filter((tests) => tests.length > 1).map((tests) => ({ tests, kills: sets.get(tests[0] ?? "")?.size ?? 0 }));
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function uncoveredCount(kills: ReadonlySet<string>, uncovered: ReadonlySet<string>): number {
|
|
94
|
+
let count = 0;
|
|
95
|
+
for (const kill of kills) if (uncovered.has(kill)) count++;
|
|
96
|
+
return count;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function bestAddition(candidates: readonly (readonly [string, ReadonlySet<string>])[], members: readonly string[], uncovered: ReadonlySet<string>): readonly [string, ReadonlySet<string>] | undefined {
|
|
100
|
+
let best: readonly [string, ReadonlySet<string>] | undefined;
|
|
101
|
+
let bestCount = 0;
|
|
102
|
+
for (const candidate of candidates) {
|
|
103
|
+
if (members.includes(candidate[0])) continue;
|
|
104
|
+
const count = uncoveredCount(candidate[1], uncovered);
|
|
105
|
+
if (count > bestCount) {
|
|
106
|
+
best = candidate;
|
|
107
|
+
bestCount = count;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return best;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function greedyCover(sets: KillSets): Cover {
|
|
114
|
+
const uncovered = new Set<string>();
|
|
115
|
+
for (const kills of sets.values()) for (const kill of kills) uncovered.add(kill);
|
|
116
|
+
const total = uncovered.size;
|
|
117
|
+
const members: string[] = [];
|
|
118
|
+
const candidates = [...sets].toSorted(([a], [b]) => (a < b ? -1 : 1));
|
|
119
|
+
while (uncovered.size > 0) {
|
|
120
|
+
const best = bestAddition(candidates, members, uncovered);
|
|
121
|
+
if (best === undefined) break;
|
|
122
|
+
members.push(best[0]);
|
|
123
|
+
for (const kill of best[1]) uncovered.delete(kill);
|
|
124
|
+
}
|
|
125
|
+
return { members, tests: sets.size, kills: total };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function testNames(run: KillRun): ReadonlyMap<string, string> {
|
|
129
|
+
const names = new Map<string, string>();
|
|
130
|
+
for (const [path, file] of run.testFiles) for (const test of file.tests) names.set(test.id, qualified(path, test.name));
|
|
131
|
+
return names;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// The bun runner writes names that already open with their file.
|
|
135
|
+
function qualified(path: string, name: string): string {
|
|
136
|
+
return path === "" || name.startsWith(`${path} > `) ? name : `${path} > ${name}`;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const byName = (a: string, b: string) => (a < b ? -1 : 1);
|
|
140
|
+
|
|
141
|
+
export function analyze(run: KillRun): Report {
|
|
142
|
+
const sets = killSetsOf(run);
|
|
143
|
+
const names = testNames(run);
|
|
144
|
+
const name = (id: string) => names.get(id) ?? id;
|
|
145
|
+
const cover = greedyCover(sets);
|
|
146
|
+
return {
|
|
147
|
+
files: [...run.files.keys()].toSorted(),
|
|
148
|
+
subsumed: findSubsumed(sets)
|
|
149
|
+
.map((one) => ({ ...one, test: name(one.test), subsumedBy: name(one.subsumedBy) }))
|
|
150
|
+
.toSorted((a, b) => byName(a.test, b.test)),
|
|
151
|
+
identical: findIdentical(sets)
|
|
152
|
+
.map((group) => ({ ...group, tests: group.tests.map(name).toSorted(byName) }))
|
|
153
|
+
.toSorted((a, b) => byName(a.tests[0] ?? "", b.tests[0] ?? "")),
|
|
154
|
+
cover: { ...cover, members: cover.members.map(name) },
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function describeTest(test: string, kills: number): string {
|
|
159
|
+
return `${JSON.stringify(test)} (${kills} kill${kills === 1 ? "" : "s"})`;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export function formatReport(report: Report): string {
|
|
163
|
+
const lines = [`subsumed-tests: ${report.files.length} file(s) mutated`];
|
|
164
|
+
for (const file of report.files) lines.push(` mutated ${file}`);
|
|
165
|
+
lines.push(`subsumed tests (${report.subsumed.length}):`);
|
|
166
|
+
for (const one of report.subsumed) lines.push(` ${describeTest(one.test, one.kills)} subsumed by ${describeTest(one.subsumedBy, one.subsumerKills)}`);
|
|
167
|
+
lines.push(`identical kill sets (${report.identical.length} group(s)):`);
|
|
168
|
+
for (const group of report.identical) lines.push(` ${group.tests.map((test) => describeTest(test, group.kills)).join(" = ")}`);
|
|
169
|
+
lines.push(`greedy cover: ${report.cover.members.length} of ${report.cover.tests} test(s) keep all ${report.cover.kills} kill(s)`);
|
|
170
|
+
for (const member of report.cover.members) lines.push(` cover ${JSON.stringify(member)}`);
|
|
171
|
+
return lines.join("\n");
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export const bailWarning = (source: string, run: KillRun): Effect.Effect<string | undefined, ReportError> => {
|
|
175
|
+
if (run.bail === "on")
|
|
176
|
+
return Effect.fail(
|
|
177
|
+
new ReportError({
|
|
178
|
+
message: `${source} was built with bail on, since its config.disableBail is not true, so each mutant records only its first killer: rerun Stryker with \`bunx stryker run --disableBail\` and pass the report it writes`,
|
|
179
|
+
}),
|
|
180
|
+
);
|
|
181
|
+
if (run.bail === "unrecorded") return Effect.succeed(`${source} records no config, so nothing shows whether bail was off: build it with \`bunx stryker run --disableBail\``);
|
|
182
|
+
return Effect.succeed(undefined);
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
export const parseArgs = Effect.fnUntraced(function* (argv: readonly string[]): Effect.fn.Return<Options, Usage> {
|
|
186
|
+
const [reportPath, ...extra] = argv.filter((arg) => !arg.startsWith("--"));
|
|
187
|
+
const flag = argv.find((arg) => arg.startsWith("--"));
|
|
188
|
+
if (flag !== undefined) return yield* new Usage({ message: `${USAGE}: unknown flag ${flag}` });
|
|
189
|
+
if (reportPath === undefined || extra.length > 0) return yield* new Usage({ message: USAGE });
|
|
190
|
+
return { reportPath };
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
const load = Effect.fn("load")(function* (path: string) {
|
|
194
|
+
const fs = yield* FileSystem.FileSystem;
|
|
195
|
+
const text = yield* fs.readFileString(path).pipe(Effect.mapError(() => new ReportError({ message: `cannot read ${path}` })));
|
|
196
|
+
const run = yield* parseKillRun(path, text);
|
|
197
|
+
const warning = yield* bailWarning(path, run);
|
|
198
|
+
if (warning !== undefined) yield* Console.error(`subsumed-tests: warning: ${warning}`);
|
|
199
|
+
return run;
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
const report = Effect.gen(function* () {
|
|
203
|
+
const options = yield* parseArgs(process.argv.slice(2));
|
|
204
|
+
yield* Console.log(formatReport(analyze(yield* load(options.reportPath))));
|
|
205
|
+
return true;
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
if (import.meta.main) runMain("subsumed-tests", report);
|