@avi2dg/checks 0.45.1 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +3 -1
- package/dist/presets/stylelint.js +8 -0
- package/docs/design.md +6 -2
- package/docs/gates/checks-advisories.md +3 -2
- package/docs/gates/checks-css-size.md +53 -0
- package/docs/gates/checks-flake.md +3 -2
- package/docs/gates/checks-frontend-syntax.md +5 -0
- package/docs/gates/checks-lint.md +2 -1
- package/docs/gates/checks-mutation-baseline.md +3 -3
- package/docs/gates/checks-mutation-compare.md +10 -8
- package/docs/gates/checks-mutation-scope.md +10 -10
- package/docs/gates/checks-mutation.md +4 -4
- package/docs/gates/checks-night-run.md +188 -0
- package/docs/gates/checks-release-notes.md +2 -1
- package/docs/gates/checks-release-pr.md +7 -1
- package/docs/gates/checks-test.md +5 -2
- package/package.json +6 -1
- package/src/core/gates.ts +3 -0
- package/src/core/main.ts +14 -0
- package/src/delivery/github.ts +7 -8
- package/src/quality/css-size.ts +69 -0
- package/src/quality/no-plain-js.ts +3 -10
- package/src/quality/presets/stylelint.ts +12 -0
- package/src/testing/mutation-compare.ts +1 -1
- package/src/testing/mutation-scope.ts +1 -1
- package/src/testing/night-run-findings.ts +74 -0
- package/src/testing/night-run.ts +253 -0
- package/src/testing/test.ts +24 -6
|
@@ -75,18 +75,23 @@ release-pr: https://github.com/acme/widget/pull/12 releases 0.4.0 from 3f2a9c81d
|
|
|
75
75
|
## When it runs
|
|
76
76
|
|
|
77
77
|
The daily release workflow below runs it once a day on the default branch.
|
|
78
|
+
A package repository runs it on every push to `main` other than a release commit.
|
|
78
79
|
Run the workflow by hand with `gh workflow run daily-release` to refresh the release pull request sooner.
|
|
79
80
|
|
|
80
81
|
## Running it in CI
|
|
81
82
|
|
|
82
83
|
A repository takes the daily release as `.github/workflows/daily-release.yml`.
|
|
84
|
+
A consuming repository pins daily-release between 03:00 and 04:00 UTC at a minute chosen so repositories do not start together.
|
|
85
|
+
A package repository released on push drops the `schedule` trigger, so every push to `main` but a release commit opens or refreshes the release pull request.
|
|
86
|
+
Its `pull-request` job's `if` reads `"github.event_name != 'push' || !startsWith(github.event.head_commit.message, 'chore: release ')"`.
|
|
87
|
+
The `pull-request` job checks out its branch's tip when it starts, so a run that starts late never rebuilds the release on an older commit.
|
|
83
88
|
The `pull-request` job runs on the schedule, and the `tag` job runs when a release commit lands on `main`, as [checks-release-tag](checks-release-tag.md) says:
|
|
84
89
|
|
|
85
90
|
```yaml
|
|
86
91
|
name: daily-release
|
|
87
92
|
on:
|
|
88
93
|
schedule:
|
|
89
|
-
- cron: "
|
|
94
|
+
- cron: "7 3 * * *"
|
|
90
95
|
workflow_dispatch:
|
|
91
96
|
push:
|
|
92
97
|
branches: [main]
|
|
@@ -107,6 +112,7 @@ jobs:
|
|
|
107
112
|
steps:
|
|
108
113
|
- uses: actions/checkout@v5
|
|
109
114
|
with:
|
|
115
|
+
ref: ${{ github.ref }}
|
|
110
116
|
fetch-depth: 0
|
|
111
117
|
- uses: oven-sh/setup-bun@v2
|
|
112
118
|
with:
|
|
@@ -47,7 +47,7 @@ A skipped test or a todo without `skipReason` fails in every environment.
|
|
|
47
47
|
It does not read a skip list from `package.json`.
|
|
48
48
|
It counts a run as `ci` when `CI` is true and as `local` otherwise.
|
|
49
49
|
|
|
50
|
-
The command takes no arguments for the default suite, except the worker count of [Parallel runs](#parallel-runs).
|
|
50
|
+
The command takes no arguments for the default suite, except the worker count of [Parallel runs](#parallel-runs) and the file `--junit` keeps the report at.
|
|
51
51
|
Use `checks-test --tier=live` or `checks-test --tier=pixel` for a named test tier.
|
|
52
52
|
Use `checks-test --tier=unit` or `checks-test --tier=e2e` to run only one level of the default suite.
|
|
53
53
|
A live or pixel run clears Bun's ignored paths, runs only `./tests/live` or `./tests/pixel`, and still checks each skip.
|
|
@@ -102,8 +102,10 @@ A timings option waits for Bun to honour timings under random order.
|
|
|
102
102
|
|
|
103
103
|
## Arguments
|
|
104
104
|
|
|
105
|
-
The default command takes no arguments except `--parallel=<workers>`, as [Parallel runs](#parallel-runs) says
|
|
105
|
+
The default command takes no arguments except `--parallel=<workers>`, as [Parallel runs](#parallel-runs) says, and `--junit=<file>`.
|
|
106
106
|
Its named tier options are `--tier=live`, `--tier=pixel`, `--tier=unit` and `--tier=e2e`, and a tier run takes `--parallel=<workers>` too.
|
|
107
|
+
`--junit=<file>` keeps the JUnit report Bun wrote at `<file>`, whether the tests passed or failed, so a night run hands it to [checks-night-run](checks-night-run.md).
|
|
108
|
+
A `package.json` script never takes `--junit`, as [checks-test-layout](checks-test-layout.md) says of `scripts.test`.
|
|
107
109
|
Each option appears at most once.
|
|
108
110
|
It refuses test filters because every test excluded by a filter would look skipped.
|
|
109
111
|
|
|
@@ -138,3 +140,4 @@ A repository that tracks no TypeScript source does not need it, as [checks-test-
|
|
|
138
140
|
- [checks-test-layout](checks-test-layout.md)
|
|
139
141
|
- [checks-flake](checks-flake.md)
|
|
140
142
|
- [checks-quarantine-clock](checks-quarantine-clock.md)
|
|
143
|
+
- [checks-night-run](checks-night-run.md)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@avi2dg/checks",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.47.0",
|
|
4
4
|
"description": "Deterministic checks shared across a set of TypeScript repositories",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
@@ -51,6 +51,8 @@
|
|
|
51
51
|
"src/testing/mutation-baseline.ts",
|
|
52
52
|
"src/testing/mutation-baseline-saved.ts",
|
|
53
53
|
"src/testing/subsumed-tests.ts",
|
|
54
|
+
"src/testing/night-run.ts",
|
|
55
|
+
"src/testing/night-run-findings.ts",
|
|
54
56
|
"src/delivery/ci-wiring.ts",
|
|
55
57
|
"src/delivery/shell-command.ts",
|
|
56
58
|
"src/delivery/workflow-references.ts",
|
|
@@ -63,6 +65,7 @@
|
|
|
63
65
|
"src/docs/prose-matchers.ts",
|
|
64
66
|
"src/quality/comments.ts",
|
|
65
67
|
"src/quality/comment-gate.ts",
|
|
68
|
+
"src/quality/css-size.ts",
|
|
66
69
|
"src/quality/frontend-syntax.ts",
|
|
67
70
|
"src/quality/browser/",
|
|
68
71
|
"src/complexity/suppressions-ratchet.ts",
|
|
@@ -155,8 +158,10 @@
|
|
|
155
158
|
"checks-mutation-baseline": "src/testing/mutation-baseline.ts",
|
|
156
159
|
"checks-mutation-scope": "src/testing/mutation-scope.ts",
|
|
157
160
|
"checks-subsumed-tests": "src/testing/subsumed-tests.ts",
|
|
161
|
+
"checks-night-run": "src/testing/night-run.ts",
|
|
158
162
|
"checks-ci-wiring": "src/delivery/ci-wiring.ts",
|
|
159
163
|
"checks-comment-gate": "src/quality/comment-gate.ts",
|
|
164
|
+
"checks-css-size": "src/quality/css-size.ts",
|
|
160
165
|
"checks-suppressions-ratchet": "src/complexity/suppressions-ratchet.ts",
|
|
161
166
|
"checks-repetition": "src/complexity/repetition.ts",
|
|
162
167
|
"checks-unused": "src/complexity/unused.ts",
|
package/src/core/gates.ts
CHANGED
|
@@ -40,6 +40,8 @@ const OXLINT_CONFIG: TrackedContent = { pathspecs: ["oxlint.config.ts"], content
|
|
|
40
40
|
|
|
41
41
|
const FRONTEND_SYNTAX_DECLARATION: TrackedContent = { pathspecs: ["frontend-syntax.json"], content: "a frontend syntax declaration" };
|
|
42
42
|
|
|
43
|
+
const STYLESHEET: TrackedContent = { pathspecs: ["*.css"], content: "a stylesheet" };
|
|
44
|
+
|
|
43
45
|
const COPY_DECLARATION: TrackedContent = { pathspecs: ["copy.json"], content: "a copy declaration" };
|
|
44
46
|
|
|
45
47
|
export const KIT_GATES = [
|
|
@@ -49,6 +51,7 @@ export const KIT_GATES = [
|
|
|
49
51
|
{ bin: "checks-comment-gate", vector: "quality", file: "comment-gate.ts", reads: "range", appliesTo: EVERY_REPOSITORY },
|
|
50
52
|
{ bin: "checks-effect-scope", vector: "quality", file: "effect-scope.ts", reads: "tree", args: ["--check"], appliesTo: OXLINT_CONFIG },
|
|
51
53
|
{ bin: "checks-frontend-syntax", vector: "quality", file: "frontend-syntax.ts", reads: "tree", appliesTo: FRONTEND_SYNTAX_DECLARATION },
|
|
54
|
+
{ bin: "checks-css-size", vector: "quality", file: "css-size.ts", reads: "tree", appliesTo: STYLESHEET },
|
|
52
55
|
{ bin: "checks-no-plain-js", vector: "quality", file: "no-plain-js.ts", reads: "tree", appliesTo: TYPESCRIPT_SOURCE },
|
|
53
56
|
{ bin: "checks-suppressions-ratchet", vector: "complexity", file: "suppressions-ratchet.ts", reads: "range", appliesTo: EVERY_REPOSITORY },
|
|
54
57
|
{ bin: "checks-ci-wiring", vector: "delivery", file: "ci-wiring.ts", reads: "tree", appliesTo: EVERY_REPOSITORY },
|
package/src/core/main.ts
CHANGED
|
@@ -26,3 +26,17 @@ export function runMain<E extends { readonly message: string }>(
|
|
|
26
26
|
);
|
|
27
27
|
BunRuntime.runMain(program, { teardown: (exit, onExit) => onExit(statusOf(exit)) });
|
|
28
28
|
}
|
|
29
|
+
|
|
30
|
+
export function runTreeGate<Outcome extends { readonly violations: readonly unknown[] }>(
|
|
31
|
+
name: string,
|
|
32
|
+
check: (root: string) => Effect.Effect<Outcome, { readonly message: string }, BunServices.BunServices>,
|
|
33
|
+
format: (result: Outcome) => string,
|
|
34
|
+
): void {
|
|
35
|
+
const gate = Effect.gen(function* () {
|
|
36
|
+
const root = process.argv[2] ?? process.cwd();
|
|
37
|
+
const result = yield* check(root);
|
|
38
|
+
yield* Console.log(format(result));
|
|
39
|
+
return result.violations.length === 0;
|
|
40
|
+
});
|
|
41
|
+
runMain(name, gate);
|
|
42
|
+
}
|
package/src/delivery/github.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Effect, Schema } from "effect";
|
|
2
2
|
import { ChildProcessSpawner } from "effect/process";
|
|
3
|
-
import { collect } from "../core/git.ts";
|
|
3
|
+
import { collect, type Feed } from "../core/git.ts";
|
|
4
4
|
|
|
5
5
|
export class GitHubFailure extends Schema.TaggedError<GitHubFailure>()("GitHubFailure", {
|
|
6
6
|
message: Schema.String,
|
|
@@ -13,17 +13,16 @@ type Decode<A> = (text: string) => Effect.Effect<A, Schema.SchemaError>;
|
|
|
13
13
|
// gh fills {owner} and {repo} from the checkout's remote, so a workflow names no repository.
|
|
14
14
|
export const REPOSITORY = "repos/{owner}/{repo}";
|
|
15
15
|
|
|
16
|
-
const
|
|
17
|
-
const failed = (reason: string): GitHubFailure => new GitHubFailure({ message:
|
|
18
|
-
const
|
|
19
|
-
const feed = body === undefined ? {} : { input: JSON.stringify(body) };
|
|
20
|
-
const { stdout, stderr, exitCode } = yield* collect("gh", ["api", "--method", method, path, ...input], undefined, feed).pipe(
|
|
21
|
-
Effect.mapError((cause) => failed(cause.message)),
|
|
22
|
-
);
|
|
16
|
+
export const gh = Effect.fn("gh")(function* (args: readonly string[], call = `gh ${args.join(" ")}`, feed: Feed = {}) {
|
|
17
|
+
const failed = (reason: string): GitHubFailure => new GitHubFailure({ message: `${call}: ${reason.trim()}` });
|
|
18
|
+
const { stdout, stderr, exitCode } = yield* collect("gh", args, undefined, feed).pipe(Effect.mapError((cause) => failed(cause.message)));
|
|
23
19
|
if (exitCode !== ChildProcessSpawner.ExitCode(0)) return yield* failed(stderr);
|
|
24
20
|
return stdout;
|
|
25
21
|
});
|
|
26
22
|
|
|
23
|
+
const gitHub = (method: Method, path: string, body?: unknown) =>
|
|
24
|
+
gh(["api", "--method", method, path, ...(body === undefined ? [] : ["--input", "-"])], `gh api ${method} ${path}`, body === undefined ? {} : { input: JSON.stringify(body) });
|
|
25
|
+
|
|
27
26
|
export function gitHubJson<A>(decode: Decode<A>, method: Method, path: string, body?: unknown): Effect.Effect<A, GitHubFailure, ChildProcessSpawner.ChildProcessSpawner> {
|
|
28
27
|
return gitHub(method, path, body).pipe(
|
|
29
28
|
Effect.flatMap((text) =>
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
import { Effect, FileSystem, Path, Schema } from "effect";
|
|
3
|
+
import { workingTreeFiles } from "../core/git.ts";
|
|
4
|
+
import { runTreeGate } from "../core/main.ts";
|
|
5
|
+
import { SOURCE_LIMITS } from "./presets/oxlint.ts";
|
|
6
|
+
|
|
7
|
+
export type Violation = {
|
|
8
|
+
readonly file: string;
|
|
9
|
+
readonly line: number | undefined;
|
|
10
|
+
readonly message: string;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
const NAME = "css-size";
|
|
14
|
+
|
|
15
|
+
const LIMIT = SOURCE_LIMITS["max-lines"];
|
|
16
|
+
|
|
17
|
+
const STYLESHEET = /\.css$/;
|
|
18
|
+
|
|
19
|
+
export function lineCount(source: string): number {
|
|
20
|
+
if (source === "") return 0;
|
|
21
|
+
const parts = source.split("\n");
|
|
22
|
+
return source.endsWith("\n") ? parts.length - 1 : parts.length;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function violationsOf(entries: readonly { readonly file: string; readonly lines: number }[]): readonly Violation[] {
|
|
26
|
+
return entries
|
|
27
|
+
.filter(({ lines }) => lines > LIMIT)
|
|
28
|
+
.toSorted((a, b) => (a.file < b.file ? -1 : 1))
|
|
29
|
+
.map(({ file, lines }) => ({
|
|
30
|
+
file,
|
|
31
|
+
line: undefined,
|
|
32
|
+
message: `${lines} lines, over the ${LIMIT}-line budget; split the stylesheet into smaller files`,
|
|
33
|
+
}));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
class CssSizeError extends Schema.TaggedError<CssSizeError>()("CssSizeError", {
|
|
37
|
+
message: Schema.String,
|
|
38
|
+
}) {}
|
|
39
|
+
|
|
40
|
+
export type Result = { readonly files: number; readonly violations: readonly Violation[] };
|
|
41
|
+
|
|
42
|
+
const listJudged = Effect.fn("listJudged")(function* (root: string) {
|
|
43
|
+
const listed = yield* workingTreeFiles(root).pipe(
|
|
44
|
+
Effect.mapError((cause) => new CssSizeError({ message: cause.message })),
|
|
45
|
+
);
|
|
46
|
+
return listed.filter((file) => STYLESHEET.test(file)).toSorted();
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
const measure = Effect.fn("measure")(function* (root: string, file: string) {
|
|
50
|
+
const fs = yield* FileSystem.FileSystem;
|
|
51
|
+
const path = yield* Path.Path;
|
|
52
|
+
const source = yield* fs.readFileString(path.join(root, file)).pipe(
|
|
53
|
+
Effect.mapError((cause) => new CssSizeError({ message: `cannot read ${file}: ${String(cause)}` })),
|
|
54
|
+
);
|
|
55
|
+
return { file, lines: lineCount(source) };
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
export const run = Effect.fn("run")(function* (root: string) {
|
|
59
|
+
const judged = yield* listJudged(root);
|
|
60
|
+
const entries = yield* Effect.forEach(judged, (file) => measure(root, file), { concurrency: "unbounded" });
|
|
61
|
+
return { files: judged.length, violations: violationsOf(entries) } satisfies Result;
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
export function report({ files, violations }: Result): string {
|
|
65
|
+
if (violations.length === 0) return `${NAME}: no stylesheet over ${LIMIT} lines among ${files} CSS file(s)`;
|
|
66
|
+
return [`${NAME}: ${violations.length} violation(s)`, ...violations.map(({ file, message }) => ` ${file}: ${message}`)].join("\n");
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if (import.meta.main) runTreeGate(NAME, run, report);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
|
-
import {
|
|
2
|
+
import { Effect, Schema } from "effect";
|
|
3
3
|
import { workingTreeFiles } from "../core/git.ts";
|
|
4
|
-
import {
|
|
4
|
+
import { runTreeGate } from "../core/main.ts";
|
|
5
5
|
|
|
6
6
|
export type Violation = {
|
|
7
7
|
readonly file: string;
|
|
@@ -61,11 +61,4 @@ export function report({ files, violations }: Result): string {
|
|
|
61
61
|
return [`${NAME}: ${violations.length} violation(s)`, ...violations.map(({ file, message }) => ` ${file}: ${message}`)].join("\n");
|
|
62
62
|
}
|
|
63
63
|
|
|
64
|
-
|
|
65
|
-
const root = process.argv[2] ?? process.cwd();
|
|
66
|
-
const result = yield* run(root);
|
|
67
|
-
yield* Console.log(report(result));
|
|
68
|
-
return result.violations.length === 0;
|
|
69
|
-
});
|
|
70
|
-
|
|
71
|
-
if (import.meta.main) runMain(NAME, noPlainJs);
|
|
64
|
+
if (import.meta.main) runTreeGate(NAME, run, report);
|
|
@@ -31,6 +31,16 @@ const SELECTOR_COMPLEXITY = {
|
|
|
31
31
|
"selector-max-compound-selectors": 3,
|
|
32
32
|
} as const satisfies Rules;
|
|
33
33
|
|
|
34
|
+
// [\s\S] without anchors passes an empty /**/, so the anchors refuse every comment text there is.
|
|
35
|
+
const COMMENT_REFUSAL = {
|
|
36
|
+
"comment-word-disallowed-list": ["/^[\\s\\S]*$/"],
|
|
37
|
+
} as const satisfies Rules;
|
|
38
|
+
|
|
39
|
+
// A non-ASCII class refuses every language at once, where naming scripts would chase each one.
|
|
40
|
+
const LANGUAGE_FREE_SELECTORS = {
|
|
41
|
+
"selector-disallowed-list": ["/[^\\x00-\\x7F]/"],
|
|
42
|
+
} as const satisfies Rules;
|
|
43
|
+
|
|
34
44
|
const COLOUR_FUNCTIONS = ["rgb", "rgba", "hsl", "hsla", "hwb", "lab", "lch", "oklab", "oklch", "color"] as const;
|
|
35
45
|
|
|
36
46
|
const DESIGN_TOKENS = {
|
|
@@ -84,6 +94,8 @@ export function defineConfig({ baseline, tokens, rules, overrides = [], ...rest
|
|
|
84
94
|
const merged: Rules = {
|
|
85
95
|
...CONVENTION_CHANGES,
|
|
86
96
|
...SELECTOR_COMPLEXITY,
|
|
97
|
+
...COMMENT_REFUSAL,
|
|
98
|
+
...LANGUAGE_FREE_SELECTORS,
|
|
87
99
|
[BASELINE_RULE]: [true, { available }],
|
|
88
100
|
...tokenRules.rules,
|
|
89
101
|
...rules,
|
|
@@ -186,7 +186,7 @@ export function readersOf(sources: ReadonlyMap<string, string>, changed: readonl
|
|
|
186
186
|
return new Map([...reached].filter(([file, via]) => file !== via));
|
|
187
187
|
}
|
|
188
188
|
|
|
189
|
-
// Names what every unit test loads that the
|
|
189
|
+
// Names what every unit test loads that the branch changes: the preload list, a preload, or a file a preload imports.
|
|
190
190
|
export function preloadChange(
|
|
191
191
|
changed: ReadonlySet<string>,
|
|
192
192
|
preloads: { readonly atBase: readonly string[]; readonly atHead: readonly string[] },
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { Option, Schema } from "effect";
|
|
2
|
+
import { OUTCOME, type ReportFile } from "./mutation-compare.ts";
|
|
3
|
+
import { NAME_SEPARATOR, type TestResult } from "./test-report.ts";
|
|
4
|
+
|
|
5
|
+
const ScoreSchema = Schema.Struct({ detected: Schema.Int, valid: Schema.Int });
|
|
6
|
+
|
|
7
|
+
export type Score = typeof ScoreSchema.Type;
|
|
8
|
+
|
|
9
|
+
export const NightRecordSchema = Schema.Struct({ scores: Schema.Record(Schema.String, ScoreSchema) });
|
|
10
|
+
|
|
11
|
+
export type NightRecord = typeof NightRecordSchema.Type;
|
|
12
|
+
|
|
13
|
+
export type TestReport = { readonly results: readonly TestResult[]; readonly crashedFiles: readonly string[] };
|
|
14
|
+
|
|
15
|
+
export type Tonight = { readonly scores: NightRecord["scores"]; readonly tests: readonly TestReport[] };
|
|
16
|
+
|
|
17
|
+
export type PreviousNight = { readonly url: string; readonly record: NightRecord };
|
|
18
|
+
|
|
19
|
+
export type Finding =
|
|
20
|
+
| { readonly kind: "mutation"; readonly file: string; readonly before: Score; readonly after: Score; readonly since: string }
|
|
21
|
+
| { readonly kind: "test"; readonly file: string; readonly name: string }
|
|
22
|
+
| { readonly kind: "crash"; readonly file: string };
|
|
23
|
+
|
|
24
|
+
function scoreOf(file: ReportFile): Score {
|
|
25
|
+
let detected = 0;
|
|
26
|
+
let valid = 0;
|
|
27
|
+
for (const mutant of file.mutants) {
|
|
28
|
+
const outcome = OUTCOME[mutant.status];
|
|
29
|
+
if (outcome === "leaves-score") continue;
|
|
30
|
+
valid += 1;
|
|
31
|
+
if (outcome === "detected") detected += 1;
|
|
32
|
+
}
|
|
33
|
+
return { detected, valid };
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function scoresOf(files: ReadonlyMap<string, ReportFile>): NightRecord["scores"] {
|
|
37
|
+
return Object.fromEntries([...files].map(([path, file]) => [path, scoreOf(file)] as const).sort(([one], [other]) => one.localeCompare(other)));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// Cross-multiplied, so two equal scores never differ by how their quotients round.
|
|
41
|
+
function fell(before: Score, after: Score): boolean {
|
|
42
|
+
return before.valid > 0 && after.valid > 0 && after.detected * before.valid < before.detected * after.valid;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function keyOf(finding: Finding): string {
|
|
46
|
+
if (finding.kind === "mutation") return `mutation:${finding.file}`;
|
|
47
|
+
if (finding.kind === "test") return `test:${finding.file}${NAME_SEPARATOR}${finding.name}`;
|
|
48
|
+
return `test:${finding.file}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function scoreFindings(previous: Option.Option<PreviousNight>, scores: NightRecord["scores"]): readonly Finding[] {
|
|
52
|
+
if (Option.isNone(previous)) return [];
|
|
53
|
+
const { url, record } = previous.value;
|
|
54
|
+
return Object.entries(scores).flatMap(([file, after]): readonly Finding[] => {
|
|
55
|
+
const before = record.scores[file];
|
|
56
|
+
return before !== undefined && fell(before, after) ? [{ kind: "mutation", file, before, after, since: url }] : [];
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function testFindings(reports: readonly TestReport[]): readonly Finding[] {
|
|
61
|
+
const failing = reports.flatMap(({ results, crashedFiles }) => [
|
|
62
|
+
...results.filter((result) => result.outcome === "failed").map((result): Finding => ({ kind: "test", file: result.file, name: result.name })),
|
|
63
|
+
...crashedFiles.map((file): Finding => ({ kind: "crash", file })),
|
|
64
|
+
]);
|
|
65
|
+
return [...new Map(failing.map((finding) => [keyOf(finding), finding])).values()];
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function findingsOf(previous: Option.Option<PreviousNight>, tonight: Tonight): readonly Finding[] {
|
|
69
|
+
return [...scoreFindings(previous, tonight.scores), ...testFindings(tonight.tests)].sort((one, other) => keyOf(one).localeCompare(keyOf(other)));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function percent(score: Score): string {
|
|
73
|
+
return `${((score.detected / score.valid) * 100).toFixed(2)}%`;
|
|
74
|
+
}
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
import { Config, Console, Context, Effect, FileSystem, Layer, Option, Path, Schema } from "effect";
|
|
3
|
+
import { ChildProcessSpawner } from "effect/process";
|
|
4
|
+
import { DEFAULT_BRANCH } from "../core/gates.ts";
|
|
5
|
+
import { runMain, Usage } from "../core/main.ts";
|
|
6
|
+
import { gh, type GitHubFailure, gitHubJson, REPOSITORY } from "../delivery/github.ts";
|
|
7
|
+
import { parseReport as parseMutationReport } from "./mutation-compare.ts";
|
|
8
|
+
import { type Finding, findingsOf, keyOf, NightRecordSchema, type NightRecord, percent, type PreviousNight, scoresOf, type TestReport } from "./night-run-findings.ts";
|
|
9
|
+
import { parseReport as parseTestReport } from "./test-report.ts";
|
|
10
|
+
|
|
11
|
+
const NAME = "checks-night-run";
|
|
12
|
+
const USAGE = `usage: ${NAME} [--mutation <mutation-report>] [--tests <junit-report>]...`;
|
|
13
|
+
export const LABEL = "night-run";
|
|
14
|
+
const LABEL_COLOR = "5319e7";
|
|
15
|
+
const LABEL_DESCRIPTION = "A regression a night run on main found";
|
|
16
|
+
const ARTIFACT = "night-run";
|
|
17
|
+
const RECORD_NAME = "night-run.json";
|
|
18
|
+
export const RECORD = `reports/${RECORD_NAME}`;
|
|
19
|
+
const PAGE_SIZE = 100;
|
|
20
|
+
const GITHUB_TITLE_LIMIT = 256;
|
|
21
|
+
|
|
22
|
+
class NightRunFailure extends Schema.TaggedError<NightRunFailure>()("NightRunFailure", {
|
|
23
|
+
message: Schema.String,
|
|
24
|
+
}) {}
|
|
25
|
+
|
|
26
|
+
export type Options = { readonly mutation: Option.Option<string>; readonly tests: readonly string[] };
|
|
27
|
+
|
|
28
|
+
export const parseArgs = Effect.fnUntraced(function* (argv: readonly string[]): Effect.fn.Return<Options, Usage> {
|
|
29
|
+
const mutation: string[] = [];
|
|
30
|
+
const tests: string[] = [];
|
|
31
|
+
for (let at = 0; at < argv.length; at += 2) {
|
|
32
|
+
const [flag, value] = [argv[at], argv[at + 1]];
|
|
33
|
+
const into = flag === "--mutation" ? mutation : flag === "--tests" ? tests : undefined;
|
|
34
|
+
if (into === undefined || value === undefined || value.startsWith("--")) return yield* new Usage({ message: USAGE });
|
|
35
|
+
into.push(value);
|
|
36
|
+
}
|
|
37
|
+
if (mutation.length > 1 || mutation.length + tests.length === 0) return yield* new Usage({ message: USAGE });
|
|
38
|
+
return { mutation: Option.fromNullishOr(mutation[0]), tests };
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
export type OpenIssue = { readonly number: number; readonly key: string };
|
|
42
|
+
|
|
43
|
+
export type Filed = { readonly key: string; readonly issue: number; readonly action: "opened" | "commented on" };
|
|
44
|
+
|
|
45
|
+
export class Issues extends Context.Service<
|
|
46
|
+
Issues,
|
|
47
|
+
{
|
|
48
|
+
readonly ensureLabel: Effect.Effect<void, GitHubFailure>;
|
|
49
|
+
readonly open: Effect.Effect<readonly OpenIssue[], GitHubFailure>;
|
|
50
|
+
readonly create: (title: string, body: string) => Effect.Effect<number, GitHubFailure>;
|
|
51
|
+
readonly comment: (issue: number, body: string) => Effect.Effect<void, GitHubFailure>;
|
|
52
|
+
}
|
|
53
|
+
>()("@avi2dg/checks/testing/Issues") {
|
|
54
|
+
static readonly gh = Layer.effect(
|
|
55
|
+
Issues,
|
|
56
|
+
Effect.gen(function* () {
|
|
57
|
+
const services = yield* Effect.context<ChildProcessSpawner.ChildProcessSpawner>();
|
|
58
|
+
const provided = <A>(effect: Effect.Effect<A, GitHubFailure, ChildProcessSpawner.ChildProcessSpawner>) => effect.pipe(Effect.provideContext(services));
|
|
59
|
+
return Issues.of({
|
|
60
|
+
ensureLabel: provided(ensureLabel),
|
|
61
|
+
open: provided(openIssues),
|
|
62
|
+
create: (title, body) => provided(gitHubJson(decodeNumbered, "POST", `${REPOSITORY}/issues`, { title, body, labels: [LABEL] })).pipe(Effect.map(({ number }) => number)),
|
|
63
|
+
comment: (issue, body) => provided(gitHubJson(decodeAny, "POST", `${REPOSITORY}/issues/${issue}/comments`, { body })).pipe(Effect.asVoid),
|
|
64
|
+
});
|
|
65
|
+
}),
|
|
66
|
+
);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const decodeNumbered = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Struct({ number: Schema.Int })));
|
|
70
|
+
const decodeAny = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
|
|
71
|
+
const decodeIssues = Schema.decodeUnknownEffect(
|
|
72
|
+
Schema.fromJsonString(Schema.Array(Schema.Struct({ number: Schema.Int, body: Schema.NullOr(Schema.String), pull_request: Schema.optionalKey(Schema.Unknown) }))),
|
|
73
|
+
);
|
|
74
|
+
const decodeKey = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.String));
|
|
75
|
+
|
|
76
|
+
const MARKER = /<!-- checks-night-run ("(?:[^"\\]|\\.)*") -->/;
|
|
77
|
+
|
|
78
|
+
// The key is JSON with every > escaped, so no key can close the HTML comment early.
|
|
79
|
+
export function marker(key: string): string {
|
|
80
|
+
return `<!-- checks-night-run ${JSON.stringify(key).replaceAll(">", "\\u003e")} -->`;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function markedKey(body: string | null): Option.Option<string> {
|
|
84
|
+
const quoted = body === null ? undefined : MARKER.exec(body)?.[1];
|
|
85
|
+
return quoted === undefined ? Option.none() : decodeKey(quoted);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const ensureLabel = gh(["label", "create", LABEL, "--color", LABEL_COLOR, "--description", LABEL_DESCRIPTION, "--force"]).pipe(Effect.asVoid);
|
|
89
|
+
|
|
90
|
+
const openIssues = Effect.gen(function* () {
|
|
91
|
+
const found: OpenIssue[] = [];
|
|
92
|
+
for (let page = 1; ; page += 1) {
|
|
93
|
+
const batch = yield* gitHubJson(decodeIssues, "GET", `${REPOSITORY}/issues?labels=${LABEL}&state=open&per_page=${PAGE_SIZE}&page=${page}`);
|
|
94
|
+
for (const issue of batch) {
|
|
95
|
+
const key = markedKey(issue.body);
|
|
96
|
+
if (issue.pull_request === undefined && Option.isSome(key)) found.push({ number: issue.number, key: key.value });
|
|
97
|
+
}
|
|
98
|
+
if (batch.length < PAGE_SIZE) return found;
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
function fullTitleOf(finding: Finding): string {
|
|
103
|
+
if (finding.kind === "mutation") return `Night run: mutation score fell in ${finding.file}`;
|
|
104
|
+
if (finding.kind === "test") return `Night run: ${finding.file} > ${finding.name} fails`;
|
|
105
|
+
return `Night run: ${finding.file} crashed its test worker`;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export function titleOf(finding: Finding): string {
|
|
109
|
+
const title = fullTitleOf(finding);
|
|
110
|
+
if (title.length <= GITHUB_TITLE_LIMIT) return title;
|
|
111
|
+
const end = GITHUB_TITLE_LIMIT - 1;
|
|
112
|
+
const splitsPair = /[\uD800-\uDBFF]/.test(title.charAt(end - 1));
|
|
113
|
+
return `${title.slice(0, splitsPair ? end - 1 : end)}…`;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function findingText(finding: Finding, run: string): string {
|
|
117
|
+
if (finding.kind === "mutation") {
|
|
118
|
+
const { file, before, after, since } = finding;
|
|
119
|
+
return [
|
|
120
|
+
`The night run ${run} scored \`${file}\` lower than the night run ${since}.`,
|
|
121
|
+
"",
|
|
122
|
+
"| File | Previous night | Tonight |",
|
|
123
|
+
"| --- | --- | --- |",
|
|
124
|
+
`| \`${file}\` | ${percent(before)}, ${before.detected} of ${before.valid} mutants detected | ${percent(after)}, ${after.detected} of ${after.valid} mutants detected |`,
|
|
125
|
+
].join("\n");
|
|
126
|
+
}
|
|
127
|
+
if (finding.kind === "test") return `The night run ${run} failed \`${finding.name}\` in \`${finding.file}\`.`;
|
|
128
|
+
return `The night run ${run} lost every result of \`${finding.file}\`, because the worker running it crashed.`;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export function issueBody(finding: Finding, run: string): string {
|
|
132
|
+
return `${findingText(finding, run)}\n\nFinding key: \`${keyOf(finding)}\`\n\n${marker(keyOf(finding))}\n`;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Nothing to file means no call at all, so a quiet night never touches the label or the issues.
|
|
136
|
+
export const fileFindings = Effect.fn("fileFindings")(function* (findings: readonly Finding[], run: string) {
|
|
137
|
+
if (findings.length === 0) return [];
|
|
138
|
+
const issues = yield* Issues;
|
|
139
|
+
yield* issues.ensureLabel;
|
|
140
|
+
const open = new Map((yield* issues.open).map((issue) => [issue.key, issue.number]));
|
|
141
|
+
const filed: Filed[] = [];
|
|
142
|
+
for (const finding of findings) {
|
|
143
|
+
const key = keyOf(finding);
|
|
144
|
+
const known = open.get(key);
|
|
145
|
+
if (known === undefined) {
|
|
146
|
+
const issue = yield* issues.create(titleOf(finding), issueBody(finding, run));
|
|
147
|
+
open.set(key, issue);
|
|
148
|
+
filed.push({ key, issue, action: "opened" });
|
|
149
|
+
} else {
|
|
150
|
+
yield* issues.comment(known, findingText(finding, run));
|
|
151
|
+
filed.push({ key, issue: known, action: "commented on" });
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return filed;
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
const required = (name: string) =>
|
|
158
|
+
Config.String(name).pipe(Effect.mapError(() => new NightRunFailure({ message: `${name} is unset, so ${NAME} runs only inside a GitHub Actions job` })));
|
|
159
|
+
|
|
160
|
+
const WORKFLOW_FILE = /\/\.github\/workflows\/([^/@]+)@/;
|
|
161
|
+
|
|
162
|
+
type Actions = { readonly runs: string; readonly run: string; readonly workflow: string; readonly ref: string };
|
|
163
|
+
|
|
164
|
+
const actionsRun = Effect.gen(function* () {
|
|
165
|
+
const runs = `${yield* required("GITHUB_SERVER_URL")}/${yield* required("GITHUB_REPOSITORY")}/actions/runs`;
|
|
166
|
+
const reference = yield* required("GITHUB_WORKFLOW_REF");
|
|
167
|
+
const workflow = WORKFLOW_FILE.exec(reference)?.[1];
|
|
168
|
+
if (workflow === undefined) return yield* new NightRunFailure({ message: `GITHUB_WORKFLOW_REF ${reference} names no workflow file` });
|
|
169
|
+
return { runs, run: `${runs}/${yield* required("GITHUB_RUN_ID")}`, workflow, ref: yield* required("GITHUB_REF") } satisfies Actions;
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
const decodeRuns = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(Schema.Struct({ databaseId: Schema.Int }))));
|
|
173
|
+
const decodeArtifacts = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Struct({ artifacts: Schema.Array(Schema.Struct({ name: Schema.String, expired: Schema.Boolean })) })));
|
|
174
|
+
const decodeRecord = Schema.decodeUnknownEffect(Schema.fromJsonString(NightRecordSchema));
|
|
175
|
+
|
|
176
|
+
const unreadable = (source: string) => (cause: { readonly message: string }) => new NightRunFailure({ message: `cannot read ${source}: ${cause.message}` });
|
|
177
|
+
|
|
178
|
+
// Only the newest successful run on main counts, so a branch dispatch never becomes the night a later run compares with.
|
|
179
|
+
const previousNight = Effect.fn("previousNight")(
|
|
180
|
+
function* ({ runs, workflow }: Actions) {
|
|
181
|
+
const listed = ["run", "list", "--workflow", workflow, "--branch", DEFAULT_BRANCH, "--status", "success", "--limit", "1", "--json", "databaseId,event,createdAt"];
|
|
182
|
+
const [newest] = yield* gh(listed).pipe(Effect.flatMap(decodeRuns), Effect.mapError(unreadable(`the runs of ${workflow}`)));
|
|
183
|
+
if (newest === undefined) return Option.none<PreviousNight>();
|
|
184
|
+
const run = newest.databaseId;
|
|
185
|
+
const { artifacts } = yield* gh(["api", `${REPOSITORY}/actions/runs/${run}/artifacts?name=${ARTIFACT}`]).pipe(
|
|
186
|
+
Effect.flatMap(decodeArtifacts),
|
|
187
|
+
Effect.mapError(unreadable(`the artifacts of run ${run}`)),
|
|
188
|
+
);
|
|
189
|
+
if (!artifacts.some((artifact) => artifact.name === ARTIFACT && !artifact.expired)) return Option.none<PreviousNight>();
|
|
190
|
+
const fs = yield* FileSystem.FileSystem;
|
|
191
|
+
const dir = yield* fs.makeTempDirectoryScoped({ prefix: "checks-night-run-" });
|
|
192
|
+
yield* gh(["run", "download", String(run), "--name", ARTIFACT, "--dir", dir]);
|
|
193
|
+
const file = (yield* Path.Path).join(dir, RECORD_NAME);
|
|
194
|
+
const record = yield* fs.readFileString(file).pipe(Effect.flatMap(decodeRecord), Effect.mapError(unreadable(`the ${ARTIFACT} artifact of run ${run}`)));
|
|
195
|
+
return Option.some<PreviousNight>({ url: `${runs}/${run}`, record });
|
|
196
|
+
},
|
|
197
|
+
Effect.scoped,
|
|
198
|
+
);
|
|
199
|
+
|
|
200
|
+
const readText = Effect.fn("readText")(function* (file: string) {
|
|
201
|
+
return yield* (yield* FileSystem.FileSystem).readFileString(file).pipe(Effect.mapError(unreadable(file)));
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
const testReport = Effect.fn("testReport")(function* (file: string) {
|
|
205
|
+
return yield* parseTestReport(yield* readText(file)).pipe(Effect.mapError(unreadable(file)));
|
|
206
|
+
});
|
|
207
|
+
|
|
208
|
+
const writeRecord = Effect.fn("writeRecord")(function* (record: NightRecord) {
|
|
209
|
+
const fs = yield* FileSystem.FileSystem;
|
|
210
|
+
yield* fs.makeDirectory((yield* Path.Path).dirname(RECORD), { recursive: true });
|
|
211
|
+
yield* fs.writeFileString(RECORD, `${JSON.stringify(record, null, 2)}\n`);
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
const tonightScores = Effect.fn("tonightScores")(function* (file: string) {
|
|
215
|
+
return scoresOf(yield* parseMutationReport(file, yield* readText(file)));
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
type Filing = { readonly onMain: true; readonly filed: readonly Filed[] } | { readonly onMain: false; readonly ref: string };
|
|
219
|
+
|
|
220
|
+
export function report(run: string, scored: Option.Option<number>, previous: Option.Option<PreviousNight>, findings: readonly Finding[], filing: Filing): string {
|
|
221
|
+
const against = Option.isNone(scored)
|
|
222
|
+
? "had no mutation report, so it compared no score"
|
|
223
|
+
: Option.isSome(previous)
|
|
224
|
+
? `compared with the night run ${previous.value.url}`
|
|
225
|
+
: `found no earlier successful run on ${DEFAULT_BRANCH} holding a ${ARTIFACT} artifact, so it compared no score`;
|
|
226
|
+
const verdict = findings.length === 0 ? "no finding" : `${findings.length} finding(s)`;
|
|
227
|
+
const listed = filing.onMain ? filing.filed.map((one) => ` ${one.action} #${one.issue} for ${one.key}`) : findings.map((finding) => ` found ${keyOf(finding)}`);
|
|
228
|
+
const recorded = !filing.onMain
|
|
229
|
+
? `${NAME}: the run is on ${filing.ref}, not ${DEFAULT_BRANCH}, so it filed no issue and recorded no score`
|
|
230
|
+
: Option.isSome(scored)
|
|
231
|
+
? `${NAME}: recorded ${scored.value} file score(s) in ${RECORD}`
|
|
232
|
+
: `${NAME}: recorded no score, since it had no mutation report`;
|
|
233
|
+
return [`${NAME}: the night run ${run} ${against}; ${verdict}`, ...listed, recorded].join("\n");
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const nightRun = Effect.gen(function* () {
|
|
237
|
+
const options = yield* parseArgs(process.argv.slice(2));
|
|
238
|
+
const actions = yield* actionsRun;
|
|
239
|
+
const scores = Option.isSome(options.mutation) ? Option.some(yield* tonightScores(options.mutation.value)) : Option.none<NightRecord["scores"]>();
|
|
240
|
+
const tests: TestReport[] = yield* Effect.forEach(options.tests, testReport);
|
|
241
|
+
const previous = Option.isSome(scores) ? yield* previousNight(actions) : Option.none<PreviousNight>();
|
|
242
|
+
const findings = findingsOf(previous, { scores: Option.getOrElse(scores, () => ({})), tests });
|
|
243
|
+
const filing: Filing = actions.ref === `refs/heads/${DEFAULT_BRANCH}` ? { onMain: true, filed: yield* fileFindings(findings, actions.run) } : { onMain: false, ref: actions.ref };
|
|
244
|
+
if (filing.onMain && Option.isSome(scores)) yield* writeRecord({ scores: scores.value });
|
|
245
|
+
yield* Console.log(report(actions.run, Option.map(scores, (tonight) => Object.keys(tonight).length), previous, findings, filing));
|
|
246
|
+
return true;
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
export function main(issues: Layer.Layer<Issues, never, ChildProcessSpawner.ChildProcessSpawner>): void {
|
|
250
|
+
runMain(NAME, nightRun.pipe(Effect.provide(issues)));
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
if (import.meta.main) main(Issues.gh);
|