@avi2dg/checks 0.69.0 → 0.71.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/docs/bins.md +6 -6
- package/docs/configs/native-settings.md +1 -1
- package/docs/design.md +2 -2
- package/docs/gates/checks-browser.md +2 -2
- package/docs/gates/checks-ci-wiring.md +3 -22
- package/docs/gates/checks-docs.md +23 -3
- package/docs/gates/checks-frontend-syntax.md +2 -2
- package/docs/gates/checks-lint-coverage.md +1 -1
- package/docs/gates/checks-lint.md +5 -5
- package/docs/gates/checks-mutation-compare.md +2 -2
- package/docs/gates/checks-quarantine-clock.md +2 -2
- package/docs/gates/checks-repetition.md +3 -3
- package/docs/gates/checks-setup.md +1 -4
- package/docs/gates/checks-suppressions-ratchet.md +3 -3
- package/docs/gates/checks-test.md +3 -3
- package/docs/gates/checks-whole-tree-run.md +281 -0
- package/docs/how-to/set-up-a-repository.md +17 -3
- package/docs/how-to/upgrade-the-kit.md +5 -2
- package/package.json +4 -5
- package/src/core/gates.ts +7 -7
- package/src/core/lint.ts +12 -13
- package/src/delivery/ci-wiring.ts +2 -7
- package/src/delivery/setup-workflow.ts +0 -25
- package/src/delivery/setup.ts +1 -2
- package/src/delivery/shell-command.ts +0 -4
- package/src/docs/doc-agents.ts +26 -3
- package/src/docs/docs.ts +62 -14
- package/src/testing/{night-run-findings.ts → whole-tree-run-findings.ts} +10 -10
- package/src/testing/{night-run.ts → whole-tree-run.ts} +44 -44
- package/docs/gates/checks-night-run.md +0 -257
- package/src/delivery/night-step.ts +0 -79
package/src/docs/docs.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env bun
|
|
2
2
|
import { Console, Effect } from "effect";
|
|
3
|
-
import { ceilingFinding, entryFindings, isAgentFile, isCeiledReadme, maintainingFinding, readmeFindings } from "./doc-agents.ts";
|
|
4
|
-
import { rootsOf, snapshotOf, unresolvedIn, type Judging, type Unresolved } from "./doc-references.ts";
|
|
3
|
+
import { ceilingFinding, entryFindings, hasManifest, isAgentFile, isCeiledReadme, maintainingFinding, missingAgentFiles, readmeFindings, type RootAgentFinding } from "./doc-agents.ts";
|
|
4
|
+
import { rootsOf, snapshotOf, type Snapshot, unresolvedIn, type Judging, type Unresolved } from "./doc-references.ts";
|
|
5
5
|
import { ADR_DIRECTORY, judge, placementOf, placementProblem, recordsOf, speaksToConsumers, type Placement, type Records } from "./doc-rules.ts";
|
|
6
6
|
import { vanishedNames } from "./doc-names.ts";
|
|
7
7
|
import { pinsAt, versionFindings } from "./doc-versions.ts";
|
|
8
8
|
import { readTexts, snapshotAt, stillMissing } from "./doc-snapshot.ts";
|
|
9
|
-
import { changedLines, changedPaths, git, pathsAt, rangeEnds, refArgs } from "../core/git.ts";
|
|
9
|
+
import { changedLines, changedPaths, git, pathsAt, pathsOf, rangeEnds, refArgs, type Change } from "../core/git.ts";
|
|
10
10
|
import { runMain } from "../core/main.ts";
|
|
11
11
|
import { proseFindings, readerOf } from "./prose-matchers.ts";
|
|
12
12
|
import { longSentences } from "./sentence-length.ts";
|
|
@@ -23,9 +23,11 @@ type Judged = {
|
|
|
23
23
|
readonly named: number;
|
|
24
24
|
readonly agents: number;
|
|
25
25
|
readonly readmes: number;
|
|
26
|
+
readonly hasManifest: boolean;
|
|
26
27
|
readonly findings: readonly Finding[];
|
|
27
28
|
readonly advisory: ReadonlyMap<string, number>;
|
|
28
29
|
readonly readmeAdvisory: readonly Finding[];
|
|
30
|
+
readonly agentFileAdvisory: readonly Finding[];
|
|
29
31
|
readonly sentenceAdvisory: readonly Finding[];
|
|
30
32
|
readonly brokenBefore: readonly Finding[];
|
|
31
33
|
};
|
|
@@ -45,6 +47,27 @@ const NAME = "docs";
|
|
|
45
47
|
const USAGE = "usage: docs.ts <ref> | <base-ref> <head-ref>";
|
|
46
48
|
const MARKDOWN = [":(glob)**/*.md"];
|
|
47
49
|
|
|
50
|
+
function agentFileFindings(path: string, text: string, tracked: Snapshot): readonly Finding[] {
|
|
51
|
+
const shaped = [ceilingFinding(text), maintainingFinding(text)].flatMap((finding) => (finding === undefined ? [] : [finding]));
|
|
52
|
+
return [...shaped, ...entryFindings(path, text, tracked)].map((finding) => ({ path, ...finding }));
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const rangeEditsDocs = Effect.fn("rangeEditsDocs")(function* (root: string, base: string, changes: readonly Change[], text: (path: string) => string) {
|
|
56
|
+
const paths = changes.flatMap((change) => pathsOf(change));
|
|
57
|
+
if (paths.some((path) => placementOf(path, text(path)).type !== "unjudged" || readerOf(path) !== undefined)) return true;
|
|
58
|
+
const gone = [...new Set(changes.flatMap((change) => (change.kind === "deleted" ? [change.path] : change.kind === "renamed" ? [change.from] : [])))].filter(
|
|
59
|
+
(path) => readerOf(path) === undefined && placementOf(path).type === "unjudged",
|
|
60
|
+
);
|
|
61
|
+
if (gone.length === 0) return false;
|
|
62
|
+
const before = yield* readTexts(root, base, gone);
|
|
63
|
+
return gone.some((path) => placementOf(path, before.get(path) ?? "").type !== "unjudged");
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
function rootAgentFindings(absent: readonly RootAgentFinding[], editsDocs: boolean): { readonly failing: readonly Finding[]; readonly advisory: readonly Finding[] } {
|
|
67
|
+
const listed = absent.map((finding) => ({ ...finding, line: undefined }));
|
|
68
|
+
return editsDocs ? { failing: listed, advisory: [] } : { failing: [], advisory: listed };
|
|
69
|
+
}
|
|
70
|
+
|
|
48
71
|
function templateFindings(path: string, text: string, placement: Placement, records: Records): readonly Finding[] {
|
|
49
72
|
const misplaced = placementProblem(placement);
|
|
50
73
|
if (misplaced !== undefined) return [{ path, line: undefined, message: misplaced }];
|
|
@@ -114,9 +137,6 @@ const runDocs = Effect.fn("runDocs")(function* (root: string, base: string, head
|
|
|
114
137
|
const versions = edited.flatMap(({ path }) =>
|
|
115
138
|
versionFindings(text(path), pins, changed.get(path)).map(({ line, message }) => ({ path, line, message })),
|
|
116
139
|
);
|
|
117
|
-
const sentenceAdvisory = edited
|
|
118
|
-
.flatMap(({ path }) => longSentences(text(path), changed.get(path)).map(({ line, message }) => ({ path, line, message })))
|
|
119
|
-
.toSorted(inPathOrder);
|
|
120
140
|
const judging = (path: string): Judging => ({ commands: !speaksToConsumers(text(path)) });
|
|
121
141
|
// A directory the range deletes still belongs to this repository, so a path under it is stale rather than another repository's.
|
|
122
142
|
const roots = rootsOf(yield* pathsAt(base, [], root));
|
|
@@ -125,12 +145,11 @@ const runDocs = Effect.fn("runDocs")(function* (root: string, base: string, head
|
|
|
125
145
|
const vanished = yield* vanishedNames(root, base, head, referenced, references.failed);
|
|
126
146
|
const agents = proseDocs.filter(({ path }) => isAgentFile(path));
|
|
127
147
|
const tracked = snapshotOf(yield* pathsAt(head, [], root), new Map(), new Map());
|
|
128
|
-
const shapes = agents.flatMap(({ path }) =>
|
|
129
|
-
[ceilingFinding(text(path)), maintainingFinding(text(path))].flatMap((finding) => (finding === undefined ? [] : [{ path, ...finding }])),
|
|
130
|
-
);
|
|
131
|
-
const entries = agents.flatMap(({ path }) => entryFindings(path, text(path), tracked).map((finding) => ({ path, ...finding })));
|
|
148
|
+
const shapes = agents.flatMap(({ path }) => agentFileFindings(path, text(path), tracked));
|
|
132
149
|
const readmes = proseDocs.filter(({ path }) => isCeiledReadme(path, tracked.files));
|
|
133
150
|
const oversize = readmes.flatMap(({ path }) => readmeFindings(text(path)).map((finding) => ({ path, ...finding })));
|
|
151
|
+
const absent = missingAgentFiles(tracked.files);
|
|
152
|
+
const rootAgents = rootAgentFindings(absent, absent.length === 0 ? false : yield* rangeEditsDocs(root, base, changes, text));
|
|
134
153
|
const advisory = new Map<string, number>();
|
|
135
154
|
for (const { path } of templated.filter((finding) => !touched.has(finding.path))) advisory.set(path, (advisory.get(path) ?? 0) + 1);
|
|
136
155
|
return {
|
|
@@ -139,6 +158,7 @@ const runDocs = Effect.fn("runDocs")(function* (root: string, base: string, head
|
|
|
139
158
|
named: referenced.size,
|
|
140
159
|
agents: agents.length,
|
|
141
160
|
readmes: readmes.filter(({ path }) => touched.has(path)).length,
|
|
161
|
+
hasManifest: hasManifest(tracked.files),
|
|
142
162
|
findings: [
|
|
143
163
|
...templated.filter((finding) => touched.has(finding.path)),
|
|
144
164
|
...prose,
|
|
@@ -146,12 +166,15 @@ const runDocs = Effect.fn("runDocs")(function* (root: string, base: string, head
|
|
|
146
166
|
...references.failing,
|
|
147
167
|
...vanished,
|
|
148
168
|
...shapes,
|
|
149
|
-
...entries,
|
|
150
169
|
...oversize.filter((finding) => touched.has(finding.path)),
|
|
170
|
+
...rootAgents.failing,
|
|
151
171
|
].toSorted(inPathOrder),
|
|
152
172
|
advisory,
|
|
153
173
|
readmeAdvisory: oversize.filter((finding) => !touched.has(finding.path)).toSorted(inPathOrder),
|
|
154
|
-
|
|
174
|
+
agentFileAdvisory: rootAgents.advisory,
|
|
175
|
+
sentenceAdvisory: edited
|
|
176
|
+
.flatMap(({ path }) => longSentences(text(path), changed.get(path)).map(({ line, message }) => ({ path, line, message })))
|
|
177
|
+
.toSorted(inPathOrder),
|
|
155
178
|
brokenBefore: references.brokenBefore.toSorted(inPathOrder),
|
|
156
179
|
} satisfies Judged;
|
|
157
180
|
});
|
|
@@ -160,7 +183,24 @@ function describe({ path, line, message }: Finding): string {
|
|
|
160
183
|
return ` ${path}${line === undefined ? "" : `:${line}`}: ${message}`;
|
|
161
184
|
}
|
|
162
185
|
|
|
163
|
-
export function report({
|
|
186
|
+
export function report({
|
|
187
|
+
held,
|
|
188
|
+
edited,
|
|
189
|
+
named,
|
|
190
|
+
agents,
|
|
191
|
+
readmes,
|
|
192
|
+
hasManifest: manifestHeld,
|
|
193
|
+
findings,
|
|
194
|
+
advisory,
|
|
195
|
+
readmeAdvisory,
|
|
196
|
+
agentFileAdvisory,
|
|
197
|
+
sentenceAdvisory,
|
|
198
|
+
brokenBefore,
|
|
199
|
+
}: Judged): string {
|
|
200
|
+
const rootAgentFiles =
|
|
201
|
+
manifestHeld && agentFileAdvisory.length === 0
|
|
202
|
+
? [`${NAME}: the repository root holds the AGENTS.md and CLAUDE.md a repository with a \`package.json\` keeps`]
|
|
203
|
+
: [];
|
|
164
204
|
const verdict =
|
|
165
205
|
findings.length === 0
|
|
166
206
|
? [
|
|
@@ -170,6 +210,7 @@ export function report({ held, edited, named, agents, readmes, findings, advisor
|
|
|
170
210
|
`${NAME}: the range breaks no path, link or command the ${named} living doc(s) or agent file(s) name`,
|
|
171
211
|
`${NAME}: the ${agents} agent file(s) hold to the ceiling, and every entry names a tracked path, a link or a command`,
|
|
172
212
|
`${NAME}: the ${readmes} README(s) the range edits hold to the 6,000-character ceiling and the 300-character table cell ceiling`,
|
|
213
|
+
...rootAgentFiles,
|
|
173
214
|
]
|
|
174
215
|
: [`${NAME}: ${findings.length} violation(s):`, ...findings.map(describe)];
|
|
175
216
|
const unconformed =
|
|
@@ -183,6 +224,13 @@ export function report({ held, edited, named, agents, readmes, findings, advisor
|
|
|
183
224
|
readmeAdvisory.length === 0
|
|
184
225
|
? []
|
|
185
226
|
: [`${NAME}: advisory, ${readmeAdvisory.length} README ceiling finding(s) in README(s) the range leaves alone:`, ...readmeAdvisory.map(describe)];
|
|
227
|
+
const missing =
|
|
228
|
+
agentFileAdvisory.length === 0
|
|
229
|
+
? []
|
|
230
|
+
: [
|
|
231
|
+
`${NAME}: advisory, ${agentFileAdvisory.length} agent file(s) missing from the repository root, which the first range that edits a doc adds:`,
|
|
232
|
+
...agentFileAdvisory.map(describe),
|
|
233
|
+
];
|
|
186
234
|
const sentences =
|
|
187
235
|
sentenceAdvisory.length === 0
|
|
188
236
|
? []
|
|
@@ -197,7 +245,7 @@ export function report({ held, edited, named, agents, readmes, findings, advisor
|
|
|
197
245
|
`${NAME}: advisory, ${brokenBefore.length} path(s), link(s) or command(s) the living docs or agent files name were broken before the range:`,
|
|
198
246
|
...brokenBefore.map(describe),
|
|
199
247
|
];
|
|
200
|
-
return [...verdict, ...unconformed, ...oversize, ...sentences, ...broken].join("\n");
|
|
248
|
+
return [...verdict, ...unconformed, ...oversize, ...missing, ...sentences, ...broken].join("\n");
|
|
201
249
|
}
|
|
202
250
|
|
|
203
251
|
const docs = Effect.gen(function* () {
|
|
@@ -6,15 +6,15 @@ const ScoreSchema = Schema.Struct({ detected: Schema.Int, valid: Schema.Int });
|
|
|
6
6
|
|
|
7
7
|
export type Score = typeof ScoreSchema.Type;
|
|
8
8
|
|
|
9
|
-
export const
|
|
9
|
+
export const WholeTreeRecordSchema = Schema.Struct({ scores: Schema.Record(Schema.String, ScoreSchema) });
|
|
10
10
|
|
|
11
|
-
export type
|
|
11
|
+
export type WholeTreeRecord = typeof WholeTreeRecordSchema.Type;
|
|
12
12
|
|
|
13
13
|
export type TestReport = { readonly results: readonly TestResult[]; readonly crashedFiles: readonly string[] };
|
|
14
14
|
|
|
15
|
-
export type
|
|
15
|
+
export type CurrentRun = { readonly scores: WholeTreeRecord["scores"]; readonly tests: readonly TestReport[] };
|
|
16
16
|
|
|
17
|
-
export type
|
|
17
|
+
export type PreviousRun = { readonly url: string; readonly record: WholeTreeRecord };
|
|
18
18
|
|
|
19
19
|
export type Finding =
|
|
20
20
|
| { readonly kind: "mutation"; readonly file: string; readonly before: Score; readonly after: Score; readonly since: string }
|
|
@@ -35,7 +35,7 @@ function scoreOf(file: ReportFile): Score {
|
|
|
35
35
|
return { detected, valid };
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
-
export function scoresOf(files: ReadonlyMap<string, ReportFile>):
|
|
38
|
+
export function scoresOf(files: ReadonlyMap<string, ReportFile>): WholeTreeRecord["scores"] {
|
|
39
39
|
return Object.fromEntries([...files].map(([path, file]) => [path, scoreOf(file)] as const).sort(([one], [other]) => one.localeCompare(other)));
|
|
40
40
|
}
|
|
41
41
|
|
|
@@ -44,7 +44,7 @@ function fell(before: Score, after: Score): boolean {
|
|
|
44
44
|
return before.valid > 0 && after.valid > 0 && after.detected * before.valid < before.detected * after.valid;
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
// A
|
|
47
|
+
// A run judges a key only from a report it read, since a report it lacks leaves every finding in it absent.
|
|
48
48
|
export type Observed = { readonly tests: boolean; readonly gates: boolean; readonly mutation: boolean; readonly mutationReports: ReadonlySet<string> };
|
|
49
49
|
|
|
50
50
|
export function keyOf(finding: Finding): string {
|
|
@@ -55,7 +55,7 @@ export function keyOf(finding: Finding): string {
|
|
|
55
55
|
return `test:${finding.file}`;
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
function scoreFindings(previous: Option.Option<
|
|
58
|
+
function scoreFindings(previous: Option.Option<PreviousRun>, scores: WholeTreeRecord["scores"]): readonly Finding[] {
|
|
59
59
|
if (Option.isNone(previous)) return [];
|
|
60
60
|
const { url, record } = previous.value;
|
|
61
61
|
return Object.entries(scores).flatMap(([file, after]): readonly Finding[] => {
|
|
@@ -72,8 +72,8 @@ function testFindings(reports: readonly TestReport[]): readonly Finding[] {
|
|
|
72
72
|
return [...new Map(failing.map((finding) => [keyOf(finding), finding])).values()];
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
-
export function findingsOf(previous: Option.Option<
|
|
76
|
-
return [...scoreFindings(previous,
|
|
75
|
+
export function findingsOf(previous: Option.Option<PreviousRun>, current: CurrentRun, others: readonly Finding[] = []): readonly Finding[] {
|
|
76
|
+
return [...scoreFindings(previous, current.scores), ...testFindings(current.tests), ...others].sort((one, other) =>
|
|
77
77
|
keyOf(one).localeCompare(keyOf(other)),
|
|
78
78
|
);
|
|
79
79
|
}
|
|
@@ -95,7 +95,7 @@ export function gatesDecided(log: string): boolean {
|
|
|
95
95
|
return FAILED_GATES.test(log) || PASSED_GATES.test(log);
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
-
// A mutation finding is a fall between two
|
|
98
|
+
// A mutation finding is a fall between two runs, so a run without it shows no score recovered.
|
|
99
99
|
export function cleared(key: string, present: ReadonlySet<string>, observed: Observed): boolean {
|
|
100
100
|
if (present.has(key)) return false;
|
|
101
101
|
if (key.startsWith("gates:")) return observed.gates;
|
|
@@ -5,21 +5,21 @@ import { DEFAULT_BRANCH } from "../core/gates.ts";
|
|
|
5
5
|
import { runMain, Usage } from "../core/main.ts";
|
|
6
6
|
import { gh, type GitHubFailure, gitHubJson, REPOSITORY } from "../delivery/github.ts";
|
|
7
7
|
import { parseReport as parseMutationReport, type ReportFile } from "./mutation-compare.ts";
|
|
8
|
-
import { cleared, type Finding, findingsOf, gatesDecided, gatesOf, keyOf,
|
|
8
|
+
import { cleared, type Finding, findingsOf, gatesDecided, gatesOf, keyOf, WholeTreeRecordSchema, noted, type Observed, type WholeTreeRecord, percent, type PreviousRun, scoresOf, type TestReport } from "./whole-tree-run-findings.ts";
|
|
9
9
|
import { parseReport as parseTestReport } from "./test-report.ts";
|
|
10
10
|
|
|
11
|
-
const NAME = "checks-
|
|
11
|
+
const NAME = "checks-whole-tree-run";
|
|
12
12
|
const USAGE = `usage: ${NAME} [--mutation <mutation-report>]... [--tests <junit-report>]... [--gates <lint-log>]`;
|
|
13
|
-
export const LABEL = "
|
|
13
|
+
export const LABEL = "whole-tree-run";
|
|
14
14
|
const LABEL_COLOR = "5319e7";
|
|
15
|
-
const LABEL_DESCRIPTION = "A regression a
|
|
16
|
-
const ARTIFACT = "
|
|
17
|
-
const RECORD_NAME = "
|
|
15
|
+
const LABEL_DESCRIPTION = "A regression a whole-tree run on main found";
|
|
16
|
+
const ARTIFACT = "whole-tree-run";
|
|
17
|
+
const RECORD_NAME = "whole-tree-run.json";
|
|
18
18
|
export const RECORD = `reports/${RECORD_NAME}`;
|
|
19
19
|
const PAGE_SIZE = 100;
|
|
20
20
|
const GITHUB_TITLE_LIMIT = 256;
|
|
21
21
|
|
|
22
|
-
class
|
|
22
|
+
class WholeTreeRunFailure extends Schema.TaggedError<WholeTreeRunFailure>()("WholeTreeRunFailure", {
|
|
23
23
|
message: Schema.String,
|
|
24
24
|
}) {}
|
|
25
25
|
|
|
@@ -81,11 +81,11 @@ const decodeIssues = Schema.decodeUnknownEffect(
|
|
|
81
81
|
const decodeKey = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.String));
|
|
82
82
|
const decodeComments = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(Schema.Struct({ body: Schema.String }))));
|
|
83
83
|
|
|
84
|
-
const MARKER = /<!-- checks-
|
|
84
|
+
const MARKER = /<!-- checks-whole-tree-run ("(?:[^"\\]|\\.)*") -->/;
|
|
85
85
|
|
|
86
86
|
// The key is JSON with every > escaped, so no key can close the HTML comment early.
|
|
87
87
|
export function marker(key: string): string {
|
|
88
|
-
return `<!-- checks-
|
|
88
|
+
return `<!-- checks-whole-tree-run ${JSON.stringify(key).replaceAll(">", "\\u003e")} -->`;
|
|
89
89
|
}
|
|
90
90
|
|
|
91
91
|
export function markedKey(body: string | null): Option.Option<string> {
|
|
@@ -117,11 +117,11 @@ const issueComments = Effect.fn("issueComments")(function* (issue: number) {
|
|
|
117
117
|
});
|
|
118
118
|
|
|
119
119
|
function fullTitleOf(finding: Finding): string {
|
|
120
|
-
if (finding.kind === "mutation") return `
|
|
121
|
-
if (finding.kind === "test") return `
|
|
122
|
-
if (finding.kind === "gates") return `
|
|
123
|
-
if (finding.kind === "mutation-report") return `
|
|
124
|
-
return `
|
|
120
|
+
if (finding.kind === "mutation") return `Whole-tree run: mutation score fell in ${finding.file}`;
|
|
121
|
+
if (finding.kind === "test") return `Whole-tree run: ${finding.file} > ${finding.name} fails`;
|
|
122
|
+
if (finding.kind === "gates") return `Whole-tree run: whole-tree gate ${finding.file} failed`;
|
|
123
|
+
if (finding.kind === "mutation-report") return `Whole-tree run: no mutation report at ${finding.file}`;
|
|
124
|
+
return `Whole-tree run: ${finding.file} crashed its test worker`;
|
|
125
125
|
}
|
|
126
126
|
|
|
127
127
|
export function titleOf(finding: Finding): string {
|
|
@@ -136,25 +136,25 @@ export function findingText(finding: Finding, run: string): string {
|
|
|
136
136
|
if (finding.kind === "mutation") {
|
|
137
137
|
const { file, before, after, since } = finding;
|
|
138
138
|
return [
|
|
139
|
-
`The
|
|
139
|
+
`The whole-tree run ${run} scored \`${file}\` lower than the whole-tree run ${since}.`,
|
|
140
140
|
"",
|
|
141
|
-
"| File | Previous
|
|
141
|
+
"| File | Previous run | This run |",
|
|
142
142
|
"| --- | --- | --- |",
|
|
143
143
|
`| \`${file}\` | ${percent(before)}, ${before.detected} of ${before.valid} mutants detected | ${percent(after)}, ${after.detected} of ${after.valid} mutants detected |`,
|
|
144
144
|
].join("\n");
|
|
145
145
|
}
|
|
146
|
-
if (finding.kind === "test") return `The
|
|
147
|
-
if (finding.kind === "gates") return `The
|
|
148
|
-
if (finding.kind === "mutation-report") return `The
|
|
149
|
-
return `The
|
|
146
|
+
if (finding.kind === "test") return `The whole-tree run ${run} failed \`${finding.name}\` in \`${finding.file}\`.`;
|
|
147
|
+
if (finding.kind === "gates") return `The whole-tree run ${run} failed whole-tree gate \`${finding.file}\`.`;
|
|
148
|
+
if (finding.kind === "mutation-report") return `The whole-tree run ${run} found no mutation report at \`${finding.file}\`: the mutation job that writes it failed, timed out or was cancelled, so no file it mutates was scored.`;
|
|
149
|
+
return `The whole-tree run ${run} lost every result of \`${finding.file}\`, because the worker running it crashed.`;
|
|
150
150
|
}
|
|
151
151
|
|
|
152
152
|
export function issueBody(finding: Finding, run: string): string {
|
|
153
153
|
return `${findingText(finding, run)}\n\nFinding key: \`${keyOf(finding)}\`\n\n${marker(keyOf(finding))}\n`;
|
|
154
154
|
}
|
|
155
155
|
|
|
156
|
-
const REPORTED = "<!-- checks-
|
|
157
|
-
const NOTED = "<!-- checks-
|
|
156
|
+
const REPORTED = "<!-- checks-whole-tree-run reported -->";
|
|
157
|
+
const NOTED = "<!-- checks-whole-tree-run noted -->";
|
|
158
158
|
|
|
159
159
|
export function reportedText(finding: Finding, run: string): string {
|
|
160
160
|
return `${findingText(finding, run)}\n\n${REPORTED}\n`;
|
|
@@ -166,11 +166,11 @@ export function alreadyNoted(comments: readonly string[]): boolean {
|
|
|
166
166
|
}
|
|
167
167
|
|
|
168
168
|
export function clearedText(key: string, run: string): string {
|
|
169
|
-
return `The
|
|
169
|
+
return `The whole-tree run ${run} no longer reports \`${key}\`, so it closes this issue.`;
|
|
170
170
|
}
|
|
171
171
|
|
|
172
172
|
export function notedText(key: string, run: string): string {
|
|
173
|
-
return `The
|
|
173
|
+
return `The whole-tree run ${run} no longer reports \`${key}\`. This issue stays open because a mutation score can recover and drop again.\n\n${NOTED}\n`;
|
|
174
174
|
}
|
|
175
175
|
|
|
176
176
|
export const fileFindings = Effect.fn("fileFindings")(function* (findings: readonly Finding[], observed: Observed, run: string) {
|
|
@@ -208,7 +208,7 @@ export const fileFindings = Effect.fn("fileFindings")(function* (findings: reado
|
|
|
208
208
|
});
|
|
209
209
|
|
|
210
210
|
const required = (name: string) =>
|
|
211
|
-
Config.String(name).pipe(Effect.mapError(() => new
|
|
211
|
+
Config.String(name).pipe(Effect.mapError(() => new WholeTreeRunFailure({ message: `${name} is unset, so ${NAME} runs only inside a GitHub Actions job` })));
|
|
212
212
|
|
|
213
213
|
const WORKFLOW_FILE = /\/\.github\/workflows\/([^/@]+)@/;
|
|
214
214
|
|
|
@@ -218,18 +218,18 @@ const actionsRun = Effect.gen(function* () {
|
|
|
218
218
|
const runs = `${yield* required("GITHUB_SERVER_URL")}/${yield* required("GITHUB_REPOSITORY")}/actions/runs`;
|
|
219
219
|
const reference = yield* required("GITHUB_WORKFLOW_REF");
|
|
220
220
|
const workflow = WORKFLOW_FILE.exec(reference)?.[1];
|
|
221
|
-
if (workflow === undefined) return yield* new
|
|
221
|
+
if (workflow === undefined) return yield* new WholeTreeRunFailure({ message: `GITHUB_WORKFLOW_REF ${reference} names no workflow file` });
|
|
222
222
|
return { runs, run: `${runs}/${yield* required("GITHUB_RUN_ID")}`, workflow, ref: yield* required("GITHUB_REF") } satisfies Actions;
|
|
223
223
|
});
|
|
224
224
|
|
|
225
225
|
const decodeRuns = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Array(Schema.Struct({ databaseId: Schema.Int }))));
|
|
226
226
|
const decodeArtifacts = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Struct({ artifacts: Schema.Array(Schema.Struct({ name: Schema.String, expired: Schema.Boolean })) })));
|
|
227
|
-
const decodeRecord = Schema.decodeUnknownEffect(Schema.fromJsonString(
|
|
227
|
+
const decodeRecord = Schema.decodeUnknownEffect(Schema.fromJsonString(WholeTreeRecordSchema));
|
|
228
228
|
|
|
229
|
-
const unreadable = (source: string) => (cause: { readonly message: string }) => new
|
|
229
|
+
const unreadable = (source: string) => (cause: { readonly message: string }) => new WholeTreeRunFailure({ message: `cannot read ${source}: ${cause.message}` });
|
|
230
230
|
|
|
231
|
-
// A skipped
|
|
232
|
-
const
|
|
231
|
+
// A skipped run records no artifact, so the newest successful run can carry no baseline.
|
|
232
|
+
const previousRun = Effect.fn("previousRun")(
|
|
233
233
|
function* ({ runs, workflow }: Actions) {
|
|
234
234
|
const listed = ["run", "list", "--workflow", workflow, "--branch", DEFAULT_BRANCH, "--status", "success", "--limit", "30", "--json", "databaseId,event,createdAt"];
|
|
235
235
|
const candidates = yield* gh(listed).pipe(Effect.flatMap(decodeRuns), Effect.mapError(unreadable(`the runs of ${workflow}`)));
|
|
@@ -241,13 +241,13 @@ const previousNight = Effect.fn("previousNight")(
|
|
|
241
241
|
);
|
|
242
242
|
if (!artifacts.some((artifact) => artifact.name === ARTIFACT && !artifact.expired)) continue;
|
|
243
243
|
const fs = yield* FileSystem.FileSystem;
|
|
244
|
-
const dir = yield* fs.makeTempDirectoryScoped({ prefix: "checks-
|
|
244
|
+
const dir = yield* fs.makeTempDirectoryScoped({ prefix: "checks-whole-tree-run-" });
|
|
245
245
|
yield* gh(["run", "download", String(run), "--name", ARTIFACT, "--dir", dir]);
|
|
246
246
|
const file = (yield* Path.Path).join(dir, RECORD_NAME);
|
|
247
247
|
const record = yield* fs.readFileString(file).pipe(Effect.flatMap(decodeRecord), Effect.mapError(unreadable(`the ${ARTIFACT} artifact of run ${run}`)));
|
|
248
|
-
return Option.some<
|
|
248
|
+
return Option.some<PreviousRun>({ url: `${runs}/${run}`, record });
|
|
249
249
|
}
|
|
250
|
-
return Option.none<
|
|
250
|
+
return Option.none<PreviousRun>();
|
|
251
251
|
},
|
|
252
252
|
Effect.scoped,
|
|
253
253
|
);
|
|
@@ -260,7 +260,7 @@ const testReport = Effect.fn("testReport")(function* (file: string) {
|
|
|
260
260
|
return yield* parseTestReport(yield* readText(file)).pipe(Effect.mapError(unreadable(file)));
|
|
261
261
|
});
|
|
262
262
|
|
|
263
|
-
const writeRecord = Effect.fn("writeRecord")(function* (record:
|
|
263
|
+
const writeRecord = Effect.fn("writeRecord")(function* (record: WholeTreeRecord) {
|
|
264
264
|
const fs = yield* FileSystem.FileSystem;
|
|
265
265
|
yield* fs.makeDirectory((yield* Path.Path).dirname(RECORD), { recursive: true });
|
|
266
266
|
yield* fs.writeFileString(RECORD, `${JSON.stringify(record, null, 2)}\n`);
|
|
@@ -282,21 +282,21 @@ function readReports(reports: readonly MutationReport[]): ReadonlySet<string> {
|
|
|
282
282
|
return new Set(reports.flatMap(({ file, files }) => (Option.isSome(files) ? [file] : [])));
|
|
283
283
|
}
|
|
284
284
|
|
|
285
|
-
function
|
|
285
|
+
function currentScores(reports: readonly MutationReport[]): Option.Option<WholeTreeRecord["scores"]> {
|
|
286
286
|
const read = reports.flatMap(({ files }) => (Option.isSome(files) ? [...files.value] : []));
|
|
287
287
|
return reports.some(({ files }) => Option.isSome(files)) ? Option.some(scoresOf(new Map(read))) : Option.none();
|
|
288
288
|
}
|
|
289
289
|
|
|
290
290
|
type Filing = { readonly onMain: true; readonly filed: readonly Filed[] } | { readonly onMain: false; readonly ref: string };
|
|
291
291
|
|
|
292
|
-
// A
|
|
292
|
+
// A run missing one report records nothing, so the next run still compares every file with the last run that scored it.
|
|
293
293
|
type Scored = { readonly files: number; readonly missing: number };
|
|
294
294
|
|
|
295
|
-
export function report(run: string, scored: Option.Option<Scored>, previous: Option.Option<
|
|
295
|
+
export function report(run: string, scored: Option.Option<Scored>, previous: Option.Option<PreviousRun>, findings: readonly Finding[], filing: Filing): string {
|
|
296
296
|
const against = Option.isNone(scored)
|
|
297
297
|
? "had no mutation report, so it compared no score"
|
|
298
298
|
: Option.isSome(previous)
|
|
299
|
-
? `compared with the
|
|
299
|
+
? `compared with the whole-tree run ${previous.value.url}`
|
|
300
300
|
: `found no earlier successful run on ${DEFAULT_BRANCH} holding a ${ARTIFACT} artifact, so it compared no score`;
|
|
301
301
|
const verdict = findings.length === 0 ? "no finding" : `${findings.length} finding(s)`;
|
|
302
302
|
const listed = filing.onMain ? filing.filed.map((one) => ` ${one.action} #${one.issue} for ${one.key}`) : findings.map((finding) => ` found ${keyOf(finding)}`);
|
|
@@ -307,30 +307,30 @@ export function report(run: string, scored: Option.Option<Scored>, previous: Opt
|
|
|
307
307
|
: scored.value.missing === 0
|
|
308
308
|
? `${NAME}: recorded ${scored.value.files} file score(s) in ${RECORD}`
|
|
309
309
|
: `${NAME}: recorded no score, since ${scored.value.missing} mutation report(s) were missing`;
|
|
310
|
-
return [`${NAME}: the
|
|
310
|
+
return [`${NAME}: the whole-tree run ${run} ${against}; ${verdict}`, ...listed, recorded].join("\n");
|
|
311
311
|
}
|
|
312
312
|
|
|
313
|
-
const
|
|
313
|
+
const wholeTreeRun = Effect.gen(function* () {
|
|
314
314
|
const options = yield* parseArgs(process.argv.slice(2));
|
|
315
315
|
const actions = yield* actionsRun;
|
|
316
316
|
const reports = yield* Effect.forEach(options.mutation, mutationReport);
|
|
317
317
|
const missing = missingReports(reports);
|
|
318
|
-
const scores =
|
|
318
|
+
const scores = currentScores(reports);
|
|
319
319
|
const tests: TestReport[] = yield* Effect.forEach(options.tests, testReport);
|
|
320
320
|
const log = Option.isSome(options.gates) ? Option.some(yield* readText(options.gates.value)) : Option.none<string>();
|
|
321
321
|
const gates: readonly Finding[] = Option.isSome(log) ? gatesOf(log.value) : [];
|
|
322
|
-
const previous = Option.isSome(scores) ? yield*
|
|
322
|
+
const previous = Option.isSome(scores) ? yield* previousRun(actions) : Option.none<PreviousRun>();
|
|
323
323
|
const observed: Observed = { tests: options.tests.length > 0, gates: Option.isSome(log) && gatesDecided(log.value), mutation: Option.isSome(previous) && missing.length === 0, mutationReports: readReports(reports) };
|
|
324
324
|
const findings = findingsOf(previous, { scores: Option.getOrElse(scores, () => ({})), tests }, [...missing, ...gates]);
|
|
325
325
|
const filing: Filing = actions.ref === `refs/heads/${DEFAULT_BRANCH}` ? { onMain: true, filed: yield* fileFindings(findings, observed, actions.run) } : { onMain: false, ref: actions.ref };
|
|
326
326
|
if (filing.onMain && Option.isSome(scores) && missing.length === 0) yield* writeRecord({ scores: scores.value });
|
|
327
|
-
const scored = Option.map(scores, (
|
|
327
|
+
const scored = Option.map(scores, (current): Scored => ({ files: Object.keys(current).length, missing: missing.length }));
|
|
328
328
|
yield* Console.log(report(actions.run, scored, previous, findings, filing));
|
|
329
329
|
return true;
|
|
330
330
|
});
|
|
331
331
|
|
|
332
332
|
export function main(issues: Layer.Layer<Issues, never, ChildProcessSpawner.ChildProcessSpawner>): void {
|
|
333
|
-
runMain(NAME,
|
|
333
|
+
runMain(NAME, wholeTreeRun.pipe(Effect.provide(issues)));
|
|
334
334
|
}
|
|
335
335
|
|
|
336
336
|
if (import.meta.main) main(Issues.gh);
|