@mindlace/chreode 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -2
- package/dist/burn-in/run.d.ts +18 -0
- package/dist/burn-in/run.js +71 -0
- package/dist/burn-in/select.d.ts +30 -0
- package/dist/burn-in/select.js +78 -0
- package/dist/burn-in/step.d.ts +10 -0
- package/dist/burn-in/step.js +14 -0
- package/dist/cli.d.ts +10 -0
- package/dist/cli.js +19 -0
- package/dist/config/types.d.ts +47 -0
- package/dist/config/types.js +12 -0
- package/dist/config/validate.js +94 -8
- package/dist/flakes/gh.d.ts +5 -0
- package/dist/flakes/gh.js +74 -0
- package/dist/flakes/ledger.d.ts +53 -0
- package/dist/flakes/ledger.js +108 -0
- package/dist/flakes/log.d.ts +39 -0
- package/dist/flakes/log.js +89 -0
- package/dist/flakes/phase.d.ts +34 -0
- package/dist/flakes/phase.js +66 -0
- package/dist/flakes/report.d.ts +14 -0
- package/dist/flakes/report.js +45 -0
- package/dist/git/version-only.d.ts +11 -0
- package/dist/git/version-only.js +56 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +1 -0
- package/dist/reporters/options.d.ts +17 -0
- package/dist/reporters/options.js +18 -0
- package/dist/reporters/playwright.d.ts +15 -0
- package/dist/reporters/playwright.js +47 -0
- package/dist/reporters/vitest.d.ts +15 -0
- package/dist/reporters/vitest.js +47 -0
- package/dist/run/stderr.d.ts +1 -0
- package/dist/run/stderr.js +4 -1
- package/dist/signoff/plan.d.ts +17 -3
- package/dist/signoff/plan.js +28 -11
- package/dist/signoff/preconditions.d.ts +3 -1
- package/dist/signoff/preconditions.js +11 -1
- package/dist/signoff/run.d.ts +5 -0
- package/dist/signoff/run.js +45 -8
- package/package.json +23 -1
package/README.md
CHANGED
|
@@ -44,6 +44,8 @@ Add `chreode.config.ts` at the repo root (see below), then:
|
|
|
44
44
|
npx chreode signoff # merge origin/main, run what changed, push, post the status
|
|
45
45
|
npx chreode signoff --dry-run # merge, then print the plan and stop
|
|
46
46
|
npx chreode protect # require the status on main (idempotent); --dry-run previews
|
|
47
|
+
npx chreode flake-report # report logged flakes to GitHub issues (signoff does this itself)
|
|
48
|
+
npx chreode burn-in --base <sha> --head <sha> # run changed test files N times, retries off
|
|
47
49
|
```
|
|
48
50
|
|
|
49
51
|
Wire them into `package.json` as `"signoff": "chreode signoff"` and
|
|
@@ -59,8 +61,9 @@ Wire them into `package.json` as `"signoff": "chreode signoff"` and
|
|
|
59
61
|
3. **Base**: `--base`, else the nearest commit on the branch carrying a `refs/notes/chreode`
|
|
60
62
|
note (the last successful signoff), else the merge-base with `origin/<baseBranch>`.
|
|
61
63
|
4. **Plan**: changed paths (both sides of renames, deletions included) → triggers (first
|
|
62
|
-
matching rule; a path no rule matches fires every trigger
|
|
63
|
-
|
|
64
|
+
matching rule; a path no rule matches fires every trigger, unless it is a version-only
|
|
65
|
+
`package.json` / `package-lock.json` bump, which fires `versionBumpTriggers`) → steps
|
|
66
|
+
(`when`, `if`, `--skip`, `supersededBy`). The plan and every skipped step's reason are printed.
|
|
64
67
|
5. **Run** the steps in order under one machine-wide lock shared by every chreode repo on the
|
|
65
68
|
machine. Each step runs in its own process group with a hard timeout; policed steps fail
|
|
66
69
|
on unexpected stderr. Children see `CHREODE_SIGNOFF=1`.
|
|
@@ -95,6 +98,96 @@ the repo to squash-only merges that delete head branches. Override either payloa
|
|
|
95
98
|
`protection` in the config. `--repo owner/name` targets another repo; `--dry-run` prints the
|
|
96
99
|
payloads.
|
|
97
100
|
|
|
101
|
+
## Flakes
|
|
102
|
+
|
|
103
|
+
A test that fails and then passes on retry is a flake, and a flake is a bug. chreode records
|
|
104
|
+
each one, files it as a GitHub issue, and fails signoff when the same flake keeps coming back.
|
|
105
|
+
The doctrine and the fix playbook are in [docs/flakes.md](docs/flakes.md).
|
|
106
|
+
|
|
107
|
+
**1. Record.** Give your runners at least one retry locally and register the reporters. They record only
|
|
108
|
+
tests that pass on a retry, so with retries off nothing is recorded. Both take
|
|
109
|
+
`{ kind?, projects? }`: `kind` (a string, or `(project) => string`) is the issue-title bucket
|
|
110
|
+
(default `'e2e'` for Playwright, the project name or `'unit'` for vitest); `projects` limits
|
|
111
|
+
recording to those runner projects.
|
|
112
|
+
|
|
113
|
+
```ts
|
|
114
|
+
// playwright.config.ts
|
|
115
|
+
export default defineConfig({
|
|
116
|
+
retries: 1,
|
|
117
|
+
use: { trace: 'retain-on-failure' }, // 'on-first-retry' records no trace of the failed attempt
|
|
118
|
+
reporter: [['list'], ['@mindlace/chreode/reporters/playwright']]
|
|
119
|
+
});
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
```ts
|
|
123
|
+
// vitest.config.ts (reporters are root-level only; keep 'default' explicit, see below)
|
|
124
|
+
import VitestFlakeReporter from '@mindlace/chreode/reporters/vitest';
|
|
125
|
+
|
|
126
|
+
export default defineConfig({
|
|
127
|
+
test: { retry: 1, reporters: ['default', new VitestFlakeReporter()] }
|
|
128
|
+
});
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
Events go to `.chreode/flakes.jsonl` (signoff tells each step the absolute path in
|
|
132
|
+
`CHREODE_FLAKE_LOG`, and the sha under test in `CHREODE_HEAD`). Add `.chreode/` to
|
|
133
|
+
`.gitignore`; when `flakes` is configured, signoff refuses to run otherwise. Registering the
|
|
134
|
+
reporters without a `flakes` block just accumulates events in `.chreode/flakes.jsonl`, unreported.
|
|
135
|
+
Filed issues carry the Playwright trace's local path, valid until the next Playwright run wipes
|
|
136
|
+
`test-results/`, not the trace itself. `playwright` and `vitest` are optional peer
|
|
137
|
+
dependencies: the reporters import only their types.
|
|
138
|
+
|
|
139
|
+
**2. Ledger.** Add `flakes: {}` to the config. After the steps, pass or fail, signoff gives
|
|
140
|
+
each flaking test one open issue labelled `flake` (the label is created if missing) and adds an
|
|
141
|
+
occurrence comment each time it recurs. At `threshold` occurrences (default 3) signoff fails,
|
|
142
|
+
any run that sees that flake again fails, listing the issue URLs in its failure output (no
|
|
143
|
+
commit status is posted), until the flake is fixed or its issue is labelled `snoozeLabel`
|
|
144
|
+
(default `flake-snoozed`). A run that sees no flake events is silent. A GitHub error is a
|
|
145
|
+
warning: the events not yet recorded stay in the log for the next run.
|
|
146
|
+
`chreode flake-report` runs the same report outside signoff and exits 1 on escalation.
|
|
147
|
+
|
|
148
|
+
**3. Burn-in.** `burnInStep({ when, timeoutMs? })` adds a step that runs every test file changed since the base
|
|
149
|
+
`repeats` times with retries off, so a new flake fails before it lands rather than after three
|
|
150
|
+
victims. Pass your test steps' triggers as `when`, and place it after them. Its stderr is policed like a
|
|
151
|
+
test step's, so it needs the same `stderrAllowlist` entries:
|
|
152
|
+
|
|
153
|
+
```ts
|
|
154
|
+
import { burnInStep, defineConfig } from '@mindlace/chreode';
|
|
155
|
+
|
|
156
|
+
export default defineConfig({
|
|
157
|
+
steps: [
|
|
158
|
+
{ label: 'test:unit', run: 'npm run test:unit', when: ['unit'], policeStderr: true },
|
|
159
|
+
{ label: 'test:e2e', run: 'npm run test:e2e', when: ['e2e'], policeStderr: true },
|
|
160
|
+
burnInStep({ when: ['unit', 'e2e'] }) // timeoutMs defaults to 10 minutes
|
|
161
|
+
],
|
|
162
|
+
flakes: {}, // threshold: 3, snoozeLabel: 'flake-snoozed', playbookUrl: chreode's docs/flakes.md on GitHub
|
|
163
|
+
burnIn: {
|
|
164
|
+
repeats: 5, // default
|
|
165
|
+
maxFiles: 12, // default; the overflow is listed, never silently dropped
|
|
166
|
+
runners: [
|
|
167
|
+
{
|
|
168
|
+
name: 'e2e',
|
|
169
|
+
match: 'e2e/**/*.spec.ts',
|
|
170
|
+
ignore: 'e2e/helpers/**',
|
|
171
|
+
// Pin a project: --repeat-each multiplies across every project otherwise.
|
|
172
|
+
command: (files, n) =>
|
|
173
|
+
`npx playwright test --project chromium --repeat-each=${n} --retries=0 ${files.join(' ')}`
|
|
174
|
+
},
|
|
175
|
+
{
|
|
176
|
+
name: 'unit',
|
|
177
|
+
match: 'src/**/*.spec.ts',
|
|
178
|
+
// vitest has no --repeat-each: return one invocation per repeat.
|
|
179
|
+
command: (files, n) =>
|
|
180
|
+
Array.from({ length: n }, () => `npx vitest --run --retry=0 ${files.join(' ')}`)
|
|
181
|
+
}
|
|
182
|
+
]
|
|
183
|
+
}
|
|
184
|
+
});
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
A changed file goes to the first runner that matches it and does not `ignore` it; files no
|
|
188
|
+
runner takes are not burned in, and deleted files never are. Commands run from the repo root
|
|
189
|
+
with inherited output; the first failure fails the step and names the files and the command.
|
|
190
|
+
|
|
98
191
|
## Config reference
|
|
99
192
|
|
|
100
193
|
```ts
|
|
@@ -108,6 +201,7 @@ export default defineConfig({
|
|
|
108
201
|
{ match: 'src/**', triggers: ['lint', 'check', 'unit', 'e2e'] },
|
|
109
202
|
{ match: 'generated/**', triggers: [] } // matched, nothing to run
|
|
110
203
|
],
|
|
204
|
+
versionBumpTriggers: ['format'], // default []: see below
|
|
111
205
|
steps: [
|
|
112
206
|
{ label: 'format', run: 'npx prettier --check .', when: ['format'], supersededBy: ['lint'] },
|
|
113
207
|
{ label: 'lint', run: 'npm run lint', when: ['lint'] },
|
|
@@ -131,6 +225,15 @@ glob, dotfiles included, or a RegExp, or an array of them) matches a changed pat
|
|
|
131
225
|
triggers. A path no rule matches fires every trigger: rules can only make runs smaller, so a
|
|
132
226
|
forgotten rule costs time, never coverage. Triggers are free-form strings.
|
|
133
227
|
|
|
228
|
+
**Version bumps** are the one exception. A `package.json` or `package-lock.json` (at any depth)
|
|
229
|
+
that matches no rule, exists on both sides, and whose only change is its version (for the
|
|
230
|
+
lockfile, the top-level `version` and `packages[""].version`; key order and formatting are
|
|
231
|
+
ignored) fires `versionBumpTriggers` instead of every trigger. The default is `[]`, so a
|
|
232
|
+
release commit that only bumps the version runs nothing. The plan prints such paths as
|
|
233
|
+
`version-only: package.json`. A rule that matches the file still wins, any other change to
|
|
234
|
+
it (or an added, deleted or unparseable file) fires everything as before, and `--full`
|
|
235
|
+
ignores the distinction.
|
|
236
|
+
|
|
134
237
|
**Steps** run in config order.
|
|
135
238
|
|
|
136
239
|
| Field | Type | Meaning |
|
|
@@ -147,8 +250,21 @@ forgotten rule costs time, never coverage. Triggers are free-form strings.
|
|
|
147
250
|
leave the checkout clean (signoff refuses to run otherwise), so use `npm ci`, which installs
|
|
148
251
|
exactly the lockfile, rather than `npm install`, which can rewrite it.
|
|
149
252
|
**`stderrAllowlist`** entries need a `reason`: allowlisting noise is a reviewed decision.
|
|
253
|
+
|
|
254
|
+
**Pin vitest's `reporters`** in any project whose test steps police stderr. With no
|
|
255
|
+
`reporters` set, vitest picks its `agent` reporter whenever it detects an AI agent (`AI_AGENT`,
|
|
256
|
+
`CLAUDECODE`, `CODEX_*` and others, via std-env), and that reporter hides console output from
|
|
257
|
+
passing tests. Signoff would then pass under an agent and fail for a human on the same commit.
|
|
258
|
+
An explicit `reporters: ['default']` (plus `VitestFlakeReporter` when flakes is on) makes every
|
|
259
|
+
run show that output, so a stray `console.error` fails signoff whoever runs it. Assert expected
|
|
260
|
+
logs in the test (`vi.spyOn(console, 'error')`) rather than allowlisting them.
|
|
261
|
+
|
|
150
262
|
Unknown keys, duplicate labels, and `supersededBy` naming a missing step are load errors.
|
|
151
263
|
|
|
264
|
+
**`flakes`** (presence turns flake tracking on) and **`burnIn`** are described under
|
|
265
|
+
[Flakes](#flakes). `threshold`, `repeats` and `maxFiles` must be positive integers, and
|
|
266
|
+
`burnIn.runners` must be non-empty.
|
|
267
|
+
|
|
152
268
|
## License
|
|
153
269
|
|
|
154
270
|
MIT
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ResolvedBurnIn } from '../config/types.js';
|
|
2
|
+
import { type Exec, type Run } from '../run/exec.js';
|
|
3
|
+
export interface BurnInDeps {
|
|
4
|
+
exec: Exec;
|
|
5
|
+
/** Inherited stdio: the test runners' own output streams through. */
|
|
6
|
+
run: Run;
|
|
7
|
+
mtime: (path: string) => number | null;
|
|
8
|
+
log: (m: string) => void;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Run every test file added or changed in base..head `repeats` times with retries off. The
|
|
12
|
+
* first failure exits 1. All of chreode's own output goes to stdout: this runs inside a
|
|
13
|
+
* policed signoff step.
|
|
14
|
+
*/
|
|
15
|
+
export declare function runBurnIn(config: ResolvedBurnIn | null, range: {
|
|
16
|
+
base: string;
|
|
17
|
+
head: string;
|
|
18
|
+
}, overrides?: Partial<BurnInDeps>): number;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { statSync } from 'node:fs';
|
|
2
|
+
import { defaultExec, defaultRun } from '../run/exec.js';
|
|
3
|
+
import { BurnInError, burnInCandidates, burnInCommands, selectBurnIn } from './select.js';
|
|
4
|
+
function mtimeOf(path) {
|
|
5
|
+
try {
|
|
6
|
+
return statSync(path).mtimeMs;
|
|
7
|
+
}
|
|
8
|
+
catch {
|
|
9
|
+
return null;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Run every test file added or changed in base..head `repeats` times with retries off. The
|
|
14
|
+
* first failure exits 1. All of chreode's own output goes to stdout: this runs inside a
|
|
15
|
+
* policed signoff step.
|
|
16
|
+
*/
|
|
17
|
+
export function runBurnIn(config, range, overrides = {}) {
|
|
18
|
+
const d = {
|
|
19
|
+
exec: defaultExec,
|
|
20
|
+
run: (command) => defaultRun(command),
|
|
21
|
+
mtime: mtimeOf,
|
|
22
|
+
log: (m) => console.log(m),
|
|
23
|
+
...overrides
|
|
24
|
+
};
|
|
25
|
+
if (config === null) {
|
|
26
|
+
d.log('[chreode] burn-in: chreode.config.ts has no burnIn block; nothing knows how to run tests');
|
|
27
|
+
return 1;
|
|
28
|
+
}
|
|
29
|
+
const candidates = burnInCandidates(range.base, range.head, d.exec, d.mtime);
|
|
30
|
+
const plan = selectBurnIn(candidates, config);
|
|
31
|
+
let commands;
|
|
32
|
+
try {
|
|
33
|
+
commands = burnInCommands(plan, config.repeats);
|
|
34
|
+
}
|
|
35
|
+
catch (err) {
|
|
36
|
+
if (err instanceof BurnInError) {
|
|
37
|
+
d.log(`[chreode] ${err.message}`);
|
|
38
|
+
return 1;
|
|
39
|
+
}
|
|
40
|
+
throw err;
|
|
41
|
+
}
|
|
42
|
+
const kept = plan.groups.reduce((n, g) => n + g.files.length, 0);
|
|
43
|
+
if (kept === 0) {
|
|
44
|
+
d.log('[chreode] burn-in: no changed test files; nothing to burn in');
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
d.log(`[chreode] burn-in: ${kept} changed test file(s) × ${config.repeats}, retries off:`);
|
|
48
|
+
for (const group of plan.groups) {
|
|
49
|
+
for (const file of group.files)
|
|
50
|
+
d.log(` ${file} (${group.runner.name})`);
|
|
51
|
+
}
|
|
52
|
+
if (plan.skipped.length > 0) {
|
|
53
|
+
d.log(`[chreode] burn-in: over the ${config.maxFiles}-file cap, NOT burned in:`);
|
|
54
|
+
for (const file of plan.skipped)
|
|
55
|
+
d.log(` ${file}`);
|
|
56
|
+
}
|
|
57
|
+
for (const c of commands) {
|
|
58
|
+
d.log(`[chreode] burn-in (${c.runner}): ${c.command}`);
|
|
59
|
+
try {
|
|
60
|
+
d.run(c.command);
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
d.log(`\n[chreode] burn-in FAILED (${c.runner}): ${c.files.join(' ')}\n` +
|
|
64
|
+
`[chreode] command: ${c.command}\n` +
|
|
65
|
+
`[chreode] A test that flakes at birth flakes forever: fix it before it lands.`);
|
|
66
|
+
return 1;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
d.log('[chreode] burn-in: every changed test file survived');
|
|
70
|
+
return 0;
|
|
71
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { BurnInRunner, ResolvedBurnIn } from '../config/types.js';
|
|
2
|
+
import type { Exec } from '../run/exec.js';
|
|
3
|
+
export declare class BurnInError extends Error {
|
|
4
|
+
constructor(message: string);
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Paths added or modified in base..head, newest mtime first so the file cap keeps the most
|
|
8
|
+
* recently touched. --no-renames lists a rename as delete + add and --diff-filter=d drops
|
|
9
|
+
* deletions, so a renamed file appears under its new name only. Files missing on disk
|
|
10
|
+
* (mtime null) are dropped.
|
|
11
|
+
*/
|
|
12
|
+
export declare function burnInCandidates(base: string, head: string, exec: Exec, mtime: (path: string) => number | null): string[];
|
|
13
|
+
/** The first runner that matches the path and does not ignore it. */
|
|
14
|
+
export declare function runnerFor(path: string, runners: readonly BurnInRunner[]): BurnInRunner | null;
|
|
15
|
+
export interface BurnInGroup {
|
|
16
|
+
runner: BurnInRunner;
|
|
17
|
+
files: string[];
|
|
18
|
+
}
|
|
19
|
+
export interface BurnInPlan {
|
|
20
|
+
groups: BurnInGroup[];
|
|
21
|
+
/** Burnable files past maxFiles: listed, never silently dropped. */
|
|
22
|
+
skipped: string[];
|
|
23
|
+
}
|
|
24
|
+
export declare function selectBurnIn(paths: readonly string[], config: ResolvedBurnIn): BurnInPlan;
|
|
25
|
+
export interface BurnInCommand {
|
|
26
|
+
runner: string;
|
|
27
|
+
files: string[];
|
|
28
|
+
command: string;
|
|
29
|
+
}
|
|
30
|
+
export declare function burnInCommands(plan: BurnInPlan, repeats: number): BurnInCommand[];
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import { matchesAny } from '../signoff/plan.js';
|
|
2
|
+
export class BurnInError extends Error {
|
|
3
|
+
constructor(message) {
|
|
4
|
+
super(message);
|
|
5
|
+
this.name = 'BurnInError';
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Paths added or modified in base..head, newest mtime first so the file cap keeps the most
|
|
10
|
+
* recently touched. --no-renames lists a rename as delete + add and --diff-filter=d drops
|
|
11
|
+
* deletions, so a renamed file appears under its new name only. Files missing on disk
|
|
12
|
+
* (mtime null) are dropped.
|
|
13
|
+
*/
|
|
14
|
+
export function burnInCandidates(base, head, exec, mtime) {
|
|
15
|
+
if (base === head)
|
|
16
|
+
return [];
|
|
17
|
+
const out = exec('git', [
|
|
18
|
+
'diff',
|
|
19
|
+
'--name-only',
|
|
20
|
+
'--no-renames',
|
|
21
|
+
'--diff-filter=d',
|
|
22
|
+
'-z',
|
|
23
|
+
`${base}..${head}`
|
|
24
|
+
]);
|
|
25
|
+
return out
|
|
26
|
+
.split('\0')
|
|
27
|
+
.filter((p) => p !== '')
|
|
28
|
+
.map((path) => ({ path, time: mtime(path) }))
|
|
29
|
+
.filter((c) => c.time !== null)
|
|
30
|
+
.sort((a, b) => b.time - a.time)
|
|
31
|
+
.map((c) => c.path);
|
|
32
|
+
}
|
|
33
|
+
/** The first runner that matches the path and does not ignore it. */
|
|
34
|
+
export function runnerFor(path, runners) {
|
|
35
|
+
for (const runner of runners) {
|
|
36
|
+
if (!matchesAny(path, runner.match))
|
|
37
|
+
continue;
|
|
38
|
+
if (runner.ignore !== undefined && matchesAny(path, runner.ignore))
|
|
39
|
+
continue;
|
|
40
|
+
return runner;
|
|
41
|
+
}
|
|
42
|
+
return null;
|
|
43
|
+
}
|
|
44
|
+
export function selectBurnIn(paths, config) {
|
|
45
|
+
const byRunner = new Map();
|
|
46
|
+
const skipped = [];
|
|
47
|
+
let kept = 0;
|
|
48
|
+
for (const path of paths) {
|
|
49
|
+
const runner = runnerFor(path, config.runners);
|
|
50
|
+
if (runner === null)
|
|
51
|
+
continue;
|
|
52
|
+
if (kept >= config.maxFiles) {
|
|
53
|
+
skipped.push(path);
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
byRunner.set(runner, [...(byRunner.get(runner) ?? []), path]);
|
|
57
|
+
kept += 1;
|
|
58
|
+
}
|
|
59
|
+
const groups = config.runners
|
|
60
|
+
.filter((runner) => byRunner.has(runner))
|
|
61
|
+
.map((runner) => ({ runner, files: byRunner.get(runner) }));
|
|
62
|
+
return { groups, skipped };
|
|
63
|
+
}
|
|
64
|
+
export function burnInCommands(plan, repeats) {
|
|
65
|
+
const out = [];
|
|
66
|
+
for (const { runner, files } of plan.groups) {
|
|
67
|
+
const produced = runner.command([...files], repeats);
|
|
68
|
+
const commands = Array.isArray(produced) ? produced : [produced];
|
|
69
|
+
const ok = commands.length > 0 && commands.every((c) => typeof c === 'string' && c.trim() !== '');
|
|
70
|
+
if (!ok) {
|
|
71
|
+
throw new BurnInError(`burnIn runner "${runner.name}": command() must return a non-empty command or a ` +
|
|
72
|
+
`non-empty array of them`);
|
|
73
|
+
}
|
|
74
|
+
for (const command of commands)
|
|
75
|
+
out.push({ runner: runner.name, files, command });
|
|
76
|
+
}
|
|
77
|
+
return out;
|
|
78
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type StepConfig } from '../config/types.js';
|
|
2
|
+
export declare const BURN_IN_LABEL = "burn-in";
|
|
3
|
+
export interface BurnInStepOptions {
|
|
4
|
+
/** The triggers of your test steps: chreode cannot tell which steps are tests. */
|
|
5
|
+
when: string[];
|
|
6
|
+
timeoutMs?: number;
|
|
7
|
+
}
|
|
8
|
+
/** A signoff step that burns in the test files changed since the base. Place it after the
|
|
9
|
+
* test steps. */
|
|
10
|
+
export declare function burnInStep(opts: BurnInStepOptions): StepConfig;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { DEFAULT_TIMEOUT_MS } from '../config/types.js';
|
|
2
|
+
export const BURN_IN_LABEL = 'burn-in';
|
|
3
|
+
/** A signoff step that burns in the test files changed since the base. Place it after the
|
|
4
|
+
* test steps. */
|
|
5
|
+
export function burnInStep(opts) {
|
|
6
|
+
return {
|
|
7
|
+
label: BURN_IN_LABEL,
|
|
8
|
+
run: ({ base, head }) => `npx chreode burn-in --base ${base} --head ${head}`,
|
|
9
|
+
when: [...opts.when],
|
|
10
|
+
if: ({ base, head }) => base !== head,
|
|
11
|
+
policeStderr: true,
|
|
12
|
+
timeoutMs: opts.timeoutMs ?? DEFAULT_TIMEOUT_MS
|
|
13
|
+
};
|
|
14
|
+
}
|
package/dist/cli.d.ts
CHANGED
|
@@ -16,11 +16,21 @@ export interface ProtectFlags {
|
|
|
16
16
|
repo?: string;
|
|
17
17
|
config?: string;
|
|
18
18
|
}
|
|
19
|
+
export interface FlakeReportFlags {
|
|
20
|
+
config?: string;
|
|
21
|
+
}
|
|
22
|
+
export interface BurnInFlags {
|
|
23
|
+
base: string;
|
|
24
|
+
head: string;
|
|
25
|
+
config?: string;
|
|
26
|
+
}
|
|
19
27
|
/** Each flag also has an env form, for harnesses that forbid flags after `--`. */
|
|
20
28
|
export declare function resolveSignoffOptions(flags: SignoffFlags, env: NodeJS.ProcessEnv): SignoffOptions;
|
|
21
29
|
export declare function buildProgram(actions: {
|
|
22
30
|
signoff: (flags: SignoffFlags) => Promise<number>;
|
|
23
31
|
protect: (flags: ProtectFlags) => Promise<number>;
|
|
32
|
+
flakeReport: (flags: FlakeReportFlags) => Promise<number>;
|
|
33
|
+
burnIn: (flags: BurnInFlags) => Promise<number>;
|
|
24
34
|
}, setExit: (code: number) => void): Command;
|
|
25
35
|
export declare function main(argv?: string[], env?: NodeJS.ProcessEnv): Promise<number>;
|
|
26
36
|
/** npm installs the bin as a symlink, so compare real paths, not the invoked path. */
|
package/dist/cli.js
CHANGED
|
@@ -3,7 +3,9 @@ import { readFileSync, realpathSync } from 'node:fs';
|
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { pathToFileURL } from 'node:url';
|
|
5
5
|
import { Command } from 'commander';
|
|
6
|
+
import { runBurnIn } from './burn-in/run.js';
|
|
6
7
|
import { CONFIG_FILE, loadConfig } from './config/load.js';
|
|
8
|
+
import { runFlakeReport } from './flakes/report.js';
|
|
7
9
|
import { parseRepo } from './github/status.js';
|
|
8
10
|
import { runProtect } from './github/protect.js';
|
|
9
11
|
import { defaultExec } from './run/exec.js';
|
|
@@ -50,6 +52,18 @@ export function buildProgram(actions, setExit) {
|
|
|
50
52
|
.option('--repo <owner/name>', 'target repository (default: gh repo view)')
|
|
51
53
|
.option('--config <path>', `config file (default: <repo root>/${CONFIG_FILE})`)
|
|
52
54
|
.action(async (flags) => setExit(await actions.protect(flags)));
|
|
55
|
+
program
|
|
56
|
+
.command('flake-report')
|
|
57
|
+
.description('report logged flakes to GitHub issues; exit 1 if any reached the threshold')
|
|
58
|
+
.option('--config <path>', `config file (default: <repo root>/${CONFIG_FILE})`)
|
|
59
|
+
.action(async (flags) => setExit(await actions.flakeReport(flags)));
|
|
60
|
+
program
|
|
61
|
+
.command('burn-in')
|
|
62
|
+
.description('run the test files changed in base..head repeatedly, with retries off')
|
|
63
|
+
.requiredOption('--base <sha>', 'diff from this commit')
|
|
64
|
+
.requiredOption('--head <sha>', 'diff to this commit')
|
|
65
|
+
.option('--config <path>', `config file (default: <repo root>/${CONFIG_FILE})`)
|
|
66
|
+
.action(async (flags) => setExit(await actions.burnIn(flags)));
|
|
53
67
|
return program;
|
|
54
68
|
}
|
|
55
69
|
/** Steps and git commands run from the repo root, whatever directory chreode was started in.
|
|
@@ -68,6 +82,11 @@ export async function main(argv = process.argv, env = process.env) {
|
|
|
68
82
|
protect: async (flags) => runProtect(await repoConfigLoader(flags.config)(), {
|
|
69
83
|
dryRun: flags.dryRun === true,
|
|
70
84
|
repo: flags.repo ? parseRepo(flags.repo) : null
|
|
85
|
+
}),
|
|
86
|
+
flakeReport: async (flags) => runFlakeReport(await repoConfigLoader(flags.config)()),
|
|
87
|
+
burnIn: async (flags) => runBurnIn((await repoConfigLoader(flags.config)()).burnIn, {
|
|
88
|
+
base: flags.base,
|
|
89
|
+
head: flags.head
|
|
71
90
|
})
|
|
72
91
|
}, (c) => {
|
|
73
92
|
code = c;
|
package/dist/config/types.d.ts
CHANGED
|
@@ -40,15 +40,53 @@ export interface ProtectionConfig {
|
|
|
40
40
|
/** Shallow-merged over the default repository merge settings. */
|
|
41
41
|
repo?: Record<string, unknown>;
|
|
42
42
|
}
|
|
43
|
+
/** Presence turns flake tracking on: retried-then-passed tests become GitHub issues. */
|
|
44
|
+
export interface FlakesConfig {
|
|
45
|
+
/** Occurrences (the issue itself counts as one) at which an unsnoozed flake fails signoff. */
|
|
46
|
+
threshold?: number;
|
|
47
|
+
/** An issue carrying this label never escalates. */
|
|
48
|
+
snoozeLabel?: string;
|
|
49
|
+
/** Linked from every filed issue. */
|
|
50
|
+
playbookUrl?: string;
|
|
51
|
+
}
|
|
52
|
+
export interface BurnInRunner {
|
|
53
|
+
/** For output. */
|
|
54
|
+
name: string;
|
|
55
|
+
match: Matcher | Matcher[];
|
|
56
|
+
ignore?: Matcher | Matcher[];
|
|
57
|
+
/** A string runs once; an array runs each in sequence (for runners with no repeat flag). */
|
|
58
|
+
command: (files: string[], repeats: number) => string | string[];
|
|
59
|
+
}
|
|
60
|
+
export interface BurnInConfig {
|
|
61
|
+
repeats?: number;
|
|
62
|
+
/** Changed test files beyond this many are listed as skipped, never silently dropped. */
|
|
63
|
+
maxFiles?: number;
|
|
64
|
+
runners: BurnInRunner[];
|
|
65
|
+
}
|
|
66
|
+
export interface ResolvedFlakes {
|
|
67
|
+
threshold: number;
|
|
68
|
+
snoozeLabel: string;
|
|
69
|
+
playbookUrl: string;
|
|
70
|
+
}
|
|
71
|
+
export interface ResolvedBurnIn {
|
|
72
|
+
repeats: number;
|
|
73
|
+
maxFiles: number;
|
|
74
|
+
runners: BurnInRunner[];
|
|
75
|
+
}
|
|
43
76
|
export interface ChreodeConfig {
|
|
44
77
|
baseBranch?: string;
|
|
45
78
|
statusContext?: string;
|
|
46
79
|
rules?: RuleConfig[];
|
|
80
|
+
/** Fired, instead of every trigger, by a package.json / package-lock.json that matched no
|
|
81
|
+
* rule and whose only change is its version. Default [] (a version bump runs nothing). */
|
|
82
|
+
versionBumpTriggers?: string[];
|
|
47
83
|
steps: StepConfig[];
|
|
48
84
|
/** Runs only when the merge of origin/<baseBranch> actually changed the branch. */
|
|
49
85
|
afterMerge?: string;
|
|
50
86
|
stderrAllowlist?: AllowlistEntry[];
|
|
51
87
|
protection?: ProtectionConfig;
|
|
88
|
+
flakes?: FlakesConfig;
|
|
89
|
+
burnIn?: BurnInConfig;
|
|
52
90
|
}
|
|
53
91
|
export interface ResolvedStep extends StepConfig {
|
|
54
92
|
supersededBy: string[];
|
|
@@ -59,13 +97,22 @@ export interface ResolvedConfig {
|
|
|
59
97
|
baseBranch: string;
|
|
60
98
|
statusContext: string;
|
|
61
99
|
rules: RuleConfig[];
|
|
100
|
+
versionBumpTriggers: string[];
|
|
62
101
|
steps: ResolvedStep[];
|
|
63
102
|
afterMerge: string | null;
|
|
64
103
|
stderrAllowlist: AllowlistEntry[];
|
|
65
104
|
protection: ProtectionConfig;
|
|
105
|
+
flakes: ResolvedFlakes | null;
|
|
106
|
+
burnIn: ResolvedBurnIn | null;
|
|
66
107
|
}
|
|
67
108
|
export declare const DEFAULT_BASE_BRANCH = "main";
|
|
68
109
|
export declare const DEFAULT_STATUS_CONTEXT = "chreode/signoff";
|
|
69
110
|
export declare const DEFAULT_TIMEOUT_MS: number;
|
|
111
|
+
export declare const DEFAULT_FLAKE_THRESHOLD = 3;
|
|
112
|
+
export declare const DEFAULT_SNOOZE_LABEL = "flake-snoozed";
|
|
113
|
+
export declare const DEFAULT_PLAYBOOK_URL = "https://github.com/mindlace/chreode/blob/main/docs/flakes.md";
|
|
114
|
+
export declare const DEFAULT_BURN_IN_REPEATS = 5;
|
|
115
|
+
export declare const DEFAULT_BURN_IN_MAX_FILES = 12;
|
|
116
|
+
export declare function resolveFlakes(config: FlakesConfig): ResolvedFlakes;
|
|
70
117
|
/** Identity helper so `chreode.config.ts` gets types. */
|
|
71
118
|
export declare function defineConfig(config: ChreodeConfig): ChreodeConfig;
|
package/dist/config/types.js
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
export const DEFAULT_BASE_BRANCH = 'main';
|
|
2
2
|
export const DEFAULT_STATUS_CONTEXT = 'chreode/signoff';
|
|
3
3
|
export const DEFAULT_TIMEOUT_MS = 10 * 60_000;
|
|
4
|
+
export const DEFAULT_FLAKE_THRESHOLD = 3;
|
|
5
|
+
export const DEFAULT_SNOOZE_LABEL = 'flake-snoozed';
|
|
6
|
+
export const DEFAULT_PLAYBOOK_URL = 'https://github.com/mindlace/chreode/blob/main/docs/flakes.md';
|
|
7
|
+
export const DEFAULT_BURN_IN_REPEATS = 5;
|
|
8
|
+
export const DEFAULT_BURN_IN_MAX_FILES = 12;
|
|
9
|
+
export function resolveFlakes(config) {
|
|
10
|
+
return {
|
|
11
|
+
threshold: config.threshold ?? DEFAULT_FLAKE_THRESHOLD,
|
|
12
|
+
snoozeLabel: config.snoozeLabel ?? DEFAULT_SNOOZE_LABEL,
|
|
13
|
+
playbookUrl: config.playbookUrl ?? DEFAULT_PLAYBOOK_URL
|
|
14
|
+
};
|
|
15
|
+
}
|
|
4
16
|
/** Identity helper so `chreode.config.ts` gets types. */
|
|
5
17
|
export function defineConfig(config) {
|
|
6
18
|
return config;
|