@mindlace/chreode 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -2
- package/dist/burn-in/run.d.ts +18 -0
- package/dist/burn-in/run.js +71 -0
- package/dist/burn-in/select.d.ts +30 -0
- package/dist/burn-in/select.js +78 -0
- package/dist/burn-in/step.d.ts +10 -0
- package/dist/burn-in/step.js +14 -0
- package/dist/cli.d.ts +10 -0
- package/dist/cli.js +19 -0
- package/dist/config/types.d.ts +47 -0
- package/dist/config/types.js +12 -0
- package/dist/config/validate.js +94 -8
- package/dist/flakes/gh.d.ts +5 -0
- package/dist/flakes/gh.js +74 -0
- package/dist/flakes/ledger.d.ts +53 -0
- package/dist/flakes/ledger.js +108 -0
- package/dist/flakes/log.d.ts +39 -0
- package/dist/flakes/log.js +89 -0
- package/dist/flakes/phase.d.ts +34 -0
- package/dist/flakes/phase.js +66 -0
- package/dist/flakes/report.d.ts +14 -0
- package/dist/flakes/report.js +45 -0
- package/dist/git/version-only.d.ts +11 -0
- package/dist/git/version-only.js +56 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.js +1 -0
- package/dist/reporters/options.d.ts +17 -0
- package/dist/reporters/options.js +18 -0
- package/dist/reporters/playwright.d.ts +15 -0
- package/dist/reporters/playwright.js +47 -0
- package/dist/reporters/vitest.d.ts +15 -0
- package/dist/reporters/vitest.js +47 -0
- package/dist/run/stderr.d.ts +1 -0
- package/dist/run/stderr.js +4 -1
- package/dist/signoff/plan.d.ts +17 -3
- package/dist/signoff/plan.js +28 -11
- package/dist/signoff/preconditions.d.ts +3 -1
- package/dist/signoff/preconditions.js +11 -1
- package/dist/signoff/run.d.ts +5 -0
- package/dist/signoff/run.js +45 -8
- package/package.json +23 -1
package/dist/config/validate.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DEFAULT_BASE_BRANCH, DEFAULT_STATUS_CONTEXT, DEFAULT_TIMEOUT_MS } from './types.js';
|
|
1
|
+
import { DEFAULT_BASE_BRANCH, DEFAULT_BURN_IN_MAX_FILES, DEFAULT_BURN_IN_REPEATS, DEFAULT_STATUS_CONTEXT, DEFAULT_TIMEOUT_MS, resolveFlakes } from './types.js';
|
|
2
2
|
export class ConfigError extends Error {
|
|
3
3
|
problems;
|
|
4
4
|
constructor(problems) {
|
|
@@ -11,10 +11,13 @@ const TOP_KEYS = new Set([
|
|
|
11
11
|
'baseBranch',
|
|
12
12
|
'statusContext',
|
|
13
13
|
'rules',
|
|
14
|
+
'versionBumpTriggers',
|
|
14
15
|
'steps',
|
|
15
16
|
'afterMerge',
|
|
16
17
|
'stderrAllowlist',
|
|
17
|
-
'protection'
|
|
18
|
+
'protection',
|
|
19
|
+
'flakes',
|
|
20
|
+
'burnIn'
|
|
18
21
|
]);
|
|
19
22
|
const RULE_KEYS = new Set(['match', 'triggers']);
|
|
20
23
|
const STEP_KEYS = new Set([
|
|
@@ -28,6 +31,10 @@ const STEP_KEYS = new Set([
|
|
|
28
31
|
]);
|
|
29
32
|
const ALLOW_KEYS = new Set(['pattern', 'reason']);
|
|
30
33
|
const PROTECTION_KEYS = new Set(['branch', 'repo']);
|
|
34
|
+
const FLAKES_KEYS = new Set(['threshold', 'snoozeLabel', 'playbookUrl']);
|
|
35
|
+
const BURN_IN_KEYS = new Set(['repeats', 'maxFiles', 'runners']);
|
|
36
|
+
const RUNNER_KEYS = new Set(['name', 'match', 'ignore', 'command']);
|
|
37
|
+
const MATCHER_PROBLEM = 'must be a glob string, a RegExp, or a non-empty array of them';
|
|
31
38
|
/** Cross-realm safe: a config evaluated by jiti may not share this realm's RegExp. */
|
|
32
39
|
export function isRegExp(v) {
|
|
33
40
|
return Object.prototype.toString.call(v) === '[object RegExp]';
|
|
@@ -44,6 +51,12 @@ function isStringArray(v) {
|
|
|
44
51
|
function isMatcher(v) {
|
|
45
52
|
return isNonEmptyString(v) || isRegExp(v);
|
|
46
53
|
}
|
|
54
|
+
function isMatcherList(v) {
|
|
55
|
+
return Array.isArray(v) ? v.length > 0 && v.every(isMatcher) : isMatcher(v);
|
|
56
|
+
}
|
|
57
|
+
function isPositiveInteger(v) {
|
|
58
|
+
return typeof v === 'number' && Number.isInteger(v) && v > 0;
|
|
59
|
+
}
|
|
47
60
|
function unknownKeys(obj, allowed, where, problems) {
|
|
48
61
|
for (const key of Object.keys(obj)) {
|
|
49
62
|
if (!allowed.has(key))
|
|
@@ -64,17 +77,23 @@ function validateRules(raw, problems) {
|
|
|
64
77
|
return;
|
|
65
78
|
}
|
|
66
79
|
unknownKeys(rule, RULE_KEYS, where, problems);
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
if (!ok) {
|
|
70
|
-
problems.push(`${where}.match: must be a glob string, a RegExp, or a non-empty array of them`);
|
|
71
|
-
}
|
|
80
|
+
if (!isMatcherList(rule.match))
|
|
81
|
+
problems.push(`${where}.match: ${MATCHER_PROBLEM}`);
|
|
72
82
|
if (!Array.isArray(rule.triggers) || !rule.triggers.every(isNonEmptyString)) {
|
|
73
83
|
problems.push(`${where}.triggers: must be an array of trigger names ([] means "matched, run nothing")`);
|
|
74
84
|
}
|
|
75
85
|
});
|
|
76
86
|
return raw;
|
|
77
87
|
}
|
|
88
|
+
function validateVersionBumpTriggers(raw, problems) {
|
|
89
|
+
if (raw === undefined)
|
|
90
|
+
return [];
|
|
91
|
+
if (!Array.isArray(raw) || !raw.every(isNonEmptyString)) {
|
|
92
|
+
problems.push('config.versionBumpTriggers: must be an array of trigger names ([] runs nothing)');
|
|
93
|
+
return [];
|
|
94
|
+
}
|
|
95
|
+
return raw;
|
|
96
|
+
}
|
|
78
97
|
function validateSteps(raw, problems) {
|
|
79
98
|
if (!Array.isArray(raw)) {
|
|
80
99
|
problems.push('config.steps: must be an array');
|
|
@@ -173,6 +192,67 @@ function validateProtection(raw, problems) {
|
|
|
173
192
|
}
|
|
174
193
|
return raw;
|
|
175
194
|
}
|
|
195
|
+
function validateFlakes(raw, problems) {
|
|
196
|
+
if (raw === undefined)
|
|
197
|
+
return null;
|
|
198
|
+
if (!isRecord(raw)) {
|
|
199
|
+
problems.push('config.flakes: must be an object ({} turns flake tracking on with the defaults)');
|
|
200
|
+
return null;
|
|
201
|
+
}
|
|
202
|
+
unknownKeys(raw, FLAKES_KEYS, 'flakes', problems);
|
|
203
|
+
if (raw.threshold !== undefined && !isPositiveInteger(raw.threshold)) {
|
|
204
|
+
problems.push('flakes.threshold: must be a positive integer');
|
|
205
|
+
}
|
|
206
|
+
for (const key of ['snoozeLabel', 'playbookUrl']) {
|
|
207
|
+
if (raw[key] !== undefined && !isNonEmptyString(raw[key])) {
|
|
208
|
+
problems.push(`flakes.${key}: must be a non-empty string`);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
return resolveFlakes(raw);
|
|
212
|
+
}
|
|
213
|
+
function validateBurnIn(raw, problems) {
|
|
214
|
+
if (raw === undefined)
|
|
215
|
+
return null;
|
|
216
|
+
if (!isRecord(raw)) {
|
|
217
|
+
problems.push('config.burnIn: must be an object');
|
|
218
|
+
return null;
|
|
219
|
+
}
|
|
220
|
+
unknownKeys(raw, BURN_IN_KEYS, 'burnIn', problems);
|
|
221
|
+
for (const key of ['repeats', 'maxFiles']) {
|
|
222
|
+
if (raw[key] !== undefined && !isPositiveInteger(raw[key])) {
|
|
223
|
+
problems.push(`burnIn.${key}: must be a positive integer`);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
const runners = raw.runners;
|
|
227
|
+
if (!Array.isArray(runners) || runners.length === 0) {
|
|
228
|
+
problems.push('burnIn.runners: must be a non-empty array');
|
|
229
|
+
}
|
|
230
|
+
else {
|
|
231
|
+
runners.forEach((runner, i) => {
|
|
232
|
+
const where = `burnIn.runners[${i}]`;
|
|
233
|
+
if (!isRecord(runner)) {
|
|
234
|
+
problems.push(`${where}: must be an object`);
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
unknownKeys(runner, RUNNER_KEYS, where, problems);
|
|
238
|
+
if (!isNonEmptyString(runner.name))
|
|
239
|
+
problems.push(`${where}.name: must be a non-empty string`);
|
|
240
|
+
if (!isMatcherList(runner.match))
|
|
241
|
+
problems.push(`${where}.match: ${MATCHER_PROBLEM}`);
|
|
242
|
+
if (runner.ignore !== undefined && !isMatcherList(runner.ignore)) {
|
|
243
|
+
problems.push(`${where}.ignore: ${MATCHER_PROBLEM}`);
|
|
244
|
+
}
|
|
245
|
+
if (typeof runner.command !== 'function') {
|
|
246
|
+
problems.push(`${where}.command: must be a function (files, repeats) => string | string[]`);
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
return {
|
|
251
|
+
repeats: raw.repeats ?? DEFAULT_BURN_IN_REPEATS,
|
|
252
|
+
maxFiles: raw.maxFiles ?? DEFAULT_BURN_IN_MAX_FILES,
|
|
253
|
+
runners: Array.isArray(runners) ? runners : []
|
|
254
|
+
};
|
|
255
|
+
}
|
|
176
256
|
export function validateConfig(raw) {
|
|
177
257
|
if (!isRecord(raw)) {
|
|
178
258
|
throw new ConfigError([
|
|
@@ -187,18 +267,24 @@ export function validateConfig(raw) {
|
|
|
187
267
|
}
|
|
188
268
|
}
|
|
189
269
|
const rules = validateRules(raw.rules, problems);
|
|
270
|
+
const versionBumpTriggers = validateVersionBumpTriggers(raw.versionBumpTriggers, problems);
|
|
190
271
|
const steps = validateSteps(raw.steps, problems);
|
|
191
272
|
const stderrAllowlist = validateAllowlist(raw.stderrAllowlist, problems);
|
|
192
273
|
const protection = validateProtection(raw.protection, problems);
|
|
274
|
+
const flakes = validateFlakes(raw.flakes, problems);
|
|
275
|
+
const burnIn = validateBurnIn(raw.burnIn, problems);
|
|
193
276
|
if (problems.length > 0)
|
|
194
277
|
throw new ConfigError(problems);
|
|
195
278
|
return {
|
|
196
279
|
baseBranch: raw.baseBranch ?? DEFAULT_BASE_BRANCH,
|
|
197
280
|
statusContext: raw.statusContext ?? DEFAULT_STATUS_CONTEXT,
|
|
198
281
|
rules,
|
|
282
|
+
versionBumpTriggers,
|
|
199
283
|
steps,
|
|
200
284
|
afterMerge: raw.afterMerge ?? null,
|
|
201
285
|
stderrAllowlist,
|
|
202
|
-
protection
|
|
286
|
+
protection,
|
|
287
|
+
flakes,
|
|
288
|
+
burnIn
|
|
203
289
|
};
|
|
204
290
|
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { type Exec } from '../run/exec.js';
|
|
2
|
+
import { type FlakeLedgerDeps } from './ledger.js';
|
|
3
|
+
/** The ledger's GitHub side over `gh`. Every call is argv (no shell), so test titles and
|
|
4
|
+
* error text reach GitHub verbatim. */
|
|
5
|
+
export declare function ghLedgerDeps(exec?: Exec): FlakeLedgerDeps;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { defaultExec, lines } from '../run/exec.js';
|
|
2
|
+
import { FLAKE_LABEL, OCCURRENCE_MARKER } from './ledger.js';
|
|
3
|
+
const LABEL_DESCRIPTION = 'A test that failed, then passed on retry (chreode)';
|
|
4
|
+
const LABEL_COLOR = 'D93F0B';
|
|
5
|
+
/** The ledger's GitHub side over `gh`. Every call is argv (no shell), so test titles and
|
|
6
|
+
* error text reach GitHub verbatim. */
|
|
7
|
+
export function ghLedgerDeps(exec = defaultExec) {
|
|
8
|
+
let labelReady = false;
|
|
9
|
+
const ensureLabel = () => {
|
|
10
|
+
if (labelReady)
|
|
11
|
+
return;
|
|
12
|
+
const labels = JSON.parse(exec('gh', ['label', 'list', '--limit', '1000', '--json', 'name']));
|
|
13
|
+
if (!labels.some((l) => l.name === FLAKE_LABEL)) {
|
|
14
|
+
exec('gh', [
|
|
15
|
+
'label',
|
|
16
|
+
'create',
|
|
17
|
+
FLAKE_LABEL,
|
|
18
|
+
'--description',
|
|
19
|
+
LABEL_DESCRIPTION,
|
|
20
|
+
'--color',
|
|
21
|
+
LABEL_COLOR
|
|
22
|
+
]);
|
|
23
|
+
}
|
|
24
|
+
labelReady = true;
|
|
25
|
+
};
|
|
26
|
+
return {
|
|
27
|
+
listOpenFlakeIssues() {
|
|
28
|
+
// A label filter, not a title search: it does not depend on search-index lag.
|
|
29
|
+
const raw = JSON.parse(exec('gh', [
|
|
30
|
+
'issue',
|
|
31
|
+
'list',
|
|
32
|
+
'--label',
|
|
33
|
+
FLAKE_LABEL,
|
|
34
|
+
'--state',
|
|
35
|
+
'open',
|
|
36
|
+
'--limit',
|
|
37
|
+
'1000',
|
|
38
|
+
'--json',
|
|
39
|
+
'number,title,labels,url'
|
|
40
|
+
]));
|
|
41
|
+
return raw.map((i) => ({
|
|
42
|
+
number: i.number,
|
|
43
|
+
title: i.title,
|
|
44
|
+
url: i.url,
|
|
45
|
+
labels: i.labels.map((l) => l.name)
|
|
46
|
+
}));
|
|
47
|
+
},
|
|
48
|
+
countOccurrenceComments(issue) {
|
|
49
|
+
const raw = JSON.parse(exec('gh', ['issue', 'view', String(issue), '--json', 'comments']));
|
|
50
|
+
return raw.comments.filter((c) => c.body.includes(OCCURRENCE_MARKER)).length;
|
|
51
|
+
},
|
|
52
|
+
createIssue(title, body) {
|
|
53
|
+
ensureLabel();
|
|
54
|
+
const out = exec('gh', [
|
|
55
|
+
'issue',
|
|
56
|
+
'create',
|
|
57
|
+
'--title',
|
|
58
|
+
title,
|
|
59
|
+
'--body',
|
|
60
|
+
body,
|
|
61
|
+
'--label',
|
|
62
|
+
FLAKE_LABEL
|
|
63
|
+
]);
|
|
64
|
+
const url = lines(out).at(-1) ?? '';
|
|
65
|
+
const m = /\/issues\/(\d+)$/.exec(url);
|
|
66
|
+
if (!m)
|
|
67
|
+
throw new Error(`gh issue create printed no issue URL: ${out.trim()}`);
|
|
68
|
+
return { number: Number(m[1]), title, url, labels: [FLAKE_LABEL] };
|
|
69
|
+
},
|
|
70
|
+
commentIssue(issue, body) {
|
|
71
|
+
exec('gh', ['issue', 'comment', String(issue), '--body', body]);
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { ResolvedFlakes } from '../config/types.js';
|
|
2
|
+
import { type FlakeEvent } from './log.js';
|
|
3
|
+
export declare const FLAKE_LABEL = "flake";
|
|
4
|
+
export declare const OCCURRENCE_MARKER = "<!-- chreode:flake-occurrence -->";
|
|
5
|
+
export declare const ISSUE_TITLE_MAX = 256;
|
|
6
|
+
export interface RunContext {
|
|
7
|
+
branch: string;
|
|
8
|
+
sha: string;
|
|
9
|
+
}
|
|
10
|
+
export interface FlakeIssue {
|
|
11
|
+
number: number;
|
|
12
|
+
title: string;
|
|
13
|
+
url: string;
|
|
14
|
+
labels: string[];
|
|
15
|
+
}
|
|
16
|
+
/** The GitHub side, injected: see gh.ts for the real one. */
|
|
17
|
+
export interface FlakeLedgerDeps {
|
|
18
|
+
/** Open issues labelled FLAKE_LABEL. */
|
|
19
|
+
listOpenFlakeIssues(): FlakeIssue[];
|
|
20
|
+
/** Comments on the issue that carry OCCURRENCE_MARKER. */
|
|
21
|
+
countOccurrenceComments(issue: number): number;
|
|
22
|
+
/** Creates an issue labelled FLAKE_LABEL. */
|
|
23
|
+
createIssue(title: string, body: string): FlakeIssue;
|
|
24
|
+
commentIssue(issue: number, body: string): void;
|
|
25
|
+
}
|
|
26
|
+
export interface LedgerEntry {
|
|
27
|
+
number: number;
|
|
28
|
+
title: string;
|
|
29
|
+
url: string;
|
|
30
|
+
/** Occurrences so far: 1 (the issue) + marker comments. */
|
|
31
|
+
count: number;
|
|
32
|
+
}
|
|
33
|
+
export interface LedgerResult {
|
|
34
|
+
filed: LedgerEntry[];
|
|
35
|
+
bumped: LedgerEntry[];
|
|
36
|
+
escalations: LedgerEntry[];
|
|
37
|
+
}
|
|
38
|
+
export declare function firstLine(err: unknown): string;
|
|
39
|
+
/** A GitHub call failed partway: `partial` was recorded, `remaining` was not. */
|
|
40
|
+
export declare class LedgerError extends Error {
|
|
41
|
+
readonly partial: LedgerResult;
|
|
42
|
+
readonly remaining: FlakeEvent[];
|
|
43
|
+
constructor(partial: LedgerResult, remaining: FlakeEvent[], cause: unknown);
|
|
44
|
+
}
|
|
45
|
+
export declare function issueTitle(event: FlakeEvent): string;
|
|
46
|
+
export declare function issueBody(event: FlakeEvent, ctx: RunContext, config: ResolvedFlakes): string;
|
|
47
|
+
export declare function occurrenceBody(event: FlakeEvent, ctx: RunContext): string;
|
|
48
|
+
/**
|
|
49
|
+
* One open issue per flaking test (exact title match); each later sighting adds a marker
|
|
50
|
+
* comment. An issue at `threshold` occurrences without the snooze label is an escalation.
|
|
51
|
+
* Any GitHub failure throws LedgerError naming what was and was not recorded.
|
|
52
|
+
*/
|
|
53
|
+
export declare function processFlakeEvents(events: readonly FlakeEvent[], ctx: RunContext, config: ResolvedFlakes, deps: FlakeLedgerDeps): LedgerResult;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { dedupeEvents } from './log.js';
|
|
2
|
+
export const FLAKE_LABEL = 'flake';
|
|
3
|
+
export const OCCURRENCE_MARKER = '<!-- chreode:flake-occurrence -->';
|
|
4
|
+
export const ISSUE_TITLE_MAX = 256;
|
|
5
|
+
export function firstLine(err) {
|
|
6
|
+
return (err instanceof Error ? err.message : String(err)).split('\n')[0] ?? '';
|
|
7
|
+
}
|
|
8
|
+
/** A GitHub call failed partway: `partial` was recorded, `remaining` was not. */
|
|
9
|
+
export class LedgerError extends Error {
|
|
10
|
+
partial;
|
|
11
|
+
remaining;
|
|
12
|
+
constructor(partial, remaining, cause) {
|
|
13
|
+
super(firstLine(cause), { cause });
|
|
14
|
+
this.partial = partial;
|
|
15
|
+
this.remaining = remaining;
|
|
16
|
+
this.name = 'LedgerError';
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
export function issueTitle(event) {
|
|
20
|
+
return `flake(${event.kind}): ${event.file} › ${event.title}`.slice(0, ISSUE_TITLE_MAX).trimEnd();
|
|
21
|
+
}
|
|
22
|
+
function longestRun(text) {
|
|
23
|
+
return Math.max(0, ...(text.match(/`+/g) ?? []).map((run) => run.length));
|
|
24
|
+
}
|
|
25
|
+
/** Inline code span that survives backticks in `text`; keeps @mentions and #refs inert. */
|
|
26
|
+
function codeSpan(text) {
|
|
27
|
+
const fence = '`'.repeat(longestRun(text) + 1);
|
|
28
|
+
const pad = text.startsWith('`') || text.endsWith('`') ? ' ' : '';
|
|
29
|
+
return `${fence}${pad}${text}${pad}${fence}`;
|
|
30
|
+
}
|
|
31
|
+
/** Fenced block whose fence is longer than any backtick run inside (minimum 3). */
|
|
32
|
+
function codeFence(text) {
|
|
33
|
+
const fence = '`'.repeat(Math.max(3, longestRun(text) + 1));
|
|
34
|
+
return [fence, text, fence];
|
|
35
|
+
}
|
|
36
|
+
function details(event, ctx) {
|
|
37
|
+
const where = event.project === '' ? event.runner : `${event.runner}, project ${event.project}`;
|
|
38
|
+
return [
|
|
39
|
+
`- **Test:** ${codeSpan(`${event.file} › ${event.title}`)} (${where})`,
|
|
40
|
+
`- **Observed:** ${event.at} on \`${ctx.branch}\` @ ${event.sha ?? ctx.sha}`,
|
|
41
|
+
...(event.tracePath ? [`- **First-attempt trace:** \`${event.tracePath}\``] : []),
|
|
42
|
+
'',
|
|
43
|
+
'**First-attempt error:**',
|
|
44
|
+
...codeFence(event.firstAttemptError)
|
|
45
|
+
].join('\n');
|
|
46
|
+
}
|
|
47
|
+
export function issueBody(event, ctx, config) {
|
|
48
|
+
return [
|
|
49
|
+
'Filed by chreode flake tracking: this test failed, then passed on retry.',
|
|
50
|
+
'',
|
|
51
|
+
details(event, ctx),
|
|
52
|
+
'',
|
|
53
|
+
`A flake is a bug, not noise. Diagnose it with the playbook: ${config.playbookUrl}`,
|
|
54
|
+
`At ${config.threshold} recorded occurrences it fails signoff until fixed; label this issue`,
|
|
55
|
+
`\`${config.snoozeLabel}\` to defer that deliberately.`
|
|
56
|
+
].join('\n');
|
|
57
|
+
}
|
|
58
|
+
export function occurrenceBody(event, ctx) {
|
|
59
|
+
return [OCCURRENCE_MARKER, '', 'Flaked again.', '', details(event, ctx)].join('\n');
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* One open issue per flaking test (exact title match); each later sighting adds a marker
|
|
63
|
+
* comment. An issue at `threshold` occurrences without the snooze label is an escalation.
|
|
64
|
+
* Any GitHub failure throws LedgerError naming what was and was not recorded.
|
|
65
|
+
*/
|
|
66
|
+
export function processFlakeEvents(events, ctx, config, deps) {
|
|
67
|
+
const result = { filed: [], bumped: [], escalations: [] };
|
|
68
|
+
const unique = dedupeEvents(events);
|
|
69
|
+
if (unique.length === 0)
|
|
70
|
+
return result;
|
|
71
|
+
let open;
|
|
72
|
+
try {
|
|
73
|
+
open = [...deps.listOpenFlakeIssues()];
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
throw new LedgerError(result, unique, err);
|
|
77
|
+
}
|
|
78
|
+
for (const [i, event] of unique.entries()) {
|
|
79
|
+
try {
|
|
80
|
+
const title = issueTitle(event);
|
|
81
|
+
const existing = open.find((candidate) => candidate.title === title);
|
|
82
|
+
let entry;
|
|
83
|
+
let labels;
|
|
84
|
+
if (existing === undefined) {
|
|
85
|
+
const created = deps.createIssue(title, issueBody(event, ctx, config));
|
|
86
|
+
// A later event whose title truncates to the same string bumps this issue.
|
|
87
|
+
open.push(created);
|
|
88
|
+
entry = { number: created.number, title, url: created.url, count: 1 };
|
|
89
|
+
labels = created.labels;
|
|
90
|
+
result.filed.push(entry);
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
const prior = deps.countOccurrenceComments(existing.number);
|
|
94
|
+
deps.commentIssue(existing.number, occurrenceBody(event, ctx));
|
|
95
|
+
entry = { number: existing.number, title, url: existing.url, count: prior + 2 };
|
|
96
|
+
labels = existing.labels;
|
|
97
|
+
result.bumped.push(entry);
|
|
98
|
+
}
|
|
99
|
+
if (entry.count >= config.threshold && !labels.includes(config.snoozeLabel)) {
|
|
100
|
+
result.escalations.push(entry);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
catch (err) {
|
|
104
|
+
throw new LedgerError(result, unique.slice(i), err);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return result;
|
|
108
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/** One retried-then-passed test, as a reporter saw it. */
|
|
2
|
+
export interface FlakeEvent {
|
|
3
|
+
/** Issue-title bucket, e.g. 'e2e', 'unit', 'client'. */
|
|
4
|
+
kind: string;
|
|
5
|
+
runner: 'playwright' | 'vitest';
|
|
6
|
+
/** Runner project name ('' if none). */
|
|
7
|
+
project: string;
|
|
8
|
+
/** Repo-relative, '/'-separated. */
|
|
9
|
+
file: string;
|
|
10
|
+
/** Describe chain joined with ' › '. */
|
|
11
|
+
title: string;
|
|
12
|
+
/** At most ERROR_LINE_CAP lines, ANSI-stripped. */
|
|
13
|
+
firstAttemptError: string;
|
|
14
|
+
/** Playwright trace of the failed attempt. */
|
|
15
|
+
tracePath?: string;
|
|
16
|
+
/** CHREODE_HEAD when run under signoff. */
|
|
17
|
+
sha?: string;
|
|
18
|
+
/** ISO timestamp. */
|
|
19
|
+
at: string;
|
|
20
|
+
}
|
|
21
|
+
/** Repo-relative. Not under test-results/: Playwright wipes its outputDir at start and would
|
|
22
|
+
* delete events an earlier vitest step wrote. */
|
|
23
|
+
export declare const FLAKE_LOG_FILE = ".chreode/flakes.jsonl";
|
|
24
|
+
export declare const ERROR_LINE_CAP = 40;
|
|
25
|
+
export declare function flakeLogPath(env?: NodeJS.ProcessEnv, cwd?: string): string;
|
|
26
|
+
/** jsonl, append-only: sequential writer processes in one run never clobber each other. */
|
|
27
|
+
export declare function appendFlakeEvent(event: FlakeEvent, logPath: string): void;
|
|
28
|
+
export declare function readFlakeEvents(logPath: string): FlakeEvent[];
|
|
29
|
+
export declare function clearFlakeLog(logPath: string): void;
|
|
30
|
+
export declare function writeFlakeEvents(logPath: string, events: readonly FlakeEvent[]): void;
|
|
31
|
+
export declare function flakeKey(event: FlakeEvent): string;
|
|
32
|
+
/** One report per test per run: the same flake seen in two projects counts once. */
|
|
33
|
+
export declare function dedupeEvents(events: readonly FlakeEvent[]): FlakeEvent[];
|
|
34
|
+
export declare function truncateError(message: string | undefined): string;
|
|
35
|
+
/** The nearest ancestor holding `.git`: a directory in a checkout, a file in a worktree. */
|
|
36
|
+
export declare function findRepoRoot(start: string, exists?: (path: string) => boolean): string | null;
|
|
37
|
+
/** Relative to the git root found from the file itself, so a runner started in a subpackage
|
|
38
|
+
* still yields repo paths; '/'-separated. */
|
|
39
|
+
export declare function repoRelative(file: string, cwd?: string, exists?: (path: string) => boolean): string;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join, relative, sep } from 'node:path';
|
|
3
|
+
import { stripAnsi } from '../run/stderr.js';
|
|
4
|
+
/** Repo-relative. Not under test-results/: Playwright wipes its outputDir at start and would
|
|
5
|
+
* delete events an earlier vitest step wrote. */
|
|
6
|
+
export const FLAKE_LOG_FILE = '.chreode/flakes.jsonl';
|
|
7
|
+
export const ERROR_LINE_CAP = 40;
|
|
8
|
+
export function flakeLogPath(env = process.env, cwd = process.cwd()) {
|
|
9
|
+
const fromEnv = env.CHREODE_FLAKE_LOG;
|
|
10
|
+
return fromEnv !== undefined && fromEnv !== '' ? fromEnv : join(cwd, FLAKE_LOG_FILE);
|
|
11
|
+
}
|
|
12
|
+
/** jsonl, append-only: sequential writer processes in one run never clobber each other. */
|
|
13
|
+
export function appendFlakeEvent(event, logPath) {
|
|
14
|
+
mkdirSync(dirname(logPath), { recursive: true });
|
|
15
|
+
appendFileSync(logPath, `${JSON.stringify(event)}\n`);
|
|
16
|
+
}
|
|
17
|
+
const REQUIRED = ['kind', 'runner', 'project', 'file', 'title', 'firstAttemptError', 'at'];
|
|
18
|
+
function isFlakeEvent(v) {
|
|
19
|
+
if (typeof v !== 'object' || v === null)
|
|
20
|
+
return false;
|
|
21
|
+
const record = v;
|
|
22
|
+
return REQUIRED.every((key) => typeof record[key] === 'string');
|
|
23
|
+
}
|
|
24
|
+
export function readFlakeEvents(logPath) {
|
|
25
|
+
if (!existsSync(logPath))
|
|
26
|
+
return [];
|
|
27
|
+
const events = [];
|
|
28
|
+
for (const line of readFileSync(logPath, 'utf8').split('\n')) {
|
|
29
|
+
if (line.trim() === '')
|
|
30
|
+
continue;
|
|
31
|
+
try {
|
|
32
|
+
const parsed = JSON.parse(line);
|
|
33
|
+
if (isFlakeEvent(parsed))
|
|
34
|
+
events.push(parsed);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
// An interrupted write: skip it rather than fail the report.
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return events;
|
|
41
|
+
}
|
|
42
|
+
export function clearFlakeLog(logPath) {
|
|
43
|
+
rmSync(logPath, { force: true });
|
|
44
|
+
}
|
|
45
|
+
export function writeFlakeEvents(logPath, events) {
|
|
46
|
+
if (events.length === 0) {
|
|
47
|
+
clearFlakeLog(logPath);
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
mkdirSync(dirname(logPath), { recursive: true });
|
|
51
|
+
writeFileSync(logPath, events.map((e) => `${JSON.stringify(e)}\n`).join(''));
|
|
52
|
+
}
|
|
53
|
+
export function flakeKey(event) {
|
|
54
|
+
return `${event.kind}/${event.file}/${event.title}`;
|
|
55
|
+
}
|
|
56
|
+
/** One report per test per run: the same flake seen in two projects counts once. */
|
|
57
|
+
export function dedupeEvents(events) {
|
|
58
|
+
const seen = new Set();
|
|
59
|
+
return events.filter((e) => {
|
|
60
|
+
const key = flakeKey(e);
|
|
61
|
+
if (seen.has(key))
|
|
62
|
+
return false;
|
|
63
|
+
seen.add(key);
|
|
64
|
+
return true;
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
export function truncateError(message) {
|
|
68
|
+
if (message === undefined || message === '')
|
|
69
|
+
return '(first attempt error unavailable)';
|
|
70
|
+
return stripAnsi(message).split('\n').slice(0, ERROR_LINE_CAP).join('\n');
|
|
71
|
+
}
|
|
72
|
+
/** The nearest ancestor holding `.git`: a directory in a checkout, a file in a worktree. */
|
|
73
|
+
export function findRepoRoot(start, exists = existsSync) {
|
|
74
|
+
let dir = start;
|
|
75
|
+
for (;;) {
|
|
76
|
+
if (exists(join(dir, '.git')))
|
|
77
|
+
return dir;
|
|
78
|
+
const parent = dirname(dir);
|
|
79
|
+
if (parent === dir)
|
|
80
|
+
return null;
|
|
81
|
+
dir = parent;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
/** Relative to the git root found from the file itself, so a runner started in a subpackage
|
|
85
|
+
* still yields repo paths; '/'-separated. */
|
|
86
|
+
export function repoRelative(file, cwd = process.cwd(), exists = existsSync) {
|
|
87
|
+
const root = findRepoRoot(dirname(file), exists) ?? cwd;
|
|
88
|
+
return relative(root, file).split(sep).join('/');
|
|
89
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { ResolvedFlakes } from '../config/types.js';
|
|
2
|
+
import { type FlakeLedgerDeps, type LedgerEntry, type LedgerResult, type RunContext } from './ledger.js';
|
|
3
|
+
export interface FlakePhaseInput {
|
|
4
|
+
logPath: string;
|
|
5
|
+
ctx: RunContext;
|
|
6
|
+
config: ResolvedFlakes;
|
|
7
|
+
ledger: FlakeLedgerDeps;
|
|
8
|
+
log: (m: string) => void;
|
|
9
|
+
warn: (m: string) => void;
|
|
10
|
+
}
|
|
11
|
+
export type FlakePhaseOutcome = {
|
|
12
|
+
kind: 'none';
|
|
13
|
+
} | {
|
|
14
|
+
kind: 'reported';
|
|
15
|
+
result: LedgerResult;
|
|
16
|
+
} | {
|
|
17
|
+
kind: 'gh-error';
|
|
18
|
+
message: string;
|
|
19
|
+
partial: LedgerResult;
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Report the logged flakes to the ledger. Recorded events leave the log; on a GitHub failure
|
|
23
|
+
* the unrecorded ones stay for the next run (an outage must not block merging, and each event
|
|
24
|
+
* keeps its own sha and time). Escalations are returned, not acted on: the caller decides.
|
|
25
|
+
*/
|
|
26
|
+
export declare function runFlakePhase(input: FlakePhaseInput): FlakePhaseOutcome;
|
|
27
|
+
/** The failure text for escalated flakes; each issue is listed once, at its highest count. */
|
|
28
|
+
export declare function escalationMessage(escalations: readonly LedgerEntry[], config: ResolvedFlakes): string;
|
|
29
|
+
/**
|
|
30
|
+
* The single place a phase outcome becomes a pass/fail decision: the failure message when any
|
|
31
|
+
* flake escalated (including ones recorded before a GitHub error), otherwise undefined.
|
|
32
|
+
* Signoff and `flake-report` both call this.
|
|
33
|
+
*/
|
|
34
|
+
export declare function escalationFailure(outcome: FlakePhaseOutcome, config: ResolvedFlakes): string | undefined;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { LedgerError, processFlakeEvents } from './ledger.js';
|
|
2
|
+
import { clearFlakeLog, readFlakeEvents, writeFlakeEvents } from './log.js';
|
|
3
|
+
function printResult(result, log) {
|
|
4
|
+
for (const f of result.filed)
|
|
5
|
+
log(`[chreode] flake filed: ${f.url} ${f.title}`);
|
|
6
|
+
for (const b of result.bumped) {
|
|
7
|
+
log(`[chreode] flake seen again (${b.count} occurrences): ${b.url} ${b.title}`);
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Report the logged flakes to the ledger. Recorded events leave the log; on a GitHub failure
|
|
12
|
+
* the unrecorded ones stay for the next run (an outage must not block merging, and each event
|
|
13
|
+
* keeps its own sha and time). Escalations are returned, not acted on: the caller decides.
|
|
14
|
+
*/
|
|
15
|
+
export function runFlakePhase(input) {
|
|
16
|
+
const events = readFlakeEvents(input.logPath);
|
|
17
|
+
if (events.length === 0)
|
|
18
|
+
return { kind: 'none' };
|
|
19
|
+
input.log(`\n[chreode] flakes: ${events.length} retried-then-passed result(s) recorded`);
|
|
20
|
+
let result;
|
|
21
|
+
try {
|
|
22
|
+
result = processFlakeEvents(events, input.ctx, input.config, input.ledger);
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
if (!(err instanceof LedgerError))
|
|
26
|
+
throw err;
|
|
27
|
+
printResult(err.partial, input.log);
|
|
28
|
+
writeFlakeEvents(input.logPath, err.remaining);
|
|
29
|
+
input.warn(`[chreode] flakes: a GitHub call failed (${err.message}); kept ` +
|
|
30
|
+
`${err.remaining.length} event(s) in ${input.logPath} for the next run`);
|
|
31
|
+
return { kind: 'gh-error', message: err.message, partial: err.partial };
|
|
32
|
+
}
|
|
33
|
+
clearFlakeLog(input.logPath);
|
|
34
|
+
printResult(result, input.log);
|
|
35
|
+
return { kind: 'reported', result };
|
|
36
|
+
}
|
|
37
|
+
function dedupeEscalations(escalations) {
|
|
38
|
+
const byIssue = new Map();
|
|
39
|
+
for (const e of escalations) {
|
|
40
|
+
const seen = byIssue.get(e.number);
|
|
41
|
+
if (!seen || e.count > seen.count)
|
|
42
|
+
byIssue.set(e.number, e);
|
|
43
|
+
}
|
|
44
|
+
return [...byIssue.values()];
|
|
45
|
+
}
|
|
46
|
+
/** The failure text for escalated flakes; each issue is listed once, at its highest count. */
|
|
47
|
+
export function escalationMessage(escalations, config) {
|
|
48
|
+
const unique = dedupeEscalations(escalations);
|
|
49
|
+
return [
|
|
50
|
+
`\n[chreode] FAILED: ${unique.length} flake(s) reached ${config.threshold} occurrences:`,
|
|
51
|
+
...unique.map((e) => ` ${e.url} (${e.count} occurrences) ${e.title}`),
|
|
52
|
+
`[chreode] Fix each one (playbook: ${config.playbookUrl}), or label its issue`,
|
|
53
|
+
`[chreode] ${config.snoozeLabel} to defer it deliberately.`
|
|
54
|
+
].join('\n');
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* The single place a phase outcome becomes a pass/fail decision: the failure message when any
|
|
58
|
+
* flake escalated (including ones recorded before a GitHub error), otherwise undefined.
|
|
59
|
+
* Signoff and `flake-report` both call this.
|
|
60
|
+
*/
|
|
61
|
+
export function escalationFailure(outcome, config) {
|
|
62
|
+
if (outcome.kind === 'none')
|
|
63
|
+
return undefined;
|
|
64
|
+
const result = outcome.kind === 'reported' ? outcome.result : outcome.partial;
|
|
65
|
+
return result.escalations.length > 0 ? escalationMessage(result.escalations, config) : undefined;
|
|
66
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type ResolvedConfig } from '../config/types.js';
|
|
2
|
+
import { type Exec } from '../run/exec.js';
|
|
3
|
+
import type { FlakeLedgerDeps } from './ledger.js';
|
|
4
|
+
export interface FlakeReportDeps {
|
|
5
|
+
exec: Exec;
|
|
6
|
+
ledger: FlakeLedgerDeps;
|
|
7
|
+
logPath: string;
|
|
8
|
+
log: (m: string) => void;
|
|
9
|
+
warn: (m: string) => void;
|
|
10
|
+
error: (m: string) => void;
|
|
11
|
+
}
|
|
12
|
+
/** `chreode flake-report`: the signoff flake phase on its own, for hosted CI or by hand.
|
|
13
|
+
* Without a `flakes` block it uses the defaults: running it is the opt-in. */
|
|
14
|
+
export declare function runFlakeReport(config: ResolvedConfig, overrides?: Partial<FlakeReportDeps>): number;
|