jev-test-filter 0.1.2 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +184 -21
- package/dist/cli.d.ts +23 -1
- package/dist/cli.js +92 -15
- package/dist/context.d.ts +18 -0
- package/dist/context.js +156 -0
- package/dist/diff.d.ts +13 -0
- package/dist/diff.js +27 -2
- package/dist/discover.js +13 -2
- package/dist/extract-rspec.d.ts +20 -0
- package/dist/extract-rspec.js +177 -0
- package/dist/filter.d.ts +3 -0
- package/dist/filter.js +17 -0
- package/dist/framework.d.ts +1 -0
- package/dist/framework.js +6 -1
- package/dist/gate.d.ts +14 -3
- package/dist/gate.js +21 -2
- package/dist/languages.js +3 -2
- package/dist/questions.d.ts +10 -15
- package/dist/questions.js +38 -1
- package/dist/run.d.ts +46 -15
- package/dist/run.js +113 -22
- package/dist/types.d.ts +125 -2
- package/package.json +7 -6
package/dist/questions.js
CHANGED
|
@@ -1,3 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One question per test.
|
|
3
|
+
*
|
|
4
|
+
* `score` rather than `choice` because the answer is an ordered conclusion:
|
|
5
|
+
* asking "how much can this change break this test" as an unordered choice
|
|
6
|
+
* discards the ordering, splits the probability mass between adjacent levels,
|
|
7
|
+
* and returns a low confidence that is indistinguishable from real
|
|
8
|
+
* uncertainty.
|
|
9
|
+
*
|
|
10
|
+
* No threshold ever appears in a question. The cutoff is a decision the gate
|
|
11
|
+
* makes from the answer; writing it into the question would mean every
|
|
12
|
+
* recalibration rewrote the question, and no run could be compared with an
|
|
13
|
+
* earlier one.
|
|
14
|
+
*
|
|
15
|
+
* The one thing a question may carry beyond the test's own identity is
|
|
16
|
+
* history from flaker (`--context`): the files this test failed with before,
|
|
17
|
+
* as one plain sentence under `instructions.history`. It is evidence, not a
|
|
18
|
+
* rule -- no count, no rate -- and it is absent, key and all, for a test the
|
|
19
|
+
* context says nothing about, so such a test is asked exactly what it was
|
|
20
|
+
* asked before contexts existed.
|
|
21
|
+
*/
|
|
22
|
+
import { rspecFullName } from "./extract-rspec.js";
|
|
1
23
|
const TASK = "The state is a git diff. Judge only the one test identified below, and only " +
|
|
2
24
|
"for whether this change can alter its outcome -- other problems with the test " +
|
|
3
25
|
"or with the change are not your concern here.";
|
|
@@ -23,10 +45,24 @@ export function displayName(t) {
|
|
|
23
45
|
return t.titlePath.join("::");
|
|
24
46
|
if (t.framework === "go")
|
|
25
47
|
return t.titlePath.join("/");
|
|
48
|
+
if (t.framework === "rspec")
|
|
49
|
+
return rspecFullName(t.titlePath);
|
|
26
50
|
const sep = t.framework === "node" ? " " : " > ";
|
|
27
51
|
return t.titlePath.join(sep);
|
|
28
52
|
}
|
|
29
|
-
|
|
53
|
+
/**
|
|
54
|
+
* The one sentence a hint becomes.
|
|
55
|
+
*
|
|
56
|
+
* A fact about the past and nothing more: nothing a model could read as a
|
|
57
|
+
* bar to clear, which is why flaker's `missed` count stays out of it. Whether
|
|
58
|
+
* THIS change can break the test is still the model's judgment.
|
|
59
|
+
*/
|
|
60
|
+
export function historySentence(files) {
|
|
61
|
+
const named = files.length <= 1 ? (files[0] ?? "") : `${files.slice(0, -1).join(", ")} or ${files[files.length - 1]}`;
|
|
62
|
+
return `This test previously failed when ${named} changed.`;
|
|
63
|
+
}
|
|
64
|
+
/** `failedWith`: the files this test failed with before, from `--context`. */
|
|
65
|
+
export function buildQuestion(t, id, failedWith = []) {
|
|
30
66
|
return {
|
|
31
67
|
type: "score",
|
|
32
68
|
instructions: {
|
|
@@ -36,6 +72,7 @@ export function buildQuestion(t, id) {
|
|
|
36
72
|
...(t.project === undefined ? {} : { playwright_project: t.project }),
|
|
37
73
|
test_name: displayName(t),
|
|
38
74
|
test_lines: t.line === t.endLine ? `${t.line}` : `${t.line}-${t.endLine}`,
|
|
75
|
+
...(failedWith.length === 0 ? {} : { history: historySentence(failedWith) }),
|
|
39
76
|
},
|
|
40
77
|
criteria: SCORE_LEVELS,
|
|
41
78
|
};
|
package/dist/run.d.ts
CHANGED
|
@@ -3,21 +3,21 @@ import type { StatePayload } from "./state.ts";
|
|
|
3
3
|
import type { GateOptions } from "./gate.ts";
|
|
4
4
|
import type { FilterArgs } from "./filter.ts";
|
|
5
5
|
import type { AskClient, Spend } from "./jev.ts";
|
|
6
|
-
import type {
|
|
6
|
+
import type { ContextLookup } from "./context.ts";
|
|
7
|
+
import type { Answer, Framework, JevContext, RunRecord, Selection, TestCase } from "./types.ts";
|
|
8
|
+
export type { RecordGate, RunRecord, RunRecordV1, RunRecordV2 } from "./types.ts";
|
|
7
9
|
/** Where a run's answers are kept, for `--replay`. */
|
|
8
10
|
export declare const RECORD_DIR = ".jev-test-filter";
|
|
9
11
|
export declare const RECORD_FILE = "last.json";
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
fallback: string | null;
|
|
20
|
-
}
|
|
12
|
+
/**
|
|
13
|
+
* One record per commit, under `RECORD_DIR`, named `<head_sha>.json`.
|
|
14
|
+
*
|
|
15
|
+
* `last.json` alone is overwritten by the next run, and the run a CI result
|
|
16
|
+
* has to be compared with is rarely the last one on this machine. Keyed by
|
|
17
|
+
* the head rather than by time because the head is what the other side of
|
|
18
|
+
* that comparison knows.
|
|
19
|
+
*/
|
|
20
|
+
export declare const RECORDS_SUBDIR = "records";
|
|
21
21
|
/**
|
|
22
22
|
* The framework the run is for.
|
|
23
23
|
*
|
|
@@ -37,6 +37,12 @@ export interface ScoreOptions {
|
|
|
37
37
|
* how much one `max_tokens_exceeded` split has to redo.
|
|
38
38
|
*/
|
|
39
39
|
batchSize?: number;
|
|
40
|
+
/**
|
|
41
|
+
* What `--context` knows. A test it skips is not asked about -- its answer
|
|
42
|
+
* stays null and the gate reports it quarantined -- and a test it has
|
|
43
|
+
* history for is asked with that history.
|
|
44
|
+
*/
|
|
45
|
+
context?: ContextLookup | null;
|
|
40
46
|
}
|
|
41
47
|
/**
|
|
42
48
|
* Ask about every test.
|
|
@@ -45,7 +51,7 @@ export interface ScoreOptions {
|
|
|
45
51
|
* null: the gate selects a test with no answer, and a zero here would silently
|
|
46
52
|
* deselect it instead.
|
|
47
53
|
*/
|
|
48
|
-
export declare function score(tests: TestCase[], state: StatePayload, { client, concurrency, batchSize }?: ScoreOptions): Promise<Map<string, Answer | null>>;
|
|
54
|
+
export declare function score(tests: TestCase[], state: StatePayload, { client, concurrency, batchSize, context }?: ScoreOptions): Promise<Map<string, Answer | null>>;
|
|
49
55
|
export interface RunOptions extends GateOptions {
|
|
50
56
|
cwd?: string;
|
|
51
57
|
base?: string | null;
|
|
@@ -61,7 +67,18 @@ export interface RunOptions extends GateOptions {
|
|
|
61
67
|
dryRun?: boolean;
|
|
62
68
|
/** When using Playwright, collect its actual test list with this runner command. */
|
|
63
69
|
playwrightCommand?: string[];
|
|
70
|
+
/**
|
|
71
|
+
* flaker's `jev-context`, already validated. Its gate values are defaults:
|
|
72
|
+
* a value set in these options wins over each.
|
|
73
|
+
*/
|
|
74
|
+
context?: JevContext | null;
|
|
64
75
|
}
|
|
76
|
+
/**
|
|
77
|
+
* The gate a run decides under: the options' own values over the context's,
|
|
78
|
+
* over the defaults. Only values actually present count, so an option passed
|
|
79
|
+
* as `undefined` does not erase the context's value.
|
|
80
|
+
*/
|
|
81
|
+
export declare function effectiveGate(opts: GateOptions, context?: JevContext | null): GateOptions;
|
|
65
82
|
export interface RunResult {
|
|
66
83
|
selection: Selection;
|
|
67
84
|
framework: Framework;
|
|
@@ -70,7 +87,21 @@ export interface RunResult {
|
|
|
70
87
|
spent: Spend | null;
|
|
71
88
|
}
|
|
72
89
|
export declare function run(opts?: RunOptions): Promise<RunResult>;
|
|
73
|
-
/**
|
|
90
|
+
/**
|
|
91
|
+
* Re-gate a recorded run, offline.
|
|
92
|
+
*
|
|
93
|
+
* Under the record's own gate by default, so a bare replay reproduces the
|
|
94
|
+
* selection the run made; a value in `opts` replaces the recorded one, which
|
|
95
|
+
* is what trying another cutoff is. A v1 record kept no gate, and replays
|
|
96
|
+
* under the defaults as it always did.
|
|
97
|
+
*/
|
|
74
98
|
export declare function replay(record: RunRecord, opts?: GateOptions): Selection;
|
|
75
|
-
|
|
99
|
+
/**
|
|
100
|
+
* Write the record to `last.json`, and to `records/<head_sha>.json` when the
|
|
101
|
+
* head is known. Returns every path written, `last.json` first.
|
|
102
|
+
*
|
|
103
|
+
* Whether a record should be written at all -- a fallback should not -- is
|
|
104
|
+
* the caller's decision, see `shouldSaveRecord`.
|
|
105
|
+
*/
|
|
106
|
+
export declare function saveRecord(cwd: string, record: RunRecord): Promise<string[]>;
|
|
76
107
|
export declare function loadRecord(path: string): Promise<RunRecord>;
|
package/dist/run.js
CHANGED
|
@@ -9,18 +9,28 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { readFile, mkdir, writeFile } from "node:fs/promises";
|
|
11
11
|
import { join } from "node:path";
|
|
12
|
-
import { loadDiff, touchesChange } from "./diff.js";
|
|
12
|
+
import { loadDiff, resolveSha, touchesChange } from "./diff.js";
|
|
13
13
|
import { discoverTests } from "./discover.js";
|
|
14
14
|
import { buildState } from "./state.js";
|
|
15
15
|
import { buildQuestion, questionId, readAnswer } from "./questions.js";
|
|
16
|
-
import { gate } from "./gate.js";
|
|
16
|
+
import { gate, gateOptions, resolveGate } from "./gate.js";
|
|
17
17
|
import { buildFilter } from "./filter.js";
|
|
18
18
|
import { listPlaywrightTests } from "./playwright.js";
|
|
19
19
|
import { Jev, mapLimit, DEFAULT_CONCURRENCY } from "./jev.js";
|
|
20
|
+
import { lookup } from "./context.js";
|
|
20
21
|
import { testId } from "./types.js";
|
|
21
22
|
/** Where a run's answers are kept, for `--replay`. */
|
|
22
23
|
export const RECORD_DIR = ".jev-test-filter";
|
|
23
24
|
export const RECORD_FILE = "last.json";
|
|
25
|
+
/**
|
|
26
|
+
* One record per commit, under `RECORD_DIR`, named `<head_sha>.json`.
|
|
27
|
+
*
|
|
28
|
+
* `last.json` alone is overwritten by the next run, and the run a CI result
|
|
29
|
+
* has to be compared with is rarely the last one on this machine. Keyed by
|
|
30
|
+
* the head rather than by time because the head is what the other side of
|
|
31
|
+
* that comparison knows.
|
|
32
|
+
*/
|
|
33
|
+
export const RECORDS_SUBDIR = "records";
|
|
24
34
|
/**
|
|
25
35
|
* The framework the run is for.
|
|
26
36
|
*
|
|
@@ -51,18 +61,22 @@ export function collect(tests, ranges) {
|
|
|
51
61
|
* null: the gate selects a test with no answer, and a zero here would silently
|
|
52
62
|
* deselect it instead.
|
|
53
63
|
*/
|
|
54
|
-
export async function score(tests, state, { client = null, concurrency = DEFAULT_CONCURRENCY, batchSize = 400 } = {}) {
|
|
64
|
+
export async function score(tests, state, { client = null, concurrency = DEFAULT_CONCURRENCY, batchSize = 400, context = null } = {}) {
|
|
55
65
|
const jev = client ?? new Jev();
|
|
56
66
|
const out = new Map();
|
|
57
67
|
tests.forEach((_, i) => out.set(questionId(i), null));
|
|
68
|
+
// Question ids stay the test's index in `tests` even with skipped tests
|
|
69
|
+
// left out, so a record's answers line up with its tests either way.
|
|
70
|
+
const asked = [];
|
|
71
|
+
tests.forEach((t, i) => {
|
|
72
|
+
if (context?.skipped(t))
|
|
73
|
+
return;
|
|
74
|
+
const id = questionId(i);
|
|
75
|
+
asked.push([id, buildQuestion(t, id, context?.failedWith(t) ?? [])]);
|
|
76
|
+
});
|
|
58
77
|
const batches = [];
|
|
59
|
-
for (let i = 0; i <
|
|
60
|
-
|
|
61
|
-
for (let j = i; j < Math.min(i + batchSize, tests.length); j += 1) {
|
|
62
|
-
const id = questionId(j);
|
|
63
|
-
batch[id] = buildQuestion(tests[j], id);
|
|
64
|
-
}
|
|
65
|
-
batches.push(batch);
|
|
78
|
+
for (let i = 0; i < asked.length; i += batchSize) {
|
|
79
|
+
batches.push(Object.fromEntries(asked.slice(i, i + batchSize)));
|
|
66
80
|
}
|
|
67
81
|
const responses = await mapLimit(batches, concurrency, (batch) => jev.askSplitting(state, batch));
|
|
68
82
|
for (const res of responses) {
|
|
@@ -73,6 +87,38 @@ export async function score(tests, state, { client = null, concurrency = DEFAULT
|
|
|
73
87
|
}
|
|
74
88
|
return out;
|
|
75
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* The gate a run decides under: the options' own values over the context's,
|
|
92
|
+
* over the defaults. Only values actually present count, so an option passed
|
|
93
|
+
* as `undefined` does not erase the context's value.
|
|
94
|
+
*/
|
|
95
|
+
export function effectiveGate(opts, context = null) {
|
|
96
|
+
const out = {};
|
|
97
|
+
const g = context?.gate;
|
|
98
|
+
if (g?.cutoff !== undefined)
|
|
99
|
+
out.cutoff = g.cutoff;
|
|
100
|
+
if (g?.unsure_below !== undefined)
|
|
101
|
+
out.unsureBelow = g.unsure_below;
|
|
102
|
+
if (g?.unsure_margin !== undefined)
|
|
103
|
+
out.unsureMargin = g.unsure_margin;
|
|
104
|
+
if (opts.cutoff !== undefined)
|
|
105
|
+
out.cutoff = opts.cutoff;
|
|
106
|
+
if (opts.unsureBelow !== undefined)
|
|
107
|
+
out.unsureBelow = opts.unsureBelow;
|
|
108
|
+
if (opts.unsureMargin !== undefined)
|
|
109
|
+
out.unsureMargin = opts.unsureMargin;
|
|
110
|
+
return out;
|
|
111
|
+
}
|
|
112
|
+
/** What labels a record: which change was judged, and under which gate. */
|
|
113
|
+
async function stamp(cwd, opts) {
|
|
114
|
+
const base = opts.base ?? null;
|
|
115
|
+
return {
|
|
116
|
+
head_sha: await resolveSha("HEAD", cwd),
|
|
117
|
+
base_sha: base === null ? null : await resolveSha(base, cwd),
|
|
118
|
+
context_digest: opts.context?.digest ?? null,
|
|
119
|
+
gate: resolveGate(effectiveGate(opts, opts.context ?? null)),
|
|
120
|
+
};
|
|
121
|
+
}
|
|
76
122
|
/**
|
|
77
123
|
* A selection is an optimization and never a correctness gate, so every
|
|
78
124
|
* failure below takes the same exit: run everything, and say why on the
|
|
@@ -106,12 +152,14 @@ export async function run(opts = {}) {
|
|
|
106
152
|
}
|
|
107
153
|
if (all.length === 0) {
|
|
108
154
|
const record = {
|
|
109
|
-
version:
|
|
155
|
+
version: 2,
|
|
110
156
|
createdAt: new Date().toISOString(),
|
|
111
157
|
base: opts.base ?? null,
|
|
158
|
+
...(await stamp(cwd, opts)),
|
|
112
159
|
framework: opts.format ?? "unknown",
|
|
113
160
|
tests: [],
|
|
114
161
|
touched: [],
|
|
162
|
+
quarantined: [],
|
|
115
163
|
answers: {},
|
|
116
164
|
fallback: discoveryFailure ?? "no tests were extracted",
|
|
117
165
|
};
|
|
@@ -120,6 +168,9 @@ export async function run(opts = {}) {
|
|
|
120
168
|
}
|
|
121
169
|
const framework = opts.format ?? pickFramework(all);
|
|
122
170
|
const touched = collect(all, diff.ranges);
|
|
171
|
+
const context = opts.context ? lookup(opts.context) : null;
|
|
172
|
+
const quarantined = new Set(context ? all.filter((t) => context.skipped(t)).map(testId) : []);
|
|
173
|
+
const gateOpts = effectiveGate(opts, opts.context ?? null);
|
|
123
174
|
const state = buildState(diff.text, diff.stat);
|
|
124
175
|
let answers = new Map();
|
|
125
176
|
let spent = null;
|
|
@@ -135,6 +186,7 @@ export async function run(opts = {}) {
|
|
|
135
186
|
client,
|
|
136
187
|
...(opts.concurrency === undefined ? {} : { concurrency: opts.concurrency }),
|
|
137
188
|
...(opts.batchSize === undefined ? {} : { batchSize: opts.batchSize }),
|
|
189
|
+
context,
|
|
138
190
|
});
|
|
139
191
|
spent = client.spent;
|
|
140
192
|
}
|
|
@@ -143,20 +195,24 @@ export async function run(opts = {}) {
|
|
|
143
195
|
}
|
|
144
196
|
// Gating is pure and offline, so it stays outside the try: a bug in the
|
|
145
197
|
// gate must not be reported to the user as a network failure.
|
|
146
|
-
selection = failure === null ? gate(all, answers, touched,
|
|
198
|
+
selection = failure === null ? gate(all, answers, touched, gateOpts, quarantined) : everything(all, failure);
|
|
147
199
|
}
|
|
148
200
|
// A truncated diff that still deselects most of the suite is a selection
|
|
149
201
|
// made from a state that was missing the change it should have judged.
|
|
150
|
-
|
|
202
|
+
// Quarantined tests were never candidates, so they count on neither side.
|
|
203
|
+
const candidates = all.length - quarantined.size;
|
|
204
|
+
if (selection.fallback === null && state.truncated && candidates > 0 && selection.selected.length / candidates < 0.5) {
|
|
151
205
|
selection = everything(all, "the diff did not fit the state budget and the selection was small");
|
|
152
206
|
}
|
|
153
207
|
const record = {
|
|
154
|
-
version:
|
|
208
|
+
version: 2,
|
|
155
209
|
createdAt: new Date().toISOString(),
|
|
156
210
|
base: opts.base ?? null,
|
|
211
|
+
...(await stamp(cwd, opts)),
|
|
157
212
|
framework,
|
|
158
213
|
tests: all,
|
|
159
214
|
touched: [...touched],
|
|
215
|
+
quarantined: [...quarantined],
|
|
160
216
|
answers: Object.fromEntries(answers),
|
|
161
217
|
fallback: selection.fallback,
|
|
162
218
|
};
|
|
@@ -165,23 +221,58 @@ export async function run(opts = {}) {
|
|
|
165
221
|
: { mode: "all", argv: [] };
|
|
166
222
|
return { selection, framework, filter, record, spent };
|
|
167
223
|
}
|
|
168
|
-
/**
|
|
224
|
+
/**
|
|
225
|
+
* Re-gate a recorded run, offline.
|
|
226
|
+
*
|
|
227
|
+
* Under the record's own gate by default, so a bare replay reproduces the
|
|
228
|
+
* selection the run made; a value in `opts` replaces the recorded one, which
|
|
229
|
+
* is what trying another cutoff is. A v1 record kept no gate, and replays
|
|
230
|
+
* under the defaults as it always did.
|
|
231
|
+
*/
|
|
169
232
|
export function replay(record, opts = {}) {
|
|
170
233
|
if (record.fallback !== null)
|
|
171
234
|
return everything(record.tests, record.fallback);
|
|
172
235
|
const answers = new Map(Object.entries(record.answers));
|
|
173
|
-
return gate(record.tests, answers, new Set(record.touched), opts);
|
|
236
|
+
return gate(record.tests, answers, new Set(record.touched), { ...gateOptions(record.gate), ...effectiveGate(opts) }, new Set(record.quarantined));
|
|
174
237
|
}
|
|
238
|
+
/**
|
|
239
|
+
* Write the record to `last.json`, and to `records/<head_sha>.json` when the
|
|
240
|
+
* head is known. Returns every path written, `last.json` first.
|
|
241
|
+
*
|
|
242
|
+
* Whether a record should be written at all -- a fallback should not -- is
|
|
243
|
+
* the caller's decision, see `shouldSaveRecord`.
|
|
244
|
+
*/
|
|
175
245
|
export async function saveRecord(cwd, record) {
|
|
176
246
|
const dir = join(cwd, RECORD_DIR);
|
|
177
247
|
await mkdir(dir, { recursive: true });
|
|
178
|
-
const
|
|
179
|
-
|
|
180
|
-
|
|
248
|
+
const text = `${JSON.stringify(record, null, 2)}\n`;
|
|
249
|
+
const paths = [join(dir, RECORD_FILE)];
|
|
250
|
+
if (record.head_sha !== null) {
|
|
251
|
+
await mkdir(join(dir, RECORDS_SUBDIR), { recursive: true });
|
|
252
|
+
paths.push(join(dir, RECORDS_SUBDIR, `${record.head_sha}.json`));
|
|
253
|
+
}
|
|
254
|
+
for (const path of paths)
|
|
255
|
+
await writeFile(path, text, "utf8");
|
|
256
|
+
return paths;
|
|
181
257
|
}
|
|
182
258
|
export async function loadRecord(path) {
|
|
183
259
|
const parsed = JSON.parse(await readFile(path, "utf8"));
|
|
184
|
-
if (parsed.version
|
|
185
|
-
|
|
186
|
-
|
|
260
|
+
if (parsed.version === 2) {
|
|
261
|
+
// Written by this version, or by hand; a missing label reads as unknown.
|
|
262
|
+
const v2 = parsed;
|
|
263
|
+
return {
|
|
264
|
+
...v2,
|
|
265
|
+
version: 2,
|
|
266
|
+
head_sha: v2.head_sha ?? null,
|
|
267
|
+
base_sha: v2.base_sha ?? null,
|
|
268
|
+
context_digest: v2.context_digest ?? null,
|
|
269
|
+
gate: v2.gate ?? resolveGate(),
|
|
270
|
+
quarantined: v2.quarantined ?? [],
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
if (parsed.version === 1) {
|
|
274
|
+
const v1 = parsed;
|
|
275
|
+
return { ...v1, head_sha: null, base_sha: null, context_digest: null, gate: null, quarantined: [] };
|
|
276
|
+
}
|
|
277
|
+
throw new Error(`unsupported record version ${String(parsed.version)}`);
|
|
187
278
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* `jest` is separate from `vitest` only so a report can name it; the two share
|
|
13
13
|
* a filter shape exactly.
|
|
14
14
|
*/
|
|
15
|
-
export type Framework = "vitest" | "jest" | "node" | "bun" | "playwright" | "rust" | "go" | "unknown";
|
|
15
|
+
export type Framework = "vitest" | "jest" | "node" | "bun" | "playwright" | "rust" | "go" | "rspec" | "unknown";
|
|
16
16
|
/** One test, as the source declares it. */
|
|
17
17
|
export interface TestCase {
|
|
18
18
|
/**
|
|
@@ -34,6 +34,14 @@ export interface TestCase {
|
|
|
34
34
|
/** Playwright project and path relative to its configured rootDir, when listed by the runner. */
|
|
35
35
|
project?: string;
|
|
36
36
|
runnerFile?: string;
|
|
37
|
+
/**
|
|
38
|
+
* The line a `file:line` filter names this test by, when it is not `line`.
|
|
39
|
+
* RSpec resolves a requested line to the nearest declaration at or above
|
|
40
|
+
* it and records a multi-line call at a line of its own choosing, so an
|
|
41
|
+
* example is named by its last line and shared examples copied into a
|
|
42
|
+
* group by the line that group's block opens on.
|
|
43
|
+
*/
|
|
44
|
+
runnerLine?: number;
|
|
37
45
|
/**
|
|
38
46
|
* The title could not be read statically -- a template with an
|
|
39
47
|
* interpolation, or a `.each` row. Such a test can never be named in a
|
|
@@ -65,7 +73,13 @@ export type Reason =
|
|
|
65
73
|
/** No usable answer came back. Selected to be safe. */
|
|
66
74
|
| "missing"
|
|
67
75
|
/** The score was under the cutoff. Not selected. */
|
|
68
|
-
| "below"
|
|
76
|
+
| "below"
|
|
77
|
+
/**
|
|
78
|
+
* The context from flaker put the test in `skip`. Not asked about and not
|
|
79
|
+
* selected, whatever the diff did to it: a quarantined test fails for
|
|
80
|
+
* reasons of its own, and running it would only say so again.
|
|
81
|
+
*/
|
|
82
|
+
| "quarantined";
|
|
69
83
|
export interface Verdict {
|
|
70
84
|
/** The question this test was asked under, e.g. `q0007`. */
|
|
71
85
|
id: string;
|
|
@@ -94,3 +108,112 @@ export interface Selection {
|
|
|
94
108
|
* confused with one another.
|
|
95
109
|
*/
|
|
96
110
|
export declare function testId(t: TestCase): string;
|
|
111
|
+
/**
|
|
112
|
+
* The gate values a run was decided under, spelled the way `jev-context`
|
|
113
|
+
* spells them so a reader that joins the two never has to translate.
|
|
114
|
+
*/
|
|
115
|
+
export interface RecordGate {
|
|
116
|
+
cutoff: number;
|
|
117
|
+
unsure_below: number;
|
|
118
|
+
unsure_margin: number;
|
|
119
|
+
}
|
|
120
|
+
/** A record as 0.1 wrote it. Still read; never written. */
|
|
121
|
+
export interface RunRecordV1 {
|
|
122
|
+
version: 1;
|
|
123
|
+
createdAt: string;
|
|
124
|
+
base: string | null;
|
|
125
|
+
framework: Framework;
|
|
126
|
+
tests: TestCase[];
|
|
127
|
+
/** `testId` of every test the diff touched. */
|
|
128
|
+
touched: string[];
|
|
129
|
+
/** Keyed by question id, which is the test's index in `tests`. */
|
|
130
|
+
answers: Record<string, Answer | null>;
|
|
131
|
+
fallback: string | null;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* A record as this version writes it: everything a run learned, and enough
|
|
135
|
+
* about where it stood to join it against something else later.
|
|
136
|
+
*
|
|
137
|
+
* The new fields are what make a record comparable. `head_sha` and
|
|
138
|
+
* `base_sha` say which change was judged, which is how a CI run of the same
|
|
139
|
+
* commit finds it; `context_digest` says which hints the questions carried,
|
|
140
|
+
* because a hint changes the question and two runs under different hints are
|
|
141
|
+
* not the same measurement; `gate` says which values turned the answers into
|
|
142
|
+
* a selection, because "this test was not selected" means nothing without
|
|
143
|
+
* the cutoff it was under.
|
|
144
|
+
*
|
|
145
|
+
* snake_case for the new fields only: they are the ones another program is
|
|
146
|
+
* meant to read, and they match the context that fed them. The old fields
|
|
147
|
+
* keep their spelling so a v1 reader's code keeps working.
|
|
148
|
+
*/
|
|
149
|
+
export interface RunRecordV2 extends Omit<RunRecordV1, "version"> {
|
|
150
|
+
version: 2;
|
|
151
|
+
/** `git rev-parse HEAD` at the time of the run; null when there was none. */
|
|
152
|
+
head_sha: string | null;
|
|
153
|
+
/** The sha `--base` resolved to; null without `--base`. */
|
|
154
|
+
base_sha: string | null;
|
|
155
|
+
/** The `digest` of the `--context` the questions were built with; null without one. */
|
|
156
|
+
context_digest: string | null;
|
|
157
|
+
gate: RecordGate;
|
|
158
|
+
/**
|
|
159
|
+
* `testId` of every test the context quarantined. Kept, rather than
|
|
160
|
+
* dropping those tests from `tests`, because a test missing from the run
|
|
161
|
+
* would be run by a filter that selects "everything"; kept by `testId`
|
|
162
|
+
* like `touched`, because the record is a snapshot of one commit.
|
|
163
|
+
*/
|
|
164
|
+
quarantined: string[];
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Any record `loadRecord` accepts, in one shape: a v1 record reads with every
|
|
168
|
+
* field it did not have as null (and nothing quarantined), so the code that
|
|
169
|
+
* replays one never has to branch on the version.
|
|
170
|
+
*/
|
|
171
|
+
export type RunRecord = RunRecordV2 | (Omit<RunRecordV2, "version" | "gate"> & {
|
|
172
|
+
version: 1;
|
|
173
|
+
gate: null;
|
|
174
|
+
});
|
|
175
|
+
/**
|
|
176
|
+
* What flaker knows about the tests, as its `jev-context` projection emits
|
|
177
|
+
* it (`flaker export --projection jev-context`). Version 1.
|
|
178
|
+
*
|
|
179
|
+
* A test is named by `file` + `title_path` (+ `project` for Playwright) and
|
|
180
|
+
* never by `testId`: the line in a `testId` moves with every edit above the
|
|
181
|
+
* test, and flaker's history is about the test, not about where it sat.
|
|
182
|
+
*/
|
|
183
|
+
export interface JevContext {
|
|
184
|
+
version: 1;
|
|
185
|
+
/**
|
|
186
|
+
* sha256 of `skip` and `tests`, computed by flaker. Kept verbatim on the
|
|
187
|
+
* record: a hint changes a question, so runs are only comparable when their
|
|
188
|
+
* digests are equal.
|
|
189
|
+
*/
|
|
190
|
+
digest: string;
|
|
191
|
+
generated_at?: string;
|
|
192
|
+
/** Defaults for the gate. A flag on the command line wins over each. */
|
|
193
|
+
gate: ContextGate | null;
|
|
194
|
+
skip: ContextSkip[];
|
|
195
|
+
tests: ContextTest[];
|
|
196
|
+
}
|
|
197
|
+
export interface ContextGate {
|
|
198
|
+
cutoff?: number;
|
|
199
|
+
unsure_below?: number;
|
|
200
|
+
unsure_margin?: number;
|
|
201
|
+
/** What the values were calibrated from. Carried, never read. */
|
|
202
|
+
basis?: unknown;
|
|
203
|
+
}
|
|
204
|
+
export interface ContextSkip {
|
|
205
|
+
file: string;
|
|
206
|
+
title_path: string[];
|
|
207
|
+
/** Absent: the test in every project. */
|
|
208
|
+
project?: string;
|
|
209
|
+
reason?: string;
|
|
210
|
+
}
|
|
211
|
+
export interface ContextTest {
|
|
212
|
+
file: string;
|
|
213
|
+
title_path: string[];
|
|
214
|
+
project?: string;
|
|
215
|
+
/** Changed files this test failed with before, most telling first. */
|
|
216
|
+
failed_with: string[];
|
|
217
|
+
/** How often a selector left it out when it then failed. Carried, never asked. */
|
|
218
|
+
missed?: number;
|
|
219
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "jev-test-filter",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.5",
|
|
4
4
|
"description": "Score every test against a git diff with Jev, and emit the filter arguments your runner already understands",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"test",
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
"playwright",
|
|
10
10
|
"test-selection",
|
|
11
11
|
"jev",
|
|
12
|
+
"rspec",
|
|
12
13
|
"ast-grep"
|
|
13
14
|
],
|
|
14
15
|
"license": "MIT",
|
|
@@ -61,13 +62,13 @@
|
|
|
61
62
|
},
|
|
62
63
|
"dependencies": {
|
|
63
64
|
"@ast-grep/lang-go": "^0.0.6",
|
|
65
|
+
"@ast-grep/lang-ruby": "^0.0.7",
|
|
64
66
|
"@ast-grep/lang-rust": "^0.0.7",
|
|
65
|
-
"@ast-grep/napi": "^0.
|
|
67
|
+
"@ast-grep/napi": "^0.51.3"
|
|
66
68
|
},
|
|
67
69
|
"devDependencies": {
|
|
68
|
-
"@
|
|
69
|
-
"
|
|
70
|
-
"
|
|
71
|
-
"typescript": "^5.9.0"
|
|
70
|
+
"@types/node": "^24",
|
|
71
|
+
"secretlint": "^14.0.1",
|
|
72
|
+
"typescript": "^7.0.2"
|
|
72
73
|
}
|
|
73
74
|
}
|