jev-test-filter 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -12
- package/dist/cli.d.ts +12 -0
- package/dist/cli.js +83 -6
- package/dist/context.d.ts +18 -0
- package/dist/context.js +156 -0
- package/dist/diff.d.ts +8 -0
- package/dist/diff.js +16 -0
- package/dist/gate.d.ts +14 -3
- package/dist/gate.js +21 -2
- package/dist/questions.d.ts +17 -1
- package/dist/questions.js +14 -1
- package/dist/run.d.ts +46 -15
- package/dist/run.js +113 -22
- package/dist/types.d.ts +116 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -215,7 +215,7 @@ like, because both mean "no arguments". The exit code tells them apart:
|
|
|
215
215
|
| --- | --- |
|
|
216
216
|
| 0 | Arguments were printed, or everything is to be run and there are none. Run the command. |
|
|
217
217
|
| 3 | Nothing was selected. Do not run the command; there is nothing to run. |
|
|
218
|
-
| 2 | Bad arguments, e.g. `unknown --format mocha
|
|
218
|
+
| 2 | Bad arguments, e.g. `unknown --format mocha`, a gate value that is not a number, or a `--context` file that is missing, malformed, or not version 1. |
|
|
219
219
|
| 1 | The tool failed, e.g. the tests span more than one framework and no `--format` was given. |
|
|
220
220
|
|
|
221
221
|
So the stdout form is used like this:
|
|
@@ -429,8 +429,8 @@ Two rules sit on top of that, and both exist to fail towards running a test
|
|
|
429
429
|
rather than skipping it:
|
|
430
430
|
|
|
431
431
|
- **Confidence routes, it does not gate.** A test that scores under the cutoff,
|
|
432
|
-
but within 1.0 of it, and that the model answered with a
|
|
433
|
-
0.5, is selected anyway. Running a test that did not need to run costs
|
|
432
|
+
but within 1.0 of it (`--unsure-margin`), and that the model answered with a
|
|
433
|
+
confidence below 0.5 (`--unsure-below`), is selected anyway. Running a test that did not need to run costs
|
|
434
434
|
seconds; skipping one that did costs a release.
|
|
435
435
|
- **No answer is not a passing grade.** A missing or malformed answer selects
|
|
436
436
|
the test.
|
|
@@ -442,7 +442,9 @@ across invocations, take it once and re-derive it with `--replay`.
|
|
|
442
442
|
|
|
443
443
|
The `reason` field on each test in `--json` says which rule decided it:
|
|
444
444
|
`touched` (the test's own body is inside the diff — never asked about),
|
|
445
|
-
`dynamic` (the title is not a literal), `scored`, `unsure`, `missing`, `below
|
|
445
|
+
`dynamic` (the title is not a literal), `scored`, `unsure`, `missing`, `below`,
|
|
446
|
+
and `quarantined` (the `--context` put the test in `skip` — never asked about,
|
|
447
|
+
never selected).
|
|
446
448
|
|
|
447
449
|
## When it runs everything anyway
|
|
448
450
|
|
|
@@ -488,11 +490,14 @@ framework with no `--format`, which exits 1 (see Known limitations).
|
|
|
488
490
|
--staged use the staged change instead of the working tree
|
|
489
491
|
--format <name> vitest | jest | node | bun | playwright | rust | go | auto (default: auto)
|
|
490
492
|
--cutoff <n> select at or above this score level (default: 2)
|
|
493
|
+
--unsure-below <n> a confidence under this counts as unsure (default: 0.5)
|
|
494
|
+
--unsure-margin <n> rescue an unsure answer this far under the cutoff (default: 1)
|
|
491
495
|
--concurrency <n> requests in flight at once (default: 32)
|
|
492
496
|
--json print the full scoring instead of the arguments
|
|
493
497
|
--dry-run extract and report without calling Jev
|
|
494
498
|
--verify-snapshots review changed text snapshots without modifying files
|
|
495
499
|
--replay <file> re-gate a recorded run offline (default: .jev-test-filter/last.json)
|
|
500
|
+
--context <file> read flaker's jev-context: quarantined tests, failure history, gate defaults
|
|
496
501
|
--exec -- <cmd...> append the arguments to <cmd...> and run it
|
|
497
502
|
-h, --help the help text
|
|
498
503
|
```
|
|
@@ -500,6 +505,11 @@ framework with no `--format`, which exits 1 (see Known limitations).
|
|
|
500
505
|
Positional arguments restrict which test files are considered, e.g.
|
|
501
506
|
`jev-test-filter --base main src/cart test/cart.test.ts`.
|
|
502
507
|
|
|
508
|
+
The three gate values come from, in order of precedence: the flag, then the
|
|
509
|
+
`--context` (or, under `--replay`, the record), then the default. A value that
|
|
510
|
+
is not a finite number is refused with exit 2 rather than read as `NaN`, which
|
|
511
|
+
would deselect every test.
|
|
512
|
+
|
|
503
513
|
## Replaying a run offline
|
|
504
514
|
|
|
505
515
|
Every run that reaches Jev writes its answers to
|
|
@@ -514,16 +524,95 @@ $ jev-test-filter --replay --cutoff 1.0 > /dev/null
|
|
|
514
524
|
jev-test-filter: 5/110 tests selected (pattern)
|
|
515
525
|
```
|
|
516
526
|
|
|
527
|
+
A replay re-gates under the gate the record was decided under, so a bare
|
|
528
|
+
`--replay` reproduces the run's selection; `--cutoff`, `--unsure-below` and
|
|
529
|
+
`--unsure-margin` replace the recorded values one by one. A record written by
|
|
530
|
+
0.1 kept no gate and replays under the defaults, as it always did.
|
|
531
|
+
|
|
517
532
|
A replay reports `"spent": null` because it spent nothing, and it needs no API
|
|
518
533
|
key. It writes no new record, and `--format` has no effect on it — the
|
|
519
|
-
framework comes from the record.
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
successful run
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
534
|
+
framework comes from the record. `--context` cannot be combined with it: the
|
|
535
|
+
record already holds the answers its context produced.
|
|
536
|
+
|
|
537
|
+
Every successful run writes the record twice: to `.jev-test-filter/last.json`,
|
|
538
|
+
which the next run replaces, and to `.jev-test-filter/records/<head_sha>.json`,
|
|
539
|
+
which only another run of the same commit replaces — that is the one a CI
|
|
540
|
+
result for the commit is compared with. **Only** a successful run writes
|
|
541
|
+
either: a run that falls back writes nothing, so `--replay` always has the
|
|
542
|
+
last scoring that actually completed to work on, and an unlucky `HTTP 529`
|
|
543
|
+
cannot destroy it. The record holds every test's title and file path, so add
|
|
544
|
+
`.jev-test-filter/` to your `.gitignore`. It never holds the API key.
|
|
545
|
+
|
|
546
|
+
### Record format (version 2)
|
|
547
|
+
|
|
548
|
+
```json
|
|
549
|
+
{
|
|
550
|
+
"version": 2,
|
|
551
|
+
"createdAt": "2026-09-24T00:00:00.000Z",
|
|
552
|
+
"base": "main",
|
|
553
|
+
"head_sha": "4c1d…",
|
|
554
|
+
"base_sha": "9e0a…",
|
|
555
|
+
"context_digest": "sha256:…",
|
|
556
|
+
"gate": { "cutoff": 2, "unsure_below": 0.5, "unsure_margin": 1 },
|
|
557
|
+
"framework": "vitest",
|
|
558
|
+
"tests": [{ "file": "test/a.test.ts", "titlePath": ["A", "b"], "line": 3, "endLine": 5, "framework": "vitest", "dynamic": false }],
|
|
559
|
+
"touched": [],
|
|
560
|
+
"quarantined": [],
|
|
561
|
+
"answers": { "q0000": { "value": 1, "confidence": 0.8 } },
|
|
562
|
+
"fallback": null
|
|
563
|
+
}
|
|
564
|
+
```
|
|
565
|
+
|
|
566
|
+
- `head_sha` is `git rev-parse HEAD`, `null` when there is no commit.
|
|
567
|
+
- `base_sha` is the commit `--base` resolved to, `null` without `--base`.
|
|
568
|
+
- `context_digest` is the `digest` of the `--context` file, `null` without one.
|
|
569
|
+
- `gate` is the values the run actually decided under, defaults filled in.
|
|
570
|
+
- `touched` and `quarantined` hold test ids (file, title path and line); the
|
|
571
|
+
answers are keyed by question id, which is the test's index in `tests`.
|
|
572
|
+
|
|
573
|
+
A version 1 record is still read, with those fields as `null` and nothing
|
|
574
|
+
quarantined. The type is exported as `RunRecordV2` from
|
|
575
|
+
`jev-test-filter/types`.
|
|
576
|
+
|
|
577
|
+
## Context from flaker
|
|
578
|
+
|
|
579
|
+
[flaker](https://github.com/mizchi/flaker) keeps the history this tool cannot
|
|
580
|
+
see, and projects it into one file for it:
|
|
581
|
+
|
|
582
|
+
```
|
|
583
|
+
flaker export --projection jev-context -o .flaker/context.json
|
|
584
|
+
jev-test-filter --base main --context .flaker/context.json --exec -- vitest run
|
|
585
|
+
```
|
|
586
|
+
|
|
587
|
+
```json
|
|
588
|
+
{
|
|
589
|
+
"version": 1,
|
|
590
|
+
"digest": "sha256:…",
|
|
591
|
+
"generated_at": "2026-09-24T00:00:00.000Z",
|
|
592
|
+
"gate": { "cutoff": 2.0, "unsure_below": 0.5, "unsure_margin": 1.0, "basis": { "records": 42 } },
|
|
593
|
+
"skip": [{ "file": "tests/a.test.ts", "title_path": ["A", "b"], "reason": "quarantined" }],
|
|
594
|
+
"tests": [{ "file": "tests/cli/init.test.ts", "title_path": ["init", "writes toml"], "failed_with": ["src/cli/config.ts"], "missed": 2 }]
|
|
595
|
+
}
|
|
596
|
+
```
|
|
597
|
+
|
|
598
|
+
- **`gate`** supplies the defaults for `--cutoff`, `--unsure-below` and
|
|
599
|
+
`--unsure-margin`. A flag still wins. The gate itself stays here: flaker
|
|
600
|
+
only suggests values, it never decides a selection.
|
|
601
|
+
- **`skip`** removes a test from the candidates. It is not asked about, is
|
|
602
|
+
not selected even if the diff touched it, and is reported with
|
|
603
|
+
`reason: "quarantined"`.
|
|
604
|
+
- **`tests[].failed_with`** is added to that test's question, and only
|
|
605
|
+
that test's, as one sentence under `instructions.history`: `This test
|
|
606
|
+
previously failed when src/cli/config.ts changed.` It is a fact, not a
|
|
607
|
+
threshold; `missed` is carried but never put into a question. A test the
|
|
608
|
+
context says nothing about is asked exactly what it was asked without one.
|
|
609
|
+
|
|
610
|
+
Entries are matched on `file` + `title_path`, and on `project` when the entry
|
|
611
|
+
names one (an entry without a `project` covers the test in every Playwright
|
|
612
|
+
project) — never on the line, which moves with every edit above the test.
|
|
613
|
+
Anything other than `version: 1`, or a shape that does not validate, exits 2
|
|
614
|
+
before a question is asked. The record keeps the context's `digest`, because
|
|
615
|
+
a hint changes the question and runs are only comparable under the same one.
|
|
527
616
|
|
|
528
617
|
## Cost and latency
|
|
529
618
|
|
package/dist/cli.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import type { SnapshotReview } from "./snapshot.ts";
|
|
3
3
|
import type { RunRecord, RunResult } from "./run.ts";
|
|
4
|
+
import type { GateOptions } from "./gate.ts";
|
|
4
5
|
import type { Framework } from "./types.ts";
|
|
5
6
|
/** Where a run leaves its answers, and what a bare `--replay` means. */
|
|
6
7
|
export declare const DEFAULT_RECORD_PATH = ".jev-test-filter/last.json";
|
|
@@ -10,11 +11,14 @@ export interface CliArgs {
|
|
|
10
11
|
paths: string[];
|
|
11
12
|
format: Framework | null;
|
|
12
13
|
cutoff: number | undefined;
|
|
14
|
+
unsureBelow: number | undefined;
|
|
15
|
+
unsureMargin: number | undefined;
|
|
13
16
|
concurrency: number | undefined;
|
|
14
17
|
json: boolean;
|
|
15
18
|
dryRun: boolean;
|
|
16
19
|
verifySnapshots: boolean;
|
|
17
20
|
replayPath: string | null;
|
|
21
|
+
contextPath: string | null;
|
|
18
22
|
exec: string[] | null;
|
|
19
23
|
help: boolean;
|
|
20
24
|
}
|
|
@@ -23,6 +27,14 @@ export interface CliArgs {
|
|
|
23
27
|
* own flags never have to be escaped past this parser.
|
|
24
28
|
*/
|
|
25
29
|
export declare function parseCliArgs(argv: string[]): CliArgs;
|
|
30
|
+
/**
|
|
31
|
+
* The gate values the command line set, and nothing else.
|
|
32
|
+
*
|
|
33
|
+
* Absent rather than defaulted, because a flag is only one of the places a
|
|
34
|
+
* gate value comes from: whatever it leaves unset falls through to the
|
|
35
|
+
* record being replayed, then to the gate's own defaults.
|
|
36
|
+
*/
|
|
37
|
+
export declare function gateFlags(args: CliArgs): GateOptions;
|
|
26
38
|
export declare function renderLine(res: RunResult): string;
|
|
27
39
|
/** The command to run, or null when nothing was selected. */
|
|
28
40
|
export declare function execArgv(cmd: string[], res: RunResult): string[] | null;
|
package/dist/cli.js
CHANGED
|
@@ -21,17 +21,37 @@
|
|
|
21
21
|
* Only this file writes to a stream or exits.
|
|
22
22
|
*/
|
|
23
23
|
import { spawn } from "node:child_process";
|
|
24
|
+
import { realpathSync } from "node:fs";
|
|
24
25
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
25
26
|
import { dirname, join } from "node:path";
|
|
27
|
+
import { fileURLToPath } from "node:url";
|
|
26
28
|
import { parseArgs } from "node:util";
|
|
27
29
|
import { displayName } from "./questions.js";
|
|
28
30
|
import { fullName } from "./filter.js";
|
|
29
31
|
import { loadDiff } from "./diff.js";
|
|
30
32
|
import { assessSnapshots, findSnapshotChangesInWorktree, loadSnapshotTestSources, snapshotTestFile } from "./snapshot.js";
|
|
33
|
+
import { loadContext } from "./context.js";
|
|
31
34
|
import { loadRecord, RECORD_DIR, RECORD_FILE, replay, run, saveRecord } from "./run.js";
|
|
32
35
|
const FORMATS = ["vitest", "jest", "node", "bun", "playwright", "rust", "go", "auto"];
|
|
33
36
|
/** Where a run leaves its answers, and what a bare `--replay` means. */
|
|
34
37
|
export const DEFAULT_RECORD_PATH = `${RECORD_DIR}/${RECORD_FILE}`;
|
|
38
|
+
/**
|
|
39
|
+
* One gate value from the command line.
|
|
40
|
+
*
|
|
41
|
+
* `Number()` alone is not enough: `Number("abc")` is NaN, and NaN compares
|
|
42
|
+
* false against every score, so a typo in `--cutoff` would deselect the whole
|
|
43
|
+
* suite without a word. An empty string is refused for the same reason
|
|
44
|
+
* `Number("")` is 0.
|
|
45
|
+
*/
|
|
46
|
+
function gateNumber(flag, raw) {
|
|
47
|
+
if (raw === undefined)
|
|
48
|
+
return undefined;
|
|
49
|
+
const text = String(raw).trim();
|
|
50
|
+
const n = Number(text);
|
|
51
|
+
if (text === "" || !Number.isFinite(n))
|
|
52
|
+
throw new Error(`${flag} needs a number, got ${JSON.stringify(String(raw))}`);
|
|
53
|
+
return n;
|
|
54
|
+
}
|
|
35
55
|
/**
|
|
36
56
|
* `--exec` takes everything after the `--` that follows it, so the runner's
|
|
37
57
|
* own flags never have to be escaped past this parser.
|
|
@@ -61,11 +81,14 @@ export function parseCliArgs(argv) {
|
|
|
61
81
|
staged: { type: "boolean", default: false },
|
|
62
82
|
format: { type: "string", default: "auto" },
|
|
63
83
|
cutoff: { type: "string" },
|
|
84
|
+
"unsure-below": { type: "string" },
|
|
85
|
+
"unsure-margin": { type: "string" },
|
|
64
86
|
concurrency: { type: "string" },
|
|
65
87
|
json: { type: "boolean", default: false },
|
|
66
88
|
"dry-run": { type: "boolean", default: false },
|
|
67
89
|
"verify-snapshots": { type: "boolean", default: false },
|
|
68
90
|
replay: { type: "string" },
|
|
91
|
+
context: { type: "string" },
|
|
69
92
|
help: { type: "boolean", short: "h", default: false },
|
|
70
93
|
},
|
|
71
94
|
});
|
|
@@ -76,21 +99,48 @@ export function parseCliArgs(argv) {
|
|
|
76
99
|
if (values["verify-snapshots"] && (exec !== null || values.replay !== undefined)) {
|
|
77
100
|
throw new Error("--verify-snapshots cannot be combined with --exec or --replay");
|
|
78
101
|
}
|
|
102
|
+
// Refused rather than ignored: a replay re-gates answers already given to
|
|
103
|
+
// questions already asked, and a snapshot review asks no per-test question,
|
|
104
|
+
// so a context could change neither -- and silently doing nothing with a
|
|
105
|
+
// flag reads as the flag having worked.
|
|
106
|
+
if (values.context !== undefined && values.replay !== undefined) {
|
|
107
|
+
throw new Error("--context cannot be combined with --replay; the record already holds the answers its context produced");
|
|
108
|
+
}
|
|
109
|
+
if (values.context !== undefined && values["verify-snapshots"]) {
|
|
110
|
+
throw new Error("--context cannot be combined with --verify-snapshots");
|
|
111
|
+
}
|
|
79
112
|
return {
|
|
80
113
|
base: values.base === undefined ? null : String(values.base),
|
|
81
114
|
staged: Boolean(values.staged),
|
|
82
115
|
paths: positionals,
|
|
83
116
|
format: format === "auto" ? null : format,
|
|
84
|
-
cutoff:
|
|
117
|
+
cutoff: gateNumber("--cutoff", values.cutoff),
|
|
118
|
+
unsureBelow: gateNumber("--unsure-below", values["unsure-below"]),
|
|
119
|
+
unsureMargin: gateNumber("--unsure-margin", values["unsure-margin"]),
|
|
85
120
|
concurrency: values.concurrency === undefined ? undefined : Number(values.concurrency),
|
|
86
121
|
json: Boolean(values.json),
|
|
87
122
|
dryRun: Boolean(values["dry-run"]),
|
|
88
123
|
verifySnapshots: Boolean(values["verify-snapshots"]),
|
|
89
124
|
replayPath: values.replay === undefined ? null : String(values.replay),
|
|
125
|
+
contextPath: values.context === undefined ? null : String(values.context),
|
|
90
126
|
exec,
|
|
91
127
|
help: Boolean(values.help),
|
|
92
128
|
};
|
|
93
129
|
}
|
|
130
|
+
/**
|
|
131
|
+
* The gate values the command line set, and nothing else.
|
|
132
|
+
*
|
|
133
|
+
* Absent rather than defaulted, because a flag is only one of the places a
|
|
134
|
+
* gate value comes from: whatever it leaves unset falls through to the
|
|
135
|
+
* record being replayed, then to the gate's own defaults.
|
|
136
|
+
*/
|
|
137
|
+
export function gateFlags(args) {
|
|
138
|
+
return {
|
|
139
|
+
...(args.cutoff === undefined ? {} : { cutoff: args.cutoff }),
|
|
140
|
+
...(args.unsureBelow === undefined ? {} : { unsureBelow: args.unsureBelow }),
|
|
141
|
+
...(args.unsureMargin === undefined ? {} : { unsureMargin: args.unsureMargin }),
|
|
142
|
+
};
|
|
143
|
+
}
|
|
94
144
|
/**
|
|
95
145
|
* Shell-safe single quoting, for a line a human will read or `eval`.
|
|
96
146
|
*
|
|
@@ -183,11 +233,14 @@ Options:
|
|
|
183
233
|
--staged use the staged change instead of the working tree
|
|
184
234
|
--format <name> vitest | jest | node | bun | playwright | rust | go | auto (default: auto)
|
|
185
235
|
--cutoff <n> select at or above this score level (default: 2)
|
|
236
|
+
--unsure-below <n> a confidence under this counts as unsure (default: 0.5)
|
|
237
|
+
--unsure-margin <n> rescue an unsure answer this far under the cutoff (default: 1)
|
|
186
238
|
--concurrency <n> requests in flight at once
|
|
187
239
|
--json print the full scoring instead of the arguments
|
|
188
240
|
--dry-run extract and report without calling Jev
|
|
189
241
|
--verify-snapshots review changed Vitest text snapshots without modifying files
|
|
190
242
|
--replay <file> re-gate a recorded run offline (default: .jev-test-filter/last.json)
|
|
243
|
+
--context <file> read flaker's jev-context: quarantined tests, failure history, gate defaults
|
|
191
244
|
--exec -- <cmd...> append the arguments to <cmd...> and run it
|
|
192
245
|
-h, --help this text
|
|
193
246
|
|
|
@@ -250,7 +303,7 @@ async function main() {
|
|
|
250
303
|
let res;
|
|
251
304
|
if (args.replayPath !== null) {
|
|
252
305
|
const record = await loadRecord(args.replayPath);
|
|
253
|
-
const selection = replay(record, args
|
|
306
|
+
const selection = replay(record, gateFlags(args));
|
|
254
307
|
const { buildFilter } = await import("./filter.js");
|
|
255
308
|
res = {
|
|
256
309
|
selection,
|
|
@@ -261,14 +314,27 @@ async function main() {
|
|
|
261
314
|
};
|
|
262
315
|
}
|
|
263
316
|
else {
|
|
317
|
+
// Read before anything is asked: a context that cannot be used is a usage
|
|
318
|
+
// error, and paying for a run it would have changed is worse than stopping.
|
|
319
|
+
let context = null;
|
|
320
|
+
if (args.contextPath !== null) {
|
|
321
|
+
try {
|
|
322
|
+
context = await loadContext(args.contextPath);
|
|
323
|
+
}
|
|
324
|
+
catch (err) {
|
|
325
|
+
process.stderr.write(`jev-test-filter: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
326
|
+
return 2;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
264
329
|
res = await run({
|
|
265
330
|
base: args.base,
|
|
266
331
|
staged: args.staged,
|
|
267
332
|
paths: args.paths,
|
|
268
333
|
format: args.format,
|
|
269
334
|
dryRun: args.dryRun,
|
|
335
|
+
context,
|
|
270
336
|
...(args.format === "playwright" && args.exec ? { playwrightCommand: args.exec } : {}),
|
|
271
|
-
...(args
|
|
337
|
+
...gateFlags(args),
|
|
272
338
|
...(args.concurrency === undefined ? {} : { concurrency: args.concurrency }),
|
|
273
339
|
});
|
|
274
340
|
if (shouldSaveRecord(res.record))
|
|
@@ -310,10 +376,21 @@ async function main() {
|
|
|
310
376
|
process.stdout.write(`${renderLine(res)}\n`);
|
|
311
377
|
return stdoutExitCode(res);
|
|
312
378
|
}
|
|
379
|
+
/** Resolve npm's bin symlink before deciding whether this module is the entrypoint. */
|
|
380
|
+
function isCliInvocation(argv1, moduleUrl) {
|
|
381
|
+
if (!argv1)
|
|
382
|
+
return false;
|
|
383
|
+
try {
|
|
384
|
+
return realpathSync(argv1) === realpathSync(fileURLToPath(moduleUrl));
|
|
385
|
+
}
|
|
386
|
+
catch {
|
|
387
|
+
return false;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
313
390
|
// Only run when invoked as a program, so the tests can import the renderers.
|
|
314
|
-
if (process.argv[1]
|
|
315
|
-
main().then((code) => process.
|
|
391
|
+
if (isCliInvocation(process.argv[1], import.meta.url)) {
|
|
392
|
+
main().then((code) => { process.exitCode = code; }, (err) => {
|
|
316
393
|
process.stderr.write(`jev-test-filter: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
317
|
-
process.
|
|
394
|
+
process.exitCode = 1;
|
|
318
395
|
});
|
|
319
396
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { JevContext, TestCase } from "./types.ts";
|
|
2
|
+
export declare const CONTEXT_VERSION = 1;
|
|
3
|
+
/** Validate a parsed `jev-context`. Throws on anything but a v1 context. */
|
|
4
|
+
export declare function parseContext(raw: unknown): JevContext;
|
|
5
|
+
export declare function loadContext(path: string): Promise<JevContext>;
|
|
6
|
+
export interface ContextLookup {
|
|
7
|
+
skipped(t: TestCase): boolean;
|
|
8
|
+
/** The files this test failed with before; empty when the context says nothing. */
|
|
9
|
+
failedWith(t: TestCase): readonly string[];
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The context indexed for per-test questions.
|
|
13
|
+
*
|
|
14
|
+
* An entry that names a project is about the test in that project only; one
|
|
15
|
+
* that names none is about the test in every project, which is how a
|
|
16
|
+
* quarantine of a test that fails in every browser is written once.
|
|
17
|
+
*/
|
|
18
|
+
export declare function lookup(ctx: JevContext): ContextLookup;
|
package/dist/context.js
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What flaker knows, read in: `--context <file>`.
|
|
3
|
+
*
|
|
4
|
+
* flaker keeps the history this tool has no way to see -- which tests are
|
|
5
|
+
* quarantined, which tests failed the last time a given file changed, which
|
|
6
|
+
* gate values its calibration settled on -- and projects it into one JSON
|
|
7
|
+
* file, `jev-context` v1. This module reads that file and answers the two
|
|
8
|
+
* questions a run asks of it, per test: is it skipped, and what has it failed
|
|
9
|
+
* with before.
|
|
10
|
+
*
|
|
11
|
+
* The gate is NOT decided here. The context only supplies default values;
|
|
12
|
+
* `gate.ts` is still the one place a score becomes a selection, so flaker
|
|
13
|
+
* never has to reimplement it and the two can never disagree.
|
|
14
|
+
*
|
|
15
|
+
* A context is validated whole and refused whole. A half-understood context
|
|
16
|
+
* would change questions in ways the digest on the record no longer
|
|
17
|
+
* describes, and a run that cannot say what it was asked is worth less than
|
|
18
|
+
* one that stops and says why.
|
|
19
|
+
*/
|
|
20
|
+
import { readFile } from "node:fs/promises";
|
|
21
|
+
export const CONTEXT_VERSION = 1;
|
|
22
|
+
function fail(what) {
|
|
23
|
+
throw new Error(`invalid context: ${what}`);
|
|
24
|
+
}
|
|
25
|
+
function isRecord(v) {
|
|
26
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
27
|
+
}
|
|
28
|
+
function strings(v, at) {
|
|
29
|
+
if (!Array.isArray(v) || !v.every((s) => typeof s === "string"))
|
|
30
|
+
fail(`${at} must be an array of strings`);
|
|
31
|
+
return [...v];
|
|
32
|
+
}
|
|
33
|
+
function optionalString(v, at) {
|
|
34
|
+
if (v === undefined)
|
|
35
|
+
return undefined;
|
|
36
|
+
if (typeof v !== "string")
|
|
37
|
+
fail(`${at} must be a string`);
|
|
38
|
+
return v;
|
|
39
|
+
}
|
|
40
|
+
function readGate(v) {
|
|
41
|
+
if (v === undefined || v === null)
|
|
42
|
+
return null;
|
|
43
|
+
if (!isRecord(v))
|
|
44
|
+
fail("gate must be an object");
|
|
45
|
+
const out = {};
|
|
46
|
+
for (const key of ["cutoff", "unsure_below", "unsure_margin"]) {
|
|
47
|
+
const n = v[key];
|
|
48
|
+
if (n === undefined || n === null)
|
|
49
|
+
continue;
|
|
50
|
+
if (typeof n !== "number" || !Number.isFinite(n))
|
|
51
|
+
fail(`gate.${key} must be a number`);
|
|
52
|
+
out[key] = n;
|
|
53
|
+
}
|
|
54
|
+
if (v.basis !== undefined)
|
|
55
|
+
out.basis = v.basis;
|
|
56
|
+
return out;
|
|
57
|
+
}
|
|
58
|
+
/** The part of an entry that names a test. */
|
|
59
|
+
function readName(v, at) {
|
|
60
|
+
if (!isRecord(v))
|
|
61
|
+
fail(`${at} must be an object`);
|
|
62
|
+
if (typeof v.file !== "string" || v.file === "")
|
|
63
|
+
fail(`${at}.file must be a non-empty string`);
|
|
64
|
+
const title_path = strings(v.title_path, `${at}.title_path`);
|
|
65
|
+
const project = optionalString(v.project, `${at}.project`);
|
|
66
|
+
return { file: v.file, title_path, ...(project === undefined ? {} : { project }) };
|
|
67
|
+
}
|
|
68
|
+
function list(v, at, read) {
|
|
69
|
+
if (v === undefined || v === null)
|
|
70
|
+
return [];
|
|
71
|
+
if (!Array.isArray(v))
|
|
72
|
+
fail(`${at} must be an array`);
|
|
73
|
+
return v.map((item, i) => read(item, `${at}[${i}]`));
|
|
74
|
+
}
|
|
75
|
+
/** Validate a parsed `jev-context`. Throws on anything but a v1 context. */
|
|
76
|
+
export function parseContext(raw) {
|
|
77
|
+
if (!isRecord(raw))
|
|
78
|
+
fail("expected a JSON object");
|
|
79
|
+
if (raw.version !== CONTEXT_VERSION) {
|
|
80
|
+
throw new Error(`unsupported context version ${String(raw.version)}; expected ${CONTEXT_VERSION}`);
|
|
81
|
+
}
|
|
82
|
+
if (typeof raw.digest !== "string" || raw.digest === "")
|
|
83
|
+
fail("digest must be a non-empty string");
|
|
84
|
+
const generated_at = optionalString(raw.generated_at, "generated_at");
|
|
85
|
+
const skip = list(raw.skip, "skip", (item, at) => {
|
|
86
|
+
const reason = optionalString(item?.reason, `${at}.reason`);
|
|
87
|
+
return { ...readName(item, at), ...(reason === undefined ? {} : { reason }) };
|
|
88
|
+
});
|
|
89
|
+
const tests = list(raw.tests, "tests", (item, at) => {
|
|
90
|
+
const name = readName(item, at);
|
|
91
|
+
const o = item;
|
|
92
|
+
const failed_with = strings(o.failed_with ?? [], `${at}.failed_with`);
|
|
93
|
+
if (o.missed !== undefined && (typeof o.missed !== "number" || !Number.isFinite(o.missed))) {
|
|
94
|
+
fail(`${at}.missed must be a number`);
|
|
95
|
+
}
|
|
96
|
+
return { ...name, failed_with, ...(o.missed === undefined ? {} : { missed: o.missed }) };
|
|
97
|
+
});
|
|
98
|
+
return {
|
|
99
|
+
version: 1,
|
|
100
|
+
digest: raw.digest,
|
|
101
|
+
...(generated_at === undefined ? {} : { generated_at }),
|
|
102
|
+
gate: readGate(raw.gate),
|
|
103
|
+
skip,
|
|
104
|
+
tests,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
export async function loadContext(path) {
|
|
108
|
+
let raw;
|
|
109
|
+
try {
|
|
110
|
+
raw = JSON.parse(await readFile(path, "utf8"));
|
|
111
|
+
}
|
|
112
|
+
catch (err) {
|
|
113
|
+
throw new Error(`could not read context ${path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
114
|
+
}
|
|
115
|
+
try {
|
|
116
|
+
return parseContext(raw);
|
|
117
|
+
}
|
|
118
|
+
catch (err) {
|
|
119
|
+
throw new Error(`${path}: ${err instanceof Error ? err.message : String(err)}`);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* The key a context entry and a test meet on. The unit separator for the
|
|
124
|
+
* reason `testId` uses it: no title a source file contains holds one.
|
|
125
|
+
*/
|
|
126
|
+
function key(file, titlePath, project) {
|
|
127
|
+
const base = `${file.replace(/^\.\//, "")}\u001f${titlePath.join("\u001f")}`;
|
|
128
|
+
return project === undefined ? base : `${base}\u001e${project}`;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* The context indexed for per-test questions.
|
|
132
|
+
*
|
|
133
|
+
* An entry that names a project is about the test in that project only; one
|
|
134
|
+
* that names none is about the test in every project, which is how a
|
|
135
|
+
* quarantine of a test that fails in every browser is written once.
|
|
136
|
+
*/
|
|
137
|
+
export function lookup(ctx) {
|
|
138
|
+
const skip = new Set(ctx.skip.map((s) => key(s.file, s.title_path, s.project)));
|
|
139
|
+
const hints = new Map();
|
|
140
|
+
for (const t of ctx.tests) {
|
|
141
|
+
if (t.failed_with.length > 0)
|
|
142
|
+
hints.set(key(t.file, t.title_path, t.project), t.failed_with);
|
|
143
|
+
}
|
|
144
|
+
const keys = (t) => t.project === undefined ? [key(t.file, t.titlePath, undefined)] : [key(t.file, t.titlePath, t.project), key(t.file, t.titlePath, undefined)];
|
|
145
|
+
return {
|
|
146
|
+
skipped: (t) => keys(t).some((k) => skip.has(k)),
|
|
147
|
+
failedWith: (t) => {
|
|
148
|
+
for (const k of keys(t)) {
|
|
149
|
+
const hit = hints.get(k);
|
|
150
|
+
if (hit)
|
|
151
|
+
return hit;
|
|
152
|
+
}
|
|
153
|
+
return [];
|
|
154
|
+
},
|
|
155
|
+
};
|
|
156
|
+
}
|
package/dist/diff.d.ts
CHANGED
|
@@ -50,3 +50,11 @@ export interface Diff {
|
|
|
50
50
|
* otherwise rename them and the parser's paths would match no file.
|
|
51
51
|
*/
|
|
52
52
|
export declare function loadDiff({ cwd, base, staged }?: DiffOptions): Promise<Diff>;
|
|
53
|
+
/**
|
|
54
|
+
* The commit a ref names, or null when it names none -- no repository, no
|
|
55
|
+
* commit yet, or a ref that does not exist.
|
|
56
|
+
*
|
|
57
|
+
* Null rather than a throw: the sha only labels a record, and a run that
|
|
58
|
+
* could select tests must not fail because it could not label them.
|
|
59
|
+
*/
|
|
60
|
+
export declare function resolveSha(ref: string, cwd?: string): Promise<string | null>;
|
package/dist/diff.js
CHANGED
|
@@ -166,3 +166,19 @@ export async function loadDiff({ cwd = process.cwd(), base = null, staged = fals
|
|
|
166
166
|
const stat = await git(["diff", "--stat", ...common, ...range], cwd);
|
|
167
167
|
return { text, stat, ranges: parseUnifiedDiff(text) };
|
|
168
168
|
}
|
|
169
|
+
/**
|
|
170
|
+
* The commit a ref names, or null when it names none -- no repository, no
|
|
171
|
+
* commit yet, or a ref that does not exist.
|
|
172
|
+
*
|
|
173
|
+
* Null rather than a throw: the sha only labels a record, and a run that
|
|
174
|
+
* could select tests must not fail because it could not label them.
|
|
175
|
+
*/
|
|
176
|
+
export async function resolveSha(ref, cwd = process.cwd()) {
|
|
177
|
+
try {
|
|
178
|
+
const out = (await git(["rev-parse", "--verify", "--quiet", `${ref}^{commit}`], cwd)).trim();
|
|
179
|
+
return /^[0-9a-f]{40,64}$/.test(out) ? out : null;
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
}
|
package/dist/gate.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Answer, Selection, TestCase, Verdict } from "./types.ts";
|
|
1
|
+
import type { Answer, RecordGate, Selection, TestCase, Verdict } from "./types.ts";
|
|
2
2
|
/**
|
|
3
3
|
* The boundary between "this change cannot alter the outcome" (level 1) and
|
|
4
4
|
* "it might" (level 2). It is a level boundary, not a tuned number, and it is
|
|
@@ -14,6 +14,14 @@ export interface GateOptions {
|
|
|
14
14
|
unsureBelow?: number;
|
|
15
15
|
unsureMargin?: number;
|
|
16
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* The values a gate will actually decide under, with every default filled
|
|
19
|
+
* in, in the spelling a record keeps them in. A record that said only "the
|
|
20
|
+
* flags that were given" would change meaning the day a default did.
|
|
21
|
+
*/
|
|
22
|
+
export declare function resolveGate({ cutoff, unsureBelow, unsureMargin, }?: GateOptions): RecordGate;
|
|
23
|
+
/** A recorded or supplied gate as options. Nothing, when there is none. */
|
|
24
|
+
export declare function gateOptions(g: RecordGate | null | undefined): GateOptions;
|
|
17
25
|
/** Decide one test. Never returns null: every test gets a side and a reason. */
|
|
18
26
|
export declare function decide(id: string, t: TestCase, answer: Answer | null, touched: boolean, { cutoff, unsureBelow, unsureMargin }?: GateOptions): Verdict;
|
|
19
27
|
/**
|
|
@@ -21,6 +29,9 @@ export declare function decide(id: string, t: TestCase, answer: Answer | null, t
|
|
|
21
29
|
*
|
|
22
30
|
* `answers` is keyed by question id, which is the test's index in `tests`;
|
|
23
31
|
* `touched` is keyed by `testId`, because it is computed from the diff before
|
|
24
|
-
* any question exists
|
|
32
|
+
* any question exists; so is `quarantined`, the tests a context put in
|
|
33
|
+
* `skip`. A quarantined test is decided before anything else, touched
|
|
34
|
+
* included: it was never asked about, and the quarantine is flaker's decision
|
|
35
|
+
* that its result means nothing right now.
|
|
25
36
|
*/
|
|
26
|
-
export declare function gate(tests: TestCase[], answers: Map<string, Answer | null>, touched: Set<string>, opts?: GateOptions): Selection;
|
|
37
|
+
export declare function gate(tests: TestCase[], answers: Map<string, Answer | null>, touched: Set<string>, opts?: GateOptions, quarantined?: ReadonlySet<string>): Selection;
|
package/dist/gate.js
CHANGED
|
@@ -27,6 +27,20 @@ export const DEFAULT_CUTOFF = 2.0;
|
|
|
27
27
|
export const DEFAULT_UNSURE_BELOW = 0.5;
|
|
28
28
|
/** How far under the cutoff an unsure answer is still rescued. */
|
|
29
29
|
export const DEFAULT_UNSURE_MARGIN = 1.0;
|
|
30
|
+
/**
|
|
31
|
+
* The values a gate will actually decide under, with every default filled
|
|
32
|
+
* in, in the spelling a record keeps them in. A record that said only "the
|
|
33
|
+
* flags that were given" would change meaning the day a default did.
|
|
34
|
+
*/
|
|
35
|
+
export function resolveGate({ cutoff = DEFAULT_CUTOFF, unsureBelow = DEFAULT_UNSURE_BELOW, unsureMargin = DEFAULT_UNSURE_MARGIN, } = {}) {
|
|
36
|
+
return { cutoff, unsure_below: unsureBelow, unsure_margin: unsureMargin };
|
|
37
|
+
}
|
|
38
|
+
/** A recorded or supplied gate as options. Nothing, when there is none. */
|
|
39
|
+
export function gateOptions(g) {
|
|
40
|
+
if (!g)
|
|
41
|
+
return {};
|
|
42
|
+
return { cutoff: g.cutoff, unsureBelow: g.unsure_below, unsureMargin: g.unsure_margin };
|
|
43
|
+
}
|
|
30
44
|
/** Decide one test. Never returns null: every test gets a side and a reason. */
|
|
31
45
|
export function decide(id, t, answer, touched, { cutoff = DEFAULT_CUTOFF, unsureBelow = DEFAULT_UNSURE_BELOW, unsureMargin = DEFAULT_UNSURE_MARGIN } = {}) {
|
|
32
46
|
if (touched)
|
|
@@ -47,11 +61,16 @@ export function decide(id, t, answer, touched, { cutoff = DEFAULT_CUTOFF, unsure
|
|
|
47
61
|
*
|
|
48
62
|
* `answers` is keyed by question id, which is the test's index in `tests`;
|
|
49
63
|
* `touched` is keyed by `testId`, because it is computed from the diff before
|
|
50
|
-
* any question exists
|
|
64
|
+
* any question exists; so is `quarantined`, the tests a context put in
|
|
65
|
+
* `skip`. A quarantined test is decided before anything else, touched
|
|
66
|
+
* included: it was never asked about, and the quarantine is flaker's decision
|
|
67
|
+
* that its result means nothing right now.
|
|
51
68
|
*/
|
|
52
|
-
export function gate(tests, answers, touched, opts = {}) {
|
|
69
|
+
export function gate(tests, answers, touched, opts = {}, quarantined = new Set()) {
|
|
53
70
|
const verdicts = tests.map((t, i) => {
|
|
54
71
|
const id = questionId(i);
|
|
72
|
+
if (quarantined.has(testId(t)))
|
|
73
|
+
return { id, test: t, answer: null, reason: "quarantined", selected: false };
|
|
55
74
|
return decide(id, t, answers.get(id) ?? null, touched.has(testId(t)), opts);
|
|
56
75
|
});
|
|
57
76
|
return {
|
package/dist/questions.d.ts
CHANGED
|
@@ -11,6 +11,13 @@
|
|
|
11
11
|
* makes from the answer; writing it into the question would mean every
|
|
12
12
|
* recalibration rewrote the question, and no run could be compared with an
|
|
13
13
|
* earlier one.
|
|
14
|
+
*
|
|
15
|
+
* The one thing a question may carry beyond the test's own identity is
|
|
16
|
+
* history from flaker (`--context`): the files this test failed with before,
|
|
17
|
+
* as one plain sentence under `instructions.history`. It is evidence, not a
|
|
18
|
+
* rule -- no count, no rate -- and it is absent, key and all, for a test the
|
|
19
|
+
* context says nothing about, so such a test is asked exactly what it was
|
|
20
|
+
* asked before contexts existed.
|
|
14
21
|
*/
|
|
15
22
|
import type { TestCase } from "./types.ts";
|
|
16
23
|
export interface ScoreQuestion {
|
|
@@ -29,7 +36,16 @@ export declare function questionId(i: number): string;
|
|
|
29
36
|
* its reporter prints it.
|
|
30
37
|
*/
|
|
31
38
|
export declare function displayName(t: TestCase): string;
|
|
32
|
-
|
|
39
|
+
/**
|
|
40
|
+
* The one sentence a hint becomes.
|
|
41
|
+
*
|
|
42
|
+
* A fact about the past and nothing more: nothing a model could read as a
|
|
43
|
+
* bar to clear, which is why flaker's `missed` count stays out of it. Whether
|
|
44
|
+
* THIS change can break the test is still the model's judgment.
|
|
45
|
+
*/
|
|
46
|
+
export declare function historySentence(files: readonly string[]): string;
|
|
47
|
+
/** `failedWith`: the files this test failed with before, from `--context`. */
|
|
48
|
+
export declare function buildQuestion(t: TestCase, id: string, failedWith?: readonly string[]): ScoreQuestion;
|
|
33
49
|
/**
|
|
34
50
|
* One answer, or null when it cannot be used.
|
|
35
51
|
*
|
package/dist/questions.js
CHANGED
|
@@ -26,7 +26,19 @@ export function displayName(t) {
|
|
|
26
26
|
const sep = t.framework === "node" ? " " : " > ";
|
|
27
27
|
return t.titlePath.join(sep);
|
|
28
28
|
}
|
|
29
|
-
|
|
29
|
+
/**
|
|
30
|
+
* The one sentence a hint becomes.
|
|
31
|
+
*
|
|
32
|
+
* A fact about the past and nothing more: nothing a model could read as a
|
|
33
|
+
* bar to clear, which is why flaker's `missed` count stays out of it. Whether
|
|
34
|
+
* THIS change can break the test is still the model's judgment.
|
|
35
|
+
*/
|
|
36
|
+
export function historySentence(files) {
|
|
37
|
+
const named = files.length <= 1 ? (files[0] ?? "") : `${files.slice(0, -1).join(", ")} or ${files[files.length - 1]}`;
|
|
38
|
+
return `This test previously failed when ${named} changed.`;
|
|
39
|
+
}
|
|
40
|
+
/** `failedWith`: the files this test failed with before, from `--context`. */
|
|
41
|
+
export function buildQuestion(t, id, failedWith = []) {
|
|
30
42
|
return {
|
|
31
43
|
type: "score",
|
|
32
44
|
instructions: {
|
|
@@ -36,6 +48,7 @@ export function buildQuestion(t, id) {
|
|
|
36
48
|
...(t.project === undefined ? {} : { playwright_project: t.project }),
|
|
37
49
|
test_name: displayName(t),
|
|
38
50
|
test_lines: t.line === t.endLine ? `${t.line}` : `${t.line}-${t.endLine}`,
|
|
51
|
+
...(failedWith.length === 0 ? {} : { history: historySentence(failedWith) }),
|
|
39
52
|
},
|
|
40
53
|
criteria: SCORE_LEVELS,
|
|
41
54
|
};
|
package/dist/run.d.ts
CHANGED
|
@@ -3,21 +3,21 @@ import type { StatePayload } from "./state.ts";
|
|
|
3
3
|
import type { GateOptions } from "./gate.ts";
|
|
4
4
|
import type { FilterArgs } from "./filter.ts";
|
|
5
5
|
import type { AskClient, Spend } from "./jev.ts";
|
|
6
|
-
import type {
|
|
6
|
+
import type { ContextLookup } from "./context.ts";
|
|
7
|
+
import type { Answer, Framework, JevContext, RunRecord, Selection, TestCase } from "./types.ts";
|
|
8
|
+
export type { RecordGate, RunRecord, RunRecordV1, RunRecordV2 } from "./types.ts";
|
|
7
9
|
/** Where a run's answers are kept, for `--replay`. */
|
|
8
10
|
export declare const RECORD_DIR = ".jev-test-filter";
|
|
9
11
|
export declare const RECORD_FILE = "last.json";
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
fallback: string | null;
|
|
20
|
-
}
|
|
12
|
+
/**
|
|
13
|
+
* One record per commit, under `RECORD_DIR`, named `<head_sha>.json`.
|
|
14
|
+
*
|
|
15
|
+
* `last.json` alone is overwritten by the next run, and the run a CI result
|
|
16
|
+
* has to be compared with is rarely the last one on this machine. Keyed by
|
|
17
|
+
* the head rather than by time because the head is what the other side of
|
|
18
|
+
* that comparison knows.
|
|
19
|
+
*/
|
|
20
|
+
export declare const RECORDS_SUBDIR = "records";
|
|
21
21
|
/**
|
|
22
22
|
* The framework the run is for.
|
|
23
23
|
*
|
|
@@ -37,6 +37,12 @@ export interface ScoreOptions {
|
|
|
37
37
|
* how much one `max_tokens_exceeded` split has to redo.
|
|
38
38
|
*/
|
|
39
39
|
batchSize?: number;
|
|
40
|
+
/**
|
|
41
|
+
* What `--context` knows. A test it skips is not asked about -- its answer
|
|
42
|
+
* stays null and the gate reports it quarantined -- and a test it has
|
|
43
|
+
* history for is asked with that history.
|
|
44
|
+
*/
|
|
45
|
+
context?: ContextLookup | null;
|
|
40
46
|
}
|
|
41
47
|
/**
|
|
42
48
|
* Ask about every test.
|
|
@@ -45,7 +51,7 @@ export interface ScoreOptions {
|
|
|
45
51
|
* null: the gate selects a test with no answer, and a zero here would silently
|
|
46
52
|
* deselect it instead.
|
|
47
53
|
*/
|
|
48
|
-
export declare function score(tests: TestCase[], state: StatePayload, { client, concurrency, batchSize }?: ScoreOptions): Promise<Map<string, Answer | null>>;
|
|
54
|
+
export declare function score(tests: TestCase[], state: StatePayload, { client, concurrency, batchSize, context }?: ScoreOptions): Promise<Map<string, Answer | null>>;
|
|
49
55
|
export interface RunOptions extends GateOptions {
|
|
50
56
|
cwd?: string;
|
|
51
57
|
base?: string | null;
|
|
@@ -61,7 +67,18 @@ export interface RunOptions extends GateOptions {
|
|
|
61
67
|
dryRun?: boolean;
|
|
62
68
|
/** When using Playwright, collect its actual test list with this runner command. */
|
|
63
69
|
playwrightCommand?: string[];
|
|
70
|
+
/**
|
|
71
|
+
* flaker's `jev-context`, already validated. Its gate values are defaults:
|
|
72
|
+
* a value set in these options wins over each.
|
|
73
|
+
*/
|
|
74
|
+
context?: JevContext | null;
|
|
64
75
|
}
|
|
76
|
+
/**
|
|
77
|
+
* The gate a run decides under: the options' own values over the context's,
|
|
78
|
+
* over the defaults. Only values actually present count, so an option passed
|
|
79
|
+
* as `undefined` does not erase the context's value.
|
|
80
|
+
*/
|
|
81
|
+
export declare function effectiveGate(opts: GateOptions, context?: JevContext | null): GateOptions;
|
|
65
82
|
export interface RunResult {
|
|
66
83
|
selection: Selection;
|
|
67
84
|
framework: Framework;
|
|
@@ -70,7 +87,21 @@ export interface RunResult {
|
|
|
70
87
|
spent: Spend | null;
|
|
71
88
|
}
|
|
72
89
|
export declare function run(opts?: RunOptions): Promise<RunResult>;
|
|
73
|
-
/**
|
|
90
|
+
/**
|
|
91
|
+
* Re-gate a recorded run, offline.
|
|
92
|
+
*
|
|
93
|
+
* Under the record's own gate by default, so a bare replay reproduces the
|
|
94
|
+
* selection the run made; a value in `opts` replaces the recorded one, which
|
|
95
|
+
* is what trying another cutoff is. A v1 record kept no gate, and replays
|
|
96
|
+
* under the defaults as it always did.
|
|
97
|
+
*/
|
|
74
98
|
export declare function replay(record: RunRecord, opts?: GateOptions): Selection;
|
|
75
|
-
|
|
99
|
+
/**
|
|
100
|
+
* Write the record to `last.json`, and to `records/<head_sha>.json` when the
|
|
101
|
+
* head is known. Returns every path written, `last.json` first.
|
|
102
|
+
*
|
|
103
|
+
* Whether a record should be written at all -- a fallback should not -- is
|
|
104
|
+
* the caller's decision, see `shouldSaveRecord`.
|
|
105
|
+
*/
|
|
106
|
+
export declare function saveRecord(cwd: string, record: RunRecord): Promise<string[]>;
|
|
76
107
|
export declare function loadRecord(path: string): Promise<RunRecord>;
|
package/dist/run.js
CHANGED
|
@@ -9,18 +9,28 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { readFile, mkdir, writeFile } from "node:fs/promises";
|
|
11
11
|
import { join } from "node:path";
|
|
12
|
-
import { loadDiff, touchesChange } from "./diff.js";
|
|
12
|
+
import { loadDiff, resolveSha, touchesChange } from "./diff.js";
|
|
13
13
|
import { discoverTests } from "./discover.js";
|
|
14
14
|
import { buildState } from "./state.js";
|
|
15
15
|
import { buildQuestion, questionId, readAnswer } from "./questions.js";
|
|
16
|
-
import { gate } from "./gate.js";
|
|
16
|
+
import { gate, gateOptions, resolveGate } from "./gate.js";
|
|
17
17
|
import { buildFilter } from "./filter.js";
|
|
18
18
|
import { listPlaywrightTests } from "./playwright.js";
|
|
19
19
|
import { Jev, mapLimit, DEFAULT_CONCURRENCY } from "./jev.js";
|
|
20
|
+
import { lookup } from "./context.js";
|
|
20
21
|
import { testId } from "./types.js";
|
|
21
22
|
/** Where a run's answers are kept, for `--replay`. */
|
|
22
23
|
export const RECORD_DIR = ".jev-test-filter";
|
|
23
24
|
export const RECORD_FILE = "last.json";
|
|
25
|
+
/**
|
|
26
|
+
* One record per commit, under `RECORD_DIR`, named `<head_sha>.json`.
|
|
27
|
+
*
|
|
28
|
+
* `last.json` alone is overwritten by the next run, and the run a CI result
|
|
29
|
+
* has to be compared with is rarely the last one on this machine. Keyed by
|
|
30
|
+
* the head rather than by time because the head is what the other side of
|
|
31
|
+
* that comparison knows.
|
|
32
|
+
*/
|
|
33
|
+
export const RECORDS_SUBDIR = "records";
|
|
24
34
|
/**
|
|
25
35
|
* The framework the run is for.
|
|
26
36
|
*
|
|
@@ -51,18 +61,22 @@ export function collect(tests, ranges) {
|
|
|
51
61
|
* null: the gate selects a test with no answer, and a zero here would silently
|
|
52
62
|
* deselect it instead.
|
|
53
63
|
*/
|
|
54
|
-
export async function score(tests, state, { client = null, concurrency = DEFAULT_CONCURRENCY, batchSize = 400 } = {}) {
|
|
64
|
+
export async function score(tests, state, { client = null, concurrency = DEFAULT_CONCURRENCY, batchSize = 400, context = null } = {}) {
|
|
55
65
|
const jev = client ?? new Jev();
|
|
56
66
|
const out = new Map();
|
|
57
67
|
tests.forEach((_, i) => out.set(questionId(i), null));
|
|
68
|
+
// Question ids stay the test's index in `tests` even with skipped tests
|
|
69
|
+
// left out, so a record's answers line up with its tests either way.
|
|
70
|
+
const asked = [];
|
|
71
|
+
tests.forEach((t, i) => {
|
|
72
|
+
if (context?.skipped(t))
|
|
73
|
+
return;
|
|
74
|
+
const id = questionId(i);
|
|
75
|
+
asked.push([id, buildQuestion(t, id, context?.failedWith(t) ?? [])]);
|
|
76
|
+
});
|
|
58
77
|
const batches = [];
|
|
59
|
-
for (let i = 0; i <
|
|
60
|
-
|
|
61
|
-
for (let j = i; j < Math.min(i + batchSize, tests.length); j += 1) {
|
|
62
|
-
const id = questionId(j);
|
|
63
|
-
batch[id] = buildQuestion(tests[j], id);
|
|
64
|
-
}
|
|
65
|
-
batches.push(batch);
|
|
78
|
+
for (let i = 0; i < asked.length; i += batchSize) {
|
|
79
|
+
batches.push(Object.fromEntries(asked.slice(i, i + batchSize)));
|
|
66
80
|
}
|
|
67
81
|
const responses = await mapLimit(batches, concurrency, (batch) => jev.askSplitting(state, batch));
|
|
68
82
|
for (const res of responses) {
|
|
@@ -73,6 +87,38 @@ export async function score(tests, state, { client = null, concurrency = DEFAULT
|
|
|
73
87
|
}
|
|
74
88
|
return out;
|
|
75
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* The gate a run decides under: the options' own values over the context's,
|
|
92
|
+
* over the defaults. Only values actually present count, so an option passed
|
|
93
|
+
* as `undefined` does not erase the context's value.
|
|
94
|
+
*/
|
|
95
|
+
export function effectiveGate(opts, context = null) {
|
|
96
|
+
const out = {};
|
|
97
|
+
const g = context?.gate;
|
|
98
|
+
if (g?.cutoff !== undefined)
|
|
99
|
+
out.cutoff = g.cutoff;
|
|
100
|
+
if (g?.unsure_below !== undefined)
|
|
101
|
+
out.unsureBelow = g.unsure_below;
|
|
102
|
+
if (g?.unsure_margin !== undefined)
|
|
103
|
+
out.unsureMargin = g.unsure_margin;
|
|
104
|
+
if (opts.cutoff !== undefined)
|
|
105
|
+
out.cutoff = opts.cutoff;
|
|
106
|
+
if (opts.unsureBelow !== undefined)
|
|
107
|
+
out.unsureBelow = opts.unsureBelow;
|
|
108
|
+
if (opts.unsureMargin !== undefined)
|
|
109
|
+
out.unsureMargin = opts.unsureMargin;
|
|
110
|
+
return out;
|
|
111
|
+
}
|
|
112
|
+
/** What labels a record: which change was judged, and under which gate. */
|
|
113
|
+
async function stamp(cwd, opts) {
|
|
114
|
+
const base = opts.base ?? null;
|
|
115
|
+
return {
|
|
116
|
+
head_sha: await resolveSha("HEAD", cwd),
|
|
117
|
+
base_sha: base === null ? null : await resolveSha(base, cwd),
|
|
118
|
+
context_digest: opts.context?.digest ?? null,
|
|
119
|
+
gate: resolveGate(effectiveGate(opts, opts.context ?? null)),
|
|
120
|
+
};
|
|
121
|
+
}
|
|
76
122
|
/**
|
|
77
123
|
* A selection is an optimization and never a correctness gate, so every
|
|
78
124
|
* failure below takes the same exit: run everything, and say why on the
|
|
@@ -106,12 +152,14 @@ export async function run(opts = {}) {
|
|
|
106
152
|
}
|
|
107
153
|
if (all.length === 0) {
|
|
108
154
|
const record = {
|
|
109
|
-
version:
|
|
155
|
+
version: 2,
|
|
110
156
|
createdAt: new Date().toISOString(),
|
|
111
157
|
base: opts.base ?? null,
|
|
158
|
+
...(await stamp(cwd, opts)),
|
|
112
159
|
framework: opts.format ?? "unknown",
|
|
113
160
|
tests: [],
|
|
114
161
|
touched: [],
|
|
162
|
+
quarantined: [],
|
|
115
163
|
answers: {},
|
|
116
164
|
fallback: discoveryFailure ?? "no tests were extracted",
|
|
117
165
|
};
|
|
@@ -120,6 +168,9 @@ export async function run(opts = {}) {
|
|
|
120
168
|
}
|
|
121
169
|
const framework = opts.format ?? pickFramework(all);
|
|
122
170
|
const touched = collect(all, diff.ranges);
|
|
171
|
+
const context = opts.context ? lookup(opts.context) : null;
|
|
172
|
+
const quarantined = new Set(context ? all.filter((t) => context.skipped(t)).map(testId) : []);
|
|
173
|
+
const gateOpts = effectiveGate(opts, opts.context ?? null);
|
|
123
174
|
const state = buildState(diff.text, diff.stat);
|
|
124
175
|
let answers = new Map();
|
|
125
176
|
let spent = null;
|
|
@@ -135,6 +186,7 @@ export async function run(opts = {}) {
|
|
|
135
186
|
client,
|
|
136
187
|
...(opts.concurrency === undefined ? {} : { concurrency: opts.concurrency }),
|
|
137
188
|
...(opts.batchSize === undefined ? {} : { batchSize: opts.batchSize }),
|
|
189
|
+
context,
|
|
138
190
|
});
|
|
139
191
|
spent = client.spent;
|
|
140
192
|
}
|
|
@@ -143,20 +195,24 @@ export async function run(opts = {}) {
|
|
|
143
195
|
}
|
|
144
196
|
// Gating is pure and offline, so it stays outside the try: a bug in the
|
|
145
197
|
// gate must not be reported to the user as a network failure.
|
|
146
|
-
selection = failure === null ? gate(all, answers, touched,
|
|
198
|
+
selection = failure === null ? gate(all, answers, touched, gateOpts, quarantined) : everything(all, failure);
|
|
147
199
|
}
|
|
148
200
|
// A truncated diff that still deselects most of the suite is a selection
|
|
149
201
|
// made from a state that was missing the change it should have judged.
|
|
150
|
-
|
|
202
|
+
// Quarantined tests were never candidates, so they count on neither side.
|
|
203
|
+
const candidates = all.length - quarantined.size;
|
|
204
|
+
if (selection.fallback === null && state.truncated && candidates > 0 && selection.selected.length / candidates < 0.5) {
|
|
151
205
|
selection = everything(all, "the diff did not fit the state budget and the selection was small");
|
|
152
206
|
}
|
|
153
207
|
const record = {
|
|
154
|
-
version:
|
|
208
|
+
version: 2,
|
|
155
209
|
createdAt: new Date().toISOString(),
|
|
156
210
|
base: opts.base ?? null,
|
|
211
|
+
...(await stamp(cwd, opts)),
|
|
157
212
|
framework,
|
|
158
213
|
tests: all,
|
|
159
214
|
touched: [...touched],
|
|
215
|
+
quarantined: [...quarantined],
|
|
160
216
|
answers: Object.fromEntries(answers),
|
|
161
217
|
fallback: selection.fallback,
|
|
162
218
|
};
|
|
@@ -165,23 +221,58 @@ export async function run(opts = {}) {
|
|
|
165
221
|
: { mode: "all", argv: [] };
|
|
166
222
|
return { selection, framework, filter, record, spent };
|
|
167
223
|
}
|
|
168
|
-
/**
|
|
224
|
+
/**
|
|
225
|
+
* Re-gate a recorded run, offline.
|
|
226
|
+
*
|
|
227
|
+
* Under the record's own gate by default, so a bare replay reproduces the
|
|
228
|
+
* selection the run made; a value in `opts` replaces the recorded one, which
|
|
229
|
+
* is what trying another cutoff is. A v1 record kept no gate, and replays
|
|
230
|
+
* under the defaults as it always did.
|
|
231
|
+
*/
|
|
169
232
|
export function replay(record, opts = {}) {
|
|
170
233
|
if (record.fallback !== null)
|
|
171
234
|
return everything(record.tests, record.fallback);
|
|
172
235
|
const answers = new Map(Object.entries(record.answers));
|
|
173
|
-
return gate(record.tests, answers, new Set(record.touched), opts);
|
|
236
|
+
return gate(record.tests, answers, new Set(record.touched), { ...gateOptions(record.gate), ...effectiveGate(opts) }, new Set(record.quarantined));
|
|
174
237
|
}
|
|
238
|
+
/**
|
|
239
|
+
* Write the record to `last.json`, and to `records/<head_sha>.json` when the
|
|
240
|
+
* head is known. Returns every path written, `last.json` first.
|
|
241
|
+
*
|
|
242
|
+
* Whether a record should be written at all -- a fallback should not -- is
|
|
243
|
+
* the caller's decision, see `shouldSaveRecord`.
|
|
244
|
+
*/
|
|
175
245
|
export async function saveRecord(cwd, record) {
|
|
176
246
|
const dir = join(cwd, RECORD_DIR);
|
|
177
247
|
await mkdir(dir, { recursive: true });
|
|
178
|
-
const
|
|
179
|
-
|
|
180
|
-
|
|
248
|
+
const text = `${JSON.stringify(record, null, 2)}\n`;
|
|
249
|
+
const paths = [join(dir, RECORD_FILE)];
|
|
250
|
+
if (record.head_sha !== null) {
|
|
251
|
+
await mkdir(join(dir, RECORDS_SUBDIR), { recursive: true });
|
|
252
|
+
paths.push(join(dir, RECORDS_SUBDIR, `${record.head_sha}.json`));
|
|
253
|
+
}
|
|
254
|
+
for (const path of paths)
|
|
255
|
+
await writeFile(path, text, "utf8");
|
|
256
|
+
return paths;
|
|
181
257
|
}
|
|
182
258
|
export async function loadRecord(path) {
|
|
183
259
|
const parsed = JSON.parse(await readFile(path, "utf8"));
|
|
184
|
-
if (parsed.version
|
|
185
|
-
|
|
186
|
-
|
|
260
|
+
if (parsed.version === 2) {
|
|
261
|
+
// Written by this version, or by hand; a missing label reads as unknown.
|
|
262
|
+
const v2 = parsed;
|
|
263
|
+
return {
|
|
264
|
+
...v2,
|
|
265
|
+
version: 2,
|
|
266
|
+
head_sha: v2.head_sha ?? null,
|
|
267
|
+
base_sha: v2.base_sha ?? null,
|
|
268
|
+
context_digest: v2.context_digest ?? null,
|
|
269
|
+
gate: v2.gate ?? resolveGate(),
|
|
270
|
+
quarantined: v2.quarantined ?? [],
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
if (parsed.version === 1) {
|
|
274
|
+
const v1 = parsed;
|
|
275
|
+
return { ...v1, head_sha: null, base_sha: null, context_digest: null, gate: null, quarantined: [] };
|
|
276
|
+
}
|
|
277
|
+
throw new Error(`unsupported record version ${String(parsed.version)}`);
|
|
187
278
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -65,7 +65,13 @@ export type Reason =
|
|
|
65
65
|
/** No usable answer came back. Selected to be safe. */
|
|
66
66
|
| "missing"
|
|
67
67
|
/** The score was under the cutoff. Not selected. */
|
|
68
|
-
| "below"
|
|
68
|
+
| "below"
|
|
69
|
+
/**
|
|
70
|
+
* The context from flaker put the test in `skip`. Not asked about and not
|
|
71
|
+
* selected, whatever the diff did to it: a quarantined test fails for
|
|
72
|
+
* reasons of its own, and running it would only say so again.
|
|
73
|
+
*/
|
|
74
|
+
| "quarantined";
|
|
69
75
|
export interface Verdict {
|
|
70
76
|
/** The question this test was asked under, e.g. `q0007`. */
|
|
71
77
|
id: string;
|
|
@@ -94,3 +100,112 @@ export interface Selection {
|
|
|
94
100
|
* confused with one another.
|
|
95
101
|
*/
|
|
96
102
|
export declare function testId(t: TestCase): string;
|
|
103
|
+
/**
|
|
104
|
+
* The gate values a run was decided under, spelled the way `jev-context`
|
|
105
|
+
* spells them so a reader that joins the two never has to translate.
|
|
106
|
+
*/
|
|
107
|
+
export interface RecordGate {
|
|
108
|
+
cutoff: number;
|
|
109
|
+
unsure_below: number;
|
|
110
|
+
unsure_margin: number;
|
|
111
|
+
}
|
|
112
|
+
/** A record as 0.1 wrote it. Still read; never written. */
|
|
113
|
+
export interface RunRecordV1 {
|
|
114
|
+
version: 1;
|
|
115
|
+
createdAt: string;
|
|
116
|
+
base: string | null;
|
|
117
|
+
framework: Framework;
|
|
118
|
+
tests: TestCase[];
|
|
119
|
+
/** `testId` of every test the diff touched. */
|
|
120
|
+
touched: string[];
|
|
121
|
+
/** Keyed by question id, which is the test's index in `tests`. */
|
|
122
|
+
answers: Record<string, Answer | null>;
|
|
123
|
+
fallback: string | null;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* A record as this version writes it: everything a run learned, and enough
|
|
127
|
+
* about where it stood to join it against something else later.
|
|
128
|
+
*
|
|
129
|
+
* The new fields are what make a record comparable. `head_sha` and
|
|
130
|
+
* `base_sha` say which change was judged, which is how a CI run of the same
|
|
131
|
+
* commit finds it; `context_digest` says which hints the questions carried,
|
|
132
|
+
* because a hint changes the question and two runs under different hints are
|
|
133
|
+
* not the same measurement; `gate` says which values turned the answers into
|
|
134
|
+
* a selection, because "this test was not selected" means nothing without
|
|
135
|
+
* the cutoff it was under.
|
|
136
|
+
*
|
|
137
|
+
* snake_case for the new fields only: they are the ones another program is
|
|
138
|
+
* meant to read, and they match the context that fed them. The old fields
|
|
139
|
+
* keep their spelling so a v1 reader's code keeps working.
|
|
140
|
+
*/
|
|
141
|
+
export interface RunRecordV2 extends Omit<RunRecordV1, "version"> {
|
|
142
|
+
version: 2;
|
|
143
|
+
/** `git rev-parse HEAD` at the time of the run; null when there was none. */
|
|
144
|
+
head_sha: string | null;
|
|
145
|
+
/** The sha `--base` resolved to; null without `--base`. */
|
|
146
|
+
base_sha: string | null;
|
|
147
|
+
/** The `digest` of the `--context` the questions were built with; null without one. */
|
|
148
|
+
context_digest: string | null;
|
|
149
|
+
gate: RecordGate;
|
|
150
|
+
/**
|
|
151
|
+
* `testId` of every test the context quarantined. Kept, rather than
|
|
152
|
+
* dropping those tests from `tests`, because a test missing from the run
|
|
153
|
+
* would be run by a filter that selects "everything"; kept by `testId`
|
|
154
|
+
* like `touched`, because the record is a snapshot of one commit.
|
|
155
|
+
*/
|
|
156
|
+
quarantined: string[];
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Any record `loadRecord` accepts, in one shape: a v1 record reads with every
|
|
160
|
+
* field it did not have as null (and nothing quarantined), so the code that
|
|
161
|
+
* replays one never has to branch on the version.
|
|
162
|
+
*/
|
|
163
|
+
export type RunRecord = RunRecordV2 | (Omit<RunRecordV2, "version" | "gate"> & {
|
|
164
|
+
version: 1;
|
|
165
|
+
gate: null;
|
|
166
|
+
});
|
|
167
|
+
/**
|
|
168
|
+
* What flaker knows about the tests, as its `jev-context` projection emits
|
|
169
|
+
* it (`flaker export --projection jev-context`). Version 1.
|
|
170
|
+
*
|
|
171
|
+
* A test is named by `file` + `title_path` (+ `project` for Playwright) and
|
|
172
|
+
* never by `testId`: the line in a `testId` moves with every edit above the
|
|
173
|
+
* test, and flaker's history is about the test, not about where it sat.
|
|
174
|
+
*/
|
|
175
|
+
export interface JevContext {
|
|
176
|
+
version: 1;
|
|
177
|
+
/**
|
|
178
|
+
* sha256 of `skip` and `tests`, computed by flaker. Kept verbatim on the
|
|
179
|
+
* record: a hint changes a question, so runs are only comparable when their
|
|
180
|
+
* digests are equal.
|
|
181
|
+
*/
|
|
182
|
+
digest: string;
|
|
183
|
+
generated_at?: string;
|
|
184
|
+
/** Defaults for the gate. A flag on the command line wins over each. */
|
|
185
|
+
gate: ContextGate | null;
|
|
186
|
+
skip: ContextSkip[];
|
|
187
|
+
tests: ContextTest[];
|
|
188
|
+
}
|
|
189
|
+
export interface ContextGate {
|
|
190
|
+
cutoff?: number;
|
|
191
|
+
unsure_below?: number;
|
|
192
|
+
unsure_margin?: number;
|
|
193
|
+
/** What the values were calibrated from. Carried, never read. */
|
|
194
|
+
basis?: unknown;
|
|
195
|
+
}
|
|
196
|
+
export interface ContextSkip {
|
|
197
|
+
file: string;
|
|
198
|
+
title_path: string[];
|
|
199
|
+
/** Absent: the test in every project. */
|
|
200
|
+
project?: string;
|
|
201
|
+
reason?: string;
|
|
202
|
+
}
|
|
203
|
+
export interface ContextTest {
|
|
204
|
+
file: string;
|
|
205
|
+
title_path: string[];
|
|
206
|
+
project?: string;
|
|
207
|
+
/** Changed files this test failed with before, most telling first. */
|
|
208
|
+
failed_with: string[];
|
|
209
|
+
/** How often a selector left it out when it then failed. Carried, never asked. */
|
|
210
|
+
missed?: number;
|
|
211
|
+
}
|