jev-test-filter 0.1.2 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +184 -21
- package/dist/cli.d.ts +23 -1
- package/dist/cli.js +92 -15
- package/dist/context.d.ts +18 -0
- package/dist/context.js +156 -0
- package/dist/diff.d.ts +13 -0
- package/dist/diff.js +27 -2
- package/dist/discover.js +13 -2
- package/dist/extract-rspec.d.ts +20 -0
- package/dist/extract-rspec.js +177 -0
- package/dist/filter.d.ts +3 -0
- package/dist/filter.js +17 -0
- package/dist/framework.d.ts +1 -0
- package/dist/framework.js +6 -1
- package/dist/gate.d.ts +14 -3
- package/dist/gate.js +21 -2
- package/dist/languages.js +3 -2
- package/dist/questions.d.ts +10 -15
- package/dist/questions.js +38 -1
- package/dist/run.d.ts +46 -15
- package/dist/run.js +113 -22
- package/dist/types.d.ts +125 -2
- package/package.json +7 -6
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
alter the outcome of every single test in the repository, and prints the filter
|
|
5
5
|
arguments your test runner already understands. It hands those arguments to the
|
|
6
6
|
runner you already use — vitest, jest, `node --test`, `bun test`, Playwright,
|
|
7
|
-
`cargo test
|
|
7
|
+
`cargo test`, `go test` or `rspec` — so the run covers the tests the change could plausibly break
|
|
8
8
|
instead of all of them.
|
|
9
9
|
|
|
10
10
|
It uses [Jev](https://typesafe.ai), TypeSafe's System One model: one shared
|
|
@@ -14,7 +14,8 @@ loop and no file reading by the model.
|
|
|
14
14
|
## Install
|
|
15
15
|
|
|
16
16
|
Requires Node 24 or newer, `git`, and a TypeSafe API key. `--format rust` and
|
|
17
|
-
`--format go` additionally need `cargo` and `go` on the `PATH
|
|
17
|
+
`--format go` additionally need `cargo` and `go` on the `PATH`; `--format
|
|
18
|
+
rspec` needs nothing beyond the `rspec` you already run.
|
|
18
19
|
|
|
19
20
|
### The command
|
|
20
21
|
|
|
@@ -215,7 +216,7 @@ like, because both mean "no arguments". The exit code tells them apart:
|
|
|
215
216
|
| --- | --- |
|
|
216
217
|
| 0 | Arguments were printed, or everything is to be run and there are none. Run the command. |
|
|
217
218
|
| 3 | Nothing was selected. Do not run the command; there is nothing to run. |
|
|
218
|
-
| 2 | Bad arguments, e.g. `unknown --format mocha
|
|
219
|
+
| 2 | Bad arguments, e.g. `unknown --format mocha`, a gate value that is not a number, or a `--context` file that is missing, malformed, or not version 1. |
|
|
219
220
|
| 1 | The tool failed, e.g. the tests span more than one framework and no `--format` was given. |
|
|
220
221
|
|
|
221
222
|
So the stdout form is used like this:
|
|
@@ -240,13 +241,19 @@ not considered in that mode. Playwright with `--format playwright --exec` uses
|
|
|
240
241
|
Playwright's own collected list instead. For source discovery, a file counts
|
|
241
242
|
as a test file when it is named
|
|
242
243
|
`*.test.*`, `*.vitest.*` or `*.spec.*` with a `.js`, `.jsx`, `.ts`, `.tsx`, `.mjs`, `.cjs`,
|
|
243
|
-
`.mts` or `.cts` extension,
|
|
244
|
-
convention and the only one `go test` compiles into a test binary.
|
|
244
|
+
`.mts` or `.cts` extension, when it ends in `_test.go`, which is Go's own
|
|
245
|
+
convention and the only one `go test` compiles into a test binary. Under
|
|
246
|
+
`--format rspec`, a file ending in `_spec.rb` counts too, which is what
|
|
247
|
+
RSpec's default `--pattern` looks for.
|
|
245
248
|
|
|
246
|
-
**Rust
|
|
249
|
+
**Rust and RSpec are never discovered automatically.** Listing a
|
|
247
250
|
crate's tests means running `cargo test -- --list`, and that builds the test
|
|
248
251
|
targets. A tool that triggers a compile nobody asked for is a tool that gets
|
|
249
252
|
removed from the workflow, so it happens only when you write `--format rust`.
|
|
253
|
+
RSpec examples cost only a parse, but a Rails application with Vitest or Jest
|
|
254
|
+
tests for its front end is common, and discovering its `_spec.rb` files unasked
|
|
255
|
+
would turn that repository's `--exec -- vitest run` into a mixed-framework
|
|
256
|
+
error. They are read only when you write `--format rspec`.
|
|
250
257
|
Every other language is read out of the source and costs nothing but a parse.
|
|
251
258
|
|
|
252
259
|
### Rust and Go
|
|
@@ -318,6 +325,41 @@ test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 3 filtered out
|
|
|
318
325
|
It emitted `-- --exact cart::tests::apply_discount::clamps_at_zero
|
|
319
326
|
cart::tests::apply_discount::halves_the_total`.
|
|
320
327
|
|
|
328
|
+
### RSpec
|
|
329
|
+
|
|
330
|
+
```
|
|
331
|
+
$ jev-test-filter --format rspec --exec -- bundle exec rspec
|
|
332
|
+
```
|
|
333
|
+
|
|
334
|
+
Examples are read out of the `*_spec.rb` sources, not listed by
|
|
335
|
+
`rspec --dry-run`: that loads every spec file, and in a Rails application it
|
|
336
|
+
boots the application. `--format rspec` is required, even in a repository with
|
|
337
|
+
no other tests. Do not give `rspec` its own paths; the tool appends the
|
|
338
|
+
locations it chose.
|
|
339
|
+
|
|
340
|
+
The same cart again, in Ruby, with the discount broken (rspec-core 3.13.6,
|
|
341
|
+
ruby 3.3.8):
|
|
342
|
+
|
|
343
|
+
```
|
|
344
|
+
$ rspec # unfiltered
|
|
345
|
+
5 examples, 1 failure
|
|
346
|
+
|
|
347
|
+
$ jev-test-filter --format rspec --exec -- rspec
|
|
348
|
+
jev-test-filter: 2/5 tests selected (locations)
|
|
349
|
+
Run options: include {:locations=>{"./spec/cart_spec.rb"=>[7, 11]}}
|
|
350
|
+
F.
|
|
351
|
+
2 examples, 1 failure
|
|
352
|
+
|
|
353
|
+
Failed examples:
|
|
354
|
+
|
|
355
|
+
rspec ./spec/cart_spec.rb:5 # Cart.apply_discount halves the total
|
|
356
|
+
```
|
|
357
|
+
|
|
358
|
+
It emitted `spec/cart_spec.rb:7:11`, the last lines of the two
|
|
359
|
+
`.apply_discount` examples; `spec/ship_spec.rb` was not loaded at all. Why
|
|
360
|
+
the last line rather than the first is in
|
|
361
|
+
[What it emits, per framework](#what-it-emits-per-framework).
|
|
362
|
+
|
|
321
363
|
`--base <ref>` compares `<ref>...HEAD`, the way a pull request does. Without
|
|
322
364
|
it, the working tree against `HEAD` is used; `--staged` uses the index.
|
|
323
365
|
|
|
@@ -337,6 +379,7 @@ before you hand-edit anything it prints.
|
|
|
337
379
|
| `playwright` | runner's test title and project | `--test-list .jev-test-filter/playwright-….txt` with `--exec`; otherwise `file:line` | exact list or positionals |
|
|
338
380
|
| `rust` | `"::"` — `cart::tests::halves_the_total` | `-- --exact <name> <name>…`, one invocation | after a literal `--` |
|
|
339
381
|
| `go` | `"/"` — `TestApplyDiscount/halves the total` | `-run '^(?:TestA\|TestB)$'` plus the `./pkg` directories | pattern **before** the packages |
|
|
382
|
+
| `rspec` | a space, none before `#`/`.`/`::` after a class — `Cart#total sums` | `spec/a_spec.rb:12:30`, one argument per file | positionals |
|
|
340
383
|
|
|
341
384
|
Bun 1.3.5 prints `Cart > totals` in its reporter, but `--test-name-pattern`
|
|
342
385
|
matches `Cart totals`. The pattern above was verified by running Bun itself.
|
|
@@ -394,15 +437,46 @@ cargo 1.98.0 and go 1.26.2:
|
|
|
394
437
|
all. `--json` still reports the per-subtest score, which is the finer signal
|
|
395
438
|
and what a reader wants to see.
|
|
396
439
|
|
|
397
|
-
|
|
398
|
-
|
|
440
|
+
And for RSpec, measured on rspec-core 3.13.6 and ruby 3.3.8:
|
|
441
|
+
|
|
442
|
+
- **An example is named by its last line.** RSpec runs, for a requested line,
|
|
443
|
+
the example or group declared nearest at or above it, and it records a
|
|
444
|
+
multi-line call at a line of its own choosing: `it "a",\n :slow do` at its
|
|
445
|
+
first line, `it(\n "a"\n) do` at its last. The first line of the second
|
|
446
|
+
shape resolves to the example *before* it — a wrong test, run green. The last
|
|
447
|
+
line cannot, because nothing is declared inside an example body.
|
|
448
|
+
- **It is never selected by name.** A full description depends on whether a
|
|
449
|
+
group was described by a class or a string (`Cart#total` against
|
|
450
|
+
`Cart #total`), and a name pattern spelled one way matches nothing spelled
|
|
451
|
+
the other. A location has no spelling, so an interpolated title, a
|
|
452
|
+
description-less one-liner (`it { is_expected.to … }`) and the rows of an
|
|
453
|
+
`.each` loop are all scored like any other test rather than selected unasked.
|
|
454
|
+
- **Shared examples** are scored as one test per inclusion. `it_behaves_like`
|
|
455
|
+
declares a nested group at its own line (the opening of its block, when
|
|
456
|
+
present), so that line runs everything it brought in, including examples
|
|
457
|
+
in a customization block. `include_examples` and `include_context` copy
|
|
458
|
+
the examples into the enclosing group
|
|
459
|
+
with the lines of the file that defined them, so it is named by the line
|
|
460
|
+
that group's block opens on, which runs the whole group.
|
|
461
|
+
- **A `shared_examples` block defined in a `_spec.rb` file is reached only
|
|
462
|
+
through that file.** Its examples are named by their own lines, which run
|
|
463
|
+
the inclusions in the same file; an `it_behaves_like` in another file is a
|
|
464
|
+
test of its own and is scored like any other. Shared examples usually live
|
|
465
|
+
in `spec/support`, which is not a spec file, so this rarely matters.
|
|
466
|
+
|
|
467
|
+
The name `--json` reports is for reading only: the class-versus-string
|
|
468
|
+
distinction is gone once a title is a string, so a group described by the
|
|
469
|
+
string `"Cart"` is joined as if it were the class.
|
|
470
|
+
|
|
471
|
+
The vitest, node:test, Bun, Playwright, Rust, Go and RSpec spellings were each
|
|
472
|
+
verified against the real runner. `jest` is emitted with the vitest shape and was not.
|
|
399
473
|
|
|
400
474
|
The `mode` field in `--json` names the emitted filter shape:
|
|
401
475
|
|
|
402
476
|
| `mode` | `argv` | Meaning |
|
|
403
477
|
| --- | --- | --- |
|
|
404
478
|
| `pattern` | flag, pattern, files | The normal case. |
|
|
405
|
-
| `locations` | `file:line`… | Playwright. |
|
|
479
|
+
| `locations` | `file:line`… | Playwright, and RSpec (`file:line:line`, one per file). |
|
|
406
480
|
| `test-list` | `--test-list <file>` | Playwright with runner discovery; `test_list` in JSON holds the file's lines. |
|
|
407
481
|
| `exact` | `--`, `--exact`, names… | Rust. Every name came from `cargo test -- --list`. |
|
|
408
482
|
| `files` | files | A name pattern could not express the selection, so whole files were chosen. This happens when a selected test has a non-literal title, or when more than 80% of the suite was selected and the alternation would not be worth it. |
|
|
@@ -429,8 +503,8 @@ Two rules sit on top of that, and both exist to fail towards running a test
|
|
|
429
503
|
rather than skipping it:
|
|
430
504
|
|
|
431
505
|
- **Confidence routes, it does not gate.** A test that scores under the cutoff,
|
|
432
|
-
but within 1.0 of it, and that the model answered with a
|
|
433
|
-
0.5, is selected anyway. Running a test that did not need to run costs
|
|
506
|
+
but within 1.0 of it (`--unsure-margin`), and that the model answered with a
|
|
507
|
+
confidence below 0.5 (`--unsure-below`), is selected anyway. Running a test that did not need to run costs
|
|
434
508
|
seconds; skipping one that did costs a release.
|
|
435
509
|
- **No answer is not a passing grade.** A missing or malformed answer selects
|
|
436
510
|
the test.
|
|
@@ -442,7 +516,9 @@ across invocations, take it once and re-derive it with `--replay`.
|
|
|
442
516
|
|
|
443
517
|
The `reason` field on each test in `--json` says which rule decided it:
|
|
444
518
|
`touched` (the test's own body is inside the diff — never asked about),
|
|
445
|
-
`dynamic` (the title is not a literal), `scored`, `unsure`, `missing`, `below
|
|
519
|
+
`dynamic` (the title is not a literal), `scored`, `unsure`, `missing`, `below`,
|
|
520
|
+
and `quarantined` (the `--context` put the test in `skip` — never asked about,
|
|
521
|
+
never selected).
|
|
446
522
|
|
|
447
523
|
## When it runs everything anyway
|
|
448
524
|
|
|
@@ -486,13 +562,16 @@ framework with no `--format`, which exits 1 (see Known limitations).
|
|
|
486
562
|
```
|
|
487
563
|
--base <ref> compare against <ref>...HEAD, as a pull request does
|
|
488
564
|
--staged use the staged change instead of the working tree
|
|
489
|
-
--format <name> vitest | jest | node | bun | playwright | rust | go | auto (default: auto)
|
|
565
|
+
--format <name> vitest | jest | node | bun | playwright | rust | go | rspec | auto (default: auto)
|
|
490
566
|
--cutoff <n> select at or above this score level (default: 2)
|
|
567
|
+
--unsure-below <n> a confidence under this counts as unsure (default: 0.5)
|
|
568
|
+
--unsure-margin <n> rescue an unsure answer this far under the cutoff (default: 1)
|
|
491
569
|
--concurrency <n> requests in flight at once (default: 32)
|
|
492
570
|
--json print the full scoring instead of the arguments
|
|
493
571
|
--dry-run extract and report without calling Jev
|
|
494
572
|
--verify-snapshots review changed text snapshots without modifying files
|
|
495
573
|
--replay <file> re-gate a recorded run offline (default: .jev-test-filter/last.json)
|
|
574
|
+
--context <file> read flaker's jev-context: quarantined tests, failure history, gate defaults
|
|
496
575
|
--exec -- <cmd...> append the arguments to <cmd...> and run it
|
|
497
576
|
-h, --help the help text
|
|
498
577
|
```
|
|
@@ -500,6 +579,11 @@ framework with no `--format`, which exits 1 (see Known limitations).
|
|
|
500
579
|
Positional arguments restrict which test files are considered, e.g.
|
|
501
580
|
`jev-test-filter --base main src/cart test/cart.test.ts`.
|
|
502
581
|
|
|
582
|
+
The three gate values come from, in order of precedence: the flag, then the
|
|
583
|
+
`--context` (or, under `--replay`, the record), then the default. A value that
|
|
584
|
+
is not a finite number is refused with exit 2 rather than read as `NaN`, which
|
|
585
|
+
would deselect every test.
|
|
586
|
+
|
|
503
587
|
## Replaying a run offline
|
|
504
588
|
|
|
505
589
|
Every run that reaches Jev writes its answers to
|
|
@@ -514,16 +598,95 @@ $ jev-test-filter --replay --cutoff 1.0 > /dev/null
|
|
|
514
598
|
jev-test-filter: 5/110 tests selected (pattern)
|
|
515
599
|
```
|
|
516
600
|
|
|
601
|
+
A replay re-gates under the gate the record was decided under, so a bare
|
|
602
|
+
`--replay` reproduces the run's selection; `--cutoff`, `--unsure-below` and
|
|
603
|
+
`--unsure-margin` replace the recorded values one by one. A record written by
|
|
604
|
+
0.1 kept no gate and replays under the defaults, as it always did.
|
|
605
|
+
|
|
517
606
|
A replay reports `"spent": null` because it spent nothing, and it needs no API
|
|
518
607
|
key. It writes no new record, and `--format` has no effect on it — the
|
|
519
|
-
framework comes from the record.
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
successful run
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
608
|
+
framework comes from the record. `--context` cannot be combined with it: the
|
|
609
|
+
record already holds the answers its context produced.
|
|
610
|
+
|
|
611
|
+
Every successful run writes the record twice: to `.jev-test-filter/last.json`,
|
|
612
|
+
which the next run replaces, and to `.jev-test-filter/records/<head_sha>.json`,
|
|
613
|
+
which only another run of the same commit replaces — that is the one a CI
|
|
614
|
+
result for the commit is compared with. **Only** a successful run writes
|
|
615
|
+
either: a run that falls back writes nothing, so `--replay` always has the
|
|
616
|
+
last scoring that actually completed to work on, and an unlucky `HTTP 529`
|
|
617
|
+
cannot destroy it. The record holds every test's title and file path, so add
|
|
618
|
+
`.jev-test-filter/` to your `.gitignore`. It never holds the API key.
|
|
619
|
+
|
|
620
|
+
### Record format (version 2)
|
|
621
|
+
|
|
622
|
+
```json
|
|
623
|
+
{
|
|
624
|
+
"version": 2,
|
|
625
|
+
"createdAt": "2026-09-24T00:00:00.000Z",
|
|
626
|
+
"base": "main",
|
|
627
|
+
"head_sha": "4c1d…",
|
|
628
|
+
"base_sha": "9e0a…",
|
|
629
|
+
"context_digest": "sha256:…",
|
|
630
|
+
"gate": { "cutoff": 2, "unsure_below": 0.5, "unsure_margin": 1 },
|
|
631
|
+
"framework": "vitest",
|
|
632
|
+
"tests": [{ "file": "test/a.test.ts", "titlePath": ["A", "b"], "line": 3, "endLine": 5, "framework": "vitest", "dynamic": false }],
|
|
633
|
+
"touched": [],
|
|
634
|
+
"quarantined": [],
|
|
635
|
+
"answers": { "q0000": { "value": 1, "confidence": 0.8 } },
|
|
636
|
+
"fallback": null
|
|
637
|
+
}
|
|
638
|
+
```
|
|
639
|
+
|
|
640
|
+
- `head_sha` is `git rev-parse HEAD`, `null` when there is no commit.
|
|
641
|
+
- `base_sha` is the commit `--base` resolved to, `null` without `--base`.
|
|
642
|
+
- `context_digest` is the `digest` of the `--context` file, `null` without one.
|
|
643
|
+
- `gate` is the values the run actually decided under, defaults filled in.
|
|
644
|
+
- `touched` and `quarantined` hold test ids (file, title path and line); the
|
|
645
|
+
answers are keyed by question id, which is the test's index in `tests`.
|
|
646
|
+
|
|
647
|
+
A version 1 record is still read, with those fields as `null` and nothing
|
|
648
|
+
quarantined. The type is exported as `RunRecordV2` from
|
|
649
|
+
`jev-test-filter/types`.
|
|
650
|
+
|
|
651
|
+
## Context from flaker
|
|
652
|
+
|
|
653
|
+
[flaker](https://github.com/mizchi/flaker) keeps the history this tool cannot
|
|
654
|
+
see, and projects it into one file for it:
|
|
655
|
+
|
|
656
|
+
```
|
|
657
|
+
flaker export --projection jev-context -o .flaker/context.json
|
|
658
|
+
jev-test-filter --base main --context .flaker/context.json --exec -- vitest run
|
|
659
|
+
```
|
|
660
|
+
|
|
661
|
+
```json
|
|
662
|
+
{
|
|
663
|
+
"version": 1,
|
|
664
|
+
"digest": "sha256:…",
|
|
665
|
+
"generated_at": "2026-09-24T00:00:00.000Z",
|
|
666
|
+
"gate": { "cutoff": 2.0, "unsure_below": 0.5, "unsure_margin": 1.0, "basis": { "records": 42 } },
|
|
667
|
+
"skip": [{ "file": "tests/a.test.ts", "title_path": ["A", "b"], "reason": "quarantined" }],
|
|
668
|
+
"tests": [{ "file": "tests/cli/init.test.ts", "title_path": ["init", "writes toml"], "failed_with": ["src/cli/config.ts"], "missed": 2 }]
|
|
669
|
+
}
|
|
670
|
+
```
|
|
671
|
+
|
|
672
|
+
- **`gate`** supplies the defaults for `--cutoff`, `--unsure-below` and
|
|
673
|
+
`--unsure-margin`. A flag still wins. The gate itself stays here: flaker
|
|
674
|
+
only suggests values, it never decides a selection.
|
|
675
|
+
- **`skip`** removes a test from the candidates. It is not asked about, is
|
|
676
|
+
not selected even if the diff touched it, and is reported with
|
|
677
|
+
`reason: "quarantined"`.
|
|
678
|
+
- **`tests[].failed_with`** is added to that test's question, and only
|
|
679
|
+
that test's, as one sentence under `instructions.history`: `This test
|
|
680
|
+
previously failed when src/cli/config.ts changed.` It is a fact, not a
|
|
681
|
+
threshold; `missed` is carried but never put into a question. A test the
|
|
682
|
+
context says nothing about is asked exactly what it was asked without one.
|
|
683
|
+
|
|
684
|
+
Entries are matched on `file` + `title_path`, and on `project` when the entry
|
|
685
|
+
names one (an entry without a `project` covers the test in every Playwright
|
|
686
|
+
project) — never on the line, which moves with every edit above the test.
|
|
687
|
+
Anything other than `version: 1`, or a shape that does not validate, exits 2
|
|
688
|
+
before a question is asked. The record keeps the context's `digest`, because
|
|
689
|
+
a hint changes the question and runs are only comparable under the same one.
|
|
527
690
|
|
|
528
691
|
## Cost and latency
|
|
529
692
|
|
package/dist/cli.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import type {
|
|
2
|
+
import type { Diff } from "./diff.ts";
|
|
3
|
+
import type { SnapshotChange, SnapshotReview, SnapshotTestSources } from "./snapshot.ts";
|
|
3
4
|
import type { RunRecord, RunResult } from "./run.ts";
|
|
5
|
+
import type { GateOptions } from "./gate.ts";
|
|
4
6
|
import type { Framework } from "./types.ts";
|
|
5
7
|
/** Where a run leaves its answers, and what a bare `--replay` means. */
|
|
6
8
|
export declare const DEFAULT_RECORD_PATH = ".jev-test-filter/last.json";
|
|
@@ -10,11 +12,14 @@ export interface CliArgs {
|
|
|
10
12
|
paths: string[];
|
|
11
13
|
format: Framework | null;
|
|
12
14
|
cutoff: number | undefined;
|
|
15
|
+
unsureBelow: number | undefined;
|
|
16
|
+
unsureMargin: number | undefined;
|
|
13
17
|
concurrency: number | undefined;
|
|
14
18
|
json: boolean;
|
|
15
19
|
dryRun: boolean;
|
|
16
20
|
verifySnapshots: boolean;
|
|
17
21
|
replayPath: string | null;
|
|
22
|
+
contextPath: string | null;
|
|
18
23
|
exec: string[] | null;
|
|
19
24
|
help: boolean;
|
|
20
25
|
}
|
|
@@ -23,6 +28,14 @@ export interface CliArgs {
|
|
|
23
28
|
* own flags never have to be escaped past this parser.
|
|
24
29
|
*/
|
|
25
30
|
export declare function parseCliArgs(argv: string[]): CliArgs;
|
|
31
|
+
/**
|
|
32
|
+
* The gate values the command line set, and nothing else.
|
|
33
|
+
*
|
|
34
|
+
* Absent rather than defaulted, because a flag is only one of the places a
|
|
35
|
+
* gate value comes from: whatever it leaves unset falls through to the
|
|
36
|
+
* record being replayed, then to the gate's own defaults.
|
|
37
|
+
*/
|
|
38
|
+
export declare function gateFlags(args: CliArgs): GateOptions;
|
|
26
39
|
export declare function renderLine(res: RunResult): string;
|
|
27
40
|
/** The command to run, or null when nothing was selected. */
|
|
28
41
|
export declare function execArgv(cmd: string[], res: RunResult): string[] | null;
|
|
@@ -52,3 +65,12 @@ export declare function stdoutExitCode(res: RunResult): number;
|
|
|
52
65
|
export declare function shouldSaveRecord(record: RunRecord): boolean;
|
|
53
66
|
export declare function renderJson(res: RunResult): string;
|
|
54
67
|
export declare function renderSnapshotReport(review: SnapshotReview): string;
|
|
68
|
+
/**
|
|
69
|
+
* The snapshot changes under review, narrowed to the paths asked for, with the
|
|
70
|
+
* test sources that explain them.
|
|
71
|
+
*/
|
|
72
|
+
export declare function collectSnapshotChanges(args: Pick<CliArgs, "base" | "staged" | "paths">, cwd?: string): Promise<{
|
|
73
|
+
diff: Diff;
|
|
74
|
+
changes: SnapshotChange[];
|
|
75
|
+
context: SnapshotTestSources;
|
|
76
|
+
}>;
|
package/dist/cli.js
CHANGED
|
@@ -23,17 +23,35 @@
|
|
|
23
23
|
import { spawn } from "node:child_process";
|
|
24
24
|
import { realpathSync } from "node:fs";
|
|
25
25
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
26
|
-
import { dirname, join } from "node:path";
|
|
26
|
+
import { dirname, join, posix } from "node:path";
|
|
27
27
|
import { fileURLToPath } from "node:url";
|
|
28
28
|
import { parseArgs } from "node:util";
|
|
29
29
|
import { displayName } from "./questions.js";
|
|
30
30
|
import { fullName } from "./filter.js";
|
|
31
31
|
import { loadDiff } from "./diff.js";
|
|
32
32
|
import { assessSnapshots, findSnapshotChangesInWorktree, loadSnapshotTestSources, snapshotTestFile } from "./snapshot.js";
|
|
33
|
+
import { loadContext } from "./context.js";
|
|
33
34
|
import { loadRecord, RECORD_DIR, RECORD_FILE, replay, run, saveRecord } from "./run.js";
|
|
34
|
-
const FORMATS = ["vitest", "jest", "node", "bun", "playwright", "rust", "go", "auto"];
|
|
35
|
+
const FORMATS = ["vitest", "jest", "node", "bun", "playwright", "rust", "go", "rspec", "auto"];
|
|
35
36
|
/** Where a run leaves its answers, and what a bare `--replay` means. */
|
|
36
37
|
export const DEFAULT_RECORD_PATH = `${RECORD_DIR}/${RECORD_FILE}`;
|
|
38
|
+
/**
|
|
39
|
+
* One gate value from the command line.
|
|
40
|
+
*
|
|
41
|
+
* `Number()` alone is not enough: `Number("abc")` is NaN, and NaN compares
|
|
42
|
+
* false against every score, so a typo in `--cutoff` would deselect the whole
|
|
43
|
+
* suite without a word. An empty string is refused for the same reason
|
|
44
|
+
* `Number("")` is 0.
|
|
45
|
+
*/
|
|
46
|
+
function gateNumber(flag, raw) {
|
|
47
|
+
if (raw === undefined)
|
|
48
|
+
return undefined;
|
|
49
|
+
const text = String(raw).trim();
|
|
50
|
+
const n = Number(text);
|
|
51
|
+
if (text === "" || !Number.isFinite(n))
|
|
52
|
+
throw new Error(`${flag} needs a number, got ${JSON.stringify(String(raw))}`);
|
|
53
|
+
return n;
|
|
54
|
+
}
|
|
37
55
|
/**
|
|
38
56
|
* `--exec` takes everything after the `--` that follows it, so the runner's
|
|
39
57
|
* own flags never have to be escaped past this parser.
|
|
@@ -63,11 +81,14 @@ export function parseCliArgs(argv) {
|
|
|
63
81
|
staged: { type: "boolean", default: false },
|
|
64
82
|
format: { type: "string", default: "auto" },
|
|
65
83
|
cutoff: { type: "string" },
|
|
84
|
+
"unsure-below": { type: "string" },
|
|
85
|
+
"unsure-margin": { type: "string" },
|
|
66
86
|
concurrency: { type: "string" },
|
|
67
87
|
json: { type: "boolean", default: false },
|
|
68
88
|
"dry-run": { type: "boolean", default: false },
|
|
69
89
|
"verify-snapshots": { type: "boolean", default: false },
|
|
70
90
|
replay: { type: "string" },
|
|
91
|
+
context: { type: "string" },
|
|
71
92
|
help: { type: "boolean", short: "h", default: false },
|
|
72
93
|
},
|
|
73
94
|
});
|
|
@@ -78,21 +99,48 @@ export function parseCliArgs(argv) {
|
|
|
78
99
|
if (values["verify-snapshots"] && (exec !== null || values.replay !== undefined)) {
|
|
79
100
|
throw new Error("--verify-snapshots cannot be combined with --exec or --replay");
|
|
80
101
|
}
|
|
102
|
+
// Refused rather than ignored: a replay re-gates answers already given to
|
|
103
|
+
// questions already asked, and a snapshot review asks no per-test question,
|
|
104
|
+
// so a context could change neither -- and silently doing nothing with a
|
|
105
|
+
// flag reads as the flag having worked.
|
|
106
|
+
if (values.context !== undefined && values.replay !== undefined) {
|
|
107
|
+
throw new Error("--context cannot be combined with --replay; the record already holds the answers its context produced");
|
|
108
|
+
}
|
|
109
|
+
if (values.context !== undefined && values["verify-snapshots"]) {
|
|
110
|
+
throw new Error("--context cannot be combined with --verify-snapshots");
|
|
111
|
+
}
|
|
81
112
|
return {
|
|
82
113
|
base: values.base === undefined ? null : String(values.base),
|
|
83
114
|
staged: Boolean(values.staged),
|
|
84
115
|
paths: positionals,
|
|
85
116
|
format: format === "auto" ? null : format,
|
|
86
|
-
cutoff:
|
|
117
|
+
cutoff: gateNumber("--cutoff", values.cutoff),
|
|
118
|
+
unsureBelow: gateNumber("--unsure-below", values["unsure-below"]),
|
|
119
|
+
unsureMargin: gateNumber("--unsure-margin", values["unsure-margin"]),
|
|
87
120
|
concurrency: values.concurrency === undefined ? undefined : Number(values.concurrency),
|
|
88
121
|
json: Boolean(values.json),
|
|
89
122
|
dryRun: Boolean(values["dry-run"]),
|
|
90
123
|
verifySnapshots: Boolean(values["verify-snapshots"]),
|
|
91
124
|
replayPath: values.replay === undefined ? null : String(values.replay),
|
|
125
|
+
contextPath: values.context === undefined ? null : String(values.context),
|
|
92
126
|
exec,
|
|
93
127
|
help: Boolean(values.help),
|
|
94
128
|
};
|
|
95
129
|
}
|
|
130
|
+
/**
|
|
131
|
+
* The gate values the command line set, and nothing else.
|
|
132
|
+
*
|
|
133
|
+
* Absent rather than defaulted, because a flag is only one of the places a
|
|
134
|
+
* gate value comes from: whatever it leaves unset falls through to the
|
|
135
|
+
* record being replayed, then to the gate's own defaults.
|
|
136
|
+
*/
|
|
137
|
+
export function gateFlags(args) {
|
|
138
|
+
return {
|
|
139
|
+
...(args.cutoff === undefined ? {} : { cutoff: args.cutoff }),
|
|
140
|
+
...(args.unsureBelow === undefined ? {} : { unsureBelow: args.unsureBelow }),
|
|
141
|
+
...(args.unsureMargin === undefined ? {} : { unsureMargin: args.unsureMargin }),
|
|
142
|
+
};
|
|
143
|
+
}
|
|
96
144
|
/**
|
|
97
145
|
* Shell-safe single quoting, for a line a human will read or `eval`.
|
|
98
146
|
*
|
|
@@ -183,13 +231,16 @@ Usage:
|
|
|
183
231
|
Options:
|
|
184
232
|
--base <ref> compare against the merge base with <ref>, as a pull request does
|
|
185
233
|
--staged use the staged change instead of the working tree
|
|
186
|
-
--format <name> vitest | jest | node | bun | playwright | rust | go | auto (default: auto)
|
|
234
|
+
--format <name> vitest | jest | node | bun | playwright | rust | go | rspec | auto (default: auto)
|
|
187
235
|
--cutoff <n> select at or above this score level (default: 2)
|
|
236
|
+
--unsure-below <n> a confidence under this counts as unsure (default: 0.5)
|
|
237
|
+
--unsure-margin <n> rescue an unsure answer this far under the cutoff (default: 1)
|
|
188
238
|
--concurrency <n> requests in flight at once
|
|
189
239
|
--json print the full scoring instead of the arguments
|
|
190
240
|
--dry-run extract and report without calling Jev
|
|
191
241
|
--verify-snapshots review changed Vitest text snapshots without modifying files
|
|
192
242
|
--replay <file> re-gate a recorded run offline (default: .jev-test-filter/last.json)
|
|
243
|
+
--context <file> read flaker's jev-context: quarantined tests, failure history, gate defaults
|
|
193
244
|
--exec -- <cmd...> append the arguments to <cmd...> and run it
|
|
194
245
|
-h, --help this text
|
|
195
246
|
|
|
@@ -201,6 +252,7 @@ Examples:
|
|
|
201
252
|
jev-test-filter --verify-snapshots --json
|
|
202
253
|
jev-test-filter --base main --format go --exec -- go test
|
|
203
254
|
jev-test-filter --base main --format rust --exec -- cargo test
|
|
255
|
+
jev-test-filter --base main --format rspec --exec -- bundle exec rspec
|
|
204
256
|
jev-test-filter --base main --json > selection.json
|
|
205
257
|
|
|
206
258
|
Rust is never discovered automatically: listing its tests builds the test
|
|
@@ -218,6 +270,26 @@ quotes. Use --exec instead of working around it.
|
|
|
218
270
|
Environment:
|
|
219
271
|
TYPESAFE_API_KEY required unless --dry-run or --replay
|
|
220
272
|
`;
|
|
273
|
+
/**
|
|
274
|
+
* The snapshot changes under review, narrowed to the paths asked for, with the
|
|
275
|
+
* test sources that explain them.
|
|
276
|
+
*/
|
|
277
|
+
export async function collectSnapshotChanges(args, cwd = process.cwd()) {
|
|
278
|
+
const diff = await loadDiff({ cwd, base: args.base, staged: args.staged });
|
|
279
|
+
const revision = args.base ? "HEAD" : args.staged ? ":" : null;
|
|
280
|
+
// The diff names files from the repository root, so they are read from there;
|
|
281
|
+
// the paths asked for are relative to cwd and are rebased to match.
|
|
282
|
+
const changes = (await findSnapshotChangesInWorktree(diff.text, diff.root, revision)).filter((change) => args.paths.length === 0 || args.paths.some((path) => {
|
|
283
|
+
const normalized = posix.normalize(`${diff.prefix}${path}`).replace(/\/$/, "");
|
|
284
|
+
if (normalized === ".")
|
|
285
|
+
return true;
|
|
286
|
+
const testFile = snapshotTestFile(change);
|
|
287
|
+
return change.file === normalized || change.file.startsWith(`${normalized}/`)
|
|
288
|
+
|| testFile === normalized || testFile?.startsWith(`${normalized}/`);
|
|
289
|
+
}));
|
|
290
|
+
const context = await loadSnapshotTestSources(diff.root, changes, 24_000, revision);
|
|
291
|
+
return { diff, changes, context };
|
|
292
|
+
}
|
|
221
293
|
async function main() {
|
|
222
294
|
let args;
|
|
223
295
|
try {
|
|
@@ -232,15 +304,7 @@ async function main() {
|
|
|
232
304
|
return 0;
|
|
233
305
|
}
|
|
234
306
|
if (args.verifySnapshots) {
|
|
235
|
-
const
|
|
236
|
-
const revision = args.base ? "HEAD" : args.staged ? ":" : null;
|
|
237
|
-
const changes = (await findSnapshotChangesInWorktree(diff.text, process.cwd(), revision)).filter((change) => args.paths.length === 0 || args.paths.some((path) => {
|
|
238
|
-
const normalized = path.replace(/^\.\//, "").replace(/\/$/, "");
|
|
239
|
-
const testFile = snapshotTestFile(change);
|
|
240
|
-
return change.file === normalized || change.file.startsWith(`${normalized}/`)
|
|
241
|
-
|| testFile === normalized || testFile?.startsWith(`${normalized}/`);
|
|
242
|
-
}));
|
|
243
|
-
const context = await loadSnapshotTestSources(process.cwd(), changes, 24_000, revision);
|
|
307
|
+
const { diff, changes, context } = await collectSnapshotChanges(args);
|
|
244
308
|
const review = args.dryRun
|
|
245
309
|
? { entries: changes.map((change) => ({ ...change, status: "unknown", score: null, confidence: null })), error: null, spent: null }
|
|
246
310
|
: await assessSnapshots(diff.text, diff.stat, undefined, changes, context);
|
|
@@ -252,7 +316,7 @@ async function main() {
|
|
|
252
316
|
let res;
|
|
253
317
|
if (args.replayPath !== null) {
|
|
254
318
|
const record = await loadRecord(args.replayPath);
|
|
255
|
-
const selection = replay(record, args
|
|
319
|
+
const selection = replay(record, gateFlags(args));
|
|
256
320
|
const { buildFilter } = await import("./filter.js");
|
|
257
321
|
res = {
|
|
258
322
|
selection,
|
|
@@ -263,14 +327,27 @@ async function main() {
|
|
|
263
327
|
};
|
|
264
328
|
}
|
|
265
329
|
else {
|
|
330
|
+
// Read before anything is asked: a context that cannot be used is a usage
|
|
331
|
+
// error, and paying for a run it would have changed is worse than stopping.
|
|
332
|
+
let context = null;
|
|
333
|
+
if (args.contextPath !== null) {
|
|
334
|
+
try {
|
|
335
|
+
context = await loadContext(args.contextPath);
|
|
336
|
+
}
|
|
337
|
+
catch (err) {
|
|
338
|
+
process.stderr.write(`jev-test-filter: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
339
|
+
return 2;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
266
342
|
res = await run({
|
|
267
343
|
base: args.base,
|
|
268
344
|
staged: args.staged,
|
|
269
345
|
paths: args.paths,
|
|
270
346
|
format: args.format,
|
|
271
347
|
dryRun: args.dryRun,
|
|
348
|
+
context,
|
|
272
349
|
...(args.format === "playwright" && args.exec ? { playwrightCommand: args.exec } : {}),
|
|
273
|
-
...(args
|
|
350
|
+
...gateFlags(args),
|
|
274
351
|
...(args.concurrency === undefined ? {} : { concurrency: args.concurrency }),
|
|
275
352
|
});
|
|
276
353
|
if (shouldSaveRecord(res.record))
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { JevContext, TestCase } from "./types.ts";
|
|
2
|
+
export declare const CONTEXT_VERSION = 1;
|
|
3
|
+
/** Validate a parsed `jev-context`. Throws on anything but a v1 context. */
|
|
4
|
+
export declare function parseContext(raw: unknown): JevContext;
|
|
5
|
+
export declare function loadContext(path: string): Promise<JevContext>;
|
|
6
|
+
export interface ContextLookup {
|
|
7
|
+
skipped(t: TestCase): boolean;
|
|
8
|
+
/** The files this test failed with before; empty when the context says nothing. */
|
|
9
|
+
failedWith(t: TestCase): readonly string[];
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* The context indexed for per-test questions.
|
|
13
|
+
*
|
|
14
|
+
* An entry that names a project is about the test in that project only; one
|
|
15
|
+
* that names none is about the test in every project, which is how a
|
|
16
|
+
* quarantine of a test that fails in every browser is written once.
|
|
17
|
+
*/
|
|
18
|
+
export declare function lookup(ctx: JevContext): ContextLookup;
|