jev-test-filter 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +590 -0
- package/dist/cargo.d.ts +30 -0
- package/dist/cargo.js +130 -0
- package/dist/cli.d.ts +49 -0
- package/dist/cli.js +265 -0
- package/dist/diff.d.ts +52 -0
- package/dist/diff.js +168 -0
- package/dist/discover.d.ts +2 -0
- package/dist/discover.js +32 -0
- package/dist/extract-go.d.ts +4 -0
- package/dist/extract-go.js +71 -0
- package/dist/extract.d.ts +15 -0
- package/dist/extract.js +146 -0
- package/dist/filter.d.ts +46 -0
- package/dist/filter.js +81 -0
- package/dist/framework.d.ts +21 -0
- package/dist/framework.js +116 -0
- package/dist/gate.d.ts +26 -0
- package/dist/gate.js +63 -0
- package/dist/jev.d.ts +216 -0
- package/dist/jev.js +390 -0
- package/dist/languages.d.ts +1 -0
- package/dist/languages.js +18 -0
- package/dist/questions.d.ts +42 -0
- package/dist/questions.js +57 -0
- package/dist/run.d.ts +74 -0
- package/dist/run.js +172 -0
- package/dist/state.d.ts +19 -0
- package/dist/state.js +55 -0
- package/dist/types.d.ts +93 -0
- package/dist/types.js +18 -0
- package/package.json +69 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 mizchi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,590 @@
|
|
|
1
|
+
# jev-test-filter
|
|
2
|
+
|
|
3
|
+
`jev-test-filter` reads a `git diff`, asks a model how much that change can
|
|
4
|
+
alter the outcome of every single test in the repository, and prints the filter
|
|
5
|
+
arguments your test runner already understands. It hands those arguments to the
|
|
6
|
+
runner you already use — vitest, jest, `node --test`, Playwright, `cargo test`
|
|
7
|
+
or `go test` — so the run covers the tests the change could plausibly break
|
|
8
|
+
instead of all of them.
|
|
9
|
+
|
|
10
|
+
It uses [Jev](https://typesafe.ai), TypeSafe's System One model: one shared
|
|
11
|
+
`state` (the diff), one question per test, one round trip. There is no agent
|
|
12
|
+
loop and no file reading by the model.
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
Requires Node 24 or newer, `git`, and a TypeSafe API key. `--format rust` and
|
|
17
|
+
`--format go` additionally need `cargo` and `go` on the `PATH`.
|
|
18
|
+
|
|
19
|
+
### The command
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
pnpm add -D jev-test-filter
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Or without installing anything:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
npx jev-test-filter --base main --exec -- vitest run
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
### The skill, for coding agents
|
|
32
|
+
|
|
33
|
+
The command is what runs; the skill is what teaches an agent to drive it —
|
|
34
|
+
which runner takes which flag and in which order, why `--exec` rather than a
|
|
35
|
+
composed shell command, and what each exit code means. Most of that is
|
|
36
|
+
knowledge an agent cannot guess and gets wrong silently, which is the whole
|
|
37
|
+
reason it is written down.
|
|
38
|
+
|
|
39
|
+
**Installing the skill does not install the command.** You want both.
|
|
40
|
+
|
|
41
|
+
As a skill for Claude Code, Codex, Cursor and the other agents the
|
|
42
|
+
[skills](https://skills.sh) CLI knows, into the current project:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
npx skills add mizchi/jev-test-filter
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
As a Claude Code plugin, which brings the same skill plus a `/filter-tests`
|
|
49
|
+
command:
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
/plugin marketplace add mizchi/jev-test-filter
|
|
53
|
+
/plugin install jev-test-filter@jev-test-filter
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### From source
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
git clone https://github.com/mizchi/jev-test-filter
|
|
60
|
+
cd jev-test-filter
|
|
61
|
+
pnpm install
|
|
62
|
+
pnpm run build
|
|
63
|
+
node dist/cli.js --help
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## Usage
|
|
67
|
+
|
|
68
|
+
There are three invocation shapes, because "wire it in" means something
|
|
69
|
+
different in a shell, in another program and in a Makefile.
|
|
70
|
+
|
|
71
|
+
**1. `--exec`: run the command for you.** This is the one to reach for.
|
|
72
|
+
Everything after `--exec --` is your command, and the selection arguments are
|
|
73
|
+
appended to it. They go straight to `spawn` with no shell in between, so
|
|
74
|
+
nothing can be re-split or re-quoted. The command's exit code becomes the
|
|
75
|
+
tool's exit code.
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
$ jev-test-filter --base main --exec -- vitest run
|
|
79
|
+
$ jev-test-filter --base main --format node --exec -- node --test
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Real output, on this repository:
|
|
83
|
+
|
|
84
|
+
```
|
|
85
|
+
$ jev-test-filter --base HEAD~1 --format node --exec -- node --test
|
|
86
|
+
jev-test-filter: 3/110 tests selected (pattern)
|
|
87
|
+
✔ parseCliArgs reads the flags and the paths (0.852375ms)
|
|
88
|
+
✔ a bare --replay means the record the last run left (0.121834ms)
|
|
89
|
+
✔ --replay still takes an explicit path either way round (0.067125ms)
|
|
90
|
+
ℹ tests 3
|
|
91
|
+
ℹ pass 3
|
|
92
|
+
ℹ fail 0
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
The commit under review changed how `--replay` parses its argument. Of 110
|
|
96
|
+
tests it picked the three about argument parsing, and left the other 107 —
|
|
97
|
+
extraction, gating, the Jev client, the Rust and Go paths — alone.
|
|
98
|
+
|
|
99
|
+
**2. `--json`: the whole scoring.** Every test, with its score, its confidence
|
|
100
|
+
and the reason it was kept or dropped. This is the shape to read, to log, or
|
|
101
|
+
to feed to another program.
|
|
102
|
+
|
|
103
|
+
```
|
|
104
|
+
$ jev-test-filter --base main --json > selection.json
|
|
105
|
+
$ jev-test-filter --base main --json
|
|
106
|
+
{
|
|
107
|
+
"framework": "node",
|
|
108
|
+
"mode": "pattern",
|
|
109
|
+
"argv": ["--test-name-pattern", "^(?:parseCliArgs reads the flags and the paths|...)$", "test/cli.test.ts"],
|
|
110
|
+
"fallback": null,
|
|
111
|
+
"selected": 3,
|
|
112
|
+
"total": 110,
|
|
113
|
+
"spent": {
|
|
114
|
+
"calls": 1,
|
|
115
|
+
"inputTokens": 23423,
|
|
116
|
+
"outputTokens": 1874,
|
|
117
|
+
"ms": 1296,
|
|
118
|
+
"retried": 0,
|
|
119
|
+
"rateLimited": 0,
|
|
120
|
+
"tokensPerSecond": 200000,
|
|
121
|
+
"splits": 0,
|
|
122
|
+
"usd": 0.000983766
|
|
123
|
+
},
|
|
124
|
+
"tests": [
|
|
125
|
+
{
|
|
126
|
+
"file": "test/cli.test.ts",
|
|
127
|
+
"name": "a bare --replay means the record the last run left",
|
|
128
|
+
"line": 89,
|
|
129
|
+
"selected": true,
|
|
130
|
+
"reason": "touched",
|
|
131
|
+
"score": 1.74,
|
|
132
|
+
"confidence": 0.19
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
"file": "test/cli.test.ts",
|
|
136
|
+
"name": "parseCliArgs reads the flags and the paths",
|
|
137
|
+
"line": 27,
|
|
138
|
+
"selected": true,
|
|
139
|
+
"reason": "unsure",
|
|
140
|
+
"score": 1.12,
|
|
141
|
+
"confidence": 0
|
|
142
|
+
},
|
|
143
|
+
{
|
|
144
|
+
"file": "test/cargo.test.ts",
|
|
145
|
+
"name": "parseCargoList takes the test lines and nothing else",
|
|
146
|
+
"line": 5,
|
|
147
|
+
"selected": false,
|
|
148
|
+
"reason": "below",
|
|
149
|
+
"score": 0.01,
|
|
150
|
+
"confidence": 0.99
|
|
151
|
+
}
|
|
152
|
+
]
|
|
153
|
+
}
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Three real entries out of the 110, and each shows a different mechanism. The
|
|
157
|
+
first scored 1.74 — under the cutoff of 2 — and ran anyway, because its own
|
|
158
|
+
body is inside the diff; `touched` costs no tokens and is decided before any
|
|
159
|
+
question is asked. The second scored 1.12 with a confidence of 0, and
|
|
160
|
+
`unsure` is the rule that confidence routes rather than gates: an uncertain
|
|
161
|
+
verdict near the cutoff runs. The third is what a clear no looks like.
|
|
162
|
+
|
|
163
|
+
**3. Arguments on stdout.** With neither `--exec` nor `--json`, the tool writes
|
|
164
|
+
the runner arguments to stdout, shell-quoted, and nothing else; progress goes
|
|
165
|
+
to stderr. Reach for this when you want to see what it decided, or when you are
|
|
166
|
+
pasting the arguments into a command by hand.
|
|
167
|
+
|
|
168
|
+
```
|
|
169
|
+
$ jev-test-filter --base HEAD~1 --format node
|
|
170
|
+
jev-test-filter: 3/110 tests selected (pattern)
|
|
171
|
+
--test-name-pattern '^(?:parseCliArgs reads the flags and the paths|a bare --replay means the record the last run left|--replay still takes an explicit path either way round)$' test/cli.test.ts
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
A test name contains spaces, so the arguments have to be quoted, and that
|
|
175
|
+
makes the line unsafe to compose with an unquoted `$(...)`:
|
|
176
|
+
|
|
177
|
+
```
|
|
178
|
+
# WRONG. The shell word-splits without re-parsing the quotes, so the runner
|
|
179
|
+
# receives several arguments where one was meant, and rejects them.
|
|
180
|
+
vitest run $(jev-test-filter --base main)
|
|
181
|
+
|
|
182
|
+
# Works, because eval re-parses the quoting.
|
|
183
|
+
eval "vitest run $(jev-test-filter --base main)"
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
An empty line on stdout is what "run everything" and "run nothing" both look
|
|
187
|
+
like, because both mean "no arguments". The exit code tells them apart:
|
|
188
|
+
|
|
189
|
+
| Exit | Meaning |
|
|
190
|
+
| --- | --- |
|
|
191
|
+
| 0 | Arguments were printed, or everything is to be run and there are none. Run the command. |
|
|
192
|
+
| 3 | Nothing was selected. Do not run the command; there is nothing to run. |
|
|
193
|
+
| 2 | Bad arguments, e.g. `unknown --format mocha`. |
|
|
194
|
+
| 1 | The tool failed, e.g. the tests span more than one framework and no `--format` was given. |
|
|
195
|
+
|
|
196
|
+
So the stdout form is used like this:
|
|
197
|
+
|
|
198
|
+
```
|
|
199
|
+
ARGS=$(jev-test-filter --base main --format node)
|
|
200
|
+
case $? in
|
|
201
|
+
0) eval "node --test $ARGS" ;;
|
|
202
|
+
3) echo "no test can be affected by this change" ;;
|
|
203
|
+
*) exit 1 ;;
|
|
204
|
+
esac
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
`--exec` and `--json` always exit 0 when the tool itself succeeded, because
|
|
208
|
+
neither is ambiguous: `--exec` simply does not start the command and prints
|
|
209
|
+
`nothing selected; not running`, and `--json` reports `"mode": "none"`.
|
|
210
|
+
|
|
211
|
+
Prefer `--exec`. It exists so that nobody has to get this right.
|
|
212
|
+
|
|
213
|
+
Tests are discovered with `git ls-files`, so untracked and ignored files are
|
|
214
|
+
never considered. A file counts as a test file when it is named
|
|
215
|
+
`*.test.*` or `*.spec.*` with a `.js`, `.jsx`, `.ts`, `.tsx`, `.mjs`, `.cjs`,
|
|
216
|
+
`.mts` or `.cts` extension, or when it ends in `_test.go`, which is Go's own
|
|
217
|
+
convention and the only one `go test` compiles into a test binary.
|
|
218
|
+
|
|
219
|
+
**Rust is the one exception: it is never discovered automatically.** Listing a
|
|
220
|
+
crate's tests means running `cargo test -- --list`, and that builds the test
|
|
221
|
+
targets. A tool that triggers a compile nobody asked for is a tool that gets
|
|
222
|
+
removed from the workflow, so it happens only when you write `--format rust`.
|
|
223
|
+
Every other language is read out of the source and costs nothing but a parse.
|
|
224
|
+
|
|
225
|
+
### Rust and Go
|
|
226
|
+
|
|
227
|
+
```
|
|
228
|
+
$ jev-test-filter --format go --exec -- go test
|
|
229
|
+
$ jev-test-filter --format rust --exec -- cargo test
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
Pass `--format` explicitly for both. For Go it is what stops a mixed
|
|
233
|
+
repository — Go tests next to vitest specs — from being an error, and for Rust
|
|
234
|
+
it is the permission to build. Do not give the runner its own package list:
|
|
235
|
+
the tool appends the packages (Go) or the exact names (Rust) it chose, so
|
|
236
|
+
`go test` and `cargo test` are the commands to hand to `--exec`.
|
|
237
|
+
`--exec -- go test ./...` is not an error, but `./...` stays in the package
|
|
238
|
+
list next to the `./pkg` the tool chose, so every package is compiled and the
|
|
239
|
+
package-level saving is lost — only the `-run` pattern still applies.
|
|
240
|
+
|
|
241
|
+
Go filters coarsely, and it is worth knowing before you read the numbers:
|
|
242
|
+
scores are per subtest, but the emitted `-run` pattern names only top-level
|
|
243
|
+
`TestXxx` functions, so **selecting one subtest runs all of that function's
|
|
244
|
+
subtests.** The reason is in [What it emits, per framework](#what-it-emits-per-framework).
|
|
245
|
+
Rust has no such limit: `--exact` takes one name per test, at any nesting.
|
|
246
|
+
|
|
247
|
+
Real output, on a two-package Go module where one edit broke one subtest
|
|
248
|
+
(go 1.26.2):
|
|
249
|
+
|
|
250
|
+
```
|
|
251
|
+
$ go test ./... -v # unfiltered: 5 tests in 2 packages
|
|
252
|
+
--- FAIL: TestApplyDiscount (0.00s)
|
|
253
|
+
--- PASS: TestApplyDiscount/clamps_at_zero (0.00s)
|
|
254
|
+
--- FAIL: TestApplyDiscount/halves_the_total (0.00s)
|
|
255
|
+
--- PASS: TestItemCount (0.00s)
|
|
256
|
+
--- PASS: TestCostIsFreeOverThreshold (0.00s)
|
|
257
|
+
--- PASS: TestCostBelowThreshold (0.00s)
|
|
258
|
+
FAIL
|
|
259
|
+
|
|
260
|
+
$ jev-test-filter --format go --exec -- go test -v
|
|
261
|
+
jev-test-filter: 2/5 tests selected (pattern)
|
|
262
|
+
--- FAIL: TestApplyDiscount (0.00s)
|
|
263
|
+
--- PASS: TestApplyDiscount/clamps_at_zero (0.00s)
|
|
264
|
+
--- FAIL: TestApplyDiscount/halves_the_total (0.00s)
|
|
265
|
+
FAIL scratchcart/cart 0.139s
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
It emitted `-run '^(?:TestApplyDiscount)$' ./cart`: the whole `ship` package
|
|
269
|
+
was skipped, and so was `TestItemCount`. The break was still run.
|
|
270
|
+
|
|
271
|
+
The same crate in Rust (cargo 1.98.0):
|
|
272
|
+
|
|
273
|
+
```
|
|
274
|
+
$ cargo test # unfiltered
|
|
275
|
+
running 5 tests
|
|
276
|
+
test cart::tests::apply_discount::clamps_at_zero ... ok
|
|
277
|
+
test cart::tests::counts_items ... ok
|
|
278
|
+
test ship::tests::charged_below_threshold ... ok
|
|
279
|
+
test ship::tests::free_over_threshold ... ok
|
|
280
|
+
test cart::tests::apply_discount::halves_the_total ... FAILED
|
|
281
|
+
test result: FAILED. 4 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out
|
|
282
|
+
|
|
283
|
+
$ jev-test-filter --format rust --exec -- cargo test
|
|
284
|
+
jev-test-filter: 2/5 tests selected (exact)
|
|
285
|
+
running 2 tests
|
|
286
|
+
test cart::tests::apply_discount::clamps_at_zero ... ok
|
|
287
|
+
test cart::tests::apply_discount::halves_the_total ... FAILED
|
|
288
|
+
test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 3 filtered out
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
It emitted `-- --exact cart::tests::apply_discount::clamps_at_zero
|
|
292
|
+
cart::tests::apply_discount::halves_the_total`.
|
|
293
|
+
|
|
294
|
+
`--base <ref>` compares `<ref>...HEAD`, the way a pull request does. Without
|
|
295
|
+
it, the working tree against `HEAD` is used; `--staged` uses the index.
|
|
296
|
+
|
|
297
|
+
## What it emits, per framework
|
|
298
|
+
|
|
299
|
+
This is the part to get right. The runners spell a full test name differently,
|
|
300
|
+
and a pattern in the wrong spelling matches nothing, runs nothing, and exits
|
|
301
|
+
zero. The tool emits the right spelling per framework, so read this table
|
|
302
|
+
before you hand-edit anything it prints.
|
|
303
|
+
|
|
304
|
+
| Framework | Full name is joined with | Selection is expressed as | Order |
|
|
305
|
+
| --- | --- | --- | --- |
|
|
306
|
+
| `vitest` | `" > "` — `Cart > totals` | `-t '^(?:a > b\|c)$'` plus the files | pattern **before** the files |
|
|
307
|
+
| `jest` | `" > "` | `-t '^(?:...)$'` plus the files | pattern **before** the files |
|
|
308
|
+
| `node` | a single space — `Cart totals` | one `--test-name-pattern '^(?:a b\|c)$'` plus the files | pattern **before** the files |
|
|
309
|
+
| `playwright` | not used | `file:line` positionals — `e2e/cart.spec.ts:12` | positionals only |
|
|
310
|
+
| `rust` | `"::"` — `cart::tests::halves_the_total` | `-- --exact <name> <name>…`, one invocation | after a literal `--` |
|
|
311
|
+
| `go` | `"/"` — `TestApplyDiscount/halves the total` | `-run '^(?:TestA\|TestB)$'` plus the `./pkg` directories | pattern **before** the packages |
|
|
312
|
+
|
|
313
|
+
Three things in that table were measured, not assumed:
|
|
314
|
+
|
|
315
|
+
- **The name pattern is emitted before the file positionals.** Node's test
|
|
316
|
+
runner silently ignores `--test-name-pattern` when it comes after a
|
|
317
|
+
positional: `node --test a.test.js --test-name-pattern X` runs the whole file
|
|
318
|
+
and exits 0. Vitest accepts either order, so one order serves both. If you
|
|
319
|
+
reorder the arguments yourself, you lose the name-level filtering without any
|
|
320
|
+
warning.
|
|
321
|
+
- **node:test gets exactly one `--test-name-pattern` flag** holding one anchored
|
|
322
|
+
alternation. Repeated flags are OR'd, and a pattern that matches a *suite*
|
|
323
|
+
name runs every test under it, so one anchored alternation of full names is
|
|
324
|
+
the only shape that selects exactly the intended tests.
|
|
325
|
+
- **Playwright is selected by `file:line`, not by `--grep`.** Playwright's
|
|
326
|
+
`--grep` matches `"<project> <file> <chain> <title>"` — the project name is
|
|
327
|
+
part of the string, so an anchored pattern would have to know it and would
|
|
328
|
+
break the moment a project is added. `npx playwright test a.spec.ts:4`
|
|
329
|
+
selects exactly one test and several positionals may be listed.
|
|
330
|
+
|
|
331
|
+
A fourth fact, if you are ever tempted to build a pattern yourself from
|
|
332
|
+
`--json`: an empty name pattern is not neutral. `vitest -t ""` runs everything,
|
|
333
|
+
while `node --test --test-name-pattern ""` does not — on Node 24.21.0 it is
|
|
334
|
+
rejected outright (`--test-name-pattern= requires an argument`), the file is
|
|
335
|
+
reported as a failing test and the process exits 1. "Select everything" has to
|
|
336
|
+
be the *absence* of the flag, which is what the tool emits.
|
|
337
|
+
|
|
338
|
+
Two more, for the two compiled languages, and both were measured on
|
|
339
|
+
cargo 1.98.0 and go 1.26.2:
|
|
340
|
+
|
|
341
|
+
- **Rust names come from cargo, never from a path this tool assembled.** The
|
|
342
|
+
names handed to `--exact` are the ones `cargo test -- --list` printed.
|
|
343
|
+
`--exact` is unforgiving: a module path reconstructed one segment wrong
|
|
344
|
+
matches nothing, the run is green, and nothing tells you a test was skipped.
|
|
345
|
+
Because that listing builds the test targets, **`--format rust` is required
|
|
346
|
+
and Rust is never discovered automatically.** ast-grep then supplies each
|
|
347
|
+
listed name's file and line by matching it to a `#[test]` function, which is
|
|
348
|
+
what lets a test whose own body sits in the diff be selected without being
|
|
349
|
+
asked about. Targets are not narrowed: a name that exists in two of them runs
|
|
350
|
+
in both, which costs time and cannot lose a test.
|
|
351
|
+
- **Go scores per subtest but filters per top-level function.** `go test -run`
|
|
352
|
+
takes one hierarchical pattern, and a second `-run` flag *replaces* the
|
|
353
|
+
first — so "all of `TestA`, but only `x` and `y` of `TestB`" cannot be said
|
|
354
|
+
at all. The shape that would cover `TestB` silently drops `TestA`'s other
|
|
355
|
+
subtests, and that under-selection is the one error this tool must not make.
|
|
356
|
+
So the emitted pattern names the top-level functions of the selected tests
|
|
357
|
+
and nothing more. **Select one subtest and you run every subtest of its
|
|
358
|
+
parent.** That is the cost: within a chosen function you get no filtering,
|
|
359
|
+
and the saving comes from the functions and packages that were not chosen at
|
|
360
|
+
all. `--json` still reports the per-subtest score, which is the finer signal
|
|
361
|
+
and what a reader wants to see.
|
|
362
|
+
|
|
363
|
+
The vitest, node:test, Playwright, Rust and Go spellings were each verified
|
|
364
|
+
against the real runner. `jest` is emitted with the vitest shape and was not.
|
|
365
|
+
|
|
366
|
+
The `mode` field in `--json` names which of five shapes came out:
|
|
367
|
+
|
|
368
|
+
| `mode` | `argv` | Meaning |
|
|
369
|
+
| --- | --- | --- |
|
|
370
|
+
| `pattern` | flag, pattern, files | The normal case. |
|
|
371
|
+
| `locations` | `file:line`… | Playwright. |
|
|
372
|
+
| `exact` | `--`, `--exact`, names… | Rust. Every name came from `cargo test -- --list`. |
|
|
373
|
+
| `files` | files | A name pattern could not express the selection, so whole files were chosen. This happens when a selected test has a non-literal title, or when more than 80% of the suite was selected and the alternation would not be worth it. |
|
|
374
|
+
| `all` | *(empty)* | Run everything. Either every test was selected, or a fail-safe fired. |
|
|
375
|
+
| `none` | *(empty)* | Nothing was selected. `--exec` does not start the command; the stdout form exits 3. |
|
|
376
|
+
|
|
377
|
+
## How a test is scored
|
|
378
|
+
|
|
379
|
+
Each test is asked one `score` question over four levels:
|
|
380
|
+
|
|
381
|
+
| Level | Name | Meaning |
|
|
382
|
+
| --- | --- | --- |
|
|
383
|
+
| 0 | `unrelated` | Running this test cannot produce a different result than before. |
|
|
384
|
+
| 1 | `unaffected` | It exercises code the change touched, but nothing in the change can alter its outcome. |
|
|
385
|
+
| 2 | `at-risk` | It could be affected: it might fail. |
|
|
386
|
+
| 3 | `likely-failing` | It directly exercises behaviour the change altered. |
|
|
387
|
+
|
|
388
|
+
The answer is continuous, not snapped to a level. A test is selected when its
|
|
389
|
+
score is at or above the cutoff, which defaults to **2.0** — the boundary
|
|
390
|
+
between "this change cannot alter the outcome" and "it might". `--cutoff`
|
|
391
|
+
moves it; a lower cutoff selects more.
|
|
392
|
+
|
|
393
|
+
Two rules sit on top of that, and both exist to fail towards running a test
|
|
394
|
+
rather than skipping it:
|
|
395
|
+
|
|
396
|
+
- **Confidence routes, it does not gate.** A test that scores under the cutoff,
|
|
397
|
+
but within 1.0 of it, and that the model answered with a confidence below
|
|
398
|
+
0.5, is selected anyway. Running a test that did not need to run costs
|
|
399
|
+
seconds; skipping one that did costs a release.
|
|
400
|
+
- **No answer is not a passing grade.** A missing or malformed answer selects
|
|
401
|
+
the test.
|
|
402
|
+
|
|
403
|
+
Scoring is not deterministic. Two runs over the same diff can disagree about a
|
|
404
|
+
test or two near the cutoff — on this repository, the same base gave 8 selected
|
|
405
|
+
tests on one run and 7 on the next. If you need a selection that is stable
|
|
406
|
+
across invocations, take it once and re-derive it with `--replay`.
|
|
407
|
+
|
|
408
|
+
The `reason` field on each test in `--json` says which rule decided it:
|
|
409
|
+
`touched` (the test's own body is inside the diff — never asked about),
|
|
410
|
+
`dynamic` (the title is not a literal), `scored`, `unsure`, `missing`, `below`.
|
|
411
|
+
|
|
412
|
+
## When it runs everything anyway
|
|
413
|
+
|
|
414
|
+
Selection is an optimization and never a correctness gate. Every one of the
|
|
415
|
+
following makes the tool give up on selecting, emit no arguments at all, and
|
|
416
|
+
print the reason on stderr:
|
|
417
|
+
|
|
418
|
+
```
|
|
419
|
+
jev-test-filter: running everything (jev failed: no API key; set TYPESAFE_API_KEY)
|
|
420
|
+
```
|
|
421
|
+
|
|
422
|
+
- **No API key.**
|
|
423
|
+
- **A Jev error, a timeout, or any other request failure.** Seen in practice:
|
|
424
|
+
`running everything (jev failed: HTTP 529: ... system_overloaded ...)`. The
|
|
425
|
+
client retries transient failures first; the fall-back is what is left after
|
|
426
|
+
those are exhausted.
|
|
427
|
+
- **A missing or malformed answer** — that test is selected. If the whole
|
|
428
|
+
request failed, every test is.
|
|
429
|
+
- **Zero tests were extracted** from the repository.
|
|
430
|
+
- **The diff did not fit the state budget and the selection was still small.**
|
|
431
|
+
A truncated diff that deselects more than half the suite was judged from a
|
|
432
|
+
state that was missing part of the change it should have judged.
|
|
433
|
+
- **`--dry-run`**, which extracts and reports without asking anything.
|
|
434
|
+
|
|
435
|
+
"Run everything" is expressed as *no arguments at all*, so `--exec -- vitest run`
|
|
436
|
+
falls back to exactly `vitest run`. Check that your bare command really does
|
|
437
|
+
run your whole suite: if your test script is `node --test "test/*.test.ts"`
|
|
438
|
+
rather than `node --test`, then `--exec -- node --test` on the fail-safe path
|
|
439
|
+
runs Node's default discovery, not your glob. There is no way to supply a glob
|
|
440
|
+
only on the fallback path: on the selected path the tool appends the files it
|
|
441
|
+
chose, and your glob would sit next to them and pull the whole suite back in.
|
|
442
|
+
|
|
443
|
+
A fail-safe is not an error: the exit code is 0 on every path above, because
|
|
444
|
+
"run everything" is a perfectly good answer. Exit 3 is not a fail-safe either —
|
|
445
|
+
it is the opposite, a complete scoring that selected nothing. The one condition
|
|
446
|
+
that really is an error is a repository whose tests span more than one
|
|
447
|
+
framework with no `--format`, which exits 1 (see Known limitations).
|
|
448
|
+
|
|
449
|
+
## Options
|
|
450
|
+
|
|
451
|
+
```
|
|
452
|
+
--base <ref> compare against <ref>...HEAD, as a pull request does
|
|
453
|
+
--staged use the staged change instead of the working tree
|
|
454
|
+
--format <name> vitest | jest | node | playwright | rust | go | auto (default: auto)
|
|
455
|
+
--cutoff <n> select at or above this score level (default: 2)
|
|
456
|
+
--concurrency <n> requests in flight at once (default: 32)
|
|
457
|
+
--json print the full scoring instead of the arguments
|
|
458
|
+
--dry-run extract and report without calling Jev
|
|
459
|
+
--replay <file> re-gate a recorded run offline (default: .jev-test-filter/last.json)
|
|
460
|
+
--exec -- <cmd...> append the arguments to <cmd...> and run it
|
|
461
|
+
-h, --help the help text
|
|
462
|
+
```
|
|
463
|
+
|
|
464
|
+
Positional arguments restrict which test files are considered, e.g.
|
|
465
|
+
`jev-test-filter --base main src/cart test/cart.test.ts`.
|
|
466
|
+
|
|
467
|
+
## Replaying a run offline
|
|
468
|
+
|
|
469
|
+
Every run that reaches Jev writes its answers to
|
|
470
|
+
`.jev-test-filter/last.json`. `--replay` re-gates that record without touching
|
|
471
|
+
the network, which is how you tune `--cutoff` for free:
|
|
472
|
+
|
|
473
|
+
```
|
|
474
|
+
$ jev-test-filter --replay > /dev/null
|
|
475
|
+
jev-test-filter: 3/110 tests selected (pattern)
|
|
476
|
+
|
|
477
|
+
$ jev-test-filter --replay --cutoff 1.0 > /dev/null
|
|
478
|
+
jev-test-filter: 5/110 tests selected (pattern)
|
|
479
|
+
```
|
|
480
|
+
|
|
481
|
+
A replay reports `"spent": null` because it spent nothing, and it needs no API
|
|
482
|
+
key. It writes no new record, and `--format` has no effect on it — the
|
|
483
|
+
framework comes from the record.
|
|
484
|
+
|
|
485
|
+
Two things to know about the record. It is one file per repository, and every
|
|
486
|
+
successful run replaces it — but **only** a successful one: a run that falls
|
|
487
|
+
back writes nothing, so `--replay` always has the last scoring that actually
|
|
488
|
+
completed to work on, and an unlucky `HTTP 529` cannot destroy it. And the
|
|
489
|
+
record holds every test's title and file path, so add `.jev-test-filter/` to
|
|
490
|
+
your `.gitignore`. It never holds the API key.
|
|
491
|
+
|
|
492
|
+
## Cost and latency
|
|
493
|
+
|
|
494
|
+
Measured on this repository: 110 tests, the diff of one commit, `--format
|
|
495
|
+
node`. One request, 23,423 input tokens, 1,874 output tokens, **1,296 ms**,
|
|
496
|
+
**$0.00098**. It selected 3 of the 110, and running the arguments it printed
|
|
497
|
+
ran exactly those 3.
|
|
498
|
+
|
|
499
|
+
That is one measurement on one repository, not a promise. Cost scales with the
|
|
500
|
+
size of the diff plus the number of tests — the diff is sent once and each test
|
|
501
|
+
is one question, at roughly fifty tokens each — and **not** with how long the
|
|
502
|
+
tests take to run.
|
|
503
|
+
|
|
504
|
+
Which is the whole argument for using it. A second and a tenth of a cent buys
|
|
505
|
+
nothing against a unit suite that finishes in 200 ms; against a Playwright
|
|
506
|
+
suite, a compiled `cargo test`, or anything that starts a database, it is
|
|
507
|
+
returned many times over. Work out what one test costs you before wiring this
|
|
508
|
+
in.
|
|
509
|
+
|
|
510
|
+
## Known limitations
|
|
511
|
+
|
|
512
|
+
- **The stdout form is shell-quoted and needs `eval`.** An unquoted
|
|
513
|
+
`runner $(jev-test-filter ...)` splits a quoted test name into several
|
|
514
|
+
arguments and the runner rejects it — vitest answers
|
|
515
|
+
`Expected a single value for option "-t, --testNamePattern <pattern>"`.
|
|
516
|
+
`--exec` passes argv straight to `spawn` with no shell in between and has no
|
|
517
|
+
such hazard, which is why it leads this document.
|
|
518
|
+
- **An empty name pattern is not neutral**, so never build one by hand from
|
|
519
|
+
`--json`: `vitest -t ""` runs everything, and
|
|
520
|
+
`node --test --test-name-pattern ""` runs nothing (on Node 24.21.0 it fails
|
|
521
|
+
to start at all). Absence of the flag is the only way to say "everything".
|
|
522
|
+
- **A repository whose tests span more than one framework needs `--format`.**
|
|
523
|
+
There is no single command that runs vitest and Playwright together, so the
|
|
524
|
+
tool refuses to guess and asks instead. It exits 1 with
|
|
525
|
+
`the selected tests span more than one framework (node, go, playwright,
|
|
526
|
+
vitest); narrow the run with a path argument or pick one with --format`.
|
|
527
|
+
This is the one failure that is not a fail-safe. This repository is itself
|
|
528
|
+
such a repository — its fixtures cover four runners — so every command above
|
|
529
|
+
that is run here passes `--format`.
|
|
530
|
+
- **A test whose title is not a literal is always selected.** An interpolated
|
|
531
|
+
template or a `.each` row has no name a pattern can hold. Such a test is kept
|
|
532
|
+
to be safe, and because a pattern applies to the whole run rather than to one
|
|
533
|
+
file, a single one of them in the selection drops the entire run to
|
|
534
|
+
file-level filtering (`mode: "files"`). Still a large saving, but less than
|
|
535
|
+
name-level.
|
|
536
|
+
- **A Go subtest whose name is not a literal cannot be named**, and its parent
|
|
537
|
+
is selected whole. That is the table-driven idiom, so it is common. Worse,
|
|
538
|
+
`go test` rewrites a subtest's spaces into underscores when it reports and
|
|
539
|
+
matches it, so two subtests named `"a b"` and `"a_b"` collide, get `#01`
|
|
540
|
+
appended, and cannot be told apart by name **at all** — not by this tool and
|
|
541
|
+
not by a `-run` pattern you write yourself. Selecting whole top-level
|
|
542
|
+
functions is the answer to both.
|
|
543
|
+
- **A Rust test a macro generated is listed by cargo but has no location.** An
|
|
544
|
+
`rstest` case, or anything else a macro produced, appears in
|
|
545
|
+
`cargo test -- --list` but matches no `#[test]` function in the source, so it
|
|
546
|
+
keeps no file and no line. No changed range can overlap it, which means it is
|
|
547
|
+
*scored* like any other test rather than selected for free when the diff
|
|
548
|
+
touches it. It is never dropped for want of a location — a missing answer
|
|
549
|
+
still selects.
|
|
550
|
+
- **`@ast-grep/lang-rust` and `@ast-grep/lang-go` ship prebuilt binaries, and
|
|
551
|
+
their install scripts are commonly skipped.** pnpm 10 blocks `postinstall`
|
|
552
|
+
by default, and both packages use one. That is harmless wherever
|
|
553
|
+
`node_modules/@ast-grep/lang-<name>/prebuilds/` already holds a build for
|
|
554
|
+
your platform — Linux and macOS on x64 and arm64, and Windows x64 — which is
|
|
555
|
+
why it works here with the script never having run. If `--format rust` or
|
|
556
|
+
`--format go` extracts nothing on some other platform, check that directory
|
|
557
|
+
first, and allow the build with `pnpm approve-builds`.
|
|
558
|
+
- **Selection is static.** Tests are read from the source with ast-grep, and
|
|
559
|
+
the model judges what the source shows. A test that reaches the changed code
|
|
560
|
+
only through a runtime indirection — a plugin registry, a dependency-injected
|
|
561
|
+
implementation chosen at startup, a fixture loaded by name — is judged on
|
|
562
|
+
what the source shows, and may be dropped.
|
|
563
|
+
|
|
564
|
+
## Environment
|
|
565
|
+
|
|
566
|
+
| Variable | Meaning |
|
|
567
|
+
| --- | --- |
|
|
568
|
+
| `TYPESAFE_API_KEY` | The API key. Required unless `--dry-run` or `--replay`. `TYPESAFEAI_API_KEY` is accepted as a second spelling. |
|
|
569
|
+
| `TYPESAFE_BASE_URL` | Override the API endpoint. Default `https://api.typesafe.ai`. `TYPESAFEAI_BASE_URL` is also accepted. |
|
|
570
|
+
| `JEV_TEST_FILTER_MODEL` | The model to ask. Default `jev-latest`. |
|
|
571
|
+
| `JEV_TEST_FILTER_TOKENS_PER_SECOND` | Client-side pacing rate. Default `200000`. |
|
|
572
|
+
| `JEV_TEST_FILTER_TOKEN_BURST` | Client-side pacing burst. Default `1200000`. |
|
|
573
|
+
|
|
574
|
+
The key is only ever sent to the API endpoint as an `Authorization` header. It
|
|
575
|
+
is never written to the record, to `--json`, or to any log line.
|
|
576
|
+
|
|
577
|
+
## Development
|
|
578
|
+
|
|
579
|
+
```
|
|
580
|
+
pnpm install
|
|
581
|
+
pkf run check # typecheck + tests
|
|
582
|
+
pkf run build
|
|
583
|
+
```
|
|
584
|
+
|
|
585
|
+
`pkf hooks install` wires the pre-push gate, which runs secretlint over the
|
|
586
|
+
outgoing diff and then `check`. `.envrc` does the same on `cd`.
|
|
587
|
+
|
|
588
|
+
## License
|
|
589
|
+
|
|
590
|
+
MIT. See [LICENSE](./LICENSE).
|
package/dist/cargo.d.ts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { TestCase } from "./types.ts";
|
|
2
|
+
/** Where a test is. */
|
|
3
|
+
export interface Location {
|
|
4
|
+
file: string;
|
|
5
|
+
line: number;
|
|
6
|
+
endLine: number;
|
|
7
|
+
}
|
|
8
|
+
export declare function isTestAttribute(text: string): boolean;
|
|
9
|
+
/** The `name: test` lines of `cargo test -- --list`, without the suffix. */
|
|
10
|
+
export declare function parseCargoList(stdout: string): string[];
|
|
11
|
+
/**
|
|
12
|
+
* The module path a file contributes, by Rust's file-to-module rules.
|
|
13
|
+
*
|
|
14
|
+
* `src/lib.rs` and `src/main.rs` are crate roots and contribute nothing;
|
|
15
|
+
* `mod.rs` names its directory; every file under `tests/`, `benches/` and
|
|
16
|
+
* `examples/` is its own crate root. A `#[path]` attribute can override all of
|
|
17
|
+
* this, and is rare enough that a name it moves simply goes unlocated.
|
|
18
|
+
*/
|
|
19
|
+
export declare function modulePrefix(file: string): string[];
|
|
20
|
+
/** Every `#[test]` function in one file, keyed by its full module path. */
|
|
21
|
+
export declare function rustTestsIn(source: string, file: string): Map<string, Location>;
|
|
22
|
+
/**
|
|
23
|
+
* Ask cargo what tests exist. This builds the test targets.
|
|
24
|
+
*
|
|
25
|
+
* `--quiet` keeps cargo's own progress off stdout; the listing is what is
|
|
26
|
+
* left. A failure to build is not something to work around -- it is reported
|
|
27
|
+
* so the run can fall back to everything.
|
|
28
|
+
*/
|
|
29
|
+
export declare function cargoList(cwd: string): Promise<string>;
|
|
30
|
+
export declare function listRustTests(cwd: string): Promise<TestCase[]>;
|