jev-test-filter 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -14
- package/dist/cli.d.ts +5 -0
- package/dist/cli.js +72 -5
- package/dist/extract.js +8 -5
- package/dist/filter.d.ts +7 -4
- package/dist/filter.js +16 -3
- package/dist/framework.js +7 -2
- package/dist/playwright.d.ts +5 -0
- package/dist/playwright.js +67 -0
- package/dist/questions.js +1 -0
- package/dist/run.d.ts +2 -0
- package/dist/run.js +18 -3
- package/dist/snapshot.d.ts +26 -0
- package/dist/snapshot.js +137 -0
- package/dist/types.d.ts +6 -3
- package/dist/types.js +2 -1
- package/package.json +5 -1
package/README.md
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
`jev-test-filter` reads a `git diff`, asks a model how much that change can
|
|
4
4
|
alter the outcome of every single test in the repository, and prints the filter
|
|
5
5
|
arguments your test runner already understands. It hands those arguments to the
|
|
6
|
-
runner you already use — vitest, jest, `node --test`,
|
|
7
|
-
or `go test` — so the run covers the tests the change could plausibly break
|
|
6
|
+
runner you already use — vitest, jest, `node --test`, `bun test`, Playwright,
|
|
7
|
+
`cargo test` or `go test` — so the run covers the tests the change could plausibly break
|
|
8
8
|
instead of all of them.
|
|
9
9
|
|
|
10
10
|
It uses [Jev](https://typesafe.ai), TypeSafe's System One model: one shared
|
|
@@ -77,8 +77,33 @@ tool's exit code.
|
|
|
77
77
|
```
|
|
78
78
|
$ jev-test-filter --base main --exec -- vitest run
|
|
79
79
|
$ jev-test-filter --base main --format node --exec -- node --test
|
|
80
|
+
$ jev-test-filter --base main --format bun --exec -- bun test
|
|
81
|
+
$ jev-test-filter --base main --format playwright --exec -- npx playwright test
|
|
80
82
|
```
|
|
81
83
|
|
|
84
|
+
For a text snapshot update, inspect the changed expectations separately:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
vitest run --update
|
|
88
|
+
jev-test-filter --verify-snapshots --json
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
`--verify-snapshots` reviews changed `.snap` files and inline snapshot edits,
|
|
92
|
+
including multiline values. It reads the working-tree diff by default,
|
|
93
|
+
does not edit snapshots, and does not run tests. Each changed file receives a
|
|
94
|
+
`plausible`, `review`, or `unknown` decision with Jev's score and confidence.
|
|
95
|
+
`review` includes low-confidence answers; a missing answer or failed request is
|
|
96
|
+
`unknown`. Jev returns structured decisions, not written explanations, so the
|
|
97
|
+
report does not claim to explain its reasoning. Image snapshots are ignored.
|
|
98
|
+
This mode is advisory: it does not fail CI on a suspicious update.
|
|
99
|
+
The associated test source is included when it fits the input budget; a
|
|
100
|
+
missing or oversized test source yields `unknown` rather than an approval.
|
|
101
|
+
Each `.snap` file is one question, so a suspicious entry marks the whole file
|
|
102
|
+
for review. A score below 1 with confidence at least 0.6 is `plausible`;
|
|
103
|
+
all other usable answers are `review`. A truncated diff is `unknown` because
|
|
104
|
+
the missing context could change the judgment. `--base <ref>` reviews committed
|
|
105
|
+
changes from that ref; the default reviews uncommitted working-tree changes.
|
|
106
|
+
|
|
82
107
|
Real output, on this repository:
|
|
83
108
|
|
|
84
109
|
```
|
|
@@ -210,9 +235,11 @@ neither is ambiguous: `--exec` simply does not start the command and prints
|
|
|
210
235
|
|
|
211
236
|
Prefer `--exec`. It exists so that nobody has to get this right.
|
|
212
237
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
238
|
+
Source-based discovery uses `git ls-files`, so untracked and ignored files are
|
|
239
|
+
not considered in that mode. Playwright with `--format playwright --exec` uses
|
|
240
|
+
Playwright's own collected list instead. For source discovery, a file counts
|
|
241
|
+
as a test file when it is named
|
|
242
|
+
`*.test.*`, `*.vitest.*` or `*.spec.*` with a `.js`, `.jsx`, `.ts`, `.tsx`, `.mjs`, `.cjs`,
|
|
216
243
|
`.mts` or `.cts` extension, or when it ends in `_test.go`, which is Go's own
|
|
217
244
|
convention and the only one `go test` compiles into a test binary.
|
|
218
245
|
|
|
@@ -306,10 +333,14 @@ before you hand-edit anything it prints.
|
|
|
306
333
|
| `vitest` | `" > "` — `Cart > totals` | `-t '^(?:a > b\|c)$'` plus the files | pattern **before** the files |
|
|
307
334
|
| `jest` | `" > "` | `-t '^(?:...)$'` plus the files | pattern **before** the files |
|
|
308
335
|
| `node` | a single space — `Cart totals` | one `--test-name-pattern '^(?:a b\|c)$'` plus the files | pattern **before** the files |
|
|
309
|
-
| `
|
|
336
|
+
| `bun` | a single space — `Cart totals` | one `--test-name-pattern '^(?:a b\|c)$'` plus the files | pattern **before** the files |
|
|
337
|
+
| `playwright` | runner's test title and project | `--test-list .jev-test-filter/playwright-….txt` with `--exec`; otherwise `file:line` | exact list or positionals |
|
|
310
338
|
| `rust` | `"::"` — `cart::tests::halves_the_total` | `-- --exact <name> <name>…`, one invocation | after a literal `--` |
|
|
311
339
|
| `go` | `"/"` — `TestApplyDiscount/halves the total` | `-run '^(?:TestA\|TestB)$'` plus the `./pkg` directories | pattern **before** the packages |
|
|
312
340
|
|
|
341
|
+
Bun 1.3.5 prints `Cart > totals` in its reporter, but `--test-name-pattern`
|
|
342
|
+
matches `Cart totals`. The pattern above was verified by running Bun itself.
|
|
343
|
+
|
|
313
344
|
Three things in that table were measured, not assumed:
|
|
314
345
|
|
|
315
346
|
- **The name pattern is emitted before the file positionals.** Node's test
|
|
@@ -322,11 +353,14 @@ Three things in that table were measured, not assumed:
|
|
|
322
353
|
alternation. Repeated flags are OR'd, and a pattern that matches a *suite*
|
|
323
354
|
name runs every test under it, so one anchored alternation of full names is
|
|
324
355
|
the only shape that selects exactly the intended tests.
|
|
325
|
-
- **Playwright
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
356
|
+
- **Playwright uses its own collected test list with `--exec`.** The tool runs
|
|
357
|
+
the given Playwright command once with `--list --reporter=json`, then scores
|
|
358
|
+
each collected test, including generated rows and project variants. It writes
|
|
359
|
+
the chosen names to `.jev-test-filter/playwright-….txt` and passes it through
|
|
360
|
+
`--test-list`. This collects tests without running browsers. Without `--exec`
|
|
361
|
+
it keeps the source-based `file:line` filter, which can run several generated
|
|
362
|
+
rows declared on one line. The tool does not use `--grep`;
|
|
363
|
+
its matching string includes the project name and file path.
|
|
330
364
|
|
|
331
365
|
A fourth fact, if you are ever tempted to build a pattern yourself from
|
|
332
366
|
`--json`: an empty name pattern is not neutral. `vitest -t ""` runs everything,
|
|
@@ -360,15 +394,16 @@ cargo 1.98.0 and go 1.26.2:
|
|
|
360
394
|
all. `--json` still reports the per-subtest score, which is the finer signal
|
|
361
395
|
and what a reader wants to see.
|
|
362
396
|
|
|
363
|
-
The vitest, node:test, Playwright, Rust and Go spellings were each verified
|
|
397
|
+
The vitest, node:test, Bun, Playwright, Rust and Go spellings were each verified
|
|
364
398
|
against the real runner. `jest` is emitted with the vitest shape and was not.
|
|
365
399
|
|
|
366
|
-
The `mode` field in `--json` names
|
|
400
|
+
The `mode` field in `--json` names the emitted filter shape:
|
|
367
401
|
|
|
368
402
|
| `mode` | `argv` | Meaning |
|
|
369
403
|
| --- | --- | --- |
|
|
370
404
|
| `pattern` | flag, pattern, files | The normal case. |
|
|
371
405
|
| `locations` | `file:line`… | Playwright. |
|
|
406
|
+
| `test-list` | `--test-list <file>` | Playwright with runner discovery; `test_list` in JSON holds the file's lines. |
|
|
372
407
|
| `exact` | `--`, `--exact`, names… | Rust. Every name came from `cargo test -- --list`. |
|
|
373
408
|
| `files` | files | A name pattern could not express the selection, so whole files were chosen. This happens when a selected test has a non-literal title, or when more than 80% of the suite was selected and the alternation would not be worth it. |
|
|
374
409
|
| `all` | *(empty)* | Run everything. Either every test was selected, or a fail-safe fired. |
|
|
@@ -451,11 +486,12 @@ framework with no `--format`, which exits 1 (see Known limitations).
|
|
|
451
486
|
```
|
|
452
487
|
--base <ref> compare against <ref>...HEAD, as a pull request does
|
|
453
488
|
--staged use the staged change instead of the working tree
|
|
454
|
-
--format <name> vitest | jest | node | playwright | rust | go | auto (default: auto)
|
|
489
|
+
--format <name> vitest | jest | node | bun | playwright | rust | go | auto (default: auto)
|
|
455
490
|
--cutoff <n> select at or above this score level (default: 2)
|
|
456
491
|
--concurrency <n> requests in flight at once (default: 32)
|
|
457
492
|
--json print the full scoring instead of the arguments
|
|
458
493
|
--dry-run extract and report without calling Jev
|
|
494
|
+
--verify-snapshots review changed text snapshots without modifying files
|
|
459
495
|
--replay <file> re-gate a recorded run offline (default: .jev-test-filter/last.json)
|
|
460
496
|
--exec -- <cmd...> append the arguments to <cmd...> and run it
|
|
461
497
|
-h, --help the help text
|
package/dist/cli.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import type { SnapshotReview } from "./snapshot.ts";
|
|
2
3
|
import type { RunRecord, RunResult } from "./run.ts";
|
|
3
4
|
import type { Framework } from "./types.ts";
|
|
4
5
|
/** Where a run leaves its answers, and what a bare `--replay` means. */
|
|
@@ -12,6 +13,7 @@ export interface CliArgs {
|
|
|
12
13
|
concurrency: number | undefined;
|
|
13
14
|
json: boolean;
|
|
14
15
|
dryRun: boolean;
|
|
16
|
+
verifySnapshots: boolean;
|
|
15
17
|
replayPath: string | null;
|
|
16
18
|
exec: string[] | null;
|
|
17
19
|
help: boolean;
|
|
@@ -24,6 +26,8 @@ export declare function parseCliArgs(argv: string[]): CliArgs;
|
|
|
24
26
|
export declare function renderLine(res: RunResult): string;
|
|
25
27
|
/** The command to run, or null when nothing was selected. */
|
|
26
28
|
export declare function execArgv(cmd: string[], res: RunResult): string[] | null;
|
|
29
|
+
/** A Playwright list must exist before its runner can consume --test-list. */
|
|
30
|
+
export declare function materializeFilter(res: RunResult, cwd?: string): Promise<void>;
|
|
27
31
|
/**
|
|
28
32
|
* The status the stdout form exits with when nothing was selected.
|
|
29
33
|
*
|
|
@@ -47,3 +51,4 @@ export declare function stdoutExitCode(res: RunResult): number;
|
|
|
47
51
|
*/
|
|
48
52
|
export declare function shouldSaveRecord(record: RunRecord): boolean;
|
|
49
53
|
export declare function renderJson(res: RunResult): string;
|
|
54
|
+
export declare function renderSnapshotReport(review: SnapshotReview): string;
|
package/dist/cli.js
CHANGED
|
@@ -21,11 +21,17 @@
|
|
|
21
21
|
* Only this file writes to a stream or exits.
|
|
22
22
|
*/
|
|
23
23
|
import { spawn } from "node:child_process";
|
|
24
|
+
import { realpathSync } from "node:fs";
|
|
25
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
26
|
+
import { dirname, join } from "node:path";
|
|
27
|
+
import { fileURLToPath } from "node:url";
|
|
24
28
|
import { parseArgs } from "node:util";
|
|
25
29
|
import { displayName } from "./questions.js";
|
|
26
30
|
import { fullName } from "./filter.js";
|
|
31
|
+
import { loadDiff } from "./diff.js";
|
|
32
|
+
import { assessSnapshots, findSnapshotChangesInWorktree, loadSnapshotTestSources, snapshotTestFile } from "./snapshot.js";
|
|
27
33
|
import { loadRecord, RECORD_DIR, RECORD_FILE, replay, run, saveRecord } from "./run.js";
|
|
28
|
-
const FORMATS = ["vitest", "jest", "node", "playwright", "rust", "go", "auto"];
|
|
34
|
+
const FORMATS = ["vitest", "jest", "node", "bun", "playwright", "rust", "go", "auto"];
|
|
29
35
|
/** Where a run leaves its answers, and what a bare `--replay` means. */
|
|
30
36
|
export const DEFAULT_RECORD_PATH = `${RECORD_DIR}/${RECORD_FILE}`;
|
|
31
37
|
/**
|
|
@@ -60,6 +66,7 @@ export function parseCliArgs(argv) {
|
|
|
60
66
|
concurrency: { type: "string" },
|
|
61
67
|
json: { type: "boolean", default: false },
|
|
62
68
|
"dry-run": { type: "boolean", default: false },
|
|
69
|
+
"verify-snapshots": { type: "boolean", default: false },
|
|
63
70
|
replay: { type: "string" },
|
|
64
71
|
help: { type: "boolean", short: "h", default: false },
|
|
65
72
|
},
|
|
@@ -68,6 +75,9 @@ export function parseCliArgs(argv) {
|
|
|
68
75
|
if (!FORMATS.includes(format)) {
|
|
69
76
|
throw new Error(`unknown --format ${format}; expected one of ${FORMATS.join(", ")}`);
|
|
70
77
|
}
|
|
78
|
+
if (values["verify-snapshots"] && (exec !== null || values.replay !== undefined)) {
|
|
79
|
+
throw new Error("--verify-snapshots cannot be combined with --exec or --replay");
|
|
80
|
+
}
|
|
71
81
|
return {
|
|
72
82
|
base: values.base === undefined ? null : String(values.base),
|
|
73
83
|
staged: Boolean(values.staged),
|
|
@@ -77,6 +87,7 @@ export function parseCliArgs(argv) {
|
|
|
77
87
|
concurrency: values.concurrency === undefined ? undefined : Number(values.concurrency),
|
|
78
88
|
json: Boolean(values.json),
|
|
79
89
|
dryRun: Boolean(values["dry-run"]),
|
|
90
|
+
verifySnapshots: Boolean(values["verify-snapshots"]),
|
|
80
91
|
replayPath: values.replay === undefined ? null : String(values.replay),
|
|
81
92
|
exec,
|
|
82
93
|
help: Boolean(values.help),
|
|
@@ -101,6 +112,14 @@ export function execArgv(cmd, res) {
|
|
|
101
112
|
return null;
|
|
102
113
|
return [...cmd, ...res.filter.argv];
|
|
103
114
|
}
|
|
115
|
+
/** A Playwright list must exist before its runner can consume --test-list. */
|
|
116
|
+
export async function materializeFilter(res, cwd = process.cwd()) {
|
|
117
|
+
if (res.filter.mode !== "test-list" || !res.filter.testList)
|
|
118
|
+
return;
|
|
119
|
+
const path = join(cwd, res.filter.argv[1]);
|
|
120
|
+
await mkdir(dirname(path), { recursive: true });
|
|
121
|
+
await writeFile(path, `${res.filter.testList.join("\n")}\n`, "utf8");
|
|
122
|
+
}
|
|
104
123
|
/**
|
|
105
124
|
* The status the stdout form exits with when nothing was selected.
|
|
106
125
|
*
|
|
@@ -132,12 +151,14 @@ export function renderJson(res) {
|
|
|
132
151
|
framework: res.framework,
|
|
133
152
|
mode: res.filter.mode,
|
|
134
153
|
argv: res.filter.argv,
|
|
154
|
+
...(res.filter.testList === undefined ? {} : { test_list: res.filter.testList }),
|
|
135
155
|
fallback: res.selection.fallback,
|
|
136
156
|
selected: res.selection.selected.length,
|
|
137
157
|
total: res.selection.all.length,
|
|
138
158
|
spent: res.spent,
|
|
139
159
|
tests: res.selection.verdicts.map((v) => ({
|
|
140
160
|
file: v.test.file,
|
|
161
|
+
...(v.test.project === undefined ? {} : { project: v.test.project }),
|
|
141
162
|
name: displayName(v.test),
|
|
142
163
|
pattern_name: fullName(v.test),
|
|
143
164
|
line: v.test.line,
|
|
@@ -148,6 +169,11 @@ export function renderJson(res) {
|
|
|
148
169
|
})),
|
|
149
170
|
}, null, 2)}\n`;
|
|
150
171
|
}
|
|
172
|
+
export function renderSnapshotReport(review) {
|
|
173
|
+
if (review.entries.length === 0)
|
|
174
|
+
return "No changed text snapshots.\n";
|
|
175
|
+
return `${review.entries.map((entry) => `${entry.status}\t${entry.file}\t${entry.kind}\tscore=${entry.score ?? "?"}\tconfidence=${entry.confidence ?? "?"}`).join("\n")}\n`;
|
|
176
|
+
}
|
|
151
177
|
const HELP = `jev-test-filter — score every test against a git diff and emit runner arguments
|
|
152
178
|
|
|
153
179
|
Usage:
|
|
@@ -157,11 +183,12 @@ Usage:
|
|
|
157
183
|
Options:
|
|
158
184
|
--base <ref> compare against the merge base with <ref>, as a pull request does
|
|
159
185
|
--staged use the staged change instead of the working tree
|
|
160
|
-
--format <name> vitest | jest | node | playwright | rust | go | auto (default: auto)
|
|
186
|
+
--format <name> vitest | jest | node | bun | playwright | rust | go | auto (default: auto)
|
|
161
187
|
--cutoff <n> select at or above this score level (default: 2)
|
|
162
188
|
--concurrency <n> requests in flight at once
|
|
163
189
|
--json print the full scoring instead of the arguments
|
|
164
190
|
--dry-run extract and report without calling Jev
|
|
191
|
+
--verify-snapshots review changed Vitest text snapshots without modifying files
|
|
165
192
|
--replay <file> re-gate a recorded run offline (default: .jev-test-filter/last.json)
|
|
166
193
|
--exec -- <cmd...> append the arguments to <cmd...> and run it
|
|
167
194
|
-h, --help this text
|
|
@@ -169,6 +196,9 @@ Options:
|
|
|
169
196
|
Examples:
|
|
170
197
|
jev-test-filter --base main --exec -- vitest run
|
|
171
198
|
jev-test-filter --base main --format node --exec -- node --test
|
|
199
|
+
jev-test-filter --base main --format bun --exec -- bun test
|
|
200
|
+
jev-test-filter --base main --format playwright --exec -- npx playwright test
|
|
201
|
+
jev-test-filter --verify-snapshots --json
|
|
172
202
|
jev-test-filter --base main --format go --exec -- go test
|
|
173
203
|
jev-test-filter --base main --format rust --exec -- cargo test
|
|
174
204
|
jev-test-filter --base main --json > selection.json
|
|
@@ -201,6 +231,24 @@ async function main() {
|
|
|
201
231
|
process.stdout.write(HELP);
|
|
202
232
|
return 0;
|
|
203
233
|
}
|
|
234
|
+
if (args.verifySnapshots) {
|
|
235
|
+
const diff = await loadDiff({ cwd: process.cwd(), base: args.base, staged: args.staged });
|
|
236
|
+
const revision = args.base ? "HEAD" : args.staged ? ":" : null;
|
|
237
|
+
const changes = (await findSnapshotChangesInWorktree(diff.text, process.cwd(), revision)).filter((change) => args.paths.length === 0 || args.paths.some((path) => {
|
|
238
|
+
const normalized = path.replace(/^\.\//, "").replace(/\/$/, "");
|
|
239
|
+
const testFile = snapshotTestFile(change);
|
|
240
|
+
return change.file === normalized || change.file.startsWith(`${normalized}/`)
|
|
241
|
+
|| testFile === normalized || testFile?.startsWith(`${normalized}/`);
|
|
242
|
+
}));
|
|
243
|
+
const context = await loadSnapshotTestSources(process.cwd(), changes, 24_000, revision);
|
|
244
|
+
const review = args.dryRun
|
|
245
|
+
? { entries: changes.map((change) => ({ ...change, status: "unknown", score: null, confidence: null })), error: null, spent: null }
|
|
246
|
+
: await assessSnapshots(diff.text, diff.stat, undefined, changes, context);
|
|
247
|
+
if (review.error)
|
|
248
|
+
process.stderr.write(`jev-test-filter: snapshot review incomplete (${review.error})\n`);
|
|
249
|
+
process.stdout.write(args.json ? `${JSON.stringify(review, null, 2)}\n` : renderSnapshotReport(review));
|
|
250
|
+
return 0;
|
|
251
|
+
}
|
|
204
252
|
let res;
|
|
205
253
|
if (args.replayPath !== null) {
|
|
206
254
|
const record = await loadRecord(args.replayPath);
|
|
@@ -221,6 +269,7 @@ async function main() {
|
|
|
221
269
|
paths: args.paths,
|
|
222
270
|
format: args.format,
|
|
223
271
|
dryRun: args.dryRun,
|
|
272
|
+
...(args.format === "playwright" && args.exec ? { playwrightCommand: args.exec } : {}),
|
|
224
273
|
...(args.cutoff === undefined ? {} : { cutoff: args.cutoff }),
|
|
225
274
|
...(args.concurrency === undefined ? {} : { concurrency: args.concurrency }),
|
|
226
275
|
});
|
|
@@ -238,6 +287,13 @@ async function main() {
|
|
|
238
287
|
process.stdout.write(renderJson(res));
|
|
239
288
|
return 0;
|
|
240
289
|
}
|
|
290
|
+
try {
|
|
291
|
+
await materializeFilter(res);
|
|
292
|
+
}
|
|
293
|
+
catch (err) {
|
|
294
|
+
process.stderr.write(`jev-test-filter: could not write Playwright test list; running everything (${err instanceof Error ? err.message : String(err)})\n`);
|
|
295
|
+
res.filter = { mode: "all", argv: [] };
|
|
296
|
+
}
|
|
241
297
|
if (args.exec) {
|
|
242
298
|
const argv = execArgv(args.exec, res);
|
|
243
299
|
if (argv === null) {
|
|
@@ -256,10 +312,21 @@ async function main() {
|
|
|
256
312
|
process.stdout.write(`${renderLine(res)}\n`);
|
|
257
313
|
return stdoutExitCode(res);
|
|
258
314
|
}
|
|
315
|
+
/** Resolve npm's bin symlink before deciding whether this module is the entrypoint. */
|
|
316
|
+
function isCliInvocation(argv1, moduleUrl) {
|
|
317
|
+
if (!argv1)
|
|
318
|
+
return false;
|
|
319
|
+
try {
|
|
320
|
+
return realpathSync(argv1) === realpathSync(fileURLToPath(moduleUrl));
|
|
321
|
+
}
|
|
322
|
+
catch {
|
|
323
|
+
return false;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
259
326
|
// Only run when invoked as a program, so the tests can import the renderers.
|
|
260
|
-
if (process.argv[1]
|
|
261
|
-
main().then((code) => process.
|
|
327
|
+
if (isCliInvocation(process.argv[1], import.meta.url)) {
|
|
328
|
+
main().then((code) => { process.exitCode = code; }, (err) => {
|
|
262
329
|
process.stderr.write(`jev-test-filter: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
263
|
-
process.
|
|
330
|
+
process.exitCode = 1;
|
|
264
331
|
});
|
|
265
332
|
}
|
package/dist/extract.js
CHANGED
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
import { parse } from "@ast-grep/napi";
|
|
12
12
|
import { langFor } from "./framework.js";
|
|
13
13
|
/** `describe(...)` and its spellings; Playwright's `test.describe` with its own modifiers. */
|
|
14
|
-
const SUITE_CALLEE = "^(x|f)?(describe|suite|context)(\\.(only|skip|todo|concurrent|sequential|shuffle|skipIf|runIf|if|each|for)(\\([
|
|
15
|
-
"|^test\\.describe(\\.(serial|parallel|only|skip|fixme|configure)(\\([
|
|
14
|
+
const SUITE_CALLEE = "^(x|f)?(describe|suite|context)(\\.(only|skip|todo|concurrent|sequential|shuffle|skipIf|runIf|if|each|for)(\\([\\s\\S]*\\))?)*$" +
|
|
15
|
+
"|^test\\.describe(\\.(serial|parallel|only|skip|fixme|configure)(\\([\\s\\S]*\\))?)*$";
|
|
16
16
|
/** The names node:test's context goes by, for `t.test("subtest", fn)`. */
|
|
17
17
|
const SUBTEST_CONTEXT = "(t|ctx|context)";
|
|
18
18
|
/** A subtest call, wherever it sits inside the parent. */
|
|
@@ -22,11 +22,11 @@ const SUBTEST_INSIDE = {
|
|
|
22
22
|
has: { field: "function", regex: `^${SUBTEST_CONTEXT}\\.test$` },
|
|
23
23
|
};
|
|
24
24
|
/** `it`/`test` with modifiers; `Deno.test`; node:test's subtest on the context. */
|
|
25
|
-
const TEST_CALLEE = "^(x|f)?(it|test)(\\.(only|skip|todo|concurrent|sequential|fails|fixme|slow|skipIf|runIf|if|todoIf|failsIf|each|for)(\\([
|
|
25
|
+
const TEST_CALLEE = "^(x|f)?(it|test)(\\.(only|skip|todo|concurrent|sequential|fails|fixme|slow|skipIf|runIf|if|todoIf|failsIf|each|for)(\\([\\s\\S]*\\))?)*$" +
|
|
26
26
|
"|^Deno\\.test(\\.(only|ignore))?$" +
|
|
27
27
|
`|^${SUBTEST_CONTEXT}\\.test$`;
|
|
28
28
|
const FUNCTION_KINDS = [{ kind: "arrow_function" }, { kind: "function_expression" }, { kind: "generator_function" }];
|
|
29
|
-
/** The title,
|
|
29
|
+
/** The title, whether literal, named in an object or function, or computed. */
|
|
30
30
|
const TITLE_ARG = {
|
|
31
31
|
field: "arguments",
|
|
32
32
|
any: [
|
|
@@ -42,6 +42,9 @@ const TITLE_ARG = {
|
|
|
42
42
|
},
|
|
43
43
|
},
|
|
44
44
|
{ has: { nthChild: 1, kind: "function_expression", has: { field: "name", pattern: "$TITLE" } } },
|
|
45
|
+
// A variable or computed title cannot be named by a runner pattern, but
|
|
46
|
+
// the test still has to be inventoried so the gate can keep its whole file.
|
|
47
|
+
{ has: { nthChild: 1, pattern: "$TITLE" } },
|
|
45
48
|
],
|
|
46
49
|
};
|
|
47
50
|
/**
|
|
@@ -81,7 +84,7 @@ const SUITE_RULE = {
|
|
|
81
84
|
{ all: [{ has: { field: "function", regex: TEST_CALLEE } }, { has: SUBTEST_INSIDE }] },
|
|
82
85
|
],
|
|
83
86
|
},
|
|
84
|
-
{ has:
|
|
87
|
+
{ has: TITLE_ARG },
|
|
85
88
|
{ has: { field: "arguments", has: { any: FUNCTION_KINDS, pattern: "$BODY" } } },
|
|
86
89
|
],
|
|
87
90
|
};
|
package/dist/filter.d.ts
CHANGED
|
@@ -9,9 +9,8 @@
|
|
|
9
9
|
* - node:test ORs repeated `--test-name-pattern` flags, and a pattern that
|
|
10
10
|
* matches a SUITE name runs every test under it. One flag, one
|
|
11
11
|
* alternation of anchored full names, is the only safe shape.
|
|
12
|
-
* - Playwright's
|
|
13
|
-
*
|
|
14
|
-
* when a project is added. `file:line` is exact and needs no escaping.
|
|
12
|
+
* - Playwright's own --list supplies project-aware names for --test-list.
|
|
13
|
+
* Source-only discovery retains file:line as its conservative fallback.
|
|
15
14
|
*/
|
|
16
15
|
import type { Framework, Selection, TestCase } from "./types.ts";
|
|
17
16
|
export type FilterMode =
|
|
@@ -25,12 +24,16 @@ export type FilterMode =
|
|
|
25
24
|
| "pattern"
|
|
26
25
|
/** `file:line` positionals. */
|
|
27
26
|
| "locations"
|
|
27
|
+
/** Playwright's own exact test list, including project and generated title. */
|
|
28
|
+
| "test-list"
|
|
28
29
|
/** Exact names, for a runner that does not take a pattern. */
|
|
29
30
|
| "exact";
|
|
30
31
|
export interface FilterArgs {
|
|
31
32
|
mode: FilterMode;
|
|
32
33
|
/** Arguments to append to the runner command, in order. */
|
|
33
34
|
argv: string[];
|
|
35
|
+
/** Lines to write to argv's --test-list file before invoking Playwright. */
|
|
36
|
+
testList?: string[];
|
|
34
37
|
}
|
|
35
38
|
export interface FilterOptions {
|
|
36
39
|
/**
|
|
@@ -41,6 +44,6 @@ export interface FilterOptions {
|
|
|
41
44
|
fileThreshold?: number;
|
|
42
45
|
}
|
|
43
46
|
export declare function escapeRegExp(s: string): string;
|
|
44
|
-
/** The full name as this
|
|
47
|
+
/** The full name as this runner's name pattern matches it. */
|
|
45
48
|
export declare function fullName(t: TestCase): string;
|
|
46
49
|
export declare function buildFilter(sel: Selection, framework: Framework, { fileThreshold }?: FilterOptions): FilterArgs;
|
package/dist/filter.js
CHANGED
|
@@ -1,18 +1,19 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
export function escapeRegExp(s) {
|
|
2
3
|
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
3
4
|
}
|
|
4
|
-
/** The full name as this
|
|
5
|
+
/** The full name as this runner's name pattern matches it. */
|
|
5
6
|
export function fullName(t) {
|
|
6
7
|
if (t.framework === "rust")
|
|
7
8
|
return t.titlePath.join("::");
|
|
8
9
|
if (t.framework === "go")
|
|
9
10
|
return t.titlePath.join("/");
|
|
10
|
-
const sep = t.framework === "node" ? " " : " > ";
|
|
11
|
+
const sep = t.framework === "node" || t.framework === "bun" ? " " : " > ";
|
|
11
12
|
return t.titlePath.join(sep);
|
|
12
13
|
}
|
|
13
14
|
/** The flag that carries a name pattern, per framework. */
|
|
14
15
|
function patternFlag(framework) {
|
|
15
|
-
if (framework === "node")
|
|
16
|
+
if (framework === "node" || framework === "bun")
|
|
16
17
|
return "--test-name-pattern";
|
|
17
18
|
if (framework === "vitest" || framework === "jest")
|
|
18
19
|
return "-t";
|
|
@@ -41,6 +42,18 @@ export function buildFilter(sel, framework, { fileThreshold = 0.8 } = {}) {
|
|
|
41
42
|
if (selected.length === all.length)
|
|
42
43
|
return { mode: "all", argv: [] };
|
|
43
44
|
if (framework === "playwright") {
|
|
45
|
+
if (selected.every((t) => t.runnerFile !== undefined)) {
|
|
46
|
+
if (selected.some((t) => t.runnerFile.includes(" > "))) {
|
|
47
|
+
return { mode: "files", argv: [...new Set(selected.map((t) => t.file))] };
|
|
48
|
+
}
|
|
49
|
+
const lines = [...new Set(selected.map((t) => {
|
|
50
|
+
const prefix = t.project && !t.project.includes("] > ") ? `[${t.project}] > ` : "";
|
|
51
|
+
const titles = t.titlePath.some((name) => name.includes(" > ")) ? [] : t.titlePath;
|
|
52
|
+
return `${prefix}${t.runnerFile}${titles.map((name) => ` > ${name}`).join("")}`;
|
|
53
|
+
}))];
|
|
54
|
+
const hash = createHash("sha256").update(lines.join("\n")).digest("hex").slice(0, 12);
|
|
55
|
+
return { mode: "test-list", argv: ["--test-list", `.jev-test-filter/playwright-${hash}.txt`], testList: lines };
|
|
56
|
+
}
|
|
44
57
|
return { mode: "locations", argv: selected.map((t) => `${t.file}:${t.line}`) };
|
|
45
58
|
}
|
|
46
59
|
if (framework === "rust") {
|
package/dist/framework.js
CHANGED
|
@@ -20,8 +20,8 @@ export function langFor(file) {
|
|
|
20
20
|
return Lang.TypeScript;
|
|
21
21
|
return Lang.JavaScript;
|
|
22
22
|
}
|
|
23
|
-
/** `foo.test.ts`, `foo.
|
|
24
|
-
const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$/;
|
|
23
|
+
/** `foo.test.ts`, `foo.vitest.ts`, `foo.spec.tsx`, and the rest of the family. */
|
|
24
|
+
const TEST_FILE = /\.(?:test|vitest|spec)\.[cm]?[jt]sx?$/;
|
|
25
25
|
/** Go's own convention, and the only one `go test` compiles into a test binary. */
|
|
26
26
|
const GO_TEST_FILE = /_test\.go$/;
|
|
27
27
|
export function isGoTestFile(file) {
|
|
@@ -84,6 +84,7 @@ function specifiers(source, file) {
|
|
|
84
84
|
export function detectFramework(source, file = "a.ts") {
|
|
85
85
|
let vitest = false;
|
|
86
86
|
let node = false;
|
|
87
|
+
let bun = false;
|
|
87
88
|
let jest = false;
|
|
88
89
|
for (const spec of specifiers(source, file)) {
|
|
89
90
|
if (spec === "@playwright/test" || spec.startsWith("@playwright/test/"))
|
|
@@ -92,6 +93,8 @@ export function detectFramework(source, file = "a.ts") {
|
|
|
92
93
|
vitest = true;
|
|
93
94
|
else if (spec === "node:test")
|
|
94
95
|
node = true;
|
|
96
|
+
else if (spec === "bun:test")
|
|
97
|
+
bun = true;
|
|
95
98
|
else if (spec === "@jest/globals")
|
|
96
99
|
jest = true;
|
|
97
100
|
}
|
|
@@ -99,6 +102,8 @@ export function detectFramework(source, file = "a.ts") {
|
|
|
99
102
|
return "vitest";
|
|
100
103
|
if (node)
|
|
101
104
|
return "node";
|
|
105
|
+
if (bun)
|
|
106
|
+
return "bun";
|
|
102
107
|
if (jest)
|
|
103
108
|
return "jest";
|
|
104
109
|
return "unknown";
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { TestCase } from "./types.ts";
|
|
2
|
+
export declare function parsePlaywrightList(raw: unknown, cwd: string): TestCase[];
|
|
3
|
+
export declare function attachPlaywrightRanges(listed: TestCase[], sourceCases: TestCase[]): TestCase[];
|
|
4
|
+
/** Collect without running browsers. The user's runner command preserves its config and project flags. */
|
|
5
|
+
export declare function listPlaywrightTests(cwd: string, command: string[]): Promise<TestCase[]>;
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/** Playwright's collected tests, including generated rows and project variants. */
|
|
2
|
+
import { execFile } from "node:child_process";
|
|
3
|
+
import { readFile } from "node:fs/promises";
|
|
4
|
+
import { relative, resolve, sep } from "node:path";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
import { extractTests } from "./extract.js";
|
|
7
|
+
const execFileAsync = promisify(execFile);
|
|
8
|
+
export function parsePlaywrightList(raw, cwd) {
|
|
9
|
+
const report = raw;
|
|
10
|
+
const rootDir = report?.config?.rootDir;
|
|
11
|
+
if (typeof rootDir !== "string" || !Array.isArray(report.suites) || (report.errors?.length ?? 0) > 0) {
|
|
12
|
+
throw new Error("invalid Playwright test list");
|
|
13
|
+
}
|
|
14
|
+
const out = [];
|
|
15
|
+
const walk = (suite, chain, root) => {
|
|
16
|
+
const titles = root ? chain : [...chain, suite.title ?? ""];
|
|
17
|
+
for (const spec of suite.specs ?? []) {
|
|
18
|
+
if (typeof spec.file !== "string" || typeof spec.title !== "string" || !Array.isArray(spec.tests)) {
|
|
19
|
+
throw new Error("invalid Playwright spec");
|
|
20
|
+
}
|
|
21
|
+
const file = relative(cwd, resolve(rootDir, spec.file)).split(sep).join("/");
|
|
22
|
+
if (file === ".." || file.startsWith("../") || file.startsWith("/")) {
|
|
23
|
+
throw new Error(`Playwright test is outside the repository: ${file}`);
|
|
24
|
+
}
|
|
25
|
+
for (const variant of spec.tests) {
|
|
26
|
+
out.push({
|
|
27
|
+
file, runnerFile: spec.file.split(sep).join("/"), titlePath: [...titles, spec.title],
|
|
28
|
+
line: spec.line ?? 0, endLine: spec.line ?? 0, framework: "playwright", dynamic: false,
|
|
29
|
+
project: variant.projectName ?? "",
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
for (const child of suite.suites ?? [])
|
|
34
|
+
walk(child, titles, false);
|
|
35
|
+
};
|
|
36
|
+
for (const suite of report.suites)
|
|
37
|
+
walk(suite, [], true);
|
|
38
|
+
return out;
|
|
39
|
+
}
|
|
40
|
+
export function attachPlaywrightRanges(listed, sourceCases) {
|
|
41
|
+
const ends = new Map();
|
|
42
|
+
for (const t of sourceCases) {
|
|
43
|
+
const key = `${t.file}:${t.line}`;
|
|
44
|
+
ends.set(key, Math.max(ends.get(key) ?? 0, t.endLine));
|
|
45
|
+
}
|
|
46
|
+
return listed.map((t) => ({ ...t, endLine: Math.max(t.endLine, ends.get(`${t.file}:${t.line}`) ?? t.endLine) }));
|
|
47
|
+
}
|
|
48
|
+
/** Collect without running browsers. The user's runner command preserves its config and project flags. */
|
|
49
|
+
export async function listPlaywrightTests(cwd, command) {
|
|
50
|
+
if (command.length === 0)
|
|
51
|
+
throw new Error("a Playwright command is required");
|
|
52
|
+
const { stdout } = await execFileAsync(command[0], [...command.slice(1), "--list", "--reporter=json"], {
|
|
53
|
+
cwd, maxBuffer: 128 * 1024 * 1024,
|
|
54
|
+
});
|
|
55
|
+
const listed = parsePlaywrightList(JSON.parse(stdout), cwd);
|
|
56
|
+
const sourceCases = [];
|
|
57
|
+
for (const file of new Set(listed.map((t) => t.file))) {
|
|
58
|
+
try {
|
|
59
|
+
sourceCases.push(...extractTests(await readFile(resolve(cwd, file), "utf8"), file, "playwright"));
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
// The runner's inventory is authoritative; a source parse only adds
|
|
63
|
+
// end lines for touched-test detection and must not hide listed tests.
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return attachPlaywrightRanges(listed, sourceCases);
|
|
67
|
+
}
|
package/dist/questions.js
CHANGED
|
@@ -33,6 +33,7 @@ export function buildQuestion(t, id) {
|
|
|
33
33
|
task: TASK,
|
|
34
34
|
subject: id,
|
|
35
35
|
test_file: t.file,
|
|
36
|
+
...(t.project === undefined ? {} : { playwright_project: t.project }),
|
|
36
37
|
test_name: displayName(t),
|
|
37
38
|
test_lines: t.line === t.endLine ? `${t.line}` : `${t.line}-${t.endLine}`,
|
|
38
39
|
},
|
package/dist/run.d.ts
CHANGED
|
@@ -59,6 +59,8 @@ export interface RunOptions extends GateOptions {
|
|
|
59
59
|
fileThreshold?: number;
|
|
60
60
|
/** Extract and gate but never call Jev; every test scores as missing. */
|
|
61
61
|
dryRun?: boolean;
|
|
62
|
+
/** When using Playwright, collect its actual test list with this runner command. */
|
|
63
|
+
playwrightCommand?: string[];
|
|
62
64
|
}
|
|
63
65
|
export interface RunResult {
|
|
64
66
|
selection: Selection;
|
package/dist/run.js
CHANGED
|
@@ -15,6 +15,7 @@ import { buildState } from "./state.js";
|
|
|
15
15
|
import { buildQuestion, questionId, readAnswer } from "./questions.js";
|
|
16
16
|
import { gate } from "./gate.js";
|
|
17
17
|
import { buildFilter } from "./filter.js";
|
|
18
|
+
import { listPlaywrightTests } from "./playwright.js";
|
|
18
19
|
import { Jev, mapLimit, DEFAULT_CONCURRENCY } from "./jev.js";
|
|
19
20
|
import { testId } from "./types.js";
|
|
20
21
|
/** Where a run's answers are kept, for `--replay`. */
|
|
@@ -88,7 +89,21 @@ function everything(tests, reason) {
|
|
|
88
89
|
export async function run(opts = {}) {
|
|
89
90
|
const cwd = opts.cwd ?? process.cwd();
|
|
90
91
|
const diff = await loadDiff({ cwd, base: opts.base ?? null, staged: opts.staged ?? false });
|
|
91
|
-
|
|
92
|
+
let all = [];
|
|
93
|
+
let discoveryFailure = null;
|
|
94
|
+
try {
|
|
95
|
+
all = opts.format === "playwright" && opts.playwrightCommand
|
|
96
|
+
? (await listPlaywrightTests(cwd, opts.playwrightCommand)).filter((t) => !opts.paths?.length || opts.paths.some((path) => {
|
|
97
|
+
const normalized = path.replace(/^\.\//, "").replace(/\/$/, "");
|
|
98
|
+
return t.file === normalized || t.file.startsWith(`${normalized}/`);
|
|
99
|
+
}))
|
|
100
|
+
: await discoverTests(cwd, opts.paths ?? [], opts.format ?? null);
|
|
101
|
+
}
|
|
102
|
+
catch (err) {
|
|
103
|
+
if (opts.format !== "playwright" || !opts.playwrightCommand)
|
|
104
|
+
throw err;
|
|
105
|
+
discoveryFailure = `Playwright test listing failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
106
|
+
}
|
|
92
107
|
if (all.length === 0) {
|
|
93
108
|
const record = {
|
|
94
109
|
version: 1,
|
|
@@ -98,9 +113,9 @@ export async function run(opts = {}) {
|
|
|
98
113
|
tests: [],
|
|
99
114
|
touched: [],
|
|
100
115
|
answers: {},
|
|
101
|
-
fallback: "no tests were extracted",
|
|
116
|
+
fallback: discoveryFailure ?? "no tests were extracted",
|
|
102
117
|
};
|
|
103
|
-
const selection = everything([],
|
|
118
|
+
const selection = everything([], record.fallback);
|
|
104
119
|
return { selection, framework: record.framework, filter: { mode: "all", argv: [] }, record, spent: null };
|
|
105
120
|
}
|
|
106
121
|
const framework = opts.format ?? pickFramework(all);
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { AskClient, Spend } from "./jev.ts";
|
|
2
|
+
export interface SnapshotChange {
|
|
3
|
+
file: string;
|
|
4
|
+
kind: "external" | "inline";
|
|
5
|
+
}
|
|
6
|
+
export interface SnapshotAssessment extends SnapshotChange {
|
|
7
|
+
status: "plausible" | "review" | "unknown";
|
|
8
|
+
score: number | null;
|
|
9
|
+
confidence: number | null;
|
|
10
|
+
}
|
|
11
|
+
export interface SnapshotReview {
|
|
12
|
+
entries: SnapshotAssessment[];
|
|
13
|
+
error: string | null;
|
|
14
|
+
spent: Spend | null;
|
|
15
|
+
}
|
|
16
|
+
export interface SnapshotTestSources {
|
|
17
|
+
sources: Record<string, string>;
|
|
18
|
+
omitted: string[];
|
|
19
|
+
}
|
|
20
|
+
export declare function snapshotTestFile(change: SnapshotChange): string | null;
|
|
21
|
+
/** Bound extra source context so the shared Jev state still fits its token budget. */
|
|
22
|
+
export declare function loadSnapshotTestSources(cwd: string, changes: SnapshotChange[], maxChars?: number, revision?: "HEAD" | ":" | null): Promise<SnapshotTestSources>;
|
|
23
|
+
export declare function findSnapshotChanges(diff: string): SnapshotChange[];
|
|
24
|
+
/** Match changed lines against complete inline assertion calls, including multiline values. */
|
|
25
|
+
export declare function findSnapshotChangesInWorktree(diff: string, cwd: string, revision?: "HEAD" | ":" | null): Promise<SnapshotChange[]>;
|
|
26
|
+
export declare function assessSnapshots(diff: string, stat: string, client?: AskClient, changes?: SnapshotChange[], context?: SnapshotTestSources): Promise<SnapshotReview>;
|
package/dist/snapshot.js
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/** Read-only review of changed text snapshots. Jev returns decisions, not prose. */
|
|
2
|
+
import { execFile } from "node:child_process";
|
|
3
|
+
import { readFile } from "node:fs/promises";
|
|
4
|
+
import { join } from "node:path";
|
|
5
|
+
import { promisify } from "node:util";
|
|
6
|
+
import { parse } from "@ast-grep/napi";
|
|
7
|
+
import { parseUnifiedDiff, splitDiffByFile, touchesChange } from "./diff.js";
|
|
8
|
+
import { langFor } from "./framework.js";
|
|
9
|
+
import { Jev } from "./jev.js";
|
|
10
|
+
import { questionId, readAnswer } from "./questions.js";
|
|
11
|
+
import { buildState } from "./state.js";
|
|
12
|
+
const execFileAsync = promisify(execFile);
|
|
13
|
+
async function readSource(cwd, file, revision) {
|
|
14
|
+
if (revision === null)
|
|
15
|
+
return readFile(join(cwd, file), "utf8");
|
|
16
|
+
const spec = revision === ":" ? `:${file}` : `HEAD:${file}`;
|
|
17
|
+
const { stdout } = await execFileAsync("git", ["show", spec], { cwd, maxBuffer: 8 * 1024 * 1024 });
|
|
18
|
+
return stdout;
|
|
19
|
+
}
|
|
20
|
+
export function snapshotTestFile(change) {
|
|
21
|
+
if (change.kind === "inline")
|
|
22
|
+
return change.file;
|
|
23
|
+
if (!/(^|\/)__snapshots__\//.test(change.file))
|
|
24
|
+
return null;
|
|
25
|
+
return change.file.replace(/(^|\/)__snapshots__\//, "$1").replace(/\.snap$/, "");
|
|
26
|
+
}
|
|
27
|
+
/** Bound extra source context so the shared Jev state still fits its token budget. */
|
|
28
|
+
export async function loadSnapshotTestSources(cwd, changes, maxChars = 24_000, revision = null) {
|
|
29
|
+
const sources = {};
|
|
30
|
+
const omitted = [];
|
|
31
|
+
let used = 0;
|
|
32
|
+
for (const file of new Set(changes.map(snapshotTestFile).filter((file) => file !== null))) {
|
|
33
|
+
try {
|
|
34
|
+
const source = await readSource(cwd, file, revision);
|
|
35
|
+
if (used + source.length > maxChars) {
|
|
36
|
+
omitted.push(file);
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
sources[file] = source;
|
|
40
|
+
used += source.length;
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
omitted.push(file);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return { sources, omitted };
|
|
47
|
+
}
|
|
48
|
+
export function findSnapshotChanges(diff) {
|
|
49
|
+
const out = [];
|
|
50
|
+
for (const [file, section] of splitDiffByFile(diff)) {
|
|
51
|
+
if (file.endsWith(".snap")) {
|
|
52
|
+
out.push({ file, kind: "external" });
|
|
53
|
+
}
|
|
54
|
+
else if (/\.(?:test|vitest|spec)\.[cm]?[jt]sx?$/.test(file)
|
|
55
|
+
&& /^[+-](?![+-]).*toMatchInlineSnapshot\s*\(/m.test(section)) {
|
|
56
|
+
out.push({ file, kind: "inline" });
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return out;
|
|
60
|
+
}
|
|
61
|
+
/** Match changed lines against complete inline assertion calls, including multiline values. */
|
|
62
|
+
export async function findSnapshotChangesInWorktree(diff, cwd, revision = null) {
|
|
63
|
+
const found = findSnapshotChanges(diff);
|
|
64
|
+
const seen = new Set(found.map((change) => change.file));
|
|
65
|
+
for (const [file, ranges] of parseUnifiedDiff(diff)) {
|
|
66
|
+
if (seen.has(file) || !/\.(?:test|vitest|spec)\.[cm]?[jt]sx?$/.test(file))
|
|
67
|
+
continue;
|
|
68
|
+
let source;
|
|
69
|
+
try {
|
|
70
|
+
source = await readSource(cwd, file, revision);
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
const root = parse(langFor(file), source).root();
|
|
76
|
+
const calls = root.findAll({ rule: {
|
|
77
|
+
kind: "call_expression",
|
|
78
|
+
has: { field: "function", regex: "\\.to(?:Match(?:Aria)?|ThrowErrorMatching)InlineSnapshot$" },
|
|
79
|
+
} });
|
|
80
|
+
if (calls.some((call) => {
|
|
81
|
+
const range = call.range();
|
|
82
|
+
return touchesChange(new Map([[file, ranges]]), file, range.start.line + 1, range.end.line + 1);
|
|
83
|
+
})) {
|
|
84
|
+
found.push({ file, kind: "inline" });
|
|
85
|
+
seen.add(file);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return found;
|
|
89
|
+
}
|
|
90
|
+
const TASK = "The state is a git diff including updated text snapshots and, when available, the associated test source. Judge whether the NEW expected snapshot values are consistent with the intended behavior visible in that context. Do not assume that a snapshot update is correct merely because a test runner generated it. Judge only the named snapshot file. If the context is insufficient, use level 1.";
|
|
91
|
+
const LEVELS = [
|
|
92
|
+
"The new expected values clearly follow the intentional behavior change visible in the diff.",
|
|
93
|
+
"The update is plausible, but the diff does not prove the new expected values are correct.",
|
|
94
|
+
"The update appears to hide an unintended behavior change or contains suspicious new expected values.",
|
|
95
|
+
"The new expected values clearly contradict the intended behavior visible in the diff.",
|
|
96
|
+
];
|
|
97
|
+
export async function assessSnapshots(diff, stat, client = new Jev(), changes = findSnapshotChanges(diff), context = { sources: {}, omitted: [] }) {
|
|
98
|
+
if (changes.length === 0)
|
|
99
|
+
return { entries: [], error: null, spent: null };
|
|
100
|
+
const state = buildState(diff, stat, { maxChars: 64_000 });
|
|
101
|
+
if (state.truncated) {
|
|
102
|
+
return {
|
|
103
|
+
entries: changes.map((change) => ({ ...change, status: "unknown", score: null, confidence: null })),
|
|
104
|
+
error: "the diff exceeded Jev's state budget",
|
|
105
|
+
spent: null,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
const questions = Object.fromEntries(changes.map((change, i) => [questionId(i), {
|
|
109
|
+
type: "score",
|
|
110
|
+
instructions: { task: TASK, subject: questionId(i), snapshot_file: change.file, snapshot_kind: change.kind },
|
|
111
|
+
criteria: LEVELS,
|
|
112
|
+
}]));
|
|
113
|
+
try {
|
|
114
|
+
const response = await client.askSplitting({ ...state, snapshot_test_sources: context.sources }, questions);
|
|
115
|
+
return {
|
|
116
|
+
entries: changes.map((change, i) => {
|
|
117
|
+
const answer = readAnswer(response.answers?.[questionId(i)]);
|
|
118
|
+
const sourceMissing = context.omitted.includes(snapshotTestFile(change) ?? "");
|
|
119
|
+
return {
|
|
120
|
+
...change,
|
|
121
|
+
status: answer === null || sourceMissing ? "unknown" : answer.value >= 1 || answer.confidence === null || answer.confidence < 0.6 ? "review" : "plausible",
|
|
122
|
+
score: answer?.value ?? null,
|
|
123
|
+
confidence: answer?.confidence ?? null,
|
|
124
|
+
};
|
|
125
|
+
}),
|
|
126
|
+
error: null,
|
|
127
|
+
spent: client.spent,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
catch (err) {
|
|
131
|
+
return {
|
|
132
|
+
entries: changes.map((change) => ({ ...change, status: "unknown", score: null, confidence: null })),
|
|
133
|
+
error: err instanceof Error ? err.message : String(err),
|
|
134
|
+
spent: client.spent,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
}
|
package/dist/types.d.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* `jest` is separate from `vitest` only so a report can name it; the two share
|
|
13
13
|
* a filter shape exactly.
|
|
14
14
|
*/
|
|
15
|
-
export type Framework = "vitest" | "jest" | "node" | "playwright" | "rust" | "go" | "unknown";
|
|
15
|
+
export type Framework = "vitest" | "jest" | "node" | "bun" | "playwright" | "rust" | "go" | "unknown";
|
|
16
16
|
/** One test, as the source declares it. */
|
|
17
17
|
export interface TestCase {
|
|
18
18
|
/**
|
|
@@ -31,11 +31,14 @@ export interface TestCase {
|
|
|
31
31
|
line: number;
|
|
32
32
|
endLine: number;
|
|
33
33
|
framework: Framework;
|
|
34
|
+
/** Playwright project and path relative to its configured rootDir, when listed by the runner. */
|
|
35
|
+
project?: string;
|
|
36
|
+
runnerFile?: string;
|
|
34
37
|
/**
|
|
35
38
|
* The title could not be read statically -- a template with an
|
|
36
39
|
* interpolation, or a `.each` row. Such a test can never be named in a
|
|
37
|
-
* `-t` pattern, so it is always selected and
|
|
38
|
-
*
|
|
40
|
+
* `-t` pattern, so it is always selected and may force file-level filtering.
|
|
41
|
+
* Playwright runner discovery resolves generated titles before scoring.
|
|
39
42
|
*/
|
|
40
43
|
dynamic: boolean;
|
|
41
44
|
}
|
package/dist/types.js
CHANGED
|
@@ -14,5 +14,6 @@
|
|
|
14
14
|
* confused with one another.
|
|
15
15
|
*/
|
|
16
16
|
export function testId(t) {
|
|
17
|
-
|
|
17
|
+
const base = `${t.file}\u001f${t.titlePath.join("\u001f")}\u001f${t.line}`;
|
|
18
|
+
return t.project === undefined ? base : `${base}\u001f${t.project}`;
|
|
18
19
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "jev-test-filter",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "Score every test against a git diff with Jev, and emit the filter arguments your runner already understands",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"test",
|
|
@@ -42,6 +42,10 @@
|
|
|
42
42
|
"types": "./dist/filter.d.ts",
|
|
43
43
|
"default": "./dist/filter.js"
|
|
44
44
|
},
|
|
45
|
+
"./snapshot": {
|
|
46
|
+
"types": "./dist/snapshot.d.ts",
|
|
47
|
+
"default": "./dist/snapshot.js"
|
|
48
|
+
},
|
|
45
49
|
"./package.json": "./package.json"
|
|
46
50
|
},
|
|
47
51
|
"files": [
|