jev-test-filter 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,57 @@
1
+ const TASK = "The state is a git diff. Judge only the one test identified below, and only " +
2
+ "for whether this change can alter its outcome -- other problems with the test " +
3
+ "or with the change are not your concern here.";
4
+ export const SCORE_LEVELS = [
5
+ "Unrelated to this change: running this test cannot produce a different result than before.",
6
+ "This test exercises code the change touched, but nothing in the change can alter its outcome.",
7
+ "This test could be affected by the change: it might fail.",
8
+ "This test directly exercises behaviour the change altered or broke: it is likely to fail.",
9
+ ];
10
+ export const SCORE_LEVEL_NAMES = ["unrelated", "unaffected", "at-risk", "likely-failing"];
11
+ /** Stable question name, so answers can be matched back by the test's index. */
12
+ export function questionId(i) {
13
+ return `q${String(i).padStart(4, "0")}`;
14
+ }
15
+ /**
16
+ * The name the test is asked about is the runner's own spelling of it, so a
17
+ * reader of `--json` can paste it into `-t` and see the same test. Playwright
18
+ * is selected by location rather than by name, so its chain is joined the way
19
+ * its reporter prints it.
20
+ */
21
+ export function displayName(t) {
22
+ if (t.framework === "rust")
23
+ return t.titlePath.join("::");
24
+ if (t.framework === "go")
25
+ return t.titlePath.join("/");
26
+ const sep = t.framework === "node" ? " " : " > ";
27
+ return t.titlePath.join(sep);
28
+ }
29
+ export function buildQuestion(t, id) {
30
+ return {
31
+ type: "score",
32
+ instructions: {
33
+ task: TASK,
34
+ subject: id,
35
+ test_file: t.file,
36
+ test_name: displayName(t),
37
+ test_lines: t.line === t.endLine ? `${t.line}` : `${t.line}-${t.endLine}`,
38
+ },
39
+ criteria: SCORE_LEVELS,
40
+ };
41
+ }
42
+ /**
43
+ * One answer, or null when it cannot be used.
44
+ *
45
+ * Null rather than zero: a malformed answer must select the test, and a zero
46
+ * would silently deselect it.
47
+ */
48
+ export function readAnswer(raw) {
49
+ if (!raw || typeof raw !== "object")
50
+ return null;
51
+ const o = raw;
52
+ const v = typeof o.score === "number" ? o.score : typeof o.value === "number" ? o.value : null;
53
+ if (v === null || !Number.isFinite(v))
54
+ return null;
55
+ const c = typeof o.confidence === "number" && Number.isFinite(o.confidence) ? o.confidence : null;
56
+ return { value: v, confidence: c };
57
+ }
package/dist/run.d.ts ADDED
@@ -0,0 +1,74 @@
1
+ import type { ChangedRanges } from "./diff.ts";
2
+ import type { StatePayload } from "./state.ts";
3
+ import type { GateOptions } from "./gate.ts";
4
+ import type { FilterArgs } from "./filter.ts";
5
+ import type { AskClient, Spend } from "./jev.ts";
6
+ import type { Answer, Framework, Selection, TestCase } from "./types.ts";
7
+ /** Where a run's answers are kept, for `--replay`. */
8
+ export declare const RECORD_DIR = ".jev-test-filter";
9
+ export declare const RECORD_FILE = "last.json";
10
+ export interface RunRecord {
11
+ version: 1;
12
+ createdAt: string;
13
+ base: string | null;
14
+ framework: Framework;
15
+ tests: TestCase[];
16
+ /** `testId` of every test the diff touched. */
17
+ touched: string[];
18
+ answers: Record<string, Answer | null>;
19
+ fallback: string | null;
20
+ }
21
+ /**
22
+ * The framework the run is for.
23
+ *
24
+ * A repository with Vitest unit tests and Playwright specs is normal, and
25
+ * there is no single command that runs both, so a mixed set is a question for
26
+ * the user rather than a guess for this function.
27
+ */
28
+ export declare function pickFramework(tests: TestCase[]): Framework;
29
+ /** The tests whose own bodies the diff touched. Keyed by `testId`. */
30
+ export declare function collect(tests: TestCase[], ranges: ChangedRanges): Set<string>;
31
+ export interface ScoreOptions {
32
+ client?: AskClient | null;
33
+ concurrency?: number;
34
+ /**
35
+ * Questions per request. The server has no documented cap on the number of
36
+ * questions and over a thousand in one request is fine, but a batch bounds
37
+ * how much one `max_tokens_exceeded` split has to redo.
38
+ */
39
+ batchSize?: number;
40
+ }
41
+ /**
42
+ * Ask about every test.
43
+ *
44
+ * Answers come back keyed by question id, and a question with no answer stays
45
+ * null: the gate selects a test with no answer, and a zero here would silently
46
+ * deselect it instead.
47
+ */
48
+ export declare function score(tests: TestCase[], state: StatePayload, { client, concurrency, batchSize }?: ScoreOptions): Promise<Map<string, Answer | null>>;
49
+ export interface RunOptions extends GateOptions {
50
+ cwd?: string;
51
+ base?: string | null;
52
+ staged?: boolean;
53
+ paths?: string[];
54
+ /** Restrict the run to one framework instead of requiring a single one. */
55
+ format?: Framework | null;
56
+ client?: AskClient | null;
57
+ concurrency?: number;
58
+ batchSize?: number;
59
+ fileThreshold?: number;
60
+ /** Extract and gate but never call Jev; every test scores as missing. */
61
+ dryRun?: boolean;
62
+ }
63
+ export interface RunResult {
64
+ selection: Selection;
65
+ framework: Framework;
66
+ filter: FilterArgs;
67
+ record: RunRecord;
68
+ spent: Spend | null;
69
+ }
70
+ export declare function run(opts?: RunOptions): Promise<RunResult>;
71
+ /** Re-gate a recorded run at today's cutoffs, offline. */
72
+ export declare function replay(record: RunRecord, opts?: GateOptions): Selection;
73
+ export declare function saveRecord(cwd: string, record: RunRecord): Promise<string>;
74
+ export declare function loadRecord(path: string): Promise<RunRecord>;
package/dist/run.js ADDED
@@ -0,0 +1,172 @@
1
+ /**
2
+ * The run: a diff and a repository in, a selection and the runner's arguments
3
+ * out.
4
+ *
5
+ * The only part of the program that touches the network and the filesystem at
6
+ * once. Everything it decides with is a pure function it calls, and
7
+ * everything it learns is written to a record, so `replay` can reach the same
8
+ * selection again for free under a different cutoff.
9
+ */
10
+ import { readFile, mkdir, writeFile } from "node:fs/promises";
11
+ import { join } from "node:path";
12
+ import { loadDiff, touchesChange } from "./diff.js";
13
+ import { discoverTests } from "./discover.js";
14
+ import { buildState } from "./state.js";
15
+ import { buildQuestion, questionId, readAnswer } from "./questions.js";
16
+ import { gate } from "./gate.js";
17
+ import { buildFilter } from "./filter.js";
18
+ import { Jev, mapLimit, DEFAULT_CONCURRENCY } from "./jev.js";
19
+ import { testId } from "./types.js";
20
+ /** Where a run's answers are kept, for `--replay`. */
21
+ export const RECORD_DIR = ".jev-test-filter";
22
+ export const RECORD_FILE = "last.json";
23
+ /**
24
+ * The framework the run is for.
25
+ *
26
+ * A repository with Vitest unit tests and Playwright specs is normal, and
27
+ * there is no single command that runs both, so a mixed set is a question for
28
+ * the user rather than a guess for this function.
29
+ */
30
+ export function pickFramework(tests) {
31
+ const kinds = [...new Set(tests.map((t) => t.framework))];
32
+ if (kinds.length === 1)
33
+ return kinds[0];
34
+ throw new Error(`the selected tests span more than one framework (${kinds.join(", ")}); ` +
35
+ `narrow the run with a path argument or pick one with --format`);
36
+ }
37
+ /** The tests whose own bodies the diff touched. Keyed by `testId`. */
38
+ export function collect(tests, ranges) {
39
+ const out = new Set();
40
+ for (const t of tests) {
41
+ if (touchesChange(ranges, t.file, t.line, t.endLine))
42
+ out.add(testId(t));
43
+ }
44
+ return out;
45
+ }
46
+ /**
47
+ * Ask about every test.
48
+ *
49
+ * Answers come back keyed by question id, and a question with no answer stays
50
+ * null: the gate selects a test with no answer, and a zero here would silently
51
+ * deselect it instead.
52
+ */
53
+ export async function score(tests, state, { client = null, concurrency = DEFAULT_CONCURRENCY, batchSize = 400 } = {}) {
54
+ const jev = client ?? new Jev();
55
+ const out = new Map();
56
+ tests.forEach((_, i) => out.set(questionId(i), null));
57
+ const batches = [];
58
+ for (let i = 0; i < tests.length; i += batchSize) {
59
+ const batch = {};
60
+ for (let j = i; j < Math.min(i + batchSize, tests.length); j += 1) {
61
+ const id = questionId(j);
62
+ batch[id] = buildQuestion(tests[j], id);
63
+ }
64
+ batches.push(batch);
65
+ }
66
+ const responses = await mapLimit(batches, concurrency, (batch) => jev.askSplitting(state, batch));
67
+ for (const res of responses) {
68
+ for (const [id, raw] of Object.entries(res.answers ?? {})) {
69
+ if (out.has(id))
70
+ out.set(id, readAnswer(raw));
71
+ }
72
+ }
73
+ return out;
74
+ }
75
+ /**
76
+ * A selection is an optimization and never a correctness gate, so every
77
+ * failure below takes the same exit: run everything, and say why on the
78
+ * result rather than in a log nobody reads.
79
+ */
80
+ function everything(tests, reason) {
81
+ return {
82
+ verdicts: tests.map((t, i) => ({ id: questionId(i), test: t, answer: null, reason: "missing", selected: true })),
83
+ selected: tests,
84
+ all: tests,
85
+ fallback: reason,
86
+ };
87
+ }
88
+ export async function run(opts = {}) {
89
+ const cwd = opts.cwd ?? process.cwd();
90
+ const diff = await loadDiff({ cwd, base: opts.base ?? null, staged: opts.staged ?? false });
91
+ const all = await discoverTests(cwd, opts.paths ?? [], opts.format ?? null);
92
+ if (all.length === 0) {
93
+ const record = {
94
+ version: 1,
95
+ createdAt: new Date().toISOString(),
96
+ base: opts.base ?? null,
97
+ framework: opts.format ?? "unknown",
98
+ tests: [],
99
+ touched: [],
100
+ answers: {},
101
+ fallback: "no tests were extracted",
102
+ };
103
+ const selection = everything([], "no tests were extracted");
104
+ return { selection, framework: record.framework, filter: { mode: "all", argv: [] }, record, spent: null };
105
+ }
106
+ const framework = opts.format ?? pickFramework(all);
107
+ const touched = collect(all, diff.ranges);
108
+ const state = buildState(diff.text, diff.stat);
109
+ let answers = new Map();
110
+ let spent = null;
111
+ let selection;
112
+ if (opts.dryRun) {
113
+ selection = everything(all, "--dry-run: no questions were asked");
114
+ }
115
+ else {
116
+ let failure = null;
117
+ try {
118
+ const client = opts.client ?? new Jev();
119
+ answers = await score(all, state, {
120
+ client,
121
+ ...(opts.concurrency === undefined ? {} : { concurrency: opts.concurrency }),
122
+ ...(opts.batchSize === undefined ? {} : { batchSize: opts.batchSize }),
123
+ });
124
+ spent = client.spent;
125
+ }
126
+ catch (err) {
127
+ failure = `jev failed: ${err instanceof Error ? err.message : String(err)}`;
128
+ }
129
+ // Gating is pure and offline, so it stays outside the try: a bug in the
130
+ // gate must not be reported to the user as a network failure.
131
+ selection = failure === null ? gate(all, answers, touched, opts) : everything(all, failure);
132
+ }
133
+ // A truncated diff that still deselects most of the suite is a selection
134
+ // made from a state that was missing the change it should have judged.
135
+ if (selection.fallback === null && state.truncated && selection.selected.length / all.length < 0.5) {
136
+ selection = everything(all, "the diff did not fit the state budget and the selection was small");
137
+ }
138
+ const record = {
139
+ version: 1,
140
+ createdAt: new Date().toISOString(),
141
+ base: opts.base ?? null,
142
+ framework,
143
+ tests: all,
144
+ touched: [...touched],
145
+ answers: Object.fromEntries(answers),
146
+ fallback: selection.fallback,
147
+ };
148
+ const filter = selection.fallback === null
149
+ ? buildFilter(selection, framework, opts.fileThreshold === undefined ? {} : { fileThreshold: opts.fileThreshold })
150
+ : { mode: "all", argv: [] };
151
+ return { selection, framework, filter, record, spent };
152
+ }
153
+ /** Re-gate a recorded run at today's cutoffs, offline. */
154
+ export function replay(record, opts = {}) {
155
+ if (record.fallback !== null)
156
+ return everything(record.tests, record.fallback);
157
+ const answers = new Map(Object.entries(record.answers));
158
+ return gate(record.tests, answers, new Set(record.touched), opts);
159
+ }
160
+ export async function saveRecord(cwd, record) {
161
+ const dir = join(cwd, RECORD_DIR);
162
+ await mkdir(dir, { recursive: true });
163
+ const path = join(dir, RECORD_FILE);
164
+ await writeFile(path, `${JSON.stringify(record, null, 2)}\n`, "utf8");
165
+ return path;
166
+ }
167
+ export async function loadRecord(path) {
168
+ const parsed = JSON.parse(await readFile(path, "utf8"));
169
+ if (parsed.version !== 1)
170
+ throw new Error(`unsupported record version ${String(parsed.version)}`);
171
+ return parsed;
172
+ }
@@ -0,0 +1,19 @@
1
+ export interface StatePayload {
2
+ reviewing: string;
3
+ changed_files: string[];
4
+ stat: string;
5
+ diff: string;
6
+ truncated: boolean;
7
+ omitted_files: string[];
8
+ }
9
+ /**
10
+ * Characters, not tokens. The state budget is 32Ki tokens and source text
11
+ * runs near 3.6 characters per token, which would put the ceiling around
12
+ * 118k; 96k leaves room for the estimate being wrong in the direction that
13
+ * costs a request.
14
+ */
15
+ export declare const DEFAULT_MAX_CHARS = 96000;
16
+ export interface StateOptions {
17
+ maxChars?: number;
18
+ }
19
+ export declare function buildState(diff: string, stat: string, { maxChars }?: StateOptions): StatePayload;
package/dist/state.js ADDED
@@ -0,0 +1,55 @@
1
+ /**
2
+ * The diff, as the one `state` every question is asked against.
3
+ *
4
+ * Jev's two ceilings are the whole request at 64Ki input tokens and the
5
+ * `state` alone at 32Ki, and the state's is the one that fills up first. A
6
+ * diff over that budget is cut by dropping whole file sections, smallest
7
+ * first, because keeping the greatest NUMBER of files is what a judgment
8
+ * about which tests a change touches actually needs -- one 4000-line
9
+ * generated file should not push nine hand-edited ones out.
10
+ *
11
+ * What was dropped is named in the state rather than silently missing, and
12
+ * `git diff --stat` is carried whole, so a question about a test in an
13
+ * omitted file is answered by a model that at least knows the file changed.
14
+ */
15
+ import { splitDiffByFile } from "./diff.js";
16
+ /**
17
+ * Characters, not tokens. The state budget is 32Ki tokens and source text
18
+ * runs near 3.6 characters per token, which would put the ceiling around
19
+ * 118k; 96k leaves room for the estimate being wrong in the direction that
20
+ * costs a request.
21
+ */
22
+ export const DEFAULT_MAX_CHARS = 96_000;
23
+ export function buildState(diff, stat, { maxChars = DEFAULT_MAX_CHARS } = {}) {
24
+ const sections = splitDiffByFile(diff);
25
+ const changed = [...sections.keys()];
26
+ const base = {
27
+ reviewing: "a git diff",
28
+ changed_files: changed,
29
+ stat,
30
+ diff,
31
+ truncated: false,
32
+ omitted_files: [],
33
+ };
34
+ if (diff.length <= maxChars)
35
+ return base;
36
+ const bySize = [...sections.entries()].sort((a, b) => a[1].length - b[1].length);
37
+ const kept = [];
38
+ let used = 0;
39
+ for (const entry of bySize) {
40
+ if (used + entry[1].length + 1 > maxChars)
41
+ continue;
42
+ kept.push(entry);
43
+ used += entry[1].length + 1;
44
+ }
45
+ const keptNames = new Set(kept.map(([f]) => f));
46
+ // Emit in the diff's own order, not in size order: a diff read out of order
47
+ // is harder to follow and carries no more information.
48
+ const ordered = changed.filter((f) => keptNames.has(f)).map((f) => sections.get(f));
49
+ return {
50
+ ...base,
51
+ diff: ordered.join("\n"),
52
+ truncated: true,
53
+ omitted_files: changed.filter((f) => !keptNames.has(f)),
54
+ };
55
+ }
@@ -0,0 +1,93 @@
1
+ /**
2
+ * The contract every other module speaks.
3
+ *
4
+ * Nothing here reaches the network or the filesystem, so a test of any layer
5
+ * can build its own inputs by hand.
6
+ */
7
+ /**
8
+ * Which runner a test belongs to. It decides how the full name is spelled and
9
+ * which flag carries the selection, so it is carried per test rather than per
10
+ * run: a repository with both Vitest unit tests and Playwright specs is normal.
11
+ *
12
+ * `jest` is separate from `vitest` only so a report can name it; the two share
13
+ * a filter shape exactly.
14
+ */
15
+ export type Framework = "vitest" | "jest" | "node" | "playwright" | "rust" | "go" | "unknown";
16
+ /** One test, as the source declares it. */
17
+ export interface TestCase {
18
+ /**
19
+ * Repository-relative, POSIX separators. Empty when the location is not
20
+ * known -- a Rust test cargo listed and no `#[test]` function could be
21
+ * matched to, such as one a macro generated.
22
+ */
23
+ file: string;
24
+ /** The enclosing suites outermost first, then the test's own title. */
25
+ titlePath: string[];
26
+ /**
27
+ * 1-based and inclusive: the range of the test call itself. Zero when the
28
+ * location is not known, which no changed range can overlap, so such a test
29
+ * is scored rather than selected for free.
30
+ */
31
+ line: number;
32
+ endLine: number;
33
+ framework: Framework;
34
+ /**
35
+ * The title could not be read statically -- a template with an
36
+ * interpolation, or a `.each` row. Such a test can never be named in a
37
+ * `-t` pattern, so it is always selected and it forces the whole run down
38
+ * to file-level filtering.
39
+ */
40
+ dynamic: boolean;
41
+ }
42
+ /** A usable answer to one question. Absent rather than zero when unusable. */
43
+ export interface Answer {
44
+ /** The score level, 0 to 3. */
45
+ value: number;
46
+ /** How sure the model is. Routes an uncertain verdict; never gates one. */
47
+ confidence: number | null;
48
+ }
49
+ /**
50
+ * Why a test ended up on the side it did. Reported, not just logged: a
51
+ * selection nobody can explain is one nobody will trust enough to leave on.
52
+ */
53
+ export type Reason =
54
+ /** The test's own body is inside the diff. Selected without asking. */
55
+ "touched"
56
+ /** The title is not statically knowable. Selected to be safe. */
57
+ | "dynamic"
58
+ /** The score reached the cutoff. */
59
+ | "scored"
60
+ /** Near the cutoff and the model was unsure. Selected to be safe. */
61
+ | "unsure"
62
+ /** No usable answer came back. Selected to be safe. */
63
+ | "missing"
64
+ /** The score was under the cutoff. Not selected. */
65
+ | "below";
66
+ export interface Verdict {
67
+ /** The question this test was asked under, e.g. `q0007`. */
68
+ id: string;
69
+ test: TestCase;
70
+ answer: Answer | null;
71
+ reason: Reason;
72
+ selected: boolean;
73
+ }
74
+ export interface Selection {
75
+ verdicts: Verdict[];
76
+ selected: TestCase[];
77
+ all: TestCase[];
78
+ /**
79
+ * Non-null when the run gave up on selecting and everything is to be run.
80
+ * The string is the reason, shown to the user.
81
+ */
82
+ fallback: string | null;
83
+ }
84
+ /**
85
+ * A stable key for one test.
86
+ *
87
+ * The line is part of it because two tests in one file may legitimately share
88
+ * a full name, and a key that collides would let one test's answer decide the
89
+ * other's fate. The unit separator cannot occur in a JavaScript string
90
+ * literal a source file actually contains, so the three parts cannot be
91
+ * confused with one another.
92
+ */
93
+ export declare function testId(t: TestCase): string;
package/dist/types.js ADDED
@@ -0,0 +1,18 @@
1
+ /**
2
+ * The contract every other module speaks.
3
+ *
4
+ * Nothing here reaches the network or the filesystem, so a test of any layer
5
+ * can build its own inputs by hand.
6
+ */
7
+ /**
8
+ * A stable key for one test.
9
+ *
10
+ * The line is part of it because two tests in one file may legitimately share
11
+ * a full name, and a key that collides would let one test's answer decide the
12
+ * other's fate. The unit separator cannot occur in a JavaScript string
13
+ * literal a source file actually contains, so the three parts cannot be
14
+ * confused with one another.
15
+ */
16
+ export function testId(t) {
17
+ return `${t.file}\u001f${t.titlePath.join("\u001f")}\u001f${t.line}`;
18
+ }
package/package.json ADDED
@@ -0,0 +1,69 @@
1
+ {
2
+ "name": "jev-test-filter",
3
+ "version": "0.1.0",
4
+ "description": "Score every test against a git diff with Jev, and emit the filter arguments your runner already understands",
5
+ "keywords": [
6
+ "test",
7
+ "vitest",
8
+ "node:test",
9
+ "playwright",
10
+ "test-selection",
11
+ "jev",
12
+ "ast-grep"
13
+ ],
14
+ "license": "MIT",
15
+ "repository": {
16
+ "type": "git",
17
+ "url": "git+https://github.com/mizchi/jev-test-filter.git"
18
+ },
19
+ "type": "module",
20
+ "engines": {
21
+ "node": ">=24"
22
+ },
23
+ "bin": {
24
+ "jev-test-filter": "dist/cli.js"
25
+ },
26
+ "main": "./dist/run.js",
27
+ "types": "./dist/run.d.ts",
28
+ "exports": {
29
+ ".": {
30
+ "types": "./dist/run.d.ts",
31
+ "default": "./dist/run.js"
32
+ },
33
+ "./types": {
34
+ "types": "./dist/types.d.ts",
35
+ "default": "./dist/types.js"
36
+ },
37
+ "./gate": {
38
+ "types": "./dist/gate.d.ts",
39
+ "default": "./dist/gate.js"
40
+ },
41
+ "./filter": {
42
+ "types": "./dist/filter.d.ts",
43
+ "default": "./dist/filter.js"
44
+ },
45
+ "./package.json": "./package.json"
46
+ },
47
+ "files": [
48
+ "dist",
49
+ "README.md",
50
+ "LICENSE"
51
+ ],
52
+ "scripts": {
53
+ "build": "tsc && node -e \"require('fs').chmodSync('dist/cli.js', 0o755)\"",
54
+ "typecheck": "tsc --noEmit",
55
+ "test": "node --test \"test/*.test.ts\"",
56
+ "prepack": "npm run build"
57
+ },
58
+ "dependencies": {
59
+ "@ast-grep/lang-go": "^0.0.6",
60
+ "@ast-grep/lang-rust": "^0.0.7",
61
+ "@ast-grep/napi": "^0.40.0"
62
+ },
63
+ "devDependencies": {
64
+ "@secretlint/secretlint-rule-preset-recommend": "^9.0.0",
65
+ "@types/node": "^24.0.0",
66
+ "secretlint": "^9.0.0",
67
+ "typescript": "^5.9.0"
68
+ }
69
+ }