@e2e-dev/decision 0.2.5-nightly-20261021080401 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -38,3 +38,24 @@ handles that only `typeSecret` fills.
38
38
 
39
39
  See the [decision models guide](https://e2e.tester.army/docs/decision-models)
40
40
  for setup, vision, gates, provider options, and limits.
41
+
42
+ ## Test filter
43
+
44
+ With `e2e run --changed-since <ref>`, `decisionFilter()` asks a decision
45
+ model whether the change since that ref can affect each test, and leaves
46
+ out the unlikely ones:
47
+
48
+ ```ts
49
+ import type { E2EConfig } from 'e2e';
50
+ import { web } from '@e2e-dev/web';
51
+ import { gateway } from 'ai';
52
+ import { decisionFilter } from '@e2e-dev/decision';
53
+
54
+ export default {
55
+ targets: [{ engine: web(), app: { url: 'http://localhost:3000' } }],
56
+ filters: [decisionFilter({ model: gateway.decisionModel('typesafe-ai/jev') })],
57
+ } satisfies E2EConfig;
58
+ ```
59
+
60
+ The model must answer `boolean` questions. See
61
+ [Test filters](https://e2e.tester.army/docs/filters) for CI setup.
package/dist/.build.json CHANGED
@@ -1 +1 @@
1
- {"version":"0.2.5-nightly-20261021080401"}
1
+ {"version":"0.3.0"}
package/dist/decide.d.ts CHANGED
@@ -1,13 +1,25 @@
1
1
  import type { StepExecutorContext } from 'e2e';
2
2
  import * as ai from 'ai';
3
+ import type { Experimental_DecisionState } from 'ai';
3
4
  import type { DecisionRequest } from './questions.ts';
4
- import type { DecisionExecutorOptions } from './types.ts';
5
+ import type { DecisionExecutorOptions, DecisionFilterOptions } from './types.ts';
6
+ type DecideResult = Awaited<ReturnType<typeof ai.experimental_decide>>;
5
7
  /**
6
- * The SDK's decide call, or INVALID_CONFIG when the installed `ai` predates
7
- * it. Read off the namespace so an older `ai` still loads this module and
8
- * the user gets the version to install instead of a link error.
8
+ * The SDK's decide call, or INVALID_CONFIG naming `caller` when the
9
+ * installed `ai` predates it. Read off the namespace so an older `ai` still
10
+ * loads this module and the user gets the version to install instead of a
11
+ * link error.
9
12
  */
10
- export declare function requireDecide(): typeof ai.experimental_decide;
13
+ export declare function requireDecide(caller: string): typeof ai.experimental_decide;
14
+ /** One call's boolean answers and what it spent. */
15
+ export interface BooleanAnswers {
16
+ /** Each answer's probability of true, by question id; undefined where the model gave none. */
17
+ readonly probabilities: Readonly<Record<string, number | undefined>>;
18
+ readonly usage: DecideResult['usage'];
19
+ readonly providerMetadata: DecideResult['providerMetadata'];
20
+ }
21
+ /** Asks `questions`, instructions by id, as boolean questions in one decide call against `state`, for the test filter. */
22
+ export declare function decideBooleans({ model, providerOptions }: DecisionFilterOptions, state: Experimental_DecisionState, questions: Readonly<Record<string, string>>, abortSignal: AbortSignal): Promise<BooleanAnswers>;
11
23
  /**
12
24
  * One gated answer: the chosen option, its probability, and the provider
13
25
  * confidence. A score answer reads its most probable level as the choice,
@@ -27,3 +39,4 @@ export interface Decision {
27
39
  * there stays a one-file change.
28
40
  */
29
41
  export declare function decide(ctx: StepExecutorContext, model: DecisionExecutorOptions['model'], request: DecisionRequest, providerOptions?: DecisionExecutorOptions['providerOptions']): Promise<Record<string, Decision>>;
42
+ export {};
package/dist/decide.js CHANGED
@@ -7,17 +7,30 @@ import { recorded } from './budget.js';
7
7
  /** Transport retries per decide call. */
8
8
  const MAX_RETRIES = 3;
9
9
  /**
10
- * The SDK's decide call, or INVALID_CONFIG when the installed `ai` predates
11
- * it. Read off the namespace so an older `ai` still loads this module and
12
- * the user gets the version to install instead of a link error.
10
+ * The SDK's decide call, or INVALID_CONFIG naming `caller` when the
11
+ * installed `ai` predates it. Read off the namespace so an older `ai` still
12
+ * loads this module and the user gets the version to install instead of a
13
+ * link error.
13
14
  */
14
- export function requireDecide() {
15
+ export function requireDecide(caller) {
15
16
  const found = ai.experimental_decide;
16
17
  if (typeof found !== 'function') {
17
- throw new ConfigurationError('INVALID_CONFIG', 'decisionExecutor() needs ai 7.0.134 or later; update the ai package');
18
+ throw new ConfigurationError('INVALID_CONFIG', `${caller}() needs ai 7.0.134 or later; update the ai package`);
18
19
  }
19
20
  return found;
20
21
  }
22
+ /** Asks `questions`, instructions by id, as boolean questions in one decide call against `state`, for the test filter. */
23
+ export async function decideBooleans({ model, providerOptions }, state, questions, abortSignal) {
24
+ const { answers, usage, providerMetadata } = await requireDecide('decisionFilter')({
25
+ model,
26
+ state,
27
+ questions: Object.fromEntries(Object.entries(questions).map(([id, instructions]) => [id, { type: 'boolean', instructions }])),
28
+ ...(providerOptions === undefined ? {} : { providerOptions }),
29
+ maxRetries: MAX_RETRIES,
30
+ abortSignal,
31
+ });
32
+ return { probabilities: Object.fromEntries(Object.keys(questions).map((id) => [id, answers[id]?.probability])), usage, providerMetadata };
33
+ }
21
34
  /**
22
35
  * Makes one decide call under the step budget and returns the answers.
23
36
  * This is the only file that touches the SDK decide API, so a rename
@@ -33,7 +46,7 @@ export async function decide(ctx, model, request, providerOptions) {
33
46
  let providerMetadata;
34
47
  try {
35
48
  ({ answers, providerMetadata } = await recorded(ctx, model, async () => {
36
- const result = await requireDecide()({
49
+ const result = await requireDecide('decisionExecutor')({
37
50
  model,
38
51
  state,
39
52
  questions: request.questions,
package/dist/executor.js CHANGED
@@ -4,6 +4,7 @@ import { decide, requireDecide } from './decide.js';
4
4
  import { perform } from './dispatch.js';
5
5
  import { actionSpace } from './elements.js';
6
6
  import { PointLocator } from './locator.js';
7
+ import { checkModel, checkProviderOptions } from './model.js';
7
8
  import { nonSecretParams } from './params.js';
8
9
  import { describe, gated, invalid, need, pick, verdictOf } from './picks.js';
9
10
  import { assertionRequest, completionRequest, operationRequest, targetRequest } from './questions.js';
@@ -17,15 +18,13 @@ export function decisionExecutor(options) {
17
18
  if (typeof options !== 'object' || options === null) {
18
19
  throw new ConfigurationError('INVALID_CONFIG', `decisionExecutor() takes an options object; ${EXAMPLE}`);
19
20
  }
20
- requireDecide();
21
- checkModel(options.model);
21
+ requireDecide('decisionExecutor');
22
+ checkModel(options.model, { caller: 'decisionExecutor', questionType: 'choice', example: EXAMPLE });
22
23
  const gates = { minProbability: options.minProbability ?? 0, minConfidence: options.minConfidence ?? 0 };
23
24
  if (!inUnit(gates.minProbability) || !inUnit(gates.minConfidence)) {
24
25
  throw new ConfigurationError('INVALID_CONFIG', 'decisionExecutor({ minProbability, minConfidence }) must be between 0 and 1');
25
26
  }
26
- if (options.providerOptions !== undefined && !isOptionsRecord(options.providerOptions)) {
27
- throw new ConfigurationError('INVALID_CONFIG', 'decisionExecutor({ providerOptions }) maps provider names to option objects, e.g. { gateway: { zeroDataRetention: true } }');
28
- }
27
+ checkProviderOptions(options.providerOptions, 'decisionExecutor');
29
28
  if (options.vision !== undefined && typeof options.vision !== 'boolean') {
30
29
  throw new ConfigurationError('INVALID_CONFIG', 'decisionExecutor({ vision }) must be a boolean');
31
30
  }
@@ -43,42 +42,6 @@ export function decisionExecutor(options) {
43
42
  }
44
43
  /** How a valid model reads, for every model error. */
45
44
  const EXAMPLE = "pass an AI SDK decision model, e.g. decisionExecutor({ model: typeSafeAi.decisionModel('jev-latest') })";
46
- /**
47
- * Rejects anything the AI SDK cannot decide with, at config load: a model id
48
- * string, a language model, or a decision model that cannot answer `choice`.
49
- */
50
- function checkModel(model) {
51
- if (typeof model === 'function') {
52
- throw new ConfigurationError('INVALID_CONFIG', `decisionExecutor({ model }) got a function, not a model; call it with a model id, ${EXAMPLE}`);
53
- }
54
- if (typeof model !== 'object' || model === null) {
55
- const got = typeof model === 'string' ? `the string ${JSON.stringify(model)}` : String(model);
56
- throw new ConfigurationError('INVALID_CONFIG', `decisionExecutor({ model }) got ${got}; ${EXAMPLE}`);
57
- }
58
- const fields = model;
59
- const name = `${String(fields['provider'])}/${String(fields['modelId'])}`;
60
- if (typeof fields['doDecide'] !== 'function' && typeof fields['doEvaluate'] !== 'function') {
61
- const kind = typeof fields['doGenerate'] === 'function' ? `the language model ${name}` : 'an object that is not a decision model';
62
- throw new ConfigurationError('INVALID_CONFIG', `decisionExecutor({ model }) got ${kind}; ${EXAMPLE}`);
63
- }
64
- if (fields['specificationVersion'] !== 'v4') {
65
- throw new ConfigurationError('INVALID_CONFIG', `decisionExecutor({ model }) got ${name}, a ${String(fields['specificationVersion'])} model; the AI SDK decides with v4 decision models`);
66
- }
67
- const supported = fields['supportedQuestionTypes'];
68
- if (!Array.isArray(supported) || !supported.includes('choice')) {
69
- throw new ConfigurationError('INVALID_CONFIG', `decisionExecutor({ model }) got ${name}, which does not answer choice questions`);
70
- }
71
- }
72
- /** A plain object whose every value is a plain object: the shape the AI SDK takes as providerOptions. */
73
- function isOptionsRecord(value) {
74
- const isRecord = (v) => {
75
- if (typeof v !== 'object' || v === null)
76
- return false;
77
- const prototype = Object.getPrototypeOf(v);
78
- return prototype === Object.prototype || prototype === null;
79
- };
80
- return isRecord(value) && Object.values(value).every(isRecord);
81
- }
82
45
  function inUnit(value) {
83
46
  return Number.isFinite(value) && value >= 0 && value <= 1;
84
47
  }
@@ -0,0 +1,14 @@
1
+ import type { TestFilter } from 'e2e';
2
+ import type { DecisionFilterOptions } from './types.ts';
3
+ /**
4
+ * A test filter named `decision` that asks the decision model, test by
5
+ * test, whether the change can affect what it checks, and whether it can
6
+ * break the setup that saves the test's session, and leaves out the tests
7
+ * unlikely on both counts. A change to the config the run loaded, any
8
+ * file named like an e2e config, or a lockfile leaves nothing out, and a
9
+ * test whose test file, setup file, or stored screenshots changed always
10
+ * runs. A test the model does not judge runs,
11
+ * and the note says why. When every call fails, as without a key, or git fails,
12
+ * the filter throws, so the run warns and runs every test.
13
+ */
14
+ export declare function decisionFilter(options: DecisionFilterOptions): TestFilter;
package/dist/filter.js ADDED
@@ -0,0 +1,411 @@
1
+ import { execFile } from 'node:child_process';
2
+ import { readFile } from 'node:fs/promises';
3
+ import path from 'node:path';
4
+ import { promisify } from 'node:util';
5
+ import { ConfigurationError } from 'e2e/engine';
6
+ import { decideBooleans, requireDecide } from './decide.js';
7
+ import { checkModel, checkProviderOptions } from './model.js';
8
+ const exec = promisify(execFile);
9
+ /** A test the model rates less likely than this to be affected is left out. */
10
+ const MIN_PROBABILITY = 0.2;
11
+ /** Most UTF-8 bytes of state, the diff and the files, one call sends: Jev takes 32k tokens of state and a question, and English text runs about 3.4 bytes a token, other scripts more. */
12
+ const MAX_STATE_BYTES = 80_000;
13
+ /** A longer diff is not sent, and the filter leaves nothing out: each call keeps room for the files. */
14
+ const MAX_DIFF_BYTES = 60_000;
15
+ /** Most questions one call asks. */
16
+ const MAX_QUESTIONS = 200;
17
+ /** Most calls in flight at once. */
18
+ const CONCURRENCY = 8;
19
+ /** Failed calls after which no more are made: the model or its key is likely down. */
20
+ const MAX_FAILURES = 3;
21
+ /** What `git diff` may print, far past MAX_DIFF_BYTES so a long diff is measured rather than lost. */
22
+ const MAX_DIFF_BUFFER = 64 * 1024 * 1024;
23
+ /** Most characters of paths one `git diff` names, well within the 32,767 of a Windows command line. */
24
+ const MAX_PATHSPEC_CHARS = 16_000;
25
+ /** Plain `git diff` output whatever the repository's config says. */
26
+ const DIFF_FLAGS = ['--no-color', '--no-ext-diff', '--no-textconv', '-U3'];
27
+ /**
28
+ * Files that can change what any test does, by name at any depth: the
29
+ * config files e2e discovers, the lockfiles of every package manager, and
30
+ * Gradle's version catalog. A change to one leaves nothing out.
31
+ */
32
+ const RUN_ALL = new Set([
33
+ 'e2e.config.ts',
34
+ 'e2e.config.mts',
35
+ 'e2e.config.js',
36
+ 'e2e.config.mjs',
37
+ 'package-lock.json',
38
+ 'npm-shrinkwrap.json',
39
+ 'pnpm-lock.yaml',
40
+ 'yarn.lock',
41
+ 'bun.lock',
42
+ 'bun.lockb',
43
+ 'deno.lock',
44
+ 'Podfile.lock',
45
+ 'Package.resolved',
46
+ 'Cartfile.resolved',
47
+ 'gradle.lockfile',
48
+ 'buildscript-gradle.lockfile',
49
+ 'libs.versions.toml',
50
+ 'Gemfile.lock',
51
+ 'poetry.lock',
52
+ 'uv.lock',
53
+ 'Pipfile.lock',
54
+ 'pdm.lock',
55
+ 'go.sum',
56
+ 'Cargo.lock',
57
+ 'composer.lock',
58
+ 'pubspec.lock',
59
+ 'mix.lock',
60
+ 'packages.lock.json',
61
+ 'flake.lock',
62
+ 'MODULE.bazel.lock',
63
+ ]);
64
+ /** What follows a test file's name in the directory of its stored screenshots. */
65
+ const SNAPSHOTS_SUFFIX = '-snapshots/';
66
+ /** How a valid model reads, for every model error. */
67
+ const EXAMPLE = "pass an AI SDK decision model, e.g. decisionFilter({ model: gateway.decisionModel('typesafe-ai/jev') })";
68
+ /**
69
+ * A test filter named `decision` that asks the decision model, test by
70
+ * test, whether the change can affect what it checks, and whether it can
71
+ * break the setup that saves the test's session, and leaves out the tests
72
+ * unlikely on both counts. A change to the config the run loaded, any
73
+ * file named like an e2e config, or a lockfile leaves nothing out, and a
74
+ * test whose test file, setup file, or stored screenshots changed always
75
+ * runs. A test the model does not judge runs,
76
+ * and the note says why. When every call fails, as without a key, or git fails,
77
+ * the filter throws, so the run warns and runs every test.
78
+ */
79
+ export function decisionFilter(options) {
80
+ if (typeof options !== 'object' || options === null) {
81
+ throw new ConfigurationError('INVALID_CONFIG', `decisionFilter() takes an options object; ${EXAMPLE}`);
82
+ }
83
+ requireDecide('decisionFilter');
84
+ checkModel(options.model, { caller: 'decisionFilter', questionType: 'boolean', example: EXAMPLE });
85
+ checkProviderOptions(options.providerOptions, 'decisionFilter');
86
+ const { model, providerOptions } = options;
87
+ const settings = { model, ...(providerOptions === undefined ? {} : { providerOptions }) };
88
+ return {
89
+ name: 'decision',
90
+ filter: (tests, context) => judge(tests, context, settings).catch((error) => {
91
+ // An abort ends with its reason, not the AbortError of a git or file read it cut short.
92
+ context.signal.throwIfAborted();
93
+ throw error;
94
+ }),
95
+ };
96
+ }
97
+ /**
98
+ * Leaves nothing out when the loaded config or a `RUN_ALL` file of the
99
+ * project changed, runs the tests the change touches directly without
100
+ * asking, and asks the model
101
+ * about the rest, and the setup files that save their sessions, against
102
+ * the redacted diff and the redacted files, in calls of at most
103
+ * `MAX_QUESTIONS`, `CONCURRENCY` at a time. A test is left out when both
104
+ * it and its setup are unlikely to be affected. A failed call leaves its
105
+ * tests in, and after `MAX_FAILURES` no more calls are made. The note names
106
+ * the tests that ran without asking, what the model did not judge, and
107
+ * what the calls spent. Throws when git or a file read fails, the first
108
+ * failure's first line when calls fail and none answers, and on an abort.
109
+ */
110
+ async function judge(given, context, options) {
111
+ const { changes, signal, log, redact } = context;
112
+ const everything = runAllFile(context);
113
+ if (everything !== undefined)
114
+ return { exclude: {}, note: `${everything} changed, so the decision filter left nothing out` };
115
+ const touched = touchedBy(context);
116
+ const tests = given.filter((test) => !touched(test.file) && !touched(test.setupFile));
117
+ const unasked = tests.length === given.length ? undefined : `${counted(given.length - tests.length, 'test')} ran without asking: the test file, setup file, or stored screenshots changed`;
118
+ // A wrapper may hand on no tests, or the change touched them all: nothing to ask, and no diff to read.
119
+ if (tests.length === 0)
120
+ return unasked === undefined ? { exclude: {} } : { exclude: {}, note: unasked };
121
+ const { modelId } = options.model;
122
+ log(`asking ${modelId} about ${testsIn(tests)}`);
123
+ const raw = await changeDiff(changes, signal);
124
+ if (raw === '')
125
+ return { exclude: {}, note: 'git printed no diff of the changed files, so the decision filter left nothing out' };
126
+ if (Buffer.byteLength(raw) > MAX_DIFF_BYTES)
127
+ return { exclude: {}, note: `the diff is over ${MAX_DIFF_BYTES / 1000} kB, so the decision filter left nothing out` };
128
+ const diff = redact(raw);
129
+ const { batches, tooLarge } = pack(await subjects(tests, context), Buffer.byteLength(diff));
130
+ const { answers, spend, failures } = await askAll(batches, diff, options, signal);
131
+ if (spend.calls === 0 && failures.length > 0)
132
+ throw new Error(firstLine(failures[0]), { cause: failures[0] });
133
+ const { exclude, missed } = verdicts(tests, answers);
134
+ return answered(exclude, [unasked, missed.length === 0 ? undefined : `${modelId} did not judge ${testsIn(missed)}: ${unjudgedBecause(failures, tooLarge)}`], spend);
135
+ }
136
+ /** The tests to leave out, each with its reason, and the tests the model did not answer for, itself or its setup. */
137
+ function verdicts(tests, answers) {
138
+ const exclude = {};
139
+ const missed = [];
140
+ for (const test of tests) {
141
+ const own = answers.tests.get(test.id);
142
+ const setup = test.setupFile === undefined ? 0 : answers.setups.get(test.setupFile);
143
+ if (own === undefined || setup === undefined)
144
+ missed.push(test);
145
+ else if (Math.max(own, setup) < MIN_PROBABILITY)
146
+ exclude[test.id] = `${Math.round(Math.max(own, setup) * 100)}% likely affected`;
147
+ }
148
+ return { exclude, missed };
149
+ }
150
+ /** Why some tests went unjudged: the first failed call's first line, and the files too large to send with the diff. */
151
+ function unjudgedBecause(failures, tooLarge) {
152
+ const why = [];
153
+ if (failures.length > 0)
154
+ why.push(firstLine(failures[0]));
155
+ if (tooLarge.length > 0)
156
+ why.push(`${tooLarge.length === 1 ? `${tooLarge[0]?.key} is` : `${counted(tooLarge.length, 'file')} are`} too large to send with the diff`);
157
+ return why.join('; ');
158
+ }
159
+ /**
160
+ * A path from the repository root, in POSIX form and NFC, as git names
161
+ * files on macOS, for a project-relative `file`; `''` for the project
162
+ * root itself when it is the repository root.
163
+ */
164
+ function repoPath({ projectRoot, changes }, file) {
165
+ return path.relative(changes.root, path.join(projectRoot, file)).split(path.sep).join('/').normalize('NFC');
166
+ }
167
+ /**
168
+ * The first changed file that leaves nothing out: the config the run
169
+ * loaded, whatever its name, or a `RUN_ALL` file that belongs to the
170
+ * project, one in the project directory or inside it, or in a directory
171
+ * above it, as a workspace root's lockfile is. A sibling project's config
172
+ * or lockfile, or one in a docs folder outside the project, goes to the
173
+ * model. Without `configFile`, from an older e2e or a config object, only
174
+ * the `RUN_ALL` names apply.
175
+ */
176
+ function runAllFile(context) {
177
+ const project = repoPath(context, '.');
178
+ const config = context.configFile?.normalize('NFC');
179
+ return context.changes.files.find((file) => {
180
+ if (file.normalize('NFC') === config)
181
+ return true;
182
+ if (!RUN_ALL.has(path.posix.basename(file)))
183
+ return false;
184
+ const directory = path.posix.dirname(file.normalize('NFC'));
185
+ return directory === '.' || project === '' || directory === project || project.startsWith(`${directory}/`) || directory.startsWith(`${project}/`);
186
+ });
187
+ }
188
+ /** Whether a project-relative file, or one of its stored screenshots, is in the change, each compared in NFC. */
189
+ function touchedBy(context) {
190
+ const changed = new Set(context.changes.files.map((file) => file.normalize('NFC')));
191
+ const screenshotted = new Set();
192
+ for (const file of changed) {
193
+ for (let at = file.indexOf(SNAPSHOTS_SUFFIX); at !== -1; at = file.indexOf(SNAPSHOTS_SUFFIX, at + 1))
194
+ screenshotted.add(file.slice(0, at));
195
+ }
196
+ return (file) => {
197
+ if (file === undefined)
198
+ return false;
199
+ const key = repoPath(context, file);
200
+ return changed.has(key) || screenshotted.has(key);
201
+ };
202
+ }
203
+ /**
204
+ * The files the model reads: each test file, then each setup file that
205
+ * saves a test's session, keyed by their redacted path from the repository
206
+ * root, as the diff names files, with their questions.
207
+ */
208
+ async function subjects(tests, context) {
209
+ const { projectRoot, signal, redact } = context;
210
+ const setups = [...new Set(tests.flatMap((test) => (test.setupFile === undefined ? [] : [test.setupFile])))];
211
+ const files = [...new Set([...tests.map((test) => test.file), ...setups])];
212
+ const read = await Promise.all(files.map((file) => readFile(path.join(projectRoot, file), { encoding: 'utf8', signal })));
213
+ const found = new Map();
214
+ files.forEach((file, i) => {
215
+ const text = redact(read[i] ?? '');
216
+ found.set(file, { key: redact(repoPath(context, file)), text, bytes: Buffer.byteLength(text), questions: [] });
217
+ });
218
+ for (const test of tests) {
219
+ const { key, questions } = found.get(test.file);
220
+ questions.push({ kind: 'test', test, text: redact(`Can the diff change what the e2e test "${test.titlePath.join(' > ')}" in ${key} checks? ${fileOf(key)}, hooks included.`) });
221
+ }
222
+ for (const file of setups) {
223
+ const { key, questions } = found.get(file);
224
+ questions.push({ kind: 'setup', file, text: `Can the diff break the e2e setup in ${key}, or the session it saves for the tests that use it? ${fileOf(key)}.` });
225
+ }
226
+ return [...found.values()];
227
+ }
228
+ /**
229
+ * Packs the subjects into calls that each stay within `MAX_STATE_BYTES`
230
+ * with the diff and ask at most `MAX_QUESTIONS`, splitting a subject's
231
+ * questions across calls where they are more. Two subjects whose keys are
232
+ * the same, as two paths that redact alike, go to separate calls, so
233
+ * neither replaces the other. A subject too large to send with the diff is
234
+ * set apart.
235
+ */
236
+ function pack(found, diffBytes) {
237
+ const batches = [];
238
+ const tooLarge = [];
239
+ let room = 0;
240
+ for (const subject of found) {
241
+ if (diffBytes + subject.bytes > MAX_STATE_BYTES) {
242
+ tooLarge.push(subject);
243
+ continue;
244
+ }
245
+ for (let start = 0; start < subject.questions.length; start += MAX_QUESTIONS) {
246
+ const chunk = subject.questions.slice(start, start + MAX_QUESTIONS);
247
+ const last = batches.at(-1);
248
+ if (last === undefined || Object.hasOwn(last.testFiles, subject.key) || subject.bytes > room || last.questions.length + chunk.length > MAX_QUESTIONS) {
249
+ batches.push({ testFiles: { [subject.key]: subject.text }, questions: chunk });
250
+ room = MAX_STATE_BYTES - diffBytes - subject.bytes;
251
+ }
252
+ else {
253
+ last.testFiles[subject.key] = subject.text;
254
+ last.questions.push(...chunk);
255
+ room -= subject.bytes;
256
+ }
257
+ }
258
+ }
259
+ return { batches, tooLarge };
260
+ }
261
+ /**
262
+ * Makes one call per batch, `CONCURRENCY` at a time. A failed call's
263
+ * questions stay unanswered; once `MAX_FAILURES` calls have failed, the
264
+ * calls in flight stop and no more start. Resolves to the answers, what
265
+ * the calls that answered spent, and the failures.
266
+ */
267
+ async function askAll(batches, diff, options, signal) {
268
+ const stop = new AbortController();
269
+ const abortSignal = AbortSignal.any([signal, stop.signal]);
270
+ const answers = { tests: new Map(), setups: new Map() };
271
+ const spend = { calls: 0, tokens: 0, costUsd: undefined, uncosted: false };
272
+ const failures = [];
273
+ let next = 0;
274
+ const work = async () => {
275
+ for (let batch = batches[next++]; batch !== undefined && !stop.signal.aborted; batch = batches[next++]) {
276
+ try {
277
+ const { probabilities, usage, providerMetadata } = await decideBooleans(options, { diff, testFiles: batch.testFiles }, Object.fromEntries(batch.questions.map((question, i) => [`t${i}`, question.text])), abortSignal);
278
+ batch.questions.forEach((question, i) => {
279
+ const probability = probabilities[`t${i}`] ?? 1;
280
+ if (question.kind === 'test')
281
+ answers.tests.set(question.test.id, probability);
282
+ else
283
+ answers.setups.set(question.file, probability);
284
+ });
285
+ const cost = readCost(providerMetadata);
286
+ spend.calls += 1;
287
+ spend.tokens += (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0);
288
+ if (cost === undefined)
289
+ spend.uncosted = true;
290
+ else
291
+ spend.costUsd = (spend.costUsd ?? 0) + cost;
292
+ }
293
+ catch (error) {
294
+ if (signal.aborted)
295
+ throw signal.reason;
296
+ // A call the stop cut short is not a failure of its own.
297
+ if (stop.signal.aborted)
298
+ return;
299
+ failures.push(error);
300
+ if (failures.length >= MAX_FAILURES)
301
+ stop.abort();
302
+ }
303
+ }
304
+ };
305
+ await Promise.all(Array.from({ length: Math.min(CONCURRENCY, batches.length) }, work));
306
+ return { answers, spend, failures };
307
+ }
308
+ /** The filter's answer: `exclude`, and a note of each of `problems` given, then what the calls spent, such as `11 calls, 307.6k tokens, $0.0129`, its cost only when every call reported one. */
309
+ function answered(exclude, problems, { calls, tokens, costUsd, uncosted }) {
310
+ const cost = costUsd === undefined || uncosted ? '' : `, ${formatCost(costUsd)}`;
311
+ const spent = calls === 0 ? undefined : `${counted(calls, 'call')}, ${formatTokens(tokens)} tokens${cost}`;
312
+ const note = [...problems, spent].filter((part) => part !== undefined).join('; ');
313
+ return note === '' ? { exclude } : { exclude, note };
314
+ }
315
+ /**
316
+ * One call's cost in USD, where the provider reports it. The Vercel AI
317
+ * Gateway's `cost` is what it bills; a route on the user's own provider key
318
+ * reports `cost: "0"`, so `marketCost`, the list price, stands in.
319
+ */
320
+ function readCost(metadata) {
321
+ const usd = (raw) => {
322
+ const value = typeof raw === 'string' ? Number(raw) : raw;
323
+ return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined;
324
+ };
325
+ const billed = usd(metadata?.['gateway']?.['cost']);
326
+ if (billed !== undefined && billed > 0)
327
+ return billed;
328
+ return usd(metadata?.['gateway']?.['marketCost']) ?? billed;
329
+ }
330
+ /** A token count: plain under a thousand, then `12.4k`, then `4.2M`. */
331
+ function formatTokens(count) {
332
+ if (count < 1_000)
333
+ return `${count}`;
334
+ if (count < 1_000_000)
335
+ return `${(count / 1_000).toFixed(1)}k`;
336
+ return `${(count / 1_000_000).toFixed(1)}M`;
337
+ }
338
+ /** USD with enough precision for sub-cent calls; `<$0.0001` for a cost that would round to nothing. */
339
+ function formatCost(costUsd) {
340
+ if (costUsd > 0 && costUsd < 0.00005)
341
+ return '<$0.0001';
342
+ return `$${costUsd.toFixed(costUsd < 0.1 ? 4 : 2)}`;
343
+ }
344
+ /** How a question points the model at the file it is about, by its key in `testFiles`. */
345
+ function fileOf(key) {
346
+ return `testFiles[${JSON.stringify(key)}] is its file`;
347
+ }
348
+ /** Tests as a note counts them: `3 tests in 2 files`. */
349
+ function testsIn(tests) {
350
+ return `${counted(tests.length, 'test')} in ${counted(new Set(tests.map((test) => test.file)).size, 'file')}`;
351
+ }
352
+ /**
353
+ * The change as one unified diff of `changes.files`, with renames found and
354
+ * a deleted file's lines left out: what the commits since `changes.base`
355
+ * change, never the working tree, then each file no commit changes by name
356
+ * alone, since CI may have written it. Stops adding
357
+ * files once the diff is past `MAX_DIFF_BYTES`.
358
+ */
359
+ async function changeDiff({ base, root, files }, signal) {
360
+ const committed = new Set((await git(root, ['diff', '--name-only', '--no-renames', '--no-ext-diff', '-z', base, 'HEAD', '--'], signal)).split('\0'));
361
+ let diff = '';
362
+ for (const paths of pathspecs(files.filter((file) => committed.has(file)))) {
363
+ diff += await git(root, ['--literal-pathspecs', 'diff', ...DIFF_FLAGS, '-M', '--irreversible-delete', base, 'HEAD', '--', ...paths], signal);
364
+ if (Buffer.byteLength(diff) > MAX_DIFF_BYTES)
365
+ return diff;
366
+ }
367
+ for (const file of files) {
368
+ if (committed.has(file))
369
+ continue;
370
+ diff += `diff --git ${diffPath(`a/${file}`)} ${diffPath(`b/${file}`)}\nnot committed, contents not read\n`;
371
+ if (Buffer.byteLength(diff) > MAX_DIFF_BYTES)
372
+ break;
373
+ }
374
+ return diff;
375
+ }
376
+ /** A path as a diff header names it: quoted, as git quotes one, when it holds a quote, a backslash, or a control character, so a name can't add lines of its own. */
377
+ function diffPath(file) {
378
+ return /["\\\p{Cc}]/u.test(file) ? JSON.stringify(file) : file;
379
+ }
380
+ /** `files` in runs whose paths together stay within `MAX_PATHSPEC_CHARS`, one `git diff` each. */
381
+ function pathspecs(files) {
382
+ const runs = [];
383
+ let run = [];
384
+ let chars = 0;
385
+ for (const file of files) {
386
+ if (run.length > 0 && chars + file.length > MAX_PATHSPEC_CHARS) {
387
+ runs.push(run);
388
+ run = [];
389
+ chars = 0;
390
+ }
391
+ run.push(file);
392
+ chars += file.length + 1;
393
+ }
394
+ if (run.length > 0)
395
+ runs.push(run);
396
+ return runs;
397
+ }
398
+ /** What `git` prints in `cwd`. */
399
+ async function git(cwd, args, signal) {
400
+ return (await exec('git', args, { cwd, signal, maxBuffer: MAX_DIFF_BUFFER, windowsHide: true })).stdout;
401
+ }
402
+ /** `count` and `noun`, plural unless the count is 1. */
403
+ function counted(count, noun) {
404
+ return `${count} ${noun}${count === 1 ? '' : 's'}`;
405
+ }
406
+ /** The first non-empty line of what `error` says. */
407
+ function firstLine(error) {
408
+ const text = error instanceof Error ? error.message : String(error);
409
+ const line = text.split('\n').find((candidate) => candidate.trim().length > 0);
410
+ return line === undefined ? (error instanceof Error ? error.name : 'an unknown error') : line.trim();
411
+ }
package/dist/index.d.ts CHANGED
@@ -1,2 +1,3 @@
1
1
  export { decisionExecutor } from './executor.ts';
2
- export type { DecisionExecutorOptions } from './types.ts';
2
+ export { decisionFilter } from './filter.ts';
3
+ export type { DecisionExecutorOptions, DecisionFilterOptions } from './types.ts';
package/dist/index.js CHANGED
@@ -1 +1,2 @@
1
1
  export { decisionExecutor } from './executor.js';
2
+ export { decisionFilter } from './filter.js';
@@ -0,0 +1,17 @@
1
+ /** What a factory needs from its decision model, and how its errors name the fix. */
2
+ export interface ModelNeeds {
3
+ /** The factory, as every error names it, e.g. `decisionExecutor`. */
4
+ readonly caller: string;
5
+ /** The question type the factory asks. */
6
+ readonly questionType: 'choice' | 'boolean';
7
+ /** How a valid model reads, ending the errors about a value that is no decision model. */
8
+ readonly example: string;
9
+ }
10
+ /**
11
+ * Rejects anything the AI SDK cannot decide with, at config load: a model id
12
+ * string, a language model, or a decision model that cannot answer the
13
+ * caller's question type.
14
+ */
15
+ export declare function checkModel(model: unknown, { caller, questionType, example }: ModelNeeds): void;
16
+ /** Rejects `providerOptions` other than provider names mapped to option objects, the shape the AI SDK takes, at config load. */
17
+ export declare function checkProviderOptions(value: unknown, caller: string): void;
package/dist/model.js ADDED
@@ -0,0 +1,44 @@
1
+ import { ConfigurationError } from 'e2e/engine';
2
+ /**
3
+ * Rejects anything the AI SDK cannot decide with, at config load: a model id
4
+ * string, a language model, or a decision model that cannot answer the
5
+ * caller's question type.
6
+ */
7
+ export function checkModel(model, { caller, questionType, example }) {
8
+ if (typeof model === 'function') {
9
+ throw new ConfigurationError('INVALID_CONFIG', `${caller}({ model }) got a function, not a model; call it with a model id, ${example}`);
10
+ }
11
+ if (typeof model !== 'object' || model === null) {
12
+ const got = typeof model === 'string' ? `the string ${JSON.stringify(model)}` : String(model);
13
+ throw new ConfigurationError('INVALID_CONFIG', `${caller}({ model }) got ${got}; ${example}`);
14
+ }
15
+ const fields = model;
16
+ const name = `${String(fields['provider'])}/${String(fields['modelId'])}`;
17
+ if (typeof fields['doDecide'] !== 'function' && typeof fields['doEvaluate'] !== 'function') {
18
+ const kind = typeof fields['doGenerate'] === 'function' ? `the language model ${name}` : 'an object that is not a decision model';
19
+ throw new ConfigurationError('INVALID_CONFIG', `${caller}({ model }) got ${kind}; ${example}`);
20
+ }
21
+ if (fields['specificationVersion'] !== 'v4') {
22
+ throw new ConfigurationError('INVALID_CONFIG', `${caller}({ model }) got ${name}, a ${String(fields['specificationVersion'])} model; the AI SDK decides with v4 decision models`);
23
+ }
24
+ const supported = fields['supportedQuestionTypes'];
25
+ if (!Array.isArray(supported) || !supported.includes(questionType)) {
26
+ throw new ConfigurationError('INVALID_CONFIG', `${caller}({ model }) got ${name}, which does not answer ${questionType} questions`);
27
+ }
28
+ }
29
+ /** Rejects `providerOptions` other than provider names mapped to option objects, the shape the AI SDK takes, at config load. */
30
+ export function checkProviderOptions(value, caller) {
31
+ if (value === undefined || isOptionsRecord(value))
32
+ return;
33
+ throw new ConfigurationError('INVALID_CONFIG', `${caller}({ providerOptions }) maps provider names to option objects, e.g. { gateway: { zeroDataRetention: true } }`);
34
+ }
35
+ /** A plain object whose every value is a plain object. */
36
+ function isOptionsRecord(value) {
37
+ const isRecord = (v) => {
38
+ if (typeof v !== 'object' || v === null)
39
+ return false;
40
+ const prototype = Object.getPrototypeOf(v);
41
+ return prototype === Object.prototype || prototype === null;
42
+ };
43
+ return isRecord(value) && Object.values(value).every(isRecord);
44
+ }
package/dist/types.d.ts CHANGED
@@ -43,3 +43,17 @@ export interface DecisionExecutorOptions {
43
43
  */
44
44
  readonly vision?: boolean;
45
45
  }
46
+ /** Options for a test filter that asks a decision model which tests a change can affect. */
47
+ export interface DecisionFilterOptions {
48
+ /**
49
+ * An AI SDK decision model that answers `boolean` questions with a
50
+ * probability, e.g. `gateway.decisionModel('typesafe-ai/jev')`, which
51
+ * reads `AI_GATEWAY_API_KEY`.
52
+ */
53
+ readonly model: Exclude<Experimental_DecisionModel, string>;
54
+ /**
55
+ * Provider options sent with every decide call, such as
56
+ * `{ gateway: { zeroDataRetention: true } }` for Vercel AI Gateway.
57
+ */
58
+ readonly providerOptions?: Parameters<typeof experimental_decide>[0]['providerOptions'];
59
+ }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@e2e-dev/decision",
3
- "version": "0.2.5-nightly-20261021080401",
4
- "description": "Decision-model executors for e2e",
3
+ "version": "0.3.0",
4
+ "description": "Decision-model executors and test filters for e2e",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
7
7
  "type": "git",
@@ -30,18 +30,18 @@
30
30
  },
31
31
  "peerDependencies": {
32
32
  "ai": "^7.0.134",
33
- "e2e": "0.21.3-nightly-20261021080401"
33
+ "e2e": ">=0.21.3 <1"
34
34
  },
35
35
  "devDependencies": {
36
36
  "@ai-sdk/openai": "4.0.90",
37
37
  "@ai-sdk/typesafe-ai": "3.0.19",
38
- "@e2e-dev/web": "0.15.3-nightly-20261021080401",
38
+ "@e2e-dev/web": "0.15.3",
39
39
  "@types/node": "26.6.2",
40
40
  "@types/pngjs": "6.0.5",
41
41
  "ai": "7.0.134",
42
42
  "ajv": "8.20.0",
43
43
  "ajv-formats": "3.0.1",
44
- "e2e": "0.21.3-nightly-20261021080401",
44
+ "e2e": "0.21.3",
45
45
  "typescript": "7.0.2",
46
46
  "vitest": "5.0.1"
47
47
  },