@e2e-dev/decision 0.2.0-nightly-20261009094406 → 0.2.0

package/README.md

@@ -1,10 +1,10 @@
 # @e2e-dev/decision
 
-Run `agent.act` and `agent.assert` through a decision model instead of an LLM:
-any AI SDK decision model that answers `choice` questions with probability
-distributions (e.g. TypeSafe Jev), plus a small language model that writes
-field values when the decision model picks `type`. Needs `ai` 7, version 7.0.128 or
-later.
+Run `agent.act` and `agent.assert` through a decision model instead of an
+LLM. Any AI SDK decision model that answers `choice` questions with
+probability distributions works, such as TypeSafe Jev or OpenAI `gpt-6-luna`
+through `@ai-sdk/openai` 4.0.90 or later. A small language model writes
+field values and upload paths. Needs `ai` 7.0.134 or later.
 
 ```ts
 import type { E2EConfig } from 'e2e';
@@ -24,21 +24,14 @@ export default {
 } satisfies E2EConfig;
 ```
 
-Set `agents.default.executor` in the e2e config. Tests stay plain natural
-language, with no params needed. One `experimental_decide` call per action
-asks the operation plus one target question per operation; operations with a
-single target dispatch without a question. The text model is also the agent's
-judgment tier (`waitFor`, `extract`); without a text model or a configured agent
-`model`, `type` is never offered.
-
-`providerOptions` go with every decide call and reach the decision model
-only; the text model keeps the agents entry's `providerOptions`.
-
-The runner authorizes every dispatched action and records every model call
-against the step budget. Secrets stay declared handles filled only through
-`typeSecret`; password fields never reach the text model. `minProbability`
-and `minConfidence` gates are off by default. A `done`/`failed` claim passes
-an independent fresh-screen check before it concludes the step.
+Tests stay plain natural language, with no params. Each turn asks the
+operation, then the target for it. Every operation the engine declares is
+offered, and every target question carries a `none` option. With
+`vision: true` and a model that takes images, such as
+`openai.decisionModel('gpt-6-luna')`, verdicts see a masked screenshot and
+`tap_at` reaches a drawn control. The runner authorizes every action and
+records every model call against the step budget. Secrets stay declared
+handles that only `typeSecret` fills.
 
 See the [decision models guide](https://e2e.tester.army/docs/decision-models)
-for setup, gates, and limits (no vision, no `navigate`, the 255-choice cap).
+for setup, vision, gates, provider options, and limits.

package/dist/.build.json

@@ -1 +1 @@
-{"version":"0.2.0-nightly-20261009094406"}
+{"version":"0.2.0"}

package/dist/budget.d.ts

@@ -0,0 +1,19 @@
+import type { StepExecutorContext } from 'e2e';
+/** What one model call came back with: its value, the tokens it used, and the model id the response named. */
+export interface Recorded<T> {
+    readonly value: T;
+    readonly usage: {
+        readonly inputTokens?: number | undefined;
+        readonly outputTokens?: number | undefined;
+    };
+    readonly modelId?: string;
+}
+/**
+ * Runs one model call against the step budget: aborted before it starts or
+ * after it returns, and recorded with its timing and tokens even when it
+ * throws. Errors propagate untouched for the caller to map.
+ */
+export declare function recorded<T>(ctx: StepExecutorContext, model: {
+    readonly provider: string;
+    readonly modelId: string;
+}, call: () => Promise<Recorded<T>>): Promise<T>;

package/dist/budget.js

@@ -0,0 +1,29 @@
+/**
+ * Runs one model call against the step budget: aborted before it starts or
+ * after it returns, and recorded with its timing and tokens even when it
+ * throws. Errors propagate untouched for the caller to map.
+ */
+export async function recorded(ctx, model, call) {
+    ctx.signal.throwIfAborted();
+    const started = performance.now();
+    const startedAt = new Date().toISOString();
+    let usage = {};
+    let modelId = model.modelId;
+    try {
+        const result = await call();
+        ctx.signal.throwIfAborted();
+        usage = result.usage;
+        modelId = result.modelId ?? modelId;
+        return result.value;
+    }
+    finally {
+        ctx.budgets.recordModelCall({
+            provider: model.provider,
+            modelId,
+            startedAt,
+            durationMs: performance.now() - started,
+            ...(usage.inputTokens === undefined ? {} : { inputTokens: usage.inputTokens }),
+            ...(usage.outputTokens === undefined ? {} : { outputTokens: usage.outputTokens }),
+        });
+    }
+}

package/dist/decide.d.ts

@@ -8,11 +8,18 @@ import type { DecisionExecutorOptions } from './types.ts';
  * the user gets the version to install instead of a link error.
  */
 export declare function requireDecide(): typeof ai.experimental_decide;
-/** One gated answer: the chosen option, its probability, and the provider confidence. */
+/**
+ * One gated answer: the chosen option, its probability, and the provider
+ * confidence. A score answer reads its most probable level as the choice,
+ * carries the probability-weighted level as `score`, and its probability
+ * is the mass on the levels within half a level of that score: a spread
+ * over two adjacent columns is a precise position, not an uncertain one.
+ */
 export interface Decision {
     readonly choice: string;
     readonly probability: number;
     readonly confidence: number;
+    readonly score?: number;
 }
 /**
  * Makes one decide call under the step budget and returns the answers.

package/dist/decide.js

@@ -1,8 +1,11 @@
 import { AgentError, isAgentError } from 'e2e/agent';
 import { ConfigurationError } from 'e2e/engine';
 import * as ai from 'ai';
-import { InvalidArgumentError, InvalidResponseDataError, JSONParseError, LoadAPIKeyError, TypeValidationError, } from 'ai';
+import { Experimental_DecisionRefusalError, InvalidArgumentError, InvalidResponseDataError, JSONParseError, LoadAPIKeyError, TypeValidationError, } from 'ai';
 import { missingKey } from './api-key.js';
+import { recorded } from './budget.js';
+/** Transport retries per decide call. */
+const MAX_RETRIES = 3;
 /**
  * The SDK's decide call, or INVALID_CONFIG when the installed `ai` predates
  * it. Read off the namespace so an older `ai` still loads this module and
@@ -11,7 +14,7 @@ import { missingKey } from './api-key.js';
 export function requireDecide() {
     const found = ai.experimental_decide;
     if (typeof found !== 'function') {
-        throw new ConfigurationError('INVALID_CONFIG', 'decisionExecutor() needs ai 7.0.128 or later; update the ai package');
+        throw new ConfigurationError('INVALID_CONFIG', 'decisionExecutor() needs ai 7.0.134 or later; update the ai package');
     }
     return found;
 }
@@ -21,46 +24,37 @@ export function requireDecide() {
  * there stays a one-file change.
  */
 export async function decide(ctx, model, request, providerOptions) {
-    ctx.signal.throwIfAborted();
-    const started = performance.now();
-    const startedAt = new Date().toISOString();
-    let inputTokens;
-    let outputTokens;
-    let modelId = model.modelId;
-    let providerMetadata;
+    const screenshot = request.screenshot;
+    // The screenshot rides beside the JSON state as a file part; a text-only provider rejects it before the request.
+    const state = screenshot === undefined
+        ? request.state
+        : [{ type: 'json', value: request.state }, { type: 'file', mediaType: screenshot.mediaType, data: screenshot.data }];
     let answers;
+    let providerMetadata;
     try {
-        const call = {
-            model,
-            state: request.state,
-            questions: request.questions,
-            ...(providerOptions === undefined ? {} : { providerOptions }),
-        };
-        const result = await requireDecide()({
-            ...call,
-            maxRetries: 0,
-            abortSignal: ctx.signal,
-        });
-        ctx.signal.throwIfAborted();
-        inputTokens = result.usage.inputTokens;
-        outputTokens = result.usage.outputTokens;
-        modelId = result.response.modelId;
-        providerMetadata = result.providerMetadata;
-        answers = result.answers;
+        ({ answers, providerMetadata } = await recorded(ctx, model, async () => {
+            const result = await requireDecide()({
+                model,
+                state,
+                questions: request.questions,
+                ...(providerOptions === undefined ? {} : { providerOptions }),
+                // Transport retries for a rate limit or a 5xx, as the runner's own adapter has; a bad answer never retries.
+                maxRetries: MAX_RETRIES,
+                abortSignal: ctx.signal,
+            });
+            return {
+                value: {
+                    answers: result.answers,
+                    providerMetadata: result.providerMetadata,
+                },
+                usage: result.usage,
+                modelId: result.response.modelId,
+            };
+        }));
     }
     catch (error) {
         throw decideError(error, ctx.signal);
     }
-    finally {
-        ctx.budgets.recordModelCall({
-            provider: model.provider,
-            modelId,
-            startedAt,
-            durationMs: performance.now() - started,
-            ...(inputTokens === undefined ? {} : { inputTokens }),
-            ...(outputTokens === undefined ? {} : { outputTokens }),
-        });
-    }
     const decisions = {};
     for (const [id, answer] of Object.entries(answers)) {
         decisions[id] = { ...selected(answer), confidence: reportedConfidence(providerMetadata, id) };
@@ -69,9 +63,24 @@ export async function decide(ctx, model, request, providerOptions) {
 }
 /** The chosen option and its probability; anything without a valid distribution is MODEL_OUTPUT_INVALID. */
 function selected(answer) {
-    if (answer === undefined || answer.type !== 'choice' || answer.probabilities === undefined) {
+    if (answer === undefined || answer.probabilities === undefined || (answer.type !== 'choice' && answer.type !== 'score')) {
         throw new AgentError('MODEL_OUTPUT_INVALID', 'The decision model returned an answer without a choice distribution.');
     }
+    if (answer.type === 'score') {
+        const score = answer.score;
+        if (typeof score !== 'number' || !Number.isFinite(score)) {
+            throw new AgentError('MODEL_OUTPUT_INVALID', 'The decision model returned a score answer without a score.');
+        }
+        let top = { choice: '', probability: -1 };
+        let nearby = 0;
+        for (const [level, probability] of Object.entries(answer.probabilities)) {
+            if (probability > top.probability)
+                top = { choice: level, probability };
+            if (Math.abs(Number(level) - score) <= 0.5)
+                nearby += probability;
+        }
+        return { choice: top.choice, probability: Math.min(1, nearby), score };
+    }
     const probability = answer.choice === undefined ? 0 : (answer.probabilities[answer.choice] ?? 0);
     if (!Number.isFinite(probability) || probability < 0 || probability > 1) {
         throw new AgentError('MODEL_OUTPUT_INVALID', 'The decision model returned a probability outside [0, 1].');
@@ -99,8 +108,32 @@ function decideError(error, signal) {
         return missingKey('decision', error);
     if (InvalidArgumentError.isInstance(error))
         return error;
+    const refusedIds = refused(error);
+    if (refusedIds !== undefined) {
+        const which = refusedIds.length === 0 ? 'a question' : refusedIds.map((id) => JSON.stringify(id)).join(', ');
+        return new AgentError('MODEL_OUTPUT_INVALID', `The decision model refused to answer ${which}.`);
+    }
     if (InvalidResponseDataError.isInstance(error) || TypeValidationError.isInstance(error) || JSONParseError.isInstance(error)) {
         return new AgentError('MODEL_OUTPUT_INVALID', 'The decision model returned an invalid answer.');
     }
     return new AgentError('MODEL_PROVIDER_FAILED', 'The decision model call failed.');
 }
+/**
+ * The question ids the decision model refused, or undefined when the error
+ * is no refusal. The SDK throws its own refusal error naming them; a
+ * provider that reports the refusal as an answer of the wrong type is read
+ * from the rejected answers.
+ */
+function refused(error) {
+    if (Experimental_DecisionRefusalError.isInstance(error))
+        return error.questionIds;
+    if (!InvalidResponseDataError.isInstance(error))
+        return undefined;
+    const answers = error.data;
+    if (typeof answers !== 'object' || answers === null)
+        return undefined;
+    const ids = Object.entries(answers)
+        .filter(([, answer]) => typeof answer === 'object' && answer !== null && answer.type === 'refusal')
+        .map(([id]) => id);
+    return ids.length === 0 ? undefined : ids;
+}

package/dist/dispatch.d.ts

@@ -0,0 +1,37 @@
+import type { StepExecutorContext } from 'e2e';
+import type { Control, Target } from './elements.ts';
+import type { Point } from './overlay.ts';
+/** Operations that take a target and nothing else. */
+export type Pointer = 'tap' | 'double_tap' | 'long_press' | 'hover' | 'secondary_tap' | 'scroll_to' | 'submit' | 'check' | 'select';
+/** One action the loop dispatches, with everything its operation needs. */
+export type Action = {
+    readonly operation: Pointer;
+    readonly target: Target;
+} | {
+    readonly operation: 'type';
+    readonly target: Target;
+    readonly text: string;
+} | {
+    readonly operation: 'typeSecret';
+    readonly target: Target;
+    readonly name: string;
+} | {
+    readonly operation: 'upload';
+    readonly target: Target;
+    readonly paths: readonly string[];
+} | {
+    readonly operation: 'drag';
+    readonly target: Target;
+    readonly destinationId: string;
+} | {
+    readonly operation: 'tap_at';
+    readonly point: Point;
+} | {
+    readonly operation: Control;
+};
+/**
+ * Performs one action through the runner's grammar: the only place that
+ * touches `ctx.actions`. Resolves to a note for the model when the engine
+ * reports something worth telling it, nothing otherwise.
+ */
+export declare function perform(ctx: StepExecutorContext, action: Action): Promise<string | undefined>;

package/dist/dispatch.js

@@ -0,0 +1,67 @@
+/**
+ * Performs one action through the runner's grammar: the only place that
+ * touches `ctx.actions`. Resolves to a note for the model when the engine
+ * reports something worth telling it, nothing otherwise.
+ */
+export async function perform(ctx, action) {
+    const { actions } = ctx;
+    switch (action.operation) {
+        case 'scroll_up':
+            await actions.scroll('up');
+            return undefined;
+        case 'scroll_down':
+            await actions.scroll('down');
+            return undefined;
+        case 'back':
+            await actions.back();
+            return undefined;
+        case 'tap':
+            await actions.tap(node(action.target));
+            return undefined;
+        case 'double_tap':
+            await actions.doubleTap(node(action.target));
+            return undefined;
+        case 'long_press':
+            await actions.longPress(node(action.target));
+            return undefined;
+        case 'hover':
+            await actions.hover(node(action.target));
+            return undefined;
+        case 'secondary_tap':
+            await actions.secondaryTap(node(action.target));
+            return undefined;
+        case 'scroll_to':
+            await actions.scrollTo(node(action.target));
+            return undefined;
+        case 'submit':
+            await actions.press(node(action.target), 'Enter');
+            return undefined;
+        case 'check':
+            await actions.check(node(action.target), !(action.target.checked ?? false));
+            return undefined;
+        case 'select':
+            await actions.select(node(action.target), action.target.optionLabel ?? '');
+            return undefined;
+        case 'type':
+            await actions.type(node(action.target), action.text);
+            return undefined;
+        case 'typeSecret':
+            await actions.typeSecret(node(action.target), action.name);
+            return undefined;
+        case 'upload':
+            await actions.upload(node(action.target), action.paths);
+            return undefined;
+        case 'drag':
+            await actions.drag(node(action.target), { id: action.destinationId });
+            return undefined;
+        case 'tap_at': {
+            // The engine's prose for a bare point ("no listed control is there")
+            // reads as a miss to a classifier; whether the page changed says more.
+            const result = await actions.tapAt(action.point);
+            return result.target === undefined ? undefined : `landed on listed control ${result.target.id}`;
+        }
+    }
+}
+function node(target) {
+    return { id: target.id };
+}

package/dist/elements.d.ts

@@ -1,6 +1,6 @@
-import type { ExecutorNode, StepExecutorContext } from 'e2e';
+import type { ExecutorNode, ExecutorPixels, StepExecutorContext } from 'e2e';
 /** Operations the executor offers on elements. */
-export type Operation = 'tap' | 'type' | 'typeSecret' | 'submit' | 'select' | 'check';
+export type Operation = 'tap' | 'type' | 'typeSecret' | 'submit' | 'select' | 'check' | 'hover' | 'secondary_tap' | 'double_tap' | 'long_press' | 'drag' | 'scroll_to' | 'upload' | 'tap_at';
 /** Viewport and history controls, always offered when the engine declares the verb. */
 export type Control = 'scroll_up' | 'scroll_down' | 'back';
 /** Terminal choices: the step claims completion, reports a product failure, or gives up. */
@@ -15,23 +15,42 @@ export interface Element {
     readonly expanded?: boolean;
     readonly operations: readonly Operation[];
 }
-/** A bound target. `run` calls `ctx.actions`; the model only ever names the key. */
+/** One target of an operation: the node it acts on, as data. The model only ever names the key. */
 export interface Target {
+    readonly id: string;
+    /** How the action reads in history, e.g. `tap Save [n4]`. */
     readonly description: string;
-    /** Text for `type`, the secret name for `typeSecret`, nothing otherwise. */
-    run(argument?: string): Promise<void>;
     /** Native-select option label, set only for `select` options; names the choice criterion. */
     readonly optionLabel?: string;
+    /** The checked state a `check` flips. */
+    readonly checked?: boolean;
+}
+/** One drop destination for `drag`, keyed like an element. */
+export interface Destination {
+    readonly id: string;
+    readonly label: string;
+    readonly role: string;
 }
 export interface ActionSpace {
     readonly elements: readonly Element[];
-    /** Per operation: target key -> bound target. Only operations with at least one target appear. */
+    /** The element behind a target key, for labels and field descriptions. */
+    element(key: string): Element | undefined;
+    /** Per operation: target key -> target. Only operations with at least one target appear. */
     readonly targets: ReadonlyMap<Operation, ReadonlyMap<string, Target>>;
-    readonly controls: ReadonlyMap<Control, Target>;
+    readonly controls: ReadonlySet<Control>;
+    /** The operation or control an answer names, typed; undefined for anything not offered. */
+    operation(choice: string): Operation | undefined;
+    control(choice: string): Control | undefined;
+    /** Where a `drag` can drop, by element key; empty when nothing can receive a drop. */
+    readonly destinations: ReadonlyMap<string, Destination>;
+    /** Whether `tap_at` is offered: a screenshot, the `tapAt` verb, and a step that locates. */
+    readonly tapAt: boolean;
     /** Elements left out to stay under the per-question cap; scrolling can bring them into view. */
     readonly omitted: number;
     /** Non-interactive page text from the tree, without node ids, clipped to 6000 chars. */
     readonly pageText: string;
+    /** Live regions (status, alert, log) as `name: text`, the lines a completion check reads first. */
+    readonly statuses: readonly string[];
     /** Stable hash of path, tree content with node ids removed, and which nodes are in view. */
     readonly fingerprint: string;
 }
@@ -43,7 +62,13 @@ interface SpaceObservation {
         readonly height: number;
     };
     readonly tree: ExecutorNode;
+    /** Granted pixels; with the `tapAt` verb and a step that locates they open `tap_at`. */
+    readonly pixels?: ExecutorPixels;
+    /** Whether the step can locate a drawn control on the pixels: the decision model scores and a text model names it. */
+    readonly locates?: boolean;
 }
 /** Builds the element table and bound targets from the newest observation. */
 export declare function actionSpace(ctx: StepExecutorContext, observation: SpaceObservation, canType: boolean): ActionSpace;
+/** Element index behind a target key (`7` for both `7` and `7:2`). */
+export declare function targetKeyIndex(key: string): string;
 export {};