e2e 0.15.1 → 0.16.0

package/dist/telemetry/events.js

@@ -1,16 +1,18 @@
 /**
  * The events e2e sends, built from facts the CLI already holds. Every
  * property is a count, a duration, a version, or a short token. The builders
- * copy no title, file, URL, instruction, message, or stack out of the report;
- * the one look at a message, in `failure-kind.ts`, yields a token from a
- * closed list.
- * The names a project declares for its engines, platforms, and model pass
- * through when they are plain tokens, so a homegrown engine counts as itself;
- * a name shaped like a path, a URL, or a sentence folds to `other`, and an
- * error code that is not an upper-case token, the shape of every runner
- * code, folds to `OTHER`. The unit tests hold the payload to that promise.
+ * copy no title, file, URL, instruction, message, or stack out of the report,
+ * the config, or an MCP session; the one look at a message, in
+ * `failure-kind.ts`, yields a token from a closed list.
+ * The names a project declares for its engines, platforms, and model, and the
+ * name an MCP client gives itself, pass through when they are plain tokens,
+ * so a homegrown engine counts as itself; a name shaped like a path, a URL,
+ * or a sentence folds to `other`, and an error code that is not an upper-case
+ * token, the shape of every runner code, folds to `OTHER`. The unit tests
+ * hold the payload to that promise.
  */
-import { GRAMMAR_ACTION_NAMES, PROJECT_TOOL_EVENT_PREFIX } from '../agent/action-names.js';
+import { GRAMMAR_ACTION_NAMES, GRAMMAR_TOOL_NAMES, PROJECT_TOOL_EVENT_PREFIX } from '../agent/action-names.js';
+import { modelLabel } from '../config/agent.js';
 import { STEP_KINDS } from '../run/steps.js';
 import { failureKind } from './failure-kind.js';
 import { errorCodeToken, plainToken } from './token.js';
@@ -20,6 +22,8 @@ export const EVENT_CLI_SESSION = 'e2e_cli_session';
 export const EVENT_RUN_COMPLETED = 'e2e_run_completed';
 /** One per `e2e init`, however it ended: what was chosen and whether anything was written. */
 export const EVENT_INIT_COMPLETED = 'e2e_init_completed';
+/** One per `open_session` an `e2e mcp` server served, sent when the session closes or its open fails. */
+export const EVENT_MCP_SESSION = 'e2e_mcp_session';
 /** The session's error code for a flag or argument commander rejected; no runner code exists for it. */
 export const USAGE_ERROR_CODE = 'CLI_USAGE';
 const SEMVER = /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/u;
@@ -80,6 +84,10 @@ function engineLabel(engine) {
 function unique(values) {
     return [...new Set(values)].toSorted();
 }
+/** Counts keyed in sorted order, so equal counts serialize equally. */
+function sortedCounts(counts) {
+    return Object.fromEntries([...counts].toSorted());
+}
 /** Every scope the run recorded, across plain results and serial groups. */
 function* scopes(report) {
     for (const result of report.run.results)
@@ -117,7 +125,7 @@ function agentActions(steps) {
             counts.set(name, (counts.get(name) ?? 0) + 1);
         }
     }
-    return Object.fromEntries([...counts].toSorted());
+    return sortedCounts(counts);
 }
 /** Whether anyone priced the model calls: a gateway that did, nobody, or no calls to price. */
 function costSource(usage, modelCalls) {
@@ -129,7 +137,50 @@ function durationMs(startedAt, finishedAt) {
     const duration = Date.parse(finishedAt) - Date.parse(startedAt);
     return Number.isFinite(duration) ? Math.max(0, duration) : null;
 }
-export function runCompletedEvent(report, flags) {
+/**
+ * Which config features a run used: counts, booleans, and the runner's own
+ * option ids, never a name, a path, or a value the project wrote.
+ */
+function configFeatures(config) {
+    const agents = [...config.agents.values()];
+    return {
+        config_workers: config.workers,
+        config_retries: config.retries,
+        config_agents: config.agents.size,
+        config_custom_executor: agents.some((agent) => agent.executor !== undefined),
+        config_separate_judge: agents.some((agent) => agent.judge !== undefined && (agent.model === undefined || modelLabel(agent.judge) !== modelLabel(agent.model))),
+        config_project_tools: new Set(agents.flatMap((agent) => Object.keys(agent.tools))).size,
+        config_credentials: config.credentials.size,
+        config_secrets: config.secrets.size,
+        config_cache_mode: config.cache.mode,
+        config_cache_store: config.cache.store === undefined ? 'file' : 'custom',
+        config_cache_strict: config.cache.strict !== false,
+        config_reporters: unique(config.reporters),
+        config_custom_reporters: config.customReporters.length,
+        config_artifact_store: config.artifactStore !== undefined,
+        config_trace_modes: unique(config.targets.map((target) => target.trace.mode)),
+        config_video_modes: unique(config.targets.map((target) => target.video.mode)),
+        config_app_commands: config.targets.filter((target) => target.app.command !== undefined).length,
+        config_environments: unique(config.targets.map((target) => target.app.environment)),
+    };
+}
+const EXPLORE_STEP_STATUSES = ['passed', 'failed', 'blocked', 'exhausted'];
+const SEVERITIES = [1, 2, 3, 4, 5];
+/** What `e2e explore` was given and found, by count: never the goal, a step's charter, or a finding's words. */
+function exploreProperties(explore) {
+    return {
+        explore_ended: explore.ended,
+        explore_max_steps: explore.budgets.maxSteps,
+        explore_timeout_ms: explore.budgets.timeoutMs,
+        explore_steps: explore.steps.length,
+        ...Object.fromEntries(EXPLORE_STEP_STATUSES.map((status) => [`explore_steps_${status}`, explore.steps.filter((step) => step.status === status).length])),
+        explore_assessed: explore.summary !== undefined,
+        explore_issues: explore.findings.filter((finding) => finding.kind === 'issue').length,
+        explore_warnings: explore.findings.filter((finding) => finding.kind === 'warning').length,
+        explore_findings_by_severity: Object.fromEntries(SEVERITIES.map((severity) => [String(severity), explore.findings.filter((finding) => finding.severity === severity).length])),
+    };
+}
+export function runCompletedEvent(report, context) {
     const { run } = report;
     const recorded = [...scopes(report)];
     const steps = recorded.flatMap((scope) => scope.steps ?? []);
@@ -144,10 +195,11 @@ export function runCompletedEvent(report, flags) {
         name: EVENT_RUN_COMPLETED,
         at: run.finishedAt,
         properties: {
+            command: context.command,
             status: run.status,
             exit_code: run.exitCode,
             duration_ms: durationMs(run.startedAt, run.finishedAt),
-            flags: [...flags],
+            flags: [...context.flags],
             tests_discovered: run.summary.discovered,
             tests_selected: run.summary.selected,
             tests_executed: run.summary.executed,
@@ -180,7 +232,49 @@ export function runCompletedEvent(report, flags) {
             primary_error_code: primary === undefined ? null : errorCodeToken(primary.code),
             error_codes: unique(errors.map((error) => errorCodeToken(error.code))).slice(0, MAX_ERROR_CODES),
             error_kinds: unique(errors.flatMap(failureKind)).slice(0, MAX_ERROR_CODES),
+            // A run whose config never loaded has no features to report.
+            ...(context.config === undefined ? {} : configFeatures(context.config)),
+            ...(run.explore === undefined ? {} : exploreProperties(run.explore)),
+        },
+    };
+}
+/**
+ * The catalog tools an MCP session serves itself, by name: the grammar and
+ * the session's own (`createSessionCatalog`). A project's tools arrive
+ * counted together; any name outside this set folds to `other`.
+ */
+const SESSION_TOOLS = new Set([...GRAMMAR_TOOL_NAMES, 'locate', 'start_recording', 'stop_recording']);
+/** One MCP session: the client, the target's platform and engine, how it ended, and its calls by the runner's tool names. */
+export function mcpSessionEvent(summary, client) {
+    const calls = new Map();
+    for (const [name, count] of summary.toolCalls) {
+        const key = SESSION_TOOLS.has(name) ? name : 'other';
+        calls.set(key, (calls.get(key) ?? 0) + count);
+    }
+    if (summary.projectToolCalls > 0)
+        calls.set('tool', summary.projectToolCalls);
+    const codes = new Map();
+    for (const [code, count] of summary.errorCodes) {
+        const key = errorCodeToken(code);
+        codes.set(key, (codes.get(key) ?? 0) + count);
+    }
+    return {
+        name: EVENT_MCP_SESSION,
+        properties: {
+            client: client === undefined ? null : plainToken(client.name),
+            client_version: client === undefined ? null : plainToken(client.version),
+            outcome: summary.outcome,
+            ended_by: summary.endedBy ?? null,
+            error_code: summary.openErrorCode === undefined ? null : errorCodeToken(summary.openErrorCode),
+            platform: summary.platform === undefined ? null : plainToken(summary.platform),
+            engine: summary.engine === undefined ? null : engineLabel(summary.engine),
+            headed: summary.headed,
+            duration_ms: Math.max(0, Math.round(summary.durationMs)),
+            sessions_open: summary.concurrent,
+            calls_total: [...calls.values()].reduce((total, count) => total + count, 0),
+            calls_failed: summary.failedCalls,
+            calls: sortedCounts(calls),
+            error_codes: sortedCounts(new Map([...codes].toSorted().slice(0, MAX_ERROR_CODES))),
         },
     };
 }
-//# sourceMappingURL=events.js.map

package/dist/telemetry/failure-kind.d.ts

@@ -13,4 +13,3 @@
 import type { ReportError } from '../report/build.ts';
 /** `CODE:kind` for an error whose code only says where it surfaced; nothing for every other code. */
 export declare function failureKind(error: ReportError): string[];
-//# sourceMappingURL=failure-kind.d.ts.map

package/dist/telemetry/failure-kind.js

@@ -41,4 +41,3 @@ export function failureKind(error) {
     const kind = FAILURE_KINDS.find(([, pattern]) => pattern.test(error.message))?.[0] ?? 'other';
     return [`${error.code}:${kind}`];
 }
-//# sourceMappingURL=failure-kind.js.map

package/dist/telemetry/posthog.d.ts

@@ -30,4 +30,3 @@ export interface PostBatchOptions {
 }
 /** Sends the batch; resolves to whether PostHog accepted it. */
 export declare function postBatch(events: readonly PostHogEvent[], options: PostBatchOptions): Promise<boolean>;
-//# sourceMappingURL=posthog.d.ts.map

package/dist/telemetry/posthog.js

@@ -26,4 +26,3 @@ export async function postBatch(events, options) {
         return false;
     }
 }
-//# sourceMappingURL=posthog.js.map

package/dist/telemetry/project.d.ts

@@ -15,4 +15,3 @@
  * working directory is not a project.
  */
 export declare function anonymousProjectId(projectRoot: string, salt: string | undefined): Promise<string | undefined>;
-//# sourceMappingURL=project.d.ts.map

package/dist/telemetry/project.js

@@ -39,4 +39,3 @@ export async function anonymousProjectId(projectRoot, salt) {
         return undefined;
     return sha256Hex(`path\n${salt}\n${projectRoot}`);
 }
-//# sourceMappingURL=project.js.map

package/dist/telemetry/store.d.ts

@@ -55,4 +55,3 @@ export declare class TelemetryStore {
     markNotified(version: number, at: string): void;
     private write;
 }
-//# sourceMappingURL=store.d.ts.map

package/dist/telemetry/store.js

@@ -163,4 +163,3 @@ export class TelemetryStore {
         }
     }
 }
-//# sourceMappingURL=store.js.map

package/dist/telemetry/telemetry.d.ts

@@ -4,8 +4,10 @@
  * One instance lives for one CLI invocation. Commands hand it events; at the
  * end of the invocation `flush` sends them in a single bounded request, so a
  * command never waits on telemetry for more than the flush budget and never
- * fails because of it. Telemetry is a CLI concern only: the runner never
- * constructs this class.
+ * fails because of it. A long-lived command (`e2e mcp`) sends what it has
+ * queued as it goes with `sendQueued`, so a client that kills it loses only
+ * what was still in progress. Telemetry is a CLI concern only: the runner
+ * never constructs this class.
  *
  * Off means off at every step. `E2E_TELEMETRY_DISABLED`, `DO_NOT_TRACK`, a
  * source checkout of the repository, an `e2e telemetry disable`, or a
@@ -26,7 +28,7 @@
 import { type TelemetryEvent } from './events.ts';
 import { anonymousProjectId } from './project.ts';
 /** Bumped when what is collected changes enough that the notice must show again. */
-export declare const NOTICE_VERSION = 2;
+export declare const NOTICE_VERSION = 3;
 export type TelemetryDisabledBy = 'E2E_TELEMETRY_DISABLED' | 'DO_NOT_TRACK' | 'checkout' | 'preference' | 'store';
 export interface TelemetryOptions {
     /** The e2e version, sent with every event. */
@@ -66,6 +68,8 @@ export declare class Telemetry {
     private command;
     private failure;
     private exitCode;
+    /** Sends `sendQueued` started that have not settled; the flush waits for them. */
+    private readonly sending;
     constructor(options: TelemetryOptions);
     /** Opened on first use, so `--version` and `--help` never touch the disk; undefined when the file cannot exist. */
     private store;
@@ -117,11 +121,18 @@ export declare class Telemetry {
     /** The session event, first in the batch; absent for an invocation that never reached a command. */
     private sessionEvent;
     /**
-     * Sends everything queued in one request, or prints it under debug. One
-     * deadline covers the project lookup still running and the request; a
-     * lost batch is the accepted cost of a command that never waits on
-     * telemetry.
+     * Sends everything queued in one request, or prints it under debug, and
+     * waits for the sends `sendQueued` started. One deadline covers the
+     * project lookup still running and the request; a lost batch is the
+     * accepted cost of a command that never waits on telemetry.
      */
     flush(maxWaitMs?: number): Promise<void>;
+    /**
+     * Sends what is queued now, without the session event, which only the
+     * flush at the end can complete. For a command that runs until its client
+     * goes away; the caller need not wait, the flush does.
+     */
+    sendQueued(maxWaitMs?: number): Promise<void>;
+    /** Sends `events` in one bounded request, or prints them under debug; nothing when off. */
+    private send;
 }
-//# sourceMappingURL=telemetry.d.ts.map

package/dist/telemetry/telemetry.js

@@ -4,8 +4,10 @@
  * One instance lives for one CLI invocation. Commands hand it events; at the
  * end of the invocation `flush` sends them in a single bounded request, so a
  * command never waits on telemetry for more than the flush budget and never
- * fails because of it. Telemetry is a CLI concern only: the runner never
- * constructs this class.
+ * fails because of it. A long-lived command (`e2e mcp`) sends what it has
+ * queued as it goes with `sendQueued`, so a client that kills it loses only
+ * what was still in progress. Telemetry is a CLI concern only: the runner
+ * never constructs this class.
  *
  * Off means off at every step. `E2E_TELEMETRY_DISABLED`, `DO_NOT_TRACK`, a
  * source checkout of the repository, an `e2e telemetry disable`, or a
@@ -34,13 +36,13 @@ import { postBatch } from './posthog.js';
 import { anonymousProjectId } from './project.js';
 import { preferencesPath, TelemetryStore, telemetryConfigDir } from './store.js';
 /** Bumped when what is collected changes enough that the notice must show again. */
-export const NOTICE_VERSION = 2;
+export const NOTICE_VERSION = 3;
 /** The longest a flush may hold the process; the project lookup and the request share it. */
 const DEFAULT_FLUSH_MS = 2_000;
 /** The one-time notice: what is collected, and the two ways out. */
 function noticeText() {
     return [
-        `${picocolors.bold('e2e collects anonymous usage telemetry')} to improve the framework: the command, the versions, the OS, and run counts. Never test names, app data, or credentials.`,
+        `${picocolors.bold('e2e collects anonymous usage telemetry')} to improve the framework: the command, the versions, the OS, and summaries of runs and MCP sessions. Never test names, app data, or credentials.`,
         `Opt out with ${picocolors.cyan('e2e telemetry disable')} or ${picocolors.cyan('E2E_TELEMETRY_DISABLED=1')}. What is sent: ${picocolors.underline(`${DOCS_URL}/telemetry`)}`,
         '',
         '',
@@ -74,6 +76,8 @@ export class Telemetry {
     command;
     failure;
     exitCode;
+    /** Sends `sendQueued` started that have not settled; the flush waits for them. */
+    sending = new Set();
     constructor(options) {
         this.version = options.version;
         this.env = options.env ?? process.env;
@@ -200,14 +204,28 @@ export class Telemetry {
         return [cliSessionEvent(this.command.name, this.command.flags, ended)];
     }
     /**
-     * Sends everything queued in one request, or prints it under debug. One
-     * deadline covers the project lookup still running and the request; a
-     * lost batch is the accepted cost of a command that never waits on
-     * telemetry.
+     * Sends everything queued in one request, or prints it under debug, and
+     * waits for the sends `sendQueued` started. One deadline covers the
+     * project lookup still running and the request; a lost batch is the
+     * accepted cost of a command that never waits on telemetry.
      */
     async flush(maxWaitMs = DEFAULT_FLUSH_MS) {
         const events = [...this.sessionEvent(), ...this.queue.splice(0)];
         this.command = undefined;
+        await Promise.all([this.send(events, maxWaitMs), ...this.sending]);
+    }
+    /**
+     * Sends what is queued now, without the session event, which only the
+     * flush at the end can complete. For a command that runs until its client
+     * goes away; the caller need not wait, the flush does.
+     */
+    sendQueued(maxWaitMs = DEFAULT_FLUSH_MS) {
+        const sending = this.send(this.queue.splice(0), maxWaitMs).finally(() => this.sending.delete(sending));
+        this.sending.add(sending);
+        return sending;
+    }
+    /** Sends `events` in one bounded request, or prints them under debug; nothing when off. */
+    async send(events, maxWaitMs) {
         if (events.length === 0)
             return;
         // A choice saved from another process while this command ran wins over the snapshot taken at its start.
@@ -250,4 +268,3 @@ export class Telemetry {
         await postBatch(items, { signal: deadline, fetch: this.fetchImpl });
     }
 }
-//# sourceMappingURL=telemetry.js.map

package/dist/telemetry/token.d.ts

@@ -9,4 +9,3 @@
 export declare function plainToken(value: string): string;
 /** A runner error code as it is, `OTHER` for a code shaped like a project's own. */
 export declare function errorCodeToken(code: string): string;
-//# sourceMappingURL=token.d.ts.map

package/dist/telemetry/token.js

@@ -15,4 +15,3 @@ export function plainToken(value) {
 export function errorCodeToken(code) {
     return ERROR_CODE.test(code) ? code : 'OTHER';
 }
-//# sourceMappingURL=token.js.map

package/dist/types.d.ts

@@ -585,31 +585,39 @@ export interface TestAPI<Fixtures = TestFixtures> {
     /** Registers a suite-instance teardown hook. */
     afterAll(fn: SuiteHookFn): void;
 }
+/**
+ * The locator matchers. An option a matcher does not take, or a flag that is
+ * not a boolean, is `INVALID_ARGUMENT` before the first read.
+ */
 export interface AsyncExpectation {
     /** Inverts the matcher. A negated matcher passes after 1000 ms of continuous truth. */
     readonly not: AsyncExpectation;
-    /** Waits for visibility. */
+    /** Waits for visibility; `visible: false` waits for hidden or absent state, as `toBeHidden`. */
     toBeVisible(options?: {
+        visible?: boolean;
         timeout?: number;
     }): Promise<void>;
     /** Waits for hidden or absent state. */
     toBeHidden(options?: {
         timeout?: number;
     }): Promise<void>;
-    /** Waits for one match to exist, visible or not. */
+    /** Waits for one match to exist, visible or not; `attached: false` waits for none. */
     toBeAttached(options?: {
+        attached?: boolean;
         timeout?: number;
     }): Promise<void>;
-    /** Waits for enabled state. */
+    /** Waits for enabled state; `enabled: false` waits for disabled state. */
     toBeEnabled(options?: {
+        enabled?: boolean;
         timeout?: number;
     }): Promise<void>;
     /** Waits for disabled state. */
     toBeDisabled(options?: {
         timeout?: number;
     }): Promise<void>;
-    /** Waits for checked state. */
+    /** Waits for checked state; `checked: false` waits for unchecked state. */
     toBeChecked(options?: {
+        checked?: boolean;
         timeout?: number;
     }): Promise<void>;
     /** Waits for selected state. */
@@ -625,19 +633,11 @@ export interface AsyncExpectation {
         timeout?: number;
     }): Promise<void>;
     /** Waits for exact normalized text; a list waits for exactly that many matches, each with its entry's text, in order. */
-    toHaveText(expected: TextMatch, options?: {
-        timeout?: number;
-    }): Promise<void>;
-    toHaveText(expected: readonly TextMatch[], options?: {
-        timeout?: number;
-    }): Promise<void>;
+    toHaveText(expected: TextMatch, options?: TextMatcherOptions): Promise<void>;
+    toHaveText(expected: readonly TextMatch[], options?: TextMatcherOptions): Promise<void>;
     /** Waits for contained normalized text; a list waits for each entry to be contained by a distinct match, in order, extra matches allowed. */
-    toContainText(expected: TextMatch, options?: {
-        timeout?: number;
-    }): Promise<void>;
-    toContainText(expected: readonly TextMatch[], options?: {
-        timeout?: number;
-    }): Promise<void>;
+    toContainText(expected: TextMatch, options?: TextMatcherOptions): Promise<void>;
+    toContainText(expected: readonly TextMatch[], options?: TextMatcherOptions): Promise<void>;
     /** Waits for a form control's value, compared as it is. */
     toHaveValue(expected: TextMatch, options?: {
         timeout?: number;
@@ -646,17 +646,20 @@ export interface AsyncExpectation {
     toHaveAttribute(name: string, options?: {
         timeout?: number;
     }): Promise<void>;
-    toHaveAttribute(name: string, value: TextMatch, options?: {
-        timeout?: number;
-    }): Promise<void>;
+    toHaveAttribute(name: string, value: TextMatch, options?: TextMatcherOptions): Promise<void>;
     /** Waits for an exact match count. */
     toHaveCount(expected: number, options?: {
         timeout?: number;
     }): Promise<void>;
     /** Waits for an accessible name. */
-    toHaveAccessibleName(expected: TextMatch, options?: {
-        timeout?: number;
-    }): Promise<void>;
+    toHaveAccessibleName(expected: TextMatch, options?: TextMatcherOptions): Promise<void>;
+}
+/** Options of a locator matcher that compares text. */
+export interface TextMatcherOptions {
+    /** Compares a string case-insensitively; on a RegExp, `true` adds the `i` flag and `false` removes it. */
+    ignoreCase?: boolean;
+    /** Assertion budget in milliseconds. */
+    timeout?: number;
 }
 /**
  * An object that carries its own expectation surface, so `expect(object)`
@@ -700,7 +703,7 @@ export interface ValueExpectation<T> {
     toBeNull(): void;
     /** Requires undefined. */
     toBeUndefined(): void;
-    /** Requires a non-nullish value. */
+    /** Requires any value but undefined; null passes. */
     toBeDefined(): void;
     /** Requires a `length` property equal to `expected`: a string, an array, or anything array-like. */
     toHaveLength(expected: number): void;
@@ -1313,4 +1316,3 @@ export interface E2EConfig {
      */
     secrets?: Readonly<Record<string, SecretConfig>>;
 }
-//# sourceMappingURL=types.d.ts.map

package/dist/types.js

@@ -3,4 +3,3 @@
  * `tests/types/sdk-types.ts` pins the parts that are easy to loosen by accident.
  */
 export {};
-//# sourceMappingURL=types.js.map

package/docs/agent-steps.mdx

@@ -0,0 +1,126 @@
+---
+title: How agent steps work
+description: What the model sees, what it may do, how a step ends, and what it costs.
+---
+
+An agent step is one `agent.act`, `agent.assert`, `agent.waitFor`, or
+`agent.extract` call. The model reads the screen, asks for actions, and ends
+the step with a verdict. The runner performs each action it allows and
+records the result. Every step has a deadline and a model-call budget.
+
+## Two sides of the contract
+
+The runner handles redaction, action authorization, budgets, secrets, and
+reporting. The executor decides which actions to request and returns a
+verdict. The built-in agent is the executor unless an agents entry sets
+`executor`. [Custom executors](/executors) get the same redacted observations
+and checked action methods.
+
+## What the model sees
+
+For `act`, the opening prompt includes the instruction, params, completed
+steps, and a redacted text snapshot of the screen's roles, names, text, and
+states. It excludes raw HTML, cookies, headers, and environment values.
+Password fields are masked. Configured secret values found on screen are
+replaced with `<secret:name>`.
+
+After an action, the model receives the changes to the screen. If most of
+the screen changed, it receives a new full snapshot. Node ids remain stable
+while their elements exist. A failed action also returns the current screen
+so the model can choose what to do next.
+
+Judgments from `assert`, `waitFor`, and `extract` see the instruction and
+current screen. They do not see completed steps or the acting agent's
+summaries. You can configure a separate [judge model](/models#a-judge-of-its-own).
+
+Use `vision: true` to include a masked screenshot or `vision: 'only'` to
+judge the screenshot alone; the [agent reference](/reference/agent#vision)
+lists the modes. No screenshot reaches the model
+[after a secret fill](/security#screenshots-after-a-secret-fill).
+
+## What the model may do
+
+The agent receives tools supported by the target's engine, such as `tap`,
+`type`, `press`, `select`, `scroll`, and `navigate`. A step with declared
+credentials can also receive `type_secret`.
+
+Keyboard-capable engines support typing and key presses into the focused
+control. Pixel tools let the agent request a screenshot and act at a point
+when the text snapshot cannot describe a control. See the
+[tool reference](/reference/agent#pixel-tools-screenshot-and-the-point-verbs).
+
+The runner checks each built-in action and counts it against the budget.
+A targeted action has a maximum timeout of 15 seconds. Secret fills also
+require a declared secret and a compatible field.
+[Project tools](/tools) add your own operations with declared budgets and
+platform support.
+
+The loop detects repeated calls and cycles. After three consecutive failed
+actions, it asks the model to change approach; after five, it asks for a
+verdict. It also requests a verdict as the budget or deadline approaches.
+
+## How a step ends
+
+An agent step returns one of three verdicts:
+
+- `passed`: the screen provides evidence that the step succeeded.
+- `failed`: the required condition is false or cannot be established.
+  Insufficient evidence produces `ASSERTION_INCONCLUSIVE`.
+- `blocked`: a prerequisite, environment problem, or execution limit
+  prevented a product verdict. The error code determines the exit code.
+
+An error code belongs to `failed` or `blocked` only. When the built-in
+agent's model concludes `passed` with one, the code is dropped, the step
+passes, and the turn's outcome notes the dropped code. A
+[custom executor](/executors) that returns one is refused with
+`MODEL_OUTPUT_INVALID`.
+
+| Blocked by | Codes | Exit |
+| --- | --- | ---: |
+| A credential the app rejected, or one that is not configured | `AUTH_CREDENTIAL_INVALID`, `AUTH_CREDENTIAL_UNAVAILABLE` | 2 |
+| An environment that is down | `ENVIRONMENT_UNAVAILABLE` | 3 |
+| Seed data or a prerequisite outside the product that is not there | `SEED_DATA_MISSING`, `TEST_SETUP_FAILED` | 2 |
+| A budget or the clock | `STEP_BUDGET_EXHAUSTED`, `STEP_TIMEOUT` | 1 |
+| An interaction the toolset cannot perform | `AUTOMATION_UNSUPPORTED` | 1 |
+
+The [errors reference](/reference/errors#agenterror) lists every code a
+verdict can carry.
+
+## Cost
+
+Every call has a model-call budget and a default timeout, listed under
+[budgets](/reference/agent#budgets). Exceeding a budget raises
+`STEP_BUDGET_EXHAUSTED`; reaching the deadline raises `STEP_TIMEOUT`. What
+drives the cost:
+
+- `assert` and `extract` make one judgment, plus one repair call if the
+  model's answer is malformed. An `extract` whose data is not on screen
+  fails with `ASSERTION_INCONCLUSIVE` rather than returning a placeholder.
+- `waitFor` skips model calls while the screen is unchanged.
+- Screenshots add input tokens on every call that carries them. `act` asks
+  for one only when the text snapshot is not enough.
+- The [replay cache](/cache) reruns passing `act` steps without model calls.
+
+Use `expect` for exact checks and keep each `act` focused on one goal. When
+the provider supports prompt caching, `--debug` shows the cached share of
+input tokens per step. Under a
+[custom executor](/executors#what-the-executor-handles), `assert` runs with
+the `act` budgets.
+
+## Where secrets go
+
+For a `Secret` param, the model receives only its name and purpose. The
+runner checks the field the model picked, then fills the value itself.
+The value never enters prompts, transcripts, reports, or AI traces. See the
+[security model](/security#secrets) for what else changes after a fill.
+
+## Next
+
+<CardGroup cols={2}>
+  <Card title="Project tools" icon="wrench" href="/tools">
+    Extend the vocabulary with your own tools.
+  </Card>
+  <Card title="Custom executors" icon="gears" href="/executors">
+    Keep the loop with your prompt, or replace it entirely.
+  </Card>
+</CardGroup>