e2e 0.17.0 → 0.18.0

package/dist/mcp/server.js

@@ -10,11 +10,11 @@ import { McpServer, ResourceTemplate } from '@modelcontextprotocol/server';
 import { StdioServerTransport } from '@modelcontextprotocol/server/stdio';
 import { readGuide, skillTopics } from '../cli/skill.js';
 import { ConfigurationError, errorMessage } from '../internal/errors.js';
+import { processSecrets } from '../run/secrecy.js';
 import { loadProjectConfig, locateProjectConfig } from './config.js';
 import { SessionHost } from './session.js';
-import { errorResult } from './tools.js';
 const INSTRUCTIONS = `e2e is a local-first end-to-end test runner; this server drives an e2e project's app (its e2e.config.ts) live.
-Call open_session (optionally with a target and a config path) to get a session, its tool catalog, and the first observation. Then call {tool, args} runs any catalog tool: observe, the grammar its engine honors, type_secret, locate, screenshot, start_recording and stop_recording when the engine records video (a video of the app for a pull request), and the project's own tools; tools lists them, tools {tool} shows one tool's arguments. close_session when done.
+Call open_session (optionally with a target, a config path, and headed: true when the user wants to watch) to get a session, its tool catalog, and the first observation. Then call {tool, args} runs any catalog tool: observe, the grammar its engine honors, type_secret, locate, screenshot, start_recording and stop_recording when the engine records video (a video of the app for a pull request), and the project's own tools; tools lists them, tools {tool} shows one tool's arguments. close_session when done.
 Several sessions can be open at once, each with its own browser or device, so parallel agents (subagents) each open their own: pass the session id from open_session to every tools, call, and close_session.
 Write deterministic tests (tests/*.e2e.ts) from what you saw and run them with the CLI: npx e2e run <file>. Resources e2e://guide and e2e://guide/{topic} hold the writing guide.`;
 /** Serves until the client disconnects or `signal` aborts; resolves with the exit code. */
@@ -26,9 +26,11 @@ export async function serveMcp(options) {
         if (!disconnected && server.isConnected())
             void server.sendLoggingMessage({ level, logger: 'e2e', data: message }).catch(() => undefined);
     };
+    const output = options.output;
     const host = new SessionHost({
         locateConfig: (configPath) => locateProjectConfig({ cwd: options.cwd, configPath: configPath ?? options.configPath }),
         loadConfig: (configPath) => loadProjectConfig(configPath, options.env),
+        withholdOutput: output === undefined ? undefined : (load) => output.withholdDuring(load),
         env: options.env,
         headed: options.headed,
         maxSessions: options.maxSessions,
@@ -44,14 +46,9 @@ export async function serveMcp(options) {
             description: spec.description,
             inputSchema: spec.inputSchema,
             annotations: { readOnlyHint: spec.readOnly, openWorldHint: false },
-        }, async (args, ctx) => {
-            // A failure is a result the agent can react to, never a protocol error.
-            try {
-                return await spec.call(args, { signal: ctx.mcpReq.signal });
-            }
-            catch (cause) {
-                return errorResult(cause);
-            }
+        }, (args, ctx) => {
+            const call = () => spec.call(args, { signal: ctx.mcpReq.signal });
+            return output === undefined ? call() : output.duringCall(call);
         });
     }
     registerGuide(server);
@@ -84,9 +81,9 @@ export async function serveMcp(options) {
             options.log(summary);
     }
     catch (cause) {
-        options.log(`session teardown failed: ${errorMessage(cause)}`);
+        options.log(`session teardown failed: ${processSecrets.redact(errorMessage(cause))}`);
     }
-    await server.close().catch((cause) => options.log(`server close failed: ${errorMessage(cause)}`));
+    await server.close().catch((cause) => options.log(`server close failed: ${processSecrets.redact(errorMessage(cause))}`));
     return 0;
 }
 /** The skill as resources, for clients that read context without a tool call. */

package/dist/mcp/session.d.ts

@@ -25,13 +25,18 @@ export interface OpenSessionOptions {
     readonly target?: string | undefined;
     /** A config file to load instead of the server's default, relative to the server's directory. */
     readonly config?: string | undefined;
+    /** Whether this session shows its UI; otherwise the server's `headed`. */
+    readonly headed?: boolean | undefined;
 }
 export interface SessionHostOptions {
     /** The absolute path of the config a session loads, without evaluating it; `configPath` overrides the server's default. */
     readonly locateConfig: (configPath: string | undefined) => string;
     /** Loads the config at an absolute path fresh for each session, so an edited config applies without a restart. */
     readonly loadConfig: (configPath: string) => Promise<LoadedConfig>;
+    /** Holds what the process prints while `load` evaluates a config and registers its secrets; by default nothing is held. */
+    readonly withholdOutput?: (<T>(load: () => Promise<T>) => Promise<T>) | undefined;
     readonly env: NodeJS.ProcessEnv;
+    /** Whether a session shows its UI when `open_session` does not say, from `--headed`. */
     readonly headed: boolean;
     /** The target every session opens on, from `--target`; a call may still name one. */
     readonly defaultTarget?: string | undefined;
@@ -54,8 +59,10 @@ export declare class SessionHost {
     constructor(options: SessionHostOptions);
     get isOpen(): boolean;
     private get maxSessions();
-    /** The server's tools: the same four whatever the project, the config, or the target. */
+    /** The server's tools: the same four whatever the project, the config, or the target, each redacting what it returns. */
     toolSpecs(): readonly McpToolSpec[];
+    /** Logs a line for the operator and the client, redacted with every secret value the process knows. */
+    private log;
     /**
      * Opens a session and returns its opening text: the summary, the catalog,
      * and the first screen. `signal` is the request's: a client that cancels

package/dist/mcp/session.js

@@ -21,7 +21,7 @@ import { LocatorEngine } from '../locator/engine.js';
 import { allocateAppPorts } from '../run/app-ports.js';
 import { SharedAppProcesses } from '../run/process-pool.js';
 import { outputLayout } from '../run/output.js';
-import { sessionSecrecy } from '../run/secrecy.js';
+import { processSecrets, registerStaticSecrets, sessionSecrecy } from '../run/secrecy.js';
 import { openStandaloneAttempt } from '../run/standalone.js';
 import { createSessionCatalog, isGrammarVerb } from './catalog.js';
 import { describeRecording, SessionRecorder } from './recording.js';
@@ -55,9 +55,13 @@ export class SessionHost {
     get maxSessions() {
         return this.options.maxSessions ?? SESSION_BOUNDS.default;
     }
-    /** The server's tools: the same four whatever the project, the config, or the target. */
+    /** The server's tools: the same four whatever the project, the config, or the target, each redacting what it returns. */
     toolSpecs() {
-        return [this.openSpec(), this.catalogSpec(), this.callSpec(), this.closeSpec()];
+        return [this.openSpec(), this.catalogSpec(), this.callSpec(), this.closeSpec()].map(redacting);
+    }
+    /** Logs a line for the operator and the client, redacted with every secret value the process knows. */
+    log(level, message) {
+        this.options.log(level, processSecrets.redact(message));
     }
     /**
      * Opens a session and returns its opening text: the summary, the catalog,
@@ -65,8 +69,9 @@ export class SessionHost {
      * it, or disconnects, aborts the open and what it started.
      */
     open(options, signal) {
-        const usage = new SessionUsage(this.sessions.liveCount, this.options.headed);
-        return this.sessions.admit((id) => this.openSession(id, options, usage, signal)).catch((cause) => {
+        const headed = options.headed ?? this.options.headed;
+        const usage = new SessionUsage(this.sessions.liveCount, headed);
+        return this.sessions.admit((id) => this.openSession(id, options, headed, usage, signal)).catch((cause) => {
             this.options.onSessionEnd?.(usage.openFailed(classifyError(cause).code));
             throw cause;
         });
@@ -99,7 +104,8 @@ export class SessionHost {
             lines.push(`The session step ended with: ${errorMessage(value.outcome.error)}`);
         for (const error of cleanupErrors)
             lines.push(`Cleanup: ${error.code}: ${error.message}`);
-        return lines.join('\n');
+        // Redacted here, not only at the tool boundary: closeAll writes it to stderr.
+        return processSecrets.redact(lines.join('\n'));
     }
     /**
      * Runs `settle` (ending the step, saving a recording), then closes the
@@ -149,7 +155,7 @@ export class SessionHost {
             return redactResult(errorResult(cause), redact);
         });
     }
-    async openSession(id, options, usage, request) {
+    async openSession(id, options, headed, usage, request) {
         // The catalog renders synchronously, so the optional SDK is loaded once
         // here when installed. Without it the catalog reads the tools' Standard
         // Schemas, and only a model-backed call needs the package.
@@ -159,7 +165,16 @@ export class SessionHost {
         // knows that session's config.
         const configPath = this.options.locateConfig(options.config);
         this.sessions.claimConfig(id, configPath);
-        const loaded = await this.options.loadConfig(configPath);
+        const withhold = this.options.withholdOutput ?? ((load) => load());
+        const loaded = await withhold(async () => {
+            const fresh = await this.options.loadConfig(configPath);
+            // Known to the process before anything can fail with one, so an open
+            // failure and what the config printed while it loaded are redacted
+            // like any other text; a session registers them only once its engine
+            // has launched.
+            registerStaticSecrets(fresh.allSecrets);
+            return fresh;
+        });
         // A session is its own run: a URL declared with port 0 gets a port here.
         const config = await allocateAppPorts(loaded);
         const target = this.resolveTarget(config, options.target);
@@ -184,12 +199,12 @@ export class SessionHost {
                 // one attempt that closes as passed, so it traces under `on` only.
                 config,
                 target: { ...target, video: { mode: 'off', source: 'default' } },
-                headed: this.options.headed,
+                headed,
                 env: this.options.env,
                 signal: abort.signal,
                 timeoutMs: ttlMs + CLOSE_GRACE_MS,
                 processes: this.apps,
-                notice: (scope, message) => this.options.log('info', `${scope}: ${message}`),
+                notice: (scope, message) => this.log('info', `${scope}: ${message}`),
             });
             // The coding agent is the brain: the step is driven from here, and the
             // configured model stays out of the way.
@@ -217,13 +232,14 @@ export class SessionHost {
                     actionTimeout: config.actionTimeout,
                     assertionTimeout: config.assertionTimeout,
                 }),
-                warn: (message) => this.options.log('warning', message),
+                warn: (message) => this.log('warning', message),
                 onActionFailed: (cause) => usage.failed(classifyError(cause).code),
             });
             const live = {
                 id,
                 target,
                 configPath: loaded.configPath,
+                headed,
                 attempt,
                 step,
                 screen,
@@ -257,8 +273,8 @@ export class SessionHost {
     endOnItsOwn(live, why, endedBy) {
         if (!this.sessions.isLive(live.id))
             return;
-        this.options.log('warning', `session ${live.id} ended: ${why}`);
-        this.close(why, live.id, endedBy).catch((cause) => this.options.log('error', `closing session ${live.id} failed: ${errorMessage(cause)}`));
+        this.log('warning', `session ${live.id} ended: ${why}`);
+        this.close(why, live.id, endedBy).catch((cause) => this.log('error', `closing session ${live.id} failed: ${errorMessage(cause)}`));
     }
     /** A recorder writing to `<output>/videos/<session>/`, when the engine records video. */
     recorder(id, config, attempt) {
@@ -315,7 +331,7 @@ export class SessionHost {
     openingText(live, config, screen) {
         const engine = live.target.engine;
         const lines = [
-            `Session ${live.id} open on target "${live.target.name}" (platform ${live.target.platform}, engine ${engine === undefined ? 'none' : `${engine.name} ${engine.version}`}), ${this.options.headed ? 'headed' : 'headless'}; config ${live.configPath}.`,
+            `Session ${live.id} open on target "${live.target.name}" (platform ${live.target.platform}, engine ${engine === undefined ? 'none' : `${engine.name} ${engine.version}`}), ${live.headed ? 'headed' : 'headless'}; config ${live.configPath}.`,
         ];
         if (live.target.app.base !== undefined) {
             lines.push(`App: ${live.target.app.base.href}.`);
@@ -385,6 +401,10 @@ export class SessionHost {
             inputSchema: z.object({
                 target: z.string().min(1).optional().describe('Target name from the config; required when the config declares several'),
                 config: z.string().min(1).optional().describe("Path to an e2e config file, relative to the server's directory; default: the nearest e2e.config.ts"),
+                headed: z
+                    .boolean()
+                    .optional()
+                    .describe('Show the browser or simulator, when the engine supports it; pass true when the user wants to watch. Default: headless, unless the server was started with --headed'),
             }).strict(),
             readOnly: false,
             call: async (args, extra) => textResult(await this.open(args, extra.signal)),
@@ -427,3 +447,24 @@ export class SessionHost {
         });
     }
 }
+/**
+ * The tool with what it returns redacted with every secret value the process
+ * knows: the static values of each config a session loaded, and every value
+ * a session resolved or derived. A failure becomes a result the agent can
+ * react to, never a protocol error, and is redacted the same way.
+ */
+function redacting(spec) {
+    return {
+        ...spec,
+        call: async (args, extra) => {
+            let result;
+            try {
+                result = await spec.call(args, extra);
+            }
+            catch (cause) {
+                result = errorResult(cause);
+            }
+            return redactResult(result, (text) => processSecrets.redact(text));
+        },
+    };
+}

package/dist/mcp/tools.js

@@ -192,13 +192,15 @@ async function resolveOutput(value) {
  * else is JSON, so a structured output is never lost.
  */
 export async function resultFromOutput(tool, output, input = undefined) {
-    if (typeof output === 'string')
-        return textResult(output);
     if (tool.toModelOutput !== undefined) {
-        const content = contentFromModelOutput(await tool.toModelOutput({ toolCallId: 'mcp', input, output }));
-        if (content !== undefined)
-            return { content };
+        const rendered = await tool.toModelOutput({ toolCallId: 'mcp', input, output });
+        const content = contentFromModelOutput(rendered);
+        if (content !== undefined) {
+            return { content, ...(rendered.type === 'error-text' || rendered.type === 'error-json' ? { isError: true } : {}) };
+        }
     }
+    if (typeof output === 'string')
+        return textResult(output);
     return textResult(output === undefined ? 'Done.' : JSON.stringify(output, null, 2));
 }
 function contentFromModelOutput(output) {

package/dist/report/list.d.ts

@@ -120,6 +120,17 @@ export declare class ListReporter implements Reporter {
      * the test's result, or the end of the run.
      */
     private testOutput;
+    /**
+     * Text user code wrote in the runner's own process (the config's top
+     * level, a test file's while it is collected, a reporter), printed like a
+     * test's output under a `stdout | runner` heading. The runner hands it
+     * here already redacted, only while this reporter shows the run.
+     */
+    processOutput(stream: 'stdout' | 'stderr', text: string): void;
+    /** Prints what `processOutput` left unfinished: the runner hands nothing more once the run's events are all dispatched. */
+    endProcessOutput(): void;
+    /** Prints the lines `text` ends under the heading of `stream` and `source`, holding the unfinished rest. */
+    private sourcedOutput;
     /** Prints finished output lines under their heading, unless the heading is the one printed last. */
     private printOutput;
     /** Prints the unfinished output of `pair`, or of every source when undefined. */

package/dist/report/list.js

@@ -272,20 +272,37 @@ export class ListReporter {
      * the test's result, or the end of the run.
      */
     testOutput(event) {
-        const { pc } = this;
-        if (event.text === '')
-            return;
         const pair = event.pair === undefined ? undefined : pairKey(event.pair.testId, event.pair.agent, event.target, event.pair.repeat);
         const running = pair === undefined ? undefined : this.pairs.get(pair);
         const source = running === undefined ? this.badge(event.target) : `${this.badge(running.group.target)} ${bounded(running.group.file)}${this.separator}${running.title}`;
-        const key = `${event.stream}\u0000${source}`;
+        this.sourcedOutput(event.stream, source, pair, event.text);
+    }
+    /**
+     * Text user code wrote in the runner's own process (the config's top
+     * level, a test file's while it is collected, a reporter), printed like a
+     * test's output under a `stdout | runner` heading. The runner hands it
+     * here already redacted, only while this reporter shows the run.
+     */
+    processOutput(stream, text) {
+        this.sourcedOutput(stream, this.pc.dim('runner'), undefined, text);
+    }
+    /** Prints what `processOutput` left unfinished: the runner hands nothing more once the run's events are all dispatched. */
+    endProcessOutput() {
+        this.flushOutput();
+    }
+    /** Prints the lines `text` ends under the heading of `stream` and `source`, holding the unfinished rest. */
+    sourcedOutput(stream, source, pair, text) {
+        const { pc } = this;
+        if (text === '')
+            return;
+        const key = `${stream}\u0000${source}`;
         let entry = this.pendingOutput.get(key);
         if (entry === undefined) {
-            const label = event.stream === 'stderr' ? pc.yellow(event.stream) : pc.dim(event.stream);
+            const label = stream === 'stderr' ? pc.yellow(stream) : pc.dim(stream);
             entry = { heading: `${label} ${pc.dim('|')} ${source}`, pair, fragment: '' };
             this.pendingOutput.set(key, entry);
         }
-        const lines = (entry.fragment + event.text).split('\n');
+        const lines = (entry.fragment + text).split('\n');
         entry.fragment = lines.pop() ?? '';
         if (entry.fragment === '')
             this.pendingOutput.delete(key);

package/dist/run/artifacts.d.ts

@@ -2,7 +2,7 @@
 import type { ArtifactStore } from '../types.ts';
 import type { ArtifactSink } from './fixtures.ts';
 import type { ArtifactRecord } from './records.ts';
-import type { SessionSecrecy } from './secrecy.ts';
+import { type SessionSecrecy } from './secrecy.ts';
 export interface AttemptArtifacts {
     /** Absolute attempt artifact directory, created eagerly. */
     readonly dir: string;
@@ -38,9 +38,9 @@ export declare function createAttemptArtifacts(options: {
     /**
      * The secrecy of the session the attempt runs on, read when a download is
      * registered; undefined (no session open yet) leaves every download as
-     * served. Once a secret reached the session (filled, or held by the
-     * engine), a text-like download is rewritten through its ledger before it
-     * is hashed or stored.
+     * served. While its ledger holds a value (`redactsRecordings`), a
+     * text-like download is rewritten through it before it is hashed or
+     * stored.
      */
     secrecy?: () => SessionSecrecy | undefined;
     /**

package/dist/run/artifacts.js

@@ -5,6 +5,7 @@ import { mkdir, readdir, readFile, rm, rmdir, stat } from 'node:fs/promises';
 import path from 'node:path';
 import { pipeline } from 'node:stream/promises';
 import { writeFileAtomic } from '../internal/atomic-write.js';
+import { redactsRecordings } from './secrecy.js';
 /**
  * How much of each kind the runner masked, unless the registration says. A
  * screenshot masks secure fields at the source; a log passes through the
@@ -60,7 +61,7 @@ export function createAttemptArtifacts(options) {
             const reportPath = path.posix.join(...options.segments, relativePath);
             const secrecy = kind === 'download' && registration?.redaction === undefined ? options.secrecy?.() : undefined;
             pending.push((async () => {
-                if (secrecy !== undefined && secrecy.exposure.redactsRecordings && isTextLike(record.mediaType)) {
+                if (secrecy !== undefined && redactsRecordings(secrecy) && isTextLike(record.mediaType)) {
                     record.redaction = await redactDownload(absolute, secrecy);
                 }
                 // Without a store the file is streamed for its size and digest only;
@@ -235,7 +236,7 @@ async function redactDownload(absolute, secrecy) {
     try {
         const bytes = await readFile(absolute);
         const text = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes);
-        const redacted = secrecy.ledger.redact(text);
+        const redacted = secrecy.ledger.redactFragments(text);
         if (redacted !== text)
             await writeFileAtomic(absolute, redacted);
         return 'complete';

package/dist/run/execute.d.ts

@@ -9,17 +9,33 @@ import type { AttemptRecordings } from '../internal/recording-modes.ts';
 import type { ArtifactStore } from '../types.ts';
 import { type ArtifactSink } from './fixtures.ts';
 import { RealmManager, type FileRef, type Realm } from './realm.ts';
-import type { AttemptRecord, ResultRecord, RunError, SerialGroupRecord } from './records.ts';
-import { type SerialHost, type SharedSerialSession } from './serial.ts';
+import type { AttemptRecord, ResultRecord, RunError, SerialGroupRecord, SerialMemberRecord } from './records.ts';
+import { type SerialAttemptRun, type SerialAttemptStart, type SerialHost, type SharedSerialSession } from './serial.ts';
 import { SessionStaging, SessionStore } from './sessions.ts';
 import { type StepProgress } from './steps.ts';
 export interface ExecutionEvents {
     onResult?(result: ResultRecord): void;
     onSerialGroup?(group: SerialGroupRecord): void;
+    /** An attempt of an ordinary or setup pair begins, its realm and `beforeAll` hooks included. */
+    onAttemptStart?(pair: TestTargetPair, attemptIndex: number): void;
+    /** One finished attempt of an ordinary or setup pair, ahead of the pair's result. */
+    onAttempt?(pair: TestTargetPair, attempt: AttemptRecord): void;
+    /** One member that finished in a serial group attempt, ahead of the group. */
+    onSerialMember?(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void;
+    /** One finished serial group attempt, ahead of the group. */
+    onSerialAttempt?(groupId: string, run: SerialAttemptRun): void;
     /** Fires before a runnable pair starts; a serial unit announces each member as it begins. */
     onPairStart?(pair: TestTargetPair): void;
     /** Live step progress of one running attempt, for reporters. */
     onProgress?(pair: TestTargetPair, progress: StepProgress): void;
+    /** An attempt's test timeout started, as its `beforeEach` hooks and body begin. */
+    onAttemptDeadline?(pair: TestTargetPair, attempt: {
+        readonly index: number;
+        readonly id: string;
+        readonly startedAt: string;
+    }): void;
+    /** The attempt `onAttemptDeadline` announced has ended, or one that never got that far has. */
+    onAttemptEnd?(): void;
     /** One line of progress the engine's `init` reported, already naming the target and worker slot. */
     onNotice?(message: string): void;
     /**
@@ -132,6 +148,10 @@ export declare class TargetExecutor implements SerialHost {
     pairStarted(pair: TestTargetPair): void;
     /** Emits one finished serial group, ahead of its members' results (SerialHost). */
     emitSerialGroup(group: SerialGroupRecord): void;
+    /** Streams one member finished in a serial group attempt (SerialHost). */
+    serialMemberFinished(groupId: string, attempt: SerialAttemptStart, member: SerialMemberRecord): void;
+    /** Streams one finished serial group attempt (SerialHost). */
+    serialAttemptFinished(groupId: string, run: SerialAttemptRun): void;
     /** Builds one engine operation context. */
     private op;
     /** Run-level errors recorded so far, in order. */
@@ -196,9 +216,9 @@ export declare class TargetExecutor implements SerialHost {
      * hands the app (basic-auth credentials). Registered for the session's and
      * the process's redaction by `resolveSecretValue`, with every value the
      * engine derives from it (the base64 credential an `Authorization` header
-     * carries) under the same name. The session's exposure rises to `engine`:
-     * its text is redacted and its trace and text downloads rewritten, while
-     * its pixels stay as they are, since nothing was typed.
+     * carries) under the same name, so its text, trace, and text downloads
+     * are redacted of them. Its pixels stay as they are, since nothing was
+     * typed.
      */
     private resolveEngineSecret;
     /**

package/dist/run/execute.js

@@ -26,7 +26,7 @@ import { isFailedStatus } from './records.js';
 import { runWithRetries } from './retry.js';
 import { runSerialUnit } from './serial.js';
 import { interruptedSkip, pairKey, pairResult, repeatSegment, unstartedResult } from './units.js';
-import { adoptSecrecy, carriedSecrecy, processSecrets, registerDerivedSecrets, redactForSession, resolveSecretValue, sessionSecrecy, staticSecretLedger } from './secrecy.js';
+import { adoptSecrecy, carriedSecrecy, processSecrets, registerDerivedSecrets, redactForSession, redactsRecordings, resolveSecretValue, sessionSecrecy, staticSecretLedger } from './secrecy.js';
 import { isSecret } from '../secrets.js';
 import { SessionStaging, SessionStore, targetIdentity } from './sessions.js';
 import { redactTraceArchives } from './trace-redaction.js';
@@ -97,6 +97,14 @@ export class TargetExecutor {
     emitSerialGroup(group) {
         this.options.events?.onSerialGroup?.(group);
     }
+    /** Streams one member finished in a serial group attempt (SerialHost). */
+    serialMemberFinished(groupId, attempt, member) {
+        this.options.events?.onSerialMember?.(groupId, attempt, member);
+    }
+    /** Streams one finished serial group attempt (SerialHost). */
+    serialAttemptFinished(groupId, run) {
+        this.options.events?.onSerialAttempt?.(groupId, run);
+    }
     /** Builds one engine operation context. */
     op(attemptId, timeoutMs, signal) {
         return { signal, timeoutMs, runId: this.options.runId, attemptId, origin: 'test' };
@@ -256,6 +264,7 @@ export class TargetExecutor {
         let realm = incomingRealm;
         let hookFailure;
         const finalStatus = await runWithRetries(pair.options.retries + 1, this.interruptSignal, async (attemptIndex) => {
+            this.options.events?.onAttemptStart?.(pair, attemptIndex);
             if (realm === null)
                 realm = await this.realms.create(file);
             const registered = findRegistered(realm, pair.test);
@@ -270,6 +279,7 @@ export class TargetExecutor {
                 kind: 'ordinary',
             });
             attempts.push(attempt);
+            this.options.events?.onAttempt?.(pair, attempt);
             if (isFailedStatus(attempt.status)) {
                 // A failed realm is never reused, but afterAll still runs for every
                 // scope whose beforeAll started in it. A body that skipped itself
@@ -316,6 +326,7 @@ export class TargetExecutor {
         const attempts = [];
         let hookFailure;
         const finalStatus = await runWithRetries(pair.options.retries + 1, this.interruptSignal, async (attemptIndex) => {
+            this.options.events?.onAttemptStart?.(pair, attemptIndex);
             const realm = attemptIndex === 0 && freshRegistration !== undefined
                 ? this.realms.adopt(freshRegistration, file)
                 : await this.realms.create(file);
@@ -334,15 +345,17 @@ export class TargetExecutor {
                 kind: 'setup',
                 staging,
             });
+            const missing = attempt.status === 'passed' ? staging.missing() : [];
+            if (missing.length > 0) {
+                attempt.status = 'failed';
+                attempt.error = serializeError(new E2EError('test', 'SESSION_CONTRACT', `setup must save each declared session exactly once; missing: [${missing.join(', ')}]`), { phase: 'body' });
+            }
             attempts.push(attempt);
+            // Sent before teardown and the session saves, as in runOrdinaryPair:
+            // a crash in either then lands on this attempt instead of erasing it.
+            this.options.events?.onAttempt?.(pair, attempt);
             await this.realms.leave(realm);
             if (attempt.status === 'passed') {
-                const missing = staging.missing();
-                if (missing.length > 0) {
-                    attempt.status = 'failed';
-                    attempt.error = serializeError(new E2EError('test', 'SESSION_CONTRACT', `setup must save each declared session exactly once; missing: [${missing.join(', ')}]`), { phase: 'body' });
-                    return attempt;
-                }
                 for (const [name, saved] of staging.entries()) {
                     await this.options.sessionStore.save(name, this.sessionIdentity, saved);
                 }
@@ -419,9 +432,9 @@ export class TargetExecutor {
      * hands the app (basic-auth credentials). Registered for the session's and
      * the process's redaction by `resolveSecretValue`, with every value the
      * engine derives from it (the base64 credential an `Authorization` header
-     * carries) under the same name. The session's exposure rises to `engine`:
-     * its text is redacted and its trace and text downloads rewritten, while
-     * its pixels stay as they are, since nothing was typed.
+     * carries) under the same name, so its text, trace, and text downloads
+     * are redacted of them. Its pixels stay as they are, since nothing was
+     * typed.
      */
     async resolveEngineSecret(session, secret, options) {
         const engine = this.target.engine;
@@ -432,7 +445,6 @@ export class TargetExecutor {
         const secrecy = sessionSecrecy(session, this.config.allSecrets);
         const plaintext = await resolveSecretValue(secret, this.config.allSecrets, secrecy.ledger);
         registerDerivedSecrets(secret.name, options?.derived?.(plaintext) ?? [], secrecy.ledger);
-        secrecy.exposure.raise('engine');
         return plaintext;
     }
     /**
@@ -479,14 +491,14 @@ export class TargetExecutor {
                 }
                 // An engine records what happened, filled secrets included, so the
                 // trace is the runner's to redact before anything hashes or stores
-                // it. Only a session a secret reached (filled, or held by the engine
-                // for an option, which a trace records the attempt opening with) can
-                // have recorded one: an unexposed trace needs no rewriting, and an
-                // exposed one is kept only once rewritten. Its screencast frames go
-                // only where pixels are withheld, after a fill.
+                // it. Any value the ledger holds may be in it, filled or not (a URL
+                // the test spelled it into, an engine option), so a trace is kept
+                // only once rewritten; only a session that knows no value skips it.
+                // Its screencast frames go only where pixels are withheld, after a
+                // fill.
                 const secrecy = sessionSecrecy(session, this.config.allSecrets);
                 let redaction = 'not-required';
-                if (secrecy.exposure.redactsRecordings) {
+                if (redactsRecordings(secrecy)) {
                     try {
                         await redactTraceArchives(artifactSink.dir, archives, secrecy.ledger, { keepFrames: !secrecy.exposure.withholdsPixels });
                     }
@@ -540,7 +552,14 @@ export class TargetExecutor {
             target: this.target.name,
             agent: pair.agent,
             attempt: attemptIndex,
-        }, () => this.executeAttempt(pair, registered, realm, attemptIndex, context));
+        }, async () => {
+            try {
+                return await this.executeAttempt(pair, registered, realm, attemptIndex, context);
+            }
+            finally {
+                this.options.events?.onAttemptEnd?.();
+            }
+        });
     }
     async executeAttempt(pair, registered, realm, attemptIndex, context) {
         const attemptId = uuidv7();
@@ -654,9 +673,13 @@ export class TargetExecutor {
         // One more look at the app the moment the failure lands: what the screen
         // held then is the evidence the message lacks. Taken before teardown, so
         // an `afterEach` that navigates away or resets state cannot replace it.
+        // Once per attempt: a screen that yielded nothing within the budget is
+        // not asked again, so a hung app costs the budget once.
+        let evidenceTried = false;
         const captureEvidence = async () => {
-            if (failure === undefined || record.failure !== undefined || openSession === null || this.interruptSignal.aborted)
+            if (failure === undefined || evidenceTried || openSession === null || this.interruptSignal.aborted)
                 return;
+            evidenceTried = true;
             const evidence = await captureFailureEvidence({
                 session: openSession,
                 error: failure,
@@ -700,6 +723,7 @@ export class TargetExecutor {
                 (await this.launchSession({ session: pair.options.session, recordings }, attemptId, artifacts.dir, attemptAbort.signal));
             openSession = session;
             const testDeadline = new Deadline(pair.options.timeout);
+            this.options.events?.onAttemptDeadline?.(pair, { index: attemptIndex, id: attemptId, startedAt });
             const budget = new AttemptBudget(attemptAbort.signal, testDeadline);
             const saveSession = context.kind !== 'setup'
                 ? undefined
@@ -936,7 +960,7 @@ export class TargetExecutor {
 function classifyAttemptStatus(failure, timedOut, interrupted) {
     if (interrupted && !timedOut)
         return 'interrupted';
-    if (timedOut || failure instanceof TestTimeoutError || failure.code === 'TEST_TIMEOUT') {
+    if (timedOut || failure.code === 'TEST_TIMEOUT') {
         return 'timed-out';
     }
     return 'failed';

package/dist/run/failure-evidence.js

@@ -14,6 +14,7 @@ import path from 'node:path';
 import { formatNode, prepareObservation } from '../agent/observation.js';
 import { truncateUtf8 } from '../internal/errors.js';
 import { isTypoOf } from '../internal/suggest.js';
+import { withAbort } from '../internal/time.js';
 /** The whole capture, observation and screenshot together, gets this long. */
 const EVIDENCE_TIMEOUT_MS = 5_000;
 /** Screen lines a locator failure lists as its nearest nodes, and the bytes each keeps; the wire schema caps both. */
@@ -37,9 +38,12 @@ export async function captureFailureEvidence(options) {
         return finish(evidence);
     const operation = options.operation(signal, EVIDENCE_TIMEOUT_MS);
     const { redact, redactCut } = options.secrecy.ledger;
+    // The engine is handed the budget, and the wait is abandoned at its end
+    // either way: an engine stuck on a hung app may never honor it.
+    const bounded = (work) => withAbort(work, signal, () => new Error('failure evidence ran out of budget'));
     let observation;
     try {
-        const raw = await options.session.observe(operation);
+        const raw = await bounded(() => options.session.observe(operation));
         observation = prepareObservation(raw, {
             redact,
             redactCut,
@@ -73,7 +77,7 @@ export async function captureFailureEvidence(options) {
     // prove a tainted viewport redacted.
     if (!options.secrecy.exposure.withholdsPixels && !signal.aborted) {
         try {
-            const relative = await options.session.artifacts.screenshot('failure', operation);
+            const relative = await bounded(() => options.session.artifacts.screenshot('failure', operation));
             evidence.screenshot = options.artifacts.register('screenshot', relative);
         }
         catch {