@langwatch/scenario 0.2.13 → 0.4.0

package/dist/index.d.mts

@@ -1,7 +1,18 @@
 import * as ai from 'ai';
-import { CoreMessage, CoreUserMessage, CoreAssistantMessage, CoreToolMessage, LanguageModel } from 'ai';
-import { z } from 'zod';
+import { CoreMessage, CoreUserMessage, CoreAssistantMessage, CoreToolMessage, LanguageModel, generateText, ModelMessage } from 'ai';
+import { z } from 'zod/v4';
+import { SpanProcessor, ReadableSpan } from '@opentelemetry/sdk-trace-base';
+import { RealtimeSession } from '@openai/agents/realtime';
 import { Observable } from 'rxjs';
+import { z as z$1 } from 'zod';
+
+/**
+ * The possible return types from an agent's `call` method.
+ * - string | CoreMessage | CoreMessage[]: Agent generated response
+ * - JudgeResult: Judge made a final decision
+ * - null: Judge wants to continue observing (no decision yet)
+ */
+type AgentReturnTypes = string | CoreMessage | CoreMessage[] | JudgeResult | null;
 
 declare enum AgentRole {
     USER = "User",
@@ -42,11 +53,6 @@ interface AgentInput {
      */
     scenarioConfig: ScenarioConfig;
 }
-/**
- * The possible return types from an agent's `call` method.
- * Can be a simple string, a single message, an array of messages, or a ScenarioResult.
- */
-type AgentReturnTypes = string | CoreMessage | CoreMessage[] | ScenarioResult;
 /**
  * Abstract base class for integrating custom agents with the Scenario framework.
  *
@@ -71,6 +77,7 @@ type AgentReturnTypes = string | CoreMessage | CoreMessage[] | ScenarioResult;
  * ```
  */
 declare abstract class AgentAdapter {
+    name?: string;
     role: AgentRole;
     /**
      * Process the input and generate a response.
@@ -88,33 +95,21 @@ declare abstract class AgentAdapter {
  * Abstract base class for user simulator agents.
  * User simulator agents are responsible for generating user messages to drive the conversation.
  */
-declare abstract class UserSimulatorAgentAdapter implements AgentAdapter {
+declare abstract class UserSimulatorAgentAdapter extends AgentAdapter {
+    name: string;
     role: AgentRole;
-    /**
-     * Process the input and generate a user message.
-     *
-     * @param input AgentInput containing conversation history, thread context, and scenario state.
-     * @returns The user's response.
-     */
-    abstract call(input: AgentInput): Promise<AgentReturnTypes>;
 }
 /**
  * Abstract base class for judge agents.
  * Judge agents are responsible for evaluating the conversation and determining success or failure.
  */
-declare abstract class JudgeAgentAdapter implements AgentAdapter {
+declare abstract class JudgeAgentAdapter extends AgentAdapter {
+    name: string;
     role: AgentRole;
     /**
      * The criteria the judge will use to evaluate the conversation.
      */
     abstract criteria: string[];
-    /**
-     * Process the input and evaluate the conversation.
-     *
-     * @param input AgentInput containing conversation history, thread context, and scenario state.
-     * @returns A ScenarioResult if the conversation should end, otherwise should continue.
-     */
-    abstract call(input: AgentInput): Promise<AgentReturnTypes>;
 }
 
 declare const DEFAULT_MAX_TURNS = 10;
@@ -249,7 +244,7 @@ interface ScenarioExecutionLike {
  * A step in a scenario script.
  * This is a function that takes the current state and an executor, and performs an action.
  */
-type ScriptStep = (state: ScenarioExecutionStateLike, executor: ScenarioExecutionLike) => Promise<void | ScenarioResult | null> | void | ScenarioResult | null;
+type ScriptStep = (state: ScenarioExecutionStateLike, executor: ScenarioExecutionLike) => Promise<void> | void;
 
 /**
  * Represents the result of a scenario execution.
@@ -348,38 +343,14 @@ interface ScenarioExecutionStateLike {
     hasToolCall(toolName: string): boolean;
 }
 
-/** Default temperature for language model inference */
-declare const DEFAULT_TEMPERATURE = 0;
 declare const scenarioProjectConfigSchema: z.ZodObject<{
     defaultModel: z.ZodOptional<z.ZodObject<{
-        model: z.ZodType<ai.LanguageModelV1, z.ZodTypeDef, ai.LanguageModelV1>;
+        model: z.ZodCustom<ai.LanguageModel, ai.LanguageModel>;
         temperature: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
-    }, "strip", z.ZodTypeAny, {
-        model: ai.LanguageModelV1;
-        temperature: number;
-        maxTokens?: number | undefined;
-    }, {
-        model: ai.LanguageModelV1;
-        temperature?: number | undefined;
-        maxTokens?: number | undefined;
-    }>>;
+    }, z.core.$strip>>;
     headless: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-}, "strict", z.ZodTypeAny, {
-    headless: boolean;
-    defaultModel?: {
-        model: ai.LanguageModelV1;
-        temperature: number;
-        maxTokens?: number | undefined;
-    } | undefined;
-}, {
-    defaultModel?: {
-        model: ai.LanguageModelV1;
-        temperature?: number | undefined;
-        maxTokens?: number | undefined;
-    } | undefined;
-    headless?: boolean | undefined;
-}>;
+}, z.core.$strict>;
 type ScenarioProjectConfig = z.infer<typeof scenarioProjectConfigSchema>;
 declare function defineConfig(config: ScenarioProjectConfig): ScenarioProjectConfig;
 
@@ -390,7 +361,6 @@ type domain_AgentReturnTypes = AgentReturnTypes;
 type domain_AgentRole = AgentRole;
 declare const domain_AgentRole: typeof AgentRole;
 declare const domain_DEFAULT_MAX_TURNS: typeof DEFAULT_MAX_TURNS;
-declare const domain_DEFAULT_TEMPERATURE: typeof DEFAULT_TEMPERATURE;
 declare const domain_DEFAULT_VERBOSE: typeof DEFAULT_VERBOSE;
 type domain_JudgeAgentAdapter = JudgeAgentAdapter;
 declare const domain_JudgeAgentAdapter: typeof JudgeAgentAdapter;
@@ -407,32 +377,33 @@ declare const domain_allAgentRoles: typeof allAgentRoles;
 declare const domain_defineConfig: typeof defineConfig;
 declare const domain_scenarioProjectConfigSchema: typeof scenarioProjectConfigSchema;
 declare namespace domain {
-  export { domain_AgentAdapter as AgentAdapter, type domain_AgentInput as AgentInput, type domain_AgentReturnTypes as AgentReturnTypes, domain_AgentRole as AgentRole, domain_DEFAULT_MAX_TURNS as DEFAULT_MAX_TURNS, domain_DEFAULT_TEMPERATURE as DEFAULT_TEMPERATURE, domain_DEFAULT_VERBOSE as DEFAULT_VERBOSE, domain_JudgeAgentAdapter as JudgeAgentAdapter, type domain_ScenarioConfig as ScenarioConfig, type domain_ScenarioConfigFinal as ScenarioConfigFinal, type domain_ScenarioExecutionLike as ScenarioExecutionLike, type domain_ScenarioExecutionStateLike as ScenarioExecutionStateLike, type domain_ScenarioProjectConfig as ScenarioProjectConfig, type domain_ScenarioResult as ScenarioResult, type domain_ScriptStep as ScriptStep, domain_UserSimulatorAgentAdapter as UserSimulatorAgentAdapter, domain_allAgentRoles as allAgentRoles, domain_defineConfig as defineConfig, domain_scenarioProjectConfigSchema as scenarioProjectConfigSchema };
+  export { domain_AgentAdapter as AgentAdapter, type domain_AgentInput as AgentInput, type domain_AgentReturnTypes as AgentReturnTypes, domain_AgentRole as AgentRole, domain_DEFAULT_MAX_TURNS as DEFAULT_MAX_TURNS, domain_DEFAULT_VERBOSE as DEFAULT_VERBOSE, domain_JudgeAgentAdapter as JudgeAgentAdapter, type domain_ScenarioConfig as ScenarioConfig, type domain_ScenarioConfigFinal as ScenarioConfigFinal, type domain_ScenarioExecutionLike as ScenarioExecutionLike, type domain_ScenarioExecutionStateLike as ScenarioExecutionStateLike, type domain_ScenarioProjectConfig as ScenarioProjectConfig, type domain_ScenarioResult as ScenarioResult, type domain_ScriptStep as ScriptStep, domain_UserSimulatorAgentAdapter as UserSimulatorAgentAdapter, domain_allAgentRoles as allAgentRoles, domain_defineConfig as defineConfig, domain_scenarioProjectConfigSchema as scenarioProjectConfigSchema };
 }
 
 /**
- * Configuration for the inference parameters of a testing agent.
+ * Schema for a language model.
  */
-interface TestingAgentInferenceConfig {
-    /**
-     * The language model to use for generating responses.
-     * If not provided, a default model will be used.
-     */
-    model?: LanguageModel;
-    /**
-     * The temperature for the language model.
-     * Defaults to 0.
-     */
-    temperature?: number;
-    /**
-     * The maximum number of tokens to generate.
-     */
-    maxTokens?: number;
-}
+declare const modelSchema: z.ZodObject<{
+    model: z.ZodCustom<LanguageModel, LanguageModel>;
+    temperature: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+    maxTokens: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+type ModelConfig = z.infer<typeof modelSchema>;
+
+/**
+ * Parameters for LLM invocation.
+ * Derived from generateText parameters for now.
+ */
+type InvokeLLMParams = Parameters<typeof generateText>[0];
+/**
+ * Result from LLM invocation.
+ * Derived from generateText return type for now.
+ */
+type InvokeLLMResult = Pick<Awaited<ReturnType<typeof generateText>>, "text" | "content" | "toolCalls" | "toolResults">;
 /**
  * General configuration for a testing agent.
  */
-interface TestingAgentConfig extends TestingAgentInferenceConfig {
+interface TestingAgentConfig extends Partial<ModelConfig> {
     /**
      * The name of the agent.
      */
@@ -464,6 +435,35 @@ interface FinishTestArgs {
     verdict: "success" | "failure" | "inconclusive";
 }
 
+interface JudgeResult {
+    success: boolean;
+    reasoning: string;
+    metCriteria: string[];
+    unmetCriteria: string[];
+}
+
+/**
+ * Collects OpenTelemetry spans for judge evaluation.
+ * Implements SpanProcessor to intercept spans as they complete.
+ */
+declare class JudgeSpanCollector implements SpanProcessor {
+    private spans;
+    onStart(): void;
+    onEnd(span: ReadableSpan): void;
+    forceFlush(): Promise<void>;
+    shutdown(): Promise<void>;
+    /**
+     * Retrieves all spans associated with a specific thread.
+     * @param threadId - The thread identifier to filter spans by
+     * @returns Array of spans for the given thread
+     */
+    getSpansForThread(threadId: string): ReadableSpan[];
+}
+/**
+ * Singleton instance of the judge span collector.
+ */
+declare const judgeSpanCollector: JudgeSpanCollector;
+
 /**
  * Configuration for the judge agent.
  */
@@ -476,6 +476,10 @@ interface JudgeAgentConfig extends TestingAgentConfig {
      * The criteria that the judge will use to evaluate the conversation.
      */
     criteria: string[];
+    /**
+     * Optional span collector for telemetry. Defaults to global singleton.
+     */
+    spanCollector?: JudgeSpanCollector;
 }
 /**
  * Agent that evaluates conversations against success criteria.
@@ -489,17 +493,16 @@ interface JudgeAgentConfig extends TestingAgentConfig {
 declare class JudgeAgent extends JudgeAgentAdapter {
     private readonly cfg;
     private logger;
+    private readonly spanCollector;
     role: AgentRole;
     criteria: string[];
+    /**
+     * LLM invocation function. Can be overridden to customize LLM behavior.
+     */
+    invokeLLM: (params: InvokeLLMParams) => Promise<InvokeLLMResult>;
     constructor(cfg: JudgeAgentConfig);
-    call(input: AgentInput): Promise<never[] | {
-        success: boolean;
-        messages: CoreMessage[];
-        reasoning: string;
-        metCriteria: string[];
-        unmetCriteria: string[];
-    }>;
-    private generateText;
+    call(input: AgentInput): Promise<JudgeResult | null>;
+    private getOpenTelemetryTracesDigest;
 }
 /**
  * Factory function for creating JudgeAgent instances.
@@ -553,15 +556,54 @@ declare class JudgeAgent extends JudgeAgentAdapter {
  */
 declare const judgeAgent: (cfg: JudgeAgentConfig) => JudgeAgent;
 
+/**
+ * Transforms OpenTelemetry spans into a complete plain-text digest for judge evaluation.
+ * Deduplicates repeated string content to reduce token usage.
+ */
+declare class JudgeSpanDigestFormatter {
+    private readonly logger;
+    private readonly deduplicator;
+    /**
+     * Formats spans into a complete digest with full content and nesting.
+     * @param spans - All spans for a thread
+     * @returns Plain text digest
+     */
+    format(spans: ReadableSpan[]): string;
+    private sortByStartTime;
+    private buildHierarchy;
+    private renderNode;
+    private getTreePrefix;
+    private getAttrIndent;
+    private cleanAttributes;
+    private formatValue;
+    private transformValue;
+    private transformString;
+    private looksLikeJson;
+    private hrTimeToMs;
+    private calculateSpanDuration;
+    private calculateTotalDuration;
+    private formatDuration;
+    private formatTimestamp;
+    private getStatusIndicator;
+    private collectErrors;
+}
+/**
+ * Singleton instance for convenience.
+ */
+declare const judgeSpanDigestFormatter: JudgeSpanDigestFormatter;
+
 declare class UserSimulatorAgent extends UserSimulatorAgentAdapter {
     private readonly cfg?;
     private logger;
+    /**
+     * LLM invocation function. Can be overridden to customize LLM behavior.
+     */
+    invokeLLM: (params: InvokeLLMParams) => Promise<InvokeLLMResult>;
     constructor(cfg?: TestingAgentConfig | undefined);
     call: (input: AgentInput) => Promise<{
         role: "user";
         content: string;
     }>;
-    private generateText;
 }
 /**
  * Agent that simulates realistic user behavior in scenario conversations.
@@ -654,14 +696,169 @@ declare class UserSimulatorAgent extends UserSimulatorAgentAdapter {
  */
 declare const userSimulatorAgent: (config?: TestingAgentConfig) => UserSimulatorAgent;
 
+/**
+ * Event emitted when an audio response is completed
+ */
+interface AudioResponseEvent {
+    transcript: string;
+    audio: string;
+}
+
+/**
+ * Realtime Agent Adapter for Scenario Testing
+ *
+ * Adapts a connected RealtimeSession to the Scenario framework interface.
+ * The session must be created and connected before passing to this adapter.
+ *
+ * This ensures we test the REAL agent, not a mock, using the same session
+ * creation pattern as the browser client.
+ */
+
+/**
+ * Configuration for RealtimeAgentAdapter
+ */
+interface RealtimeAgentAdapterConfig {
+    /**
+     * The role of the agent
+     */
+    role: AgentRole;
+    /**
+     * A connected RealtimeSession instance
+     *
+     * The session should be created using your agent's session creator function
+     * and connected before passing to this adapter.
+     *
+     * @example
+     * ```typescript
+     * const session = createVegetarianRecipeSession();
+     * await session.connect({ apiKey: process.env.OPENAI_API_KEY });
+     * const adapter = new RealtimeAgentAdapter({
+     *   session,
+     *   role: AgentRole.AGENT,
+     *   agentName: "Vegetarian Recipe Assistant"
+     * });
+     * ```
+     */
+    session: RealtimeSession;
+    /**
+     * Name of the agent (for logging/identification)
+     */
+    agentName: string;
+    /**
+     * Timeout for waiting for agent response (ms)
+     * @default 30000
+     */
+    responseTimeout?: number;
+}
+/**
+ * Adapter that connects Scenario testing framework to OpenAI Realtime API
+ *
+ * This adapter wraps a connected RealtimeSession to provide the Scenario
+ * framework interface. The session must be created and connected externally,
+ * ensuring the same session creation pattern is used in both browser and tests.
+ *
+ * @example
+ * ```typescript
+ * // In beforeAll
+ * const session = createVegetarianRecipeSession();
+ * await session.connect({ apiKey: process.env.OPENAI_API_KEY });
+ * const adapter = new RealtimeAgentAdapter({
+ *   session,
+ *   role: AgentRole.AGENT
+ * });
+ *
+ * // In test
+ * await scenario.run({
+ *   agents: [adapter, scenario.userSimulatorAgent()],
+ *   script: [scenario.user("quick recipe"), scenario.agent()]
+ * });
+ *
+ * // In afterAll
+ * session.close();
+ * ```
+ */
+declare class RealtimeAgentAdapter extends AgentAdapter {
+    private config;
+    role: AgentRole;
+    name: string;
+    private session;
+    private eventHandler;
+    private messageProcessor;
+    private responseFormatter;
+    private audioEvents;
+    /**
+     * Creates a new RealtimeAgentAdapter instance
+     *
+     * The session can be either connected or unconnected.
+     * If unconnected, call connect() with an API key before use.
+     *
+     * @param config - Configuration for the realtime agent adapter
+     */
+    constructor(config: RealtimeAgentAdapterConfig);
+    /**
+     * Get the connect method from the session
+     */
+    connect(params?: Parameters<RealtimeSession["connect"]>[0] | undefined): Promise<void>;
+    /**
+     * Closes the session connection
+     */
+    disconnect(): Promise<void>;
+    /**
+     * Process input and generate response (implements AgentAdapter interface)
+     *
+     * This is called by Scenario framework for each agent turn.
+     * Handles both text and audio input, returns audio message with transcript.
+     *
+     * @param input - Scenario agent input with message history
+     * @returns Agent response as audio message or text
+     */
+    call(input: AgentInput): Promise<AgentReturnTypes>;
+    /**
+     * Handles the initial response when no user message exists
+     */
+    private handleInitialResponse;
+    /**
+     * Handles audio input from the user
+     */
+    private handleAudioInput;
+    /**
+     * Handles text input from the user
+     */
+    private handleTextInput;
+    /**
+     * Subscribe to audio response events
+     *
+     * @param callback - Function called when an audio response completes
+     */
+    onAudioResponse(callback: (event: AudioResponseEvent) => void): void;
+    /**
+     * Remove audio response listener
+     *
+     * @param callback - The callback function to remove
+     */
+    offAudioResponse(callback: (event: AudioResponseEvent) => void): void;
+}
+
+type agents_AudioResponseEvent = AudioResponseEvent;
 type agents_FinishTestArgs = FinishTestArgs;
+type agents_InvokeLLMParams = InvokeLLMParams;
+type agents_InvokeLLMResult = InvokeLLMResult;
 type agents_JudgeAgentConfig = JudgeAgentConfig;
+type agents_JudgeResult = JudgeResult;
+type agents_JudgeSpanCollector = JudgeSpanCollector;
+declare const agents_JudgeSpanCollector: typeof JudgeSpanCollector;
+type agents_JudgeSpanDigestFormatter = JudgeSpanDigestFormatter;
+declare const agents_JudgeSpanDigestFormatter: typeof JudgeSpanDigestFormatter;
+type agents_RealtimeAgentAdapter = RealtimeAgentAdapter;
+declare const agents_RealtimeAgentAdapter: typeof RealtimeAgentAdapter;
+type agents_RealtimeAgentAdapterConfig = RealtimeAgentAdapterConfig;
 type agents_TestingAgentConfig = TestingAgentConfig;
-type agents_TestingAgentInferenceConfig = TestingAgentInferenceConfig;
 declare const agents_judgeAgent: typeof judgeAgent;
+declare const agents_judgeSpanCollector: typeof judgeSpanCollector;
+declare const agents_judgeSpanDigestFormatter: typeof judgeSpanDigestFormatter;
 declare const agents_userSimulatorAgent: typeof userSimulatorAgent;
 declare namespace agents {
-  export { type agents_FinishTestArgs as FinishTestArgs, type agents_JudgeAgentConfig as JudgeAgentConfig, type agents_TestingAgentConfig as TestingAgentConfig, type agents_TestingAgentInferenceConfig as TestingAgentInferenceConfig, agents_judgeAgent as judgeAgent, agents_userSimulatorAgent as userSimulatorAgent };
+  export { type agents_AudioResponseEvent as AudioResponseEvent, type agents_FinishTestArgs as FinishTestArgs, type agents_InvokeLLMParams as InvokeLLMParams, type agents_InvokeLLMResult as InvokeLLMResult, type agents_JudgeAgentConfig as JudgeAgentConfig, type agents_JudgeResult as JudgeResult, agents_JudgeSpanCollector as JudgeSpanCollector, agents_JudgeSpanDigestFormatter as JudgeSpanDigestFormatter, agents_RealtimeAgentAdapter as RealtimeAgentAdapter, type agents_RealtimeAgentAdapterConfig as RealtimeAgentAdapterConfig, type agents_TestingAgentConfig as TestingAgentConfig, agents_judgeAgent as judgeAgent, agents_judgeSpanCollector as judgeSpanCollector, agents_judgeSpanDigestFormatter as judgeSpanDigestFormatter, agents_userSimulatorAgent as userSimulatorAgent };
 }
 
 /**
@@ -690,26 +887,26 @@ declare enum ScenarioRunStatus {
  * Discriminated union of all possible scenario event types.
  * Enables type-safe handling of different event types based on the 'type' field.
  */
-declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
-    timestamp: z.ZodNumber;
-    rawEvent: z.ZodOptional<z.ZodAny>;
-    batchRunId: z.ZodString;
-    scenarioId: z.ZodString;
-    scenarioRunId: z.ZodString;
-    scenarioSetId: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+declare const scenarioEventSchema: z$1.ZodDiscriminatedUnion<"type", [z$1.ZodObject<{
+    timestamp: z$1.ZodNumber;
+    rawEvent: z$1.ZodOptional<z$1.ZodAny>;
+    batchRunId: z$1.ZodString;
+    scenarioId: z$1.ZodString;
+    scenarioRunId: z$1.ZodString;
+    scenarioSetId: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodString>>;
 } & {
-    type: z.ZodLiteral<ScenarioEventType.RUN_STARTED>;
-    metadata: z.ZodObject<{
-        name: z.ZodOptional<z.ZodString>;
-        description: z.ZodOptional<z.ZodString>;
-    }, "strip", z.ZodTypeAny, {
-        name?: string | undefined;
+    type: z$1.ZodLiteral<ScenarioEventType.RUN_STARTED>;
+    metadata: z$1.ZodObject<{
+        name: z$1.ZodOptional<z$1.ZodString>;
+        description: z$1.ZodOptional<z$1.ZodString>;
+    }, "strip", z$1.ZodTypeAny, {
         description?: string | undefined;
-    }, {
         name?: string | undefined;
+    }, {
         description?: string | undefined;
+        name?: string | undefined;
     }>;
-}, "strip", z.ZodTypeAny, {
+}, "strip", z$1.ZodTypeAny, {
     type: ScenarioEventType.RUN_STARTED;
     timestamp: number;
     batchRunId: string;
@@ -717,8 +914,8 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
     scenarioRunId: string;
     scenarioSetId: string;
     metadata: {
-        name?: string | undefined;
         description?: string | undefined;
+        name?: string | undefined;
     };
     rawEvent?: any;
 }, {
@@ -728,41 +925,41 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
     scenarioId: string;
     scenarioRunId: string;
     metadata: {
-        name?: string | undefined;
         description?: string | undefined;
+        name?: string | undefined;
     };
     rawEvent?: any;
     scenarioSetId?: string | undefined;
-}>, z.ZodObject<{
-    timestamp: z.ZodNumber;
-    rawEvent: z.ZodOptional<z.ZodAny>;
-    batchRunId: z.ZodString;
-    scenarioId: z.ZodString;
-    scenarioRunId: z.ZodString;
-    scenarioSetId: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+}>, z$1.ZodObject<{
+    timestamp: z$1.ZodNumber;
+    rawEvent: z$1.ZodOptional<z$1.ZodAny>;
+    batchRunId: z$1.ZodString;
+    scenarioId: z$1.ZodString;
+    scenarioRunId: z$1.ZodString;
+    scenarioSetId: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodString>>;
 } & {
-    type: z.ZodLiteral<ScenarioEventType.RUN_FINISHED>;
-    status: z.ZodNativeEnum<typeof ScenarioRunStatus>;
-    results: z.ZodNullable<z.ZodOptional<z.ZodObject<{
-        verdict: z.ZodNativeEnum<typeof Verdict>;
-        reasoning: z.ZodOptional<z.ZodString>;
-        metCriteria: z.ZodArray<z.ZodString, "many">;
-        unmetCriteria: z.ZodArray<z.ZodString, "many">;
-        error: z.ZodOptional<z.ZodString>;
-    }, "strip", z.ZodTypeAny, {
+    type: z$1.ZodLiteral<ScenarioEventType.RUN_FINISHED>;
+    status: z$1.ZodNativeEnum<typeof ScenarioRunStatus>;
+    results: z$1.ZodNullable<z$1.ZodOptional<z$1.ZodObject<{
+        verdict: z$1.ZodNativeEnum<typeof Verdict>;
+        reasoning: z$1.ZodOptional<z$1.ZodString>;
+        metCriteria: z$1.ZodArray<z$1.ZodString, "many">;
+        unmetCriteria: z$1.ZodArray<z$1.ZodString, "many">;
+        error: z$1.ZodOptional<z$1.ZodString>;
+    }, "strip", z$1.ZodTypeAny, {
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     }, {
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     }>>>;
-}, "strip", z.ZodTypeAny, {
+}, "strip", z$1.ZodTypeAny, {
     type: ScenarioEventType.RUN_FINISHED;
     status: ScenarioRunStatus;
     timestamp: number;
@@ -775,8 +972,8 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     } | null | undefined;
 }, {
     type: ScenarioEventType.RUN_FINISHED;
@@ -791,19 +988,19 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     } | null | undefined;
-}>, z.ZodObject<{
-    messages: z.ZodArray<z.ZodDiscriminatedUnion<"role", [z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+}>, z$1.ZodObject<{
+    messages: z$1.ZodArray<z$1.ZodDiscriminatedUnion<"role", [z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"developer">;
-        content: z.ZodString;
-    }>, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"developer">;
+        content: z$1.ZodString;
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "developer";
         content: string;
@@ -813,15 +1010,15 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         role: "developer";
         content: string;
         name?: string | undefined;
-    }>, z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+    }>, z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"system">;
-        content: z.ZodString;
-    }>, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"system">;
+        content: z$1.ZodString;
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "system";
         content: string;
@@ -831,28 +1028,28 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         role: "system";
         content: string;
         name?: string | undefined;
-    }>, z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+    }>, z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"assistant">;
-        content: z.ZodOptional<z.ZodString>;
-        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
-            id: z.ZodString;
-            type: z.ZodLiteral<"function">;
-            function: z.ZodObject<{
-                name: z.ZodString;
-                arguments: z.ZodString;
-            }, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"assistant">;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        toolCalls: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
+            id: z$1.ZodString;
+            type: z$1.ZodLiteral<"function">;
+            function: z$1.ZodObject<{
+                name: z$1.ZodString;
+                arguments: z$1.ZodString;
+            }, "strip", z$1.ZodTypeAny, {
                 name: string;
                 arguments: string;
             }, {
                 name: string;
                 arguments: string;
             }>;
-        }, "strip", z.ZodTypeAny, {
+        }, "strip", z$1.ZodTypeAny, {
             function: {
                 name: string;
                 arguments: string;
@@ -867,7 +1064,7 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
             type: "function";
             id: string;
         }>, "many">>;
-    }>, "strip", z.ZodTypeAny, {
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "assistant";
         name?: string | undefined;
@@ -893,15 +1090,15 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
             type: "function";
             id: string;
         }[] | undefined;
-    }>, z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+    }>, z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"user">;
-        content: z.ZodString;
-    }>, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"user">;
+        content: z$1.ZodString;
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "user";
         content: string;
@@ -911,12 +1108,12 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         role: "user";
         content: string;
         name?: string | undefined;
-    }>, z.ZodObject<{
-        id: z.ZodString;
-        content: z.ZodString;
-        role: z.ZodLiteral<"tool">;
-        toolCallId: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
+    }>, z$1.ZodObject<{
+        id: z$1.ZodString;
+        content: z$1.ZodString;
+        role: z$1.ZodLiteral<"tool">;
+        toolCallId: z$1.ZodString;
+    }, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "tool";
         content: string;
@@ -928,14 +1125,14 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         toolCallId: string;
     }>]>, "many">;
 } & {
-    timestamp: z.ZodNumber;
-    rawEvent: z.ZodOptional<z.ZodAny>;
-    batchRunId: z.ZodString;
-    scenarioId: z.ZodString;
-    scenarioRunId: z.ZodString;
-    scenarioSetId: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-    type: z.ZodLiteral<ScenarioEventType.MESSAGE_SNAPSHOT>;
-}, "strip", z.ZodTypeAny, {
+    timestamp: z$1.ZodNumber;
+    rawEvent: z$1.ZodOptional<z$1.ZodAny>;
+    batchRunId: z$1.ZodString;
+    scenarioId: z$1.ZodString;
+    scenarioRunId: z$1.ZodString;
+    scenarioSetId: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodString>>;
+    type: z$1.ZodLiteral<ScenarioEventType.MESSAGE_SNAPSHOT>;
+}, "strip", z$1.ZodTypeAny, {
     type: ScenarioEventType.MESSAGE_SNAPSHOT;
     messages: ({
         id: string;
@@ -1020,7 +1217,7 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
     rawEvent?: any;
     scenarioSetId?: string | undefined;
 }>]>;
-type ScenarioEvent = z.infer<typeof scenarioEventSchema>;
+type ScenarioEvent = z$1.infer<typeof scenarioEventSchema>;
 
 /**
  * Manages the execution of a single scenario test.
@@ -1107,8 +1304,12 @@ type ScenarioEvent = z.infer<typeof scenarioEventSchema>;
  * ```
  */
 declare class ScenarioExecution implements ScenarioExecutionLike {
+    /** LangWatch tracer for scenario execution */
+    private tracer;
     /** The current state of the scenario execution */
     private state;
+    /** The final result of the scenario execution, set when a conclusion is reached */
+    private _result?;
     /** Logger for debugging and monitoring */
     private logger;
     /** Finalized configuration with all defaults applied */
@@ -1127,10 +1328,10 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * Key: agent index, Value: array of pending messages for that agent
      */
     private pendingMessages;
-    /** Intermediate result set by agents that make final decisions */
-    private partialResult;
     /** Accumulated execution time for each agent (for performance tracking) */
     private agentTimes;
+    /** Current turn span for trace context management */
+    private currentTurnSpan?;
     /** Timestamp when execution started (for total time calculation) */
     private totalStartTime;
     /** Event stream for monitoring scenario progress */
@@ -1155,9 +1356,9 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
     /**
      * Gets the complete conversation history as an array of messages.
      *
-     * @returns Array of CoreMessage objects representing the full conversation
+     * @returns Array of ModelMessage objects representing the full conversation
      */
-    get messages(): CoreMessage[];
+    get messages(): ModelMessage[];
     /**
      * Gets the unique identifier for the conversation thread.
      * This ID is used to maintain conversation context across multiple runs.
@@ -1165,6 +1366,20 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * @returns The thread identifier string
      */
     get threadId(): string;
+    /**
+     * Gets the result of the scenario execution if it has been set.
+     *
+     * @returns The scenario result or undefined if not yet set
+     */
+    get result(): ScenarioResult | undefined;
+    /**
+     * Sets the result of the scenario execution.
+     * This is called when the scenario reaches a conclusion (success or failure).
+     * Automatically includes messages, totalTime, and agentTime from the current execution context.
+     *
+     * @param result - The final scenario result (without messages/timing, which will be added automatically)
+     */
+    private setResult;
     /**
      * The total elapsed time for the scenario execution.
      */
@@ -1207,30 +1422,25 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * - Progress to the next turn if needed
      * - Find the next agent that should act
      * - Execute that agent's response
-     * - Return either new messages or a final scenario result
+     * - Set the result if the scenario concludes
      *
      * Note: This method is primarily for debugging or custom execution flows. Most users
      * will use `execute()` to run the entire scenario automatically.
      *
-     * @returns A promise that resolves with either:
-     *   - Array of new messages added during the agent interaction, or
-     *   - A final ScenarioResult if the interaction concludes the scenario
-     * @throws Error if no result is returned from the step
+     * After calling this method, check `this.result` to see if the scenario has concluded.
      *
      * @example
      * ```typescript
      * const execution = new ScenarioExecution(config, script);
      *
      * // Execute one agent interaction at a time
-     * const messages = await execution.step();
-     * if (Array.isArray(messages)) {
-     *   console.log('New messages:', messages);
-     * } else {
-     *   console.log('Scenario finished:', messages.success);
+     * await execution.step();
+     * if (execution.result) {
+     *   console.log('Scenario finished:', execution.result.success);
      * }
      * ```
      */
-    step(): Promise<CoreMessage[] | ScenarioResult>;
+    step(): Promise<void>;
     private _step;
     /**
      * Calls a specific agent to generate a response or make a decision.
@@ -1249,15 +1459,12 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * After the agent responds:
      * - Performance timing is recorded
      * - Pending messages for this agent are cleared (they've been processed)
-     * - If the agent returns a ScenarioResult, it's returned immediately
+     * - If the agent returns a ScenarioResult, it's set on this.result
      * - Otherwise, the agent's messages are added to the conversation and broadcast
      *
      * @param idx - The index of the agent in the agents array
      * @param role - The role the agent is being asked to play (USER, AGENT, or JUDGE)
      * @param judgmentRequest - Whether this is a judgment request (for judge agents)
-     * @returns A promise that resolves with either:
-     *   - Array of messages if the agent generated a response, or
-     *   - ScenarioResult if the agent made a final decision
      * @throws Error if the agent call fails
      */
     private callAgent;
@@ -1270,7 +1477,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * - "assistant" messages are routed to AGENT role agents
      * - Other message types are added directly to the conversation
      *
-     * @param message - The CoreMessage to add to the conversation
+     * @param message - The ModelMessage to add to the conversation
      *
      * @example
      * ```typescript
@@ -1280,7 +1487,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * });
      * ```
      */
-    message(message: CoreMessage): Promise<void>;
+    message(message: ModelMessage): Promise<void>;
     /**
      * Executes a user turn in the conversation.
      *
@@ -1290,7 +1497,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      *
      * This method is part of the ScenarioExecutionLike interface used by script steps.
      *
-     * @param content - Optional content for the user's message. Can be a string or CoreMessage.
+     * @param content - Optional content for the user's message. Can be a string or ModelMessage.
      *                 If not provided, the user simulator agent will generate the content.
      *
      * @example
@@ -1301,14 +1508,14 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * // Let user simulator generate content
      * await execution.user();
      *
-     * // Use a CoreMessage object
+     * // Use a ModelMessage object
      * await execution.user({
      *   role: "user",
      *   content: "Tell me a joke"
      * });
      * ```
      */
-    user(content?: string | CoreMessage): Promise<void>;
+    user(content?: string | ModelMessage): Promise<void>;
     /**
      * Executes an agent turn in the conversation.
      *
@@ -1318,7 +1525,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      *
      * This method is part of the ScenarioExecutionLike interface used by script steps.
      *
-     * @param content - Optional content for the agent's response. Can be a string or CoreMessage.
+     * @param content - Optional content for the agent's response. Can be a string or ModelMessage.
      *                 If not provided, the agent under test will generate the response.
      *
      * @example
@@ -1329,14 +1536,14 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * // Use provided content
      * await execution.agent("The weather is sunny today!");
      *
-     * // Use a CoreMessage object
+     * // Use a ModelMessage object
      * await execution.agent({
      *   role: "assistant",
      *   content: "I'm here to help you with weather information."
      * });
      * ```
      */
-    agent(content?: string | CoreMessage): Promise<void>;
+    agent(content?: string | ModelMessage): Promise<void>;
     /**
      * Invokes the judge agent to evaluate the current state of the conversation.
      *
@@ -1364,7 +1571,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * const result = await execution.judge("Please consider the user's satisfaction level");
      * ```
      */
-    judge(content?: string | CoreMessage): Promise<ScenarioResult | null>;
+    judge(content?: string | ModelMessage): Promise<ScenarioResult | null>;
     /**
      * Lets the scenario proceed automatically for a specified number of turns.
      *
@@ -1472,49 +1679,6 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * ```
      */
     addAgentTime(agentIdx: number, time: number): void;
-    /**
-     * Checks if a partial result has been set for the scenario.
-     *
-     * This method is used internally to determine if a scenario has already reached
-     * a conclusion (success or failure) but hasn't been finalized yet. Partial results
-     * are typically set by agents that make final decisions (like judge agents) and
-     * are later finalized with the complete message history.
-     *
-     * @returns True if a partial result exists, false otherwise
-     *
-     * @example
-     * ```typescript
-     * // This is typically used internally by the execution engine
-     * if (execution.hasResult()) {
-     *   console.log('Scenario has reached a conclusion');
-     * }
-     * ```
-     */
-    hasResult(): boolean;
-    /**
-     * Sets a partial result for the scenario.
-     *
-     * This method is used internally to store intermediate results that may be
-     * finalized later with the complete message history. Partial results are typically
-     * created by agents that make final decisions (like judge agents) and contain
-     * the success/failure status, reasoning, and criteria evaluation, but not the
-     * complete message history.
-     *
-     * @param result - The partial result without the messages field. Should include
-     *                success status, reasoning, and criteria evaluation.
-     *
-     * @example
-     * ```typescript
-     * // This is typically called internally by agents that make final decisions
-     * execution.setResult({
-     *   success: true,
-     *   reasoning: "Agent provided accurate weather information",
-     *   metCriteria: ["Provides accurate weather data"],
-     *   unmetCriteria: []
-     * });
-     * ```
-     */
-    setResult(result: Omit<ScenarioResult, "messages">): void;
     /**
      * Internal method to handle script step calls to agents.
      *
@@ -1527,7 +1691,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * - Progress to a new turn if no agent is available
      * - Execute the agent with the provided content or let it generate content
      * - Handle judgment requests for judge agents
-     * - Return a final result if the agent makes a decision
+     * - Set the result if the agent makes a decision
      *
      * @param role - The role of the agent to call (USER, AGENT, or JUDGE)
      * @param content - Optional content to use instead of letting the agent generate it
@@ -1551,6 +1715,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * - Starts the first turn
      * - Records the start time for performance tracking
      * - Clears any pending messages
+     * - Clears the result from any previous execution
      */
     private reset;
     private nextAgentForRole;
@@ -1575,7 +1740,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      *
      * This method is called when the scenario execution reaches the maximum number
      * of turns without reaching a conclusion. It creates a failure result with
-     * appropriate reasoning and includes performance metrics.
+     * appropriate reasoning and includes performance metrics, then sets it on this.result.
      *
      * The result includes:
      * - All messages from the conversation
@@ -1585,7 +1750,6 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * - Total execution time and agent response times
      *
      * @param errorMessage - Optional custom error message to use instead of the default
-     * @returns A ScenarioResult indicating failure due to reaching max turns
      */
     private reachedMaxTurns;
     private getJudgeAgent;
@@ -1692,12 +1856,25 @@ declare class ScenarioExecutionState implements ScenarioExecutionStateLike {
      * Adds a message to the conversation history.
      *
      * @param message - The message to add.
+     * @param traceId - Optional trace ID to associate with the message.
      */
-    addMessage(message: CoreMessage): void;
-    lastMessage(): CoreMessage;
-    lastUserMessage(): CoreUserMessage;
-    lastAgentMessage(): CoreAssistantMessage;
-    lastToolCall(toolName: string): CoreToolMessage;
+    addMessage(message: CoreMessage & {
+        traceId?: string;
+    }): void;
+    lastMessage(): ai.ModelMessage & {
+        id: string;
+        traceId?: string;
+    };
+    lastUserMessage(): ai.UserModelMessage & {
+        id: string;
+        traceId?: string;
+    };
+    lastAgentMessage(): CoreAssistantMessage & {
+        traceId?: string;
+    };
+    lastToolCall(toolName: string): CoreToolMessage & {
+        traceId?: string;
+    };
     hasToolCall(toolName: string): boolean;
 }
 
@@ -1871,4 +2048,4 @@ declare namespace script {
 type ScenarioApi = typeof agents & typeof domain & typeof execution & typeof runner & typeof script;
 declare const scenario: ScenarioApi;
 
-export { AgentAdapter, type AgentInput, type AgentReturnTypes, AgentRole, DEFAULT_MAX_TURNS, DEFAULT_TEMPERATURE, DEFAULT_VERBOSE, type FinishTestArgs, JudgeAgentAdapter, type JudgeAgentConfig, type ScenarioConfig, type ScenarioConfigFinal, ScenarioExecution, type ScenarioExecutionLike, ScenarioExecutionState, type ScenarioExecutionStateLike, type ScenarioProjectConfig, type ScenarioResult, type ScriptStep, type StateChangeEvent, StateChangeEventType, type TestingAgentConfig, type TestingAgentInferenceConfig, UserSimulatorAgentAdapter, agent, allAgentRoles, scenario as default, defineConfig, fail, judge, judgeAgent, message, proceed, run, scenario, scenarioProjectConfigSchema, succeed, user, userSimulatorAgent };
+export { AgentAdapter, type AgentInput, type AgentReturnTypes, AgentRole, type AudioResponseEvent, DEFAULT_MAX_TURNS, DEFAULT_VERBOSE, type FinishTestArgs, type InvokeLLMParams, type InvokeLLMResult, JudgeAgentAdapter, type JudgeAgentConfig, type JudgeResult, JudgeSpanCollector, JudgeSpanDigestFormatter, RealtimeAgentAdapter, type RealtimeAgentAdapterConfig, type ScenarioConfig, type ScenarioConfigFinal, ScenarioExecution, type ScenarioExecutionLike, ScenarioExecutionState, type ScenarioExecutionStateLike, type ScenarioProjectConfig, type ScenarioResult, type ScriptStep, type StateChangeEvent, StateChangeEventType, type TestingAgentConfig, UserSimulatorAgentAdapter, agent, allAgentRoles, scenario as default, defineConfig, fail, judge, judgeAgent, judgeSpanCollector, judgeSpanDigestFormatter, message, proceed, run, scenario, scenarioProjectConfigSchema, succeed, user, userSimulatorAgent };