@langwatch/scenario 0.2.0-prerelease.0

package/dist/index.d.mts

@@ -0,0 +1,1193 @@
+import * as ai from 'ai';
+import { CoreMessage, CoreToolMessage, LanguageModel } from 'ai';
+import { Observable } from 'rxjs';
+import { z } from 'zod';
+
+interface ScenarioResult {
+    success: boolean;
+    messages: CoreMessage[];
+    reasoning?: string;
+    passedCriteria: string[];
+    failedCriteria: string[];
+    totalTime?: number;
+    agentTime?: number;
+}
+interface ScenarioExecutionStateLike {
+    history: CoreMessage[];
+    historyWithoutLastMessage: CoreMessage[];
+    historyWithoutLastUserMessage: CoreMessage[];
+    threadId: string;
+    turn: number | null;
+    agents: AgentAdapter[];
+    pendingRolesOnTurn: AgentRole[];
+    pendingAgentsOnTurn: AgentAdapter[];
+    partialResult: Omit<ScenarioResult, "messages"> | null;
+    totalTime: number;
+    agentTimes: Map<number, number>;
+    addMessage(message: CoreMessage, fromAgentIdx?: number): void;
+    addMessages(messages: CoreMessage[], fromAgentIdx?: number): void;
+    setThreadId(threadId: string): void;
+    setAgents(agents: AgentAdapter[]): void;
+    appendMessage(role: CoreMessage["role"], content: string): void;
+    appendUserMessage(content: string): void;
+    appendAssistantMessage(content: string): void;
+    getPendingMessages(agentIdx: number): CoreMessage[];
+    clearPendingMessages(agentIdx: number): void;
+    newTurn(): void;
+    removePendingRole(role: AgentRole): void;
+    removeLastPendingRole(): void;
+    removePendingAgent(agent: AgentAdapter): void;
+    getNextAgentForRole(role: AgentRole): {
+        index: number;
+        agent: AgentAdapter;
+    } | null;
+    addAgentTime(agentIdx: number, time: number): void;
+    hasResult(): boolean;
+    setResult(result: Omit<ScenarioResult, "messages">): void;
+    readonly lastMessage: CoreMessage | undefined;
+    readonly lastUserMessage: CoreMessage | undefined;
+    readonly lastAssistantMessage: CoreMessage | undefined;
+    readonly lastToolCall: CoreToolMessage | undefined;
+    getLastToolCallByToolName(toolName: string): CoreToolMessage | undefined;
+    hasToolCall(toolName: string): boolean;
+}
+
+declare const scenarioProjectConfigSchema: z.ZodObject<{
+    defaultModel: z.ZodOptional<z.ZodObject<{
+        model: z.ZodType<ai.LanguageModelV1, z.ZodTypeDef, ai.LanguageModelV1>;
+        temperature: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
+        maxTokens: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        model: ai.LanguageModelV1;
+        temperature: number;
+        maxTokens?: number | undefined;
+    }, {
+        model: ai.LanguageModelV1;
+        temperature?: number | undefined;
+        maxTokens?: number | undefined;
+    }>>;
+    langwatchEndpoint: z.ZodOptional<z.ZodString>;
+    langwatchApiKey: z.ZodOptional<z.ZodString>;
+}, "strict", z.ZodTypeAny, {
+    defaultModel?: {
+        model: ai.LanguageModelV1;
+        temperature: number;
+        maxTokens?: number | undefined;
+    } | undefined;
+    langwatchEndpoint?: string | undefined;
+    langwatchApiKey?: string | undefined;
+}, {
+    defaultModel?: {
+        model: ai.LanguageModelV1;
+        temperature?: number | undefined;
+        maxTokens?: number | undefined;
+    } | undefined;
+    langwatchEndpoint?: string | undefined;
+    langwatchApiKey?: string | undefined;
+}>;
+type ScenarioProjectConfig = z.infer<typeof scenarioProjectConfigSchema>;
+declare function defineConfig(config: ScenarioProjectConfig): ScenarioProjectConfig;
+
+/**
+ * Configuration for a scenario.
+ */
+interface ScenarioConfig {
+    /**
+     * Optional unique identifier for the scenario.
+     * If not provided, a UUID will be generated.
+     */
+    id?: string;
+    /**
+     * The name of the scenario.
+     */
+    name: string;
+    /**
+     * A description of what the scenario tests.
+     */
+    description: string;
+    /**
+     * The agents participating in the scenario.
+     */
+    agents: AgentAdapter[];
+    /**
+     * The script of steps to execute for the scenario.
+     */
+    script?: ScriptStep[];
+    /**
+     * Whether to output verbose logging. Defaults to false.
+     */
+    verbose?: boolean;
+    /**
+     * The maximum number of turns to execute. Defaults to 20.
+     */
+    maxTurns?: number;
+    /**
+     * Optional thread ID to use for the conversation.
+     * If not provided, a new thread will be created.
+     */
+    threadId?: string;
+}
+/**
+ * Final, normalized scenario configuration.
+ * All optional fields are filled with default values.
+ * @internal
+ */
+interface ScenarioConfigFinal extends Omit<ScenarioConfig, "id" | "script" | "threadId" | "verbose" | "maxTurns"> {
+    id: string;
+    script: ScriptStep[];
+    verbose: boolean;
+    maxTurns: number;
+    threadId: string;
+}
+/**
+ * The execution context for a scenario script.
+ * This provides the functions to control the flow of the scenario.
+ */
+interface ScenarioExecutionLike {
+    /**
+     * The history of messages in the conversation.
+     */
+    readonly history: CoreMessage[];
+    /**
+     * The ID of the conversation thread.
+     */
+    readonly threadId: string;
+    /**
+     * Adds a message to the conversation.
+     * @param message The message to add.
+     */
+    message(message: CoreMessage): Promise<void>;
+    /**
+     * Adds a user message to the conversation.
+     * If no content is provided, the user simulator will generate a message.
+     * @param content The content of the user message.
+     */
+    user(content?: string | CoreMessage): Promise<void>;
+    /**
+     * Adds an agent message to the conversation.
+     * If no content is provided, the agent under test will generate a message.
+     * @param content The content of the agent message.
+     */
+    agent(content?: string | CoreMessage): Promise<void>;
+    /**
+     * Invokes the judge agent to evaluate the current state.
+     * @param content Optional message to the judge.
+     * @returns The result of the scenario if the judge makes a final decision.
+     */
+    judge(content?: string | CoreMessage): Promise<ScenarioResult | null>;
+    /**
+     * Proceeds with the scenario automatically for a number of turns.
+     * @param turns The number of turns to proceed. Defaults to running until the scenario ends.
+     * @param onTurn Optional callback executed at the end of each turn.
+     * @param onStep Optional callback executed after each agent interaction.
+     * @returns The result of the scenario if it ends.
+     */
+    proceed(turns?: number, onTurn?: (state: ScenarioExecutionStateLike) => void | Promise<void>, onStep?: (state: ScenarioExecutionStateLike) => void | Promise<void>): Promise<ScenarioResult | null>;
+    /**
+     * Ends the scenario with a success.
+     * @param reasoning Optional reasoning for the success.
+     * @returns The final result of the scenario.
+     */
+    succeed(reasoning?: string): Promise<ScenarioResult>;
+    /**
+     * Ends the scenario with a failure.
+     * @param reasoning Optional reasoning for the failure.
+     * @returns The final result of the scenario.
+     */
+    fail(reasoning?: string): Promise<ScenarioResult>;
+}
+/**
+ * A step in a scenario script.
+ * This is a function that takes the current state and an executor, and performs an action.
+ */
+type ScriptStep = (state: ScenarioExecutionStateLike, executor: ScenarioExecutionLike) => Promise<void | ScenarioResult | null> | void | ScenarioResult | null;
+
+declare enum AgentRole {
+    USER = "User",
+    AGENT = "Agent",
+    JUDGE = "Judge"
+}
+declare const allAgentRoles: readonly [AgentRole.USER, AgentRole.AGENT, AgentRole.JUDGE];
+/**
+ * Input provided to an agent's `call` method.
+ */
+interface AgentInput {
+    /**
+     * A unique identifier for the conversation thread.
+     */
+    threadId: string;
+    /**
+     * The full history of messages in the conversation.
+     */
+    messages: CoreMessage[];
+    /**
+     * New messages added since the last time this agent was called.
+     */
+    newMessages: CoreMessage[];
+    /**
+     * The role the agent is being asked to play in this turn.
+     */
+    requestedRole: AgentRole;
+    /**
+     * Whether a judgment is being requested in this turn.
+     */
+    judgmentRequest: boolean;
+    /**
+     * The current state of the scenario execution.
+     */
+    scenarioState: ScenarioExecutionStateLike;
+    /**
+     * The configuration for the current scenario.
+     */
+    scenarioConfig: ScenarioConfig;
+}
+/**
+ * The possible return types from an agent's `call` method.
+ * Can be a simple string, a single message, an array of messages, or a ScenarioResult.
+ */
+type AgentReturnTypes = string | CoreMessage | CoreMessage[] | ScenarioResult;
+/**
+ * Abstract base class for integrating custom agents with the Scenario framework.
+ *
+ * This adapter pattern allows you to wrap any existing agent implementation
+ * (LLM calls, agent frameworks, or complex multi-step systems) to work with
+ * the Scenario testing framework. The adapter receives structured input about
+ * the conversation state and returns responses in a standardized format.
+ *
+ * @example
+ * ```typescript
+ * class MyAgent extends AgentAdapter {
+ *   role = AgentRole.AGENT;
+ *
+ *   async call(input: AgentInput): Promise<AgentReturnTypes> {
+ *     const userMessage = input.messages.find(m => m.role === 'user');
+ *     if (userMessage) {
+ *       return `You said: ${userMessage.content}`;
+ *     }
+ *     return "Hello!";
+ *   }
+ * }
+ * ```
+ */
+declare abstract class AgentAdapter {
+    role: AgentRole;
+    constructor(input: AgentInput);
+    /**
+     * Process the input and generate a response.
+     *
+     * This is the main method that your agent implementation must provide.
+     * It receives structured information about the current conversation state
+     * and must return a response in one of the supported formats.
+     *
+     * @param input AgentInput containing conversation history, thread context, and scenario state.
+     * @returns The agent's response.
+     */
+    abstract call(input: AgentInput): Promise<AgentReturnTypes>;
+}
+/**
+ * Abstract base class for user simulator agents.
+ * User simulator agents are responsible for generating user messages to drive the conversation.
+ */
+declare abstract class UserSimulatorAgentAdapter implements AgentAdapter {
+    role: AgentRole;
+    constructor(input: AgentInput);
+    /**
+     * Process the input and generate a user message.
+     *
+     * @param input AgentInput containing conversation history, thread context, and scenario state.
+     * @returns The user's response.
+     */
+    abstract call(input: AgentInput): Promise<AgentReturnTypes>;
+}
+/**
+ * Abstract base class for judge agents.
+ * Judge agents are responsible for evaluating the conversation and determining success or failure.
+ */
+declare abstract class JudgeAgentAdapter implements AgentAdapter {
+    role: AgentRole;
+    /**
+     * The criteria the judge will use to evaluate the conversation.
+     */
+    abstract criteria: string[];
+    constructor(input: AgentInput);
+    /**
+     * Process the input and evaluate the conversation.
+     *
+     * @param input AgentInput containing conversation history, thread context, and scenario state.
+     * @returns A ScenarioResult if the conversation should end, otherwise should continue.
+     */
+    abstract call(input: AgentInput): Promise<AgentReturnTypes>;
+}
+
+/**
+ * Scenario script DSL (Domain Specific Language) module.
+ *
+ * This module provides a collection of functions that form a declarative language
+ * for controlling scenario execution flow. These functions can be used to create
+ * scripts that precisely control how conversations unfold, when evaluations occur,
+ * and when scenarios should succeed or fail.
+ */
+
+/**
+ * Add a specific message to the conversation.
+ *
+ * This function allows you to inject any CoreMessage compatible message directly
+ * into the conversation at a specific point in the script. Useful for
+ * simulating tool responses, system messages, or specific conversational states.
+ *
+ * @param message The message to add to the conversation.
+ * @returns A ScriptStep function that can be used in scenario scripts.
+ */
+declare const message: (message: CoreMessage) => ScriptStep;
+/**
+ * Generate or specify an agent response in the conversation.
+ *
+ * If content is provided, it will be used as the agent response. If no content
+ * is provided, the agent under test will be called to generate its response
+ * based on the current conversation state.
+ *
+ * @param content Optional agent response content. Can be a string or full message object.
+ *                If undefined, the agent under test will generate content automatically.
+ * @returns A ScriptStep function that can be used in scenario scripts.
+ */
+declare const agent: (content?: string | CoreMessage) => ScriptStep;
+/**
+ * Invoke the judge agent to evaluate the current conversation state.
+ *
+ * This function forces the judge agent to make a decision about whether
+ * the scenario should continue or end with a success/failure verdict.
+ * The judge will evaluate based on its configured criteria.
+ *
+ * @param content Optional message content for the judge. Usually undefined to let
+ *                the judge evaluate based on its criteria.
+ * @returns A ScriptStep function that can be used in scenario scripts.
+ */
+declare const judge: (content?: string | CoreMessage) => ScriptStep;
+/**
+ * Generate or specify a user message in the conversation.
+ *
+ * If content is provided, it will be used as the user message. If no content
+ * is provided, the user simulator agent will automatically generate an
+ * appropriate message based on the scenario context.
+ *
+ * @param content Optional user message content. Can be a string or full message object.
+ *                If undefined, the user simulator will generate content automatically.
+ * @returns A ScriptStep function that can be used in scenario scripts.
+ */
+declare const user: (content?: string | CoreMessage) => ScriptStep;
+/**
+ * Let the scenario proceed automatically for a specified number of turns.
+ *
+ * This function allows the scenario to run automatically with the normal
+ * agent interaction flow (user -> agent -> judge evaluation). You can
+ * optionally provide callbacks to execute custom logic at each turn or step.
+ *
+ * @param turns Number of turns to proceed automatically. If undefined, proceeds until
+ *              the judge agent decides to end the scenario or max_turns is reached.
+ * @param onTurn Optional callback function called at the end of each turn.
+ * @param onStep Optional callback function called after each agent interaction.
+ * @returns A ScriptStep function that can be used in scenario scripts.
+ */
+declare const proceed: (turns?: number, onTurn?: (state: ScenarioExecutionStateLike) => void | Promise<void>, onStep?: (state: ScenarioExecutionStateLike) => void | Promise<void>) => ScriptStep;
+/**
+ * End the scenario with a success verdict.
+ *
+ * This function immediately concludes the scenario and marks it as successful.
+ *
+ * @param reasoning Optional explanation for why the scenario succeeded.
+ * @returns A ScriptStep function that can be used in scenario scripts.
+ */
+declare const succeed: (reasoning?: string) => ScriptStep;
+/**
+ * End the scenario with a failure verdict.
+ *
+ * This function immediately concludes the scenario and marks it as failed.
+ *
+ * @param reasoning Optional explanation for why the scenario failed.
+ * @returns A ScriptStep function that can be used in scenario scripts.
+ */
+declare const fail: (reasoning?: string) => ScriptStep;
+
+/**
+ * The type of a scenario event.
+ */
+declare enum ScenarioEventType {
+    /**
+     * A scenario run has started.
+     */
+    RUN_STARTED = "SCENARIO_RUN_STARTED",
+    /**
+     * A scenario run has finished.
+     */
+    RUN_FINISHED = "SCENARIO_RUN_FINISHED",
+    /**
+     * A snapshot of the messages in a scenario.
+     */
+    MESSAGE_SNAPSHOT = "SCENARIO_MESSAGE_SNAPSHOT"
+}
+/**
+ * The status of a scenario run.
+ */
+declare enum ScenarioRunStatus {
+    /**
+     * The scenario completed successfully.
+     */
+    SUCCESS = "SUCCESS",
+    /**
+     * The scenario failed with an error.
+     */
+    ERROR = "ERROR",
+    /**
+     * The scenario was cancelled.
+     */
+    CANCELLED = "CANCELLED",
+    /**
+     * The scenario is in progress.
+     */
+    IN_PROGRESS = "IN_PROGRESS",
+    /**
+     * The scenario is pending execution.
+     */
+    PENDING = "PENDING",
+    /**
+     * The scenario failed.
+     */
+    FAILED = "FAILED"
+}
+declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
+    timestamp: z.ZodOptional<z.ZodNumber>;
+    rawEvent: z.ZodOptional<z.ZodAny>;
+    batchRunId: z.ZodString;
+    scenarioId: z.ZodString;
+    scenarioRunId: z.ZodString;
+} & {
+    type: z.ZodLiteral<ScenarioEventType.RUN_STARTED>;
+    metadata: z.ZodObject<{
+        name: z.ZodString;
+        description: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        name: string;
+        description?: string | undefined;
+    }, {
+        name: string;
+        description?: string | undefined;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    type: ScenarioEventType.RUN_STARTED;
+    batchRunId: string;
+    scenarioId: string;
+    scenarioRunId: string;
+    metadata: {
+        name: string;
+        description?: string | undefined;
+    };
+    timestamp?: number | undefined;
+    rawEvent?: any;
+}, {
+    type: ScenarioEventType.RUN_STARTED;
+    batchRunId: string;
+    scenarioId: string;
+    scenarioRunId: string;
+    metadata: {
+        name: string;
+        description?: string | undefined;
+    };
+    timestamp?: number | undefined;
+    rawEvent?: any;
+}>, z.ZodObject<{
+    timestamp: z.ZodOptional<z.ZodNumber>;
+    rawEvent: z.ZodOptional<z.ZodAny>;
+    batchRunId: z.ZodString;
+    scenarioId: z.ZodString;
+    scenarioRunId: z.ZodString;
+} & {
+    type: z.ZodLiteral<ScenarioEventType.RUN_FINISHED>;
+    status: z.ZodNativeEnum<typeof ScenarioRunStatus>;
+}, "strip", z.ZodTypeAny, {
+    type: ScenarioEventType.RUN_FINISHED;
+    status: ScenarioRunStatus;
+    batchRunId: string;
+    scenarioId: string;
+    scenarioRunId: string;
+    timestamp?: number | undefined;
+    rawEvent?: any;
+}, {
+    type: ScenarioEventType.RUN_FINISHED;
+    status: ScenarioRunStatus;
+    batchRunId: string;
+    scenarioId: string;
+    scenarioRunId: string;
+    timestamp?: number | undefined;
+    rawEvent?: any;
+}>, z.ZodObject<{
+    messages: z.ZodArray<z.ZodDiscriminatedUnion<"role", [z.ZodObject<z.objectUtil.extendShape<{
+        id: z.ZodString;
+        role: z.ZodString;
+        content: z.ZodOptional<z.ZodString>;
+        name: z.ZodOptional<z.ZodString>;
+    }, {
+        role: z.ZodLiteral<"developer">;
+        content: z.ZodString;
+    }>, "strip", z.ZodTypeAny, {
+        id: string;
+        role: "developer";
+        content: string;
+        name?: string | undefined;
+    }, {
+        id: string;
+        role: "developer";
+        content: string;
+        name?: string | undefined;
+    }>, z.ZodObject<z.objectUtil.extendShape<{
+        id: z.ZodString;
+        role: z.ZodString;
+        content: z.ZodOptional<z.ZodString>;
+        name: z.ZodOptional<z.ZodString>;
+    }, {
+        role: z.ZodLiteral<"system">;
+        content: z.ZodString;
+    }>, "strip", z.ZodTypeAny, {
+        id: string;
+        role: "system";
+        content: string;
+        name?: string | undefined;
+    }, {
+        id: string;
+        role: "system";
+        content: string;
+        name?: string | undefined;
+    }>, z.ZodObject<z.objectUtil.extendShape<{
+        id: z.ZodString;
+        role: z.ZodString;
+        content: z.ZodOptional<z.ZodString>;
+        name: z.ZodOptional<z.ZodString>;
+    }, {
+        role: z.ZodLiteral<"assistant">;
+        content: z.ZodOptional<z.ZodString>;
+        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            type: z.ZodLiteral<"function">;
+            function: z.ZodObject<{
+                name: z.ZodString;
+                arguments: z.ZodString;
+            }, "strip", z.ZodTypeAny, {
+                name: string;
+                arguments: string;
+            }, {
+                name: string;
+                arguments: string;
+            }>;
+        }, "strip", z.ZodTypeAny, {
+            function: {
+                name: string;
+                arguments: string;
+            };
+            type: "function";
+            id: string;
+        }, {
+            function: {
+                name: string;
+                arguments: string;
+            };
+            type: "function";
+            id: string;
+        }>, "many">>;
+    }>, "strip", z.ZodTypeAny, {
+        id: string;
+        role: "assistant";
+        name?: string | undefined;
+        content?: string | undefined;
+        toolCalls?: {
+            function: {
+                name: string;
+                arguments: string;
+            };
+            type: "function";
+            id: string;
+        }[] | undefined;
+    }, {
+        id: string;
+        role: "assistant";
+        name?: string | undefined;
+        content?: string | undefined;
+        toolCalls?: {
+            function: {
+                name: string;
+                arguments: string;
+            };
+            type: "function";
+            id: string;
+        }[] | undefined;
+    }>, z.ZodObject<z.objectUtil.extendShape<{
+        id: z.ZodString;
+        role: z.ZodString;
+        content: z.ZodOptional<z.ZodString>;
+        name: z.ZodOptional<z.ZodString>;
+    }, {
+        role: z.ZodLiteral<"user">;
+        content: z.ZodString;
+    }>, "strip", z.ZodTypeAny, {
+        id: string;
+        role: "user";
+        content: string;
+        name?: string | undefined;
+    }, {
+        id: string;
+        role: "user";
+        content: string;
+        name?: string | undefined;
+    }>, z.ZodObject<{
+        id: z.ZodString;
+        content: z.ZodString;
+        role: z.ZodLiteral<"tool">;
+        toolCallId: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        role: "tool";
+        content: string;
+        toolCallId: string;
+    }, {
+        id: string;
+        role: "tool";
+        content: string;
+        toolCallId: string;
+    }>]>, "many">;
+} & {
+    timestamp: z.ZodOptional<z.ZodNumber>;
+    rawEvent: z.ZodOptional<z.ZodAny>;
+    batchRunId: z.ZodString;
+    scenarioId: z.ZodString;
+    scenarioRunId: z.ZodString;
+    type: z.ZodLiteral<ScenarioEventType.MESSAGE_SNAPSHOT>;
+}, "strip", z.ZodTypeAny, {
+    messages: ({
+        id: string;
+        role: "developer";
+        content: string;
+        name?: string | undefined;
+    } | {
+        id: string;
+        role: "system";
+        content: string;
+        name?: string | undefined;
+    } | {
+        id: string;
+        role: "assistant";
+        name?: string | undefined;
+        content?: string | undefined;
+        toolCalls?: {
+            function: {
+                name: string;
+                arguments: string;
+            };
+            type: "function";
+            id: string;
+        }[] | undefined;
+    } | {
+        id: string;
+        role: "user";
+        content: string;
+        name?: string | undefined;
+    } | {
+        id: string;
+        role: "tool";
+        content: string;
+        toolCallId: string;
+    })[];
+    type: ScenarioEventType.MESSAGE_SNAPSHOT;
+    batchRunId: string;
+    scenarioId: string;
+    scenarioRunId: string;
+    timestamp?: number | undefined;
+    rawEvent?: any;
+}, {
+    messages: ({
+        id: string;
+        role: "developer";
+        content: string;
+        name?: string | undefined;
+    } | {
+        id: string;
+        role: "system";
+        content: string;
+        name?: string | undefined;
+    } | {
+        id: string;
+        role: "assistant";
+        name?: string | undefined;
+        content?: string | undefined;
+        toolCalls?: {
+            function: {
+                name: string;
+                arguments: string;
+            };
+            type: "function";
+            id: string;
+        }[] | undefined;
+    } | {
+        id: string;
+        role: "user";
+        content: string;
+        name?: string | undefined;
+    } | {
+        id: string;
+        role: "tool";
+        content: string;
+        toolCallId: string;
+    })[];
+    type: ScenarioEventType.MESSAGE_SNAPSHOT;
+    batchRunId: string;
+    scenarioId: string;
+    scenarioRunId: string;
+    timestamp?: number | undefined;
+    rawEvent?: any;
+}>]>;
+/**
+ * A union of all possible scenario events.
+ */
+type ScenarioEvent = z.infer<typeof scenarioEventSchema>;
+
+/**
+ * Manages the execution of a single scenario.
+ *
+ * This class orchestrates the interaction between agents, executes the script,
+ * and manages the scenario's state. It also emits events that can be subscribed to
+ * for observing the scenario's progress.
+ *
+ * @example
+ * ```typescript
+ * import { scenario, user, agent, succeed, judge } from "@getscenario/scenario";
+ *
+ * const myScenario = scenario(
+ *   {
+ *     name: "My First Scenario",
+ *     description: "A simple test of the agent's greeting.",
+ *     agents: [
+ *       scenario.userSimulatorAgent(),
+ *       scenario.judgeAgent({
+ *         criteria: [
+ *           "Agent should respond with a greeting",
+ *           "Agent should ask for the user's name",
+ *           "Agent should respond with a farewell",
+ *         ],
+ *       }),
+ *     ],
+ *   },
+ *   [
+ *     user("Hello"),
+ *     agent("Hi, how can I help you?"),
+ *     succeed("Agent responded correctly."),
+ *   ]
+ * );
+ *
+ * const execution = new ScenarioExecution(myScenario.config, myScenario.script);
+ *
+ * execution.events$.subscribe(event => {
+ *   console.log("Scenario event:", event);
+ * });
+ *
+ * const result = await execution.execute();
+ * console.log("Scenario result:", result.success);
+ * ```
+ */
+declare class ScenarioExecution implements ScenarioExecutionLike {
+    private state;
+    private eventSubject;
+    private logger;
+    private config;
+    /**
+     * An observable stream of events that occur during the scenario execution.
+     * Subscribe to this to monitor the progress of the scenario in real-time.
+     */
+    readonly events$: Observable<ScenarioEvent>;
+    /**
+     * Creates a new ScenarioExecution instance.
+     * @param config The scenario configuration.
+     * @param script The script steps to execute.
+     */
+    constructor(config: ScenarioConfig, script: ScriptStep[]);
+    /**
+     * The history of messages in the conversation.
+     */
+    get history(): CoreMessage[];
+    /**
+     * The unique identifier for the conversation thread.
+     */
+    get threadId(): string;
+    /**
+     * Executes the entire scenario from start to finish.
+     * This will run through the script and any automatic proceeding logic until a
+     * final result (success, failure, or error) is determined.
+     * @returns A promise that resolves with the final result of the scenario.
+     */
+    execute(): Promise<ScenarioResult>;
+    /**
+     * Executes a single step in the scenario.
+     * A step usually corresponds to a single agent's turn. This method is useful
+     * for manually controlling the scenario's progress.
+     * @returns A promise that resolves with the new messages added during the step, or a final scenario result if the step concludes the scenario.
+     */
+    step(): Promise<CoreMessage[] | ScenarioResult>;
+    private _step;
+    private callAgent;
+    private nextAgentForRole;
+    private reachedMaxTurns;
+    private getJudgeAgent;
+    private consumeUntilRole;
+    private scriptCallAgent;
+    /**
+     * Adds a message to the conversation history.
+     * This is part of the `ScenarioExecutionLike` interface used by script steps.
+     * @param message The message to add.
+     */
+    message(message: CoreMessage): Promise<void>;
+    /**
+     * Executes a user turn.
+     * If content is provided, it's used as the user's message.
+     * If not, the user simulator agent is called to generate a message.
+     * This is part of the `ScenarioExecutionLike` interface used by script steps.
+     * @param content The optional content of the user's message.
+     */
+    user(content?: string | CoreMessage): Promise<void>;
+    /**
+     * Executes an agent turn.
+     * If content is provided, it's used as the agent's message.
+     * If not, the agent under test is called to generate a response.
+     * This is part of the `ScenarioExecutionLike` interface used by script steps.
+     * @param content The optional content of the agent's message.
+     */
+    agent(content?: string | CoreMessage): Promise<void>;
+    /**
+     * Invokes the judge agent to evaluate the current state of the conversation.
+     * This is part of the `ScenarioExecutionLike` interface used by script steps.
+     * @param content Optional message to pass to the judge.
+     * @returns A promise that resolves with the scenario result if the judge makes a final decision, otherwise null.
+     */
+    judge(content?: string | CoreMessage): Promise<ScenarioResult | null>;
+    /**
+     * Lets the scenario proceed automatically for a specified number of turns.
+     * This simulates the natural flow of conversation between agents.
+     * This is part of the `ScenarioExecutionLike` interface used by script steps.
+     * @param turns The number of turns to proceed. If undefined, runs until a conclusion or max turns is reached.
+     * @param onTurn A callback executed at the end of each turn.
+     * @param onStep A callback executed after each agent interaction.
+     * @returns A promise that resolves with the scenario result if a conclusion is reached.
+     */
+    proceed(turns?: number, onTurn?: (state: ScenarioExecutionStateLike) => void | Promise<void>, onStep?: (state: ScenarioExecutionStateLike) => void | Promise<void>): Promise<ScenarioResult | null>;
+    /**
+     * Immediately ends the scenario with a success verdict.
+     * This is part of the `ScenarioExecutionLike` interface used by script steps.
+     * @param reasoning An optional explanation for the success.
+     * @returns A promise that resolves with the final successful scenario result.
+     */
+    succeed(reasoning?: string): Promise<ScenarioResult>;
+    /**
+     * Immediately ends the scenario with a failure verdict.
+     * This is part of the `ScenarioExecutionLike` interface used by script steps.
+     * @param reasoning An optional explanation for the failure.
+     * @returns A promise that resolves with the final failed scenario result.
+     */
+    fail(reasoning?: string): Promise<ScenarioResult>;
+    private reset;
+    /**
+     * Emits an event to the event stream for external consumption.
+     */
+    private emitEvent;
+    /**
+     * Creates base event properties shared across all scenario events.
+     */
+    private makeBaseEvent;
+    /**
+     * Emits a run started event to indicate scenario execution has begun.
+     */
+    private emitRunStarted;
+    /**
+     * Emits a message snapshot event containing current conversation history.
+     */
+    private emitMessageSnapshot;
+    /**
+     * Emits a run finished event with the final execution status.
+     */
+    private emitRunFinished;
+}
+
+/**
+ * Manages the state of a scenario execution.
+ * This class implements the ScenarioExecutionStateLike interface and provides
+ * the internal logic for tracking conversation history, turns, results, and
+ * other related information.
+ */
+declare class ScenarioExecutionState implements ScenarioExecutionStateLike {
+    private _history;
+    private _turn;
+    private _partialResult;
+    private _threadId;
+    private _agents;
+    private _pendingMessages;
+    private _pendingRolesOnTurn;
+    private _pendingAgentsOnTurn;
+    private _agentTimes;
+    private _totalStartTime;
+    /**
+     * Creates a new ScenarioExecutionState.
+     */
+    constructor();
+    setThreadId(threadId: string): void;
+    setAgents(agents: AgentAdapter[]): void;
+    appendMessage(role: CoreMessage["role"], content: string): void;
+    appendUserMessage(content: string): void;
+    appendAssistantMessage(content: string): void;
+    addMessage(message: CoreMessage, fromAgentIdx?: number): void;
+    addMessages(messages: CoreMessage[], fromAgentIdx?: number): void;
+    getPendingMessages(agentIdx: number): CoreMessage[];
+    clearPendingMessages(agentIdx: number): void;
+    newTurn(): void;
+    removePendingRole(role: AgentRole): void;
+    removePendingAgent(agent: AgentAdapter): void;
+    getNextAgentForRole(role: AgentRole): {
+        index: number;
+        agent: AgentAdapter;
+    } | null;
+    addAgentTime(agentIdx: number, time: number): void;
+    hasResult(): boolean;
+    setResult(result: Omit<ScenarioResult, "messages">): void;
+    get lastMessage(): CoreMessage | undefined;
+    get lastUserMessage(): CoreMessage | undefined;
+    get lastAssistantMessage(): CoreMessage | undefined;
+    get lastToolCall(): CoreToolMessage | undefined;
+    getLastToolCallByToolName(toolName: string): CoreToolMessage | undefined;
+    hasToolCall(toolName: string): boolean;
+    get history(): CoreMessage[];
+    get historyWithoutLastMessage(): CoreMessage[];
+    get historyWithoutLastUserMessage(): CoreMessage[];
+    get turn(): number | null;
+    set turn(turn: number);
+    get threadId(): string;
+    get agents(): AgentAdapter[];
+    get pendingRolesOnTurn(): AgentRole[];
+    set pendingRolesOnTurn(roles: AgentRole[]);
+    get pendingAgentsOnTurn(): AgentAdapter[];
+    set pendingAgentsOnTurn(agents: AgentAdapter[]);
+    get partialResult(): Omit<ScenarioResult, "messages"> | null;
+    get totalTime(): number;
+    get agentTimes(): Map<number, number>;
+    removeLastPendingRole(): void;
+}
+
+/**
+ * High-level interface for running a scenario test.
+ *
+ * This is the main entry point for executing scenario tests. It creates a
+ * ScenarioExecution instance and runs it.
+ *
+ * @param cfg Configuration for the scenario test.
+ * @param cfg.name Human-readable name for the scenario.
+ * @param cfg.description Detailed description of what the scenario tests.
+ * @param cfg.agents List of agent adapters (agent under test, user simulator, judge).
+ * @param cfg.maxTurns Maximum conversation turns before timeout (default: 10).
+ * @param cfg.verbose Show detailed output during execution.
+ * @param cfg.script Optional script steps to control scenario flow.
+ * @param cfg.threadId Optional ID for the conversation thread.
+ * @returns A promise that resolves with the ScenarioResult containing the test outcome,
+ *          conversation history, success/failure status, and detailed reasoning.
+ *
+ * @example
+ * ```typescript
+ * import { run, AgentAdapter, AgentRole, user, agent } from '@langwatch/scenario';
+ *
+ * const myAgent: AgentAdapter = {
+ *   role: AgentRole.AGENT,
+ *   async call(input) {
+ *     return `The user said: ${input.messages.at(-1)?.content}`;
+ *   }
+ * };
+ *
+ * async function main() {
+ *   const result = await run({
+ *     name: "Customer Service Test",
+ *     description: "A simple test to see if the agent responds.",
+ *     agents: [myAgent],
+ *     script: [
+ *       user("Hello, world!"),
+ *       agent(),
+ *     ],
+ *   });
+ *
+ *   if (result.success) {
+ *     console.log("Scenario passed!");
+ *   } else {
+ *     console.error(`Scenario failed: ${result.reasoning}`);
+ *   }
+ * }
+ *
+ * main();
+ * ```
+ */
+declare function run(cfg: ScenarioConfig): Promise<ScenarioResult>;
+
+/**
+ * Configuration for the inference parameters of a testing agent.
+ */
+interface TestingAgentInferenceConfig {
+    /**
+     * The language model to use for generating responses.
+     * If not provided, a default model will be used.
+     */
+    model?: LanguageModel;
+    /**
+     * The temperature for the language model.
+     * Defaults to 0.
+     */
+    temperature?: number;
+    /**
+     * The maximum number of tokens to generate.
+     */
+    maxTokens?: number;
+}
+/**
+ * General configuration for a testing agent.
+ */
+interface TestingAgentConfig extends TestingAgentInferenceConfig {
+    /**
+     * The name of the agent.
+     */
+    name?: string;
+}
+/**
+ * The arguments for finishing a test, used by the judge agent's tool.
+ */
+interface FinishTestArgs {
+    /**
+     * A record of the criteria and their results.
+     */
+    criteria: Record<string, "true" | "false" | "inconclusive">;
+    /**
+     * The reasoning behind the verdict.
+     */
+    reasoning: string;
+    /**
+     * The final verdict of the test.
+     */
+    verdict: "success" | "failure" | "inconclusive";
+}
+
+/**
+ * Configuration for the judge agent.
+ */
+interface JudgeAgentConfig extends TestingAgentConfig {
+    /**
+     * A custom system prompt to override the default behavior of the judge.
+     */
+    systemPrompt?: string;
+    /**
+     * The criteria that the judge will use to evaluate the conversation.
+     */
+    criteria: string[];
+}
+/**
+ * Agent that evaluates conversations against success criteria.
+ *
+ * The JudgeAgent watches conversations in real-time and makes decisions about
+ * whether the agent under test is meeting the specified criteria. It can either
+ * allow the conversation to continue or end it with a success/failure verdict.
+ *
+ * The judge uses function calling to make structured decisions and provides
+ * detailed reasoning for its verdicts. It evaluates each criterion independently
+ * and provides comprehensive feedback about what worked and what didn't.
+ *
+ * @param cfg Configuration for the judge agent.
+ * @param cfg.criteria List of success criteria to evaluate against.
+ * @param cfg.model Optional The language model to use for generating responses.
+ * @param cfg.temperature Optional The temperature to use for the model.
+ * @param cfg.maxTokens Optional The maximum number of tokens to generate.
+ * @param cfg.systemPrompt Optional Custom system prompt to override default judge behavior.
+ *
+ * @example
+ * ```typescript
+ * import { run, judgeAgent, AgentRole, user, agent, AgentAdapter } from '@langwatch/scenario';
+ *
+ * const myAgent: AgentAdapter = {
+ *   role: AgentRole.AGENT,
+ *   async call(input) {
+ *     return `The user said: ${input.messages.at(-1)?.content}`;
+ *   }
+ * };
+ *
+ * async function main() {
+ *   const result = await run({
+ *     name: "Judge Agent Test",
+ *     description: "A simple test to see if the judge agent works.",
+ *     agents: [
+ *       myAgent,
+ *       judgeAgent({
+ *         criteria: ["The agent must respond to the user."],
+ *       }),
+ *     ],
+ *     script: [
+ *       user("Hello!"),
+ *       agent(),
+ *     ],
+ *   });
+ * }
+ * main();
+ * ```
+ */
+declare const judgeAgent: (cfg: JudgeAgentConfig) => {
+    role: AgentRole.JUDGE;
+    criteria: string[];
+    call: (input: AgentInput) => Promise<never[] | {
+        success: boolean;
+        messages: CoreMessage[];
+        reasoning: string;
+        passedCriteria: string[];
+        failedCriteria: string[];
+    }>;
+};
+
+/**
+ * Agent that simulates realistic user behavior in scenario conversations.
+ *
+ * This agent generates user messages that are appropriate for the given scenario
+ * context, simulating how a real human user would interact with the agent under test.
+ * It uses an LLM to generate natural, contextually relevant user inputs that help
+ * drive the conversation forward according to the scenario description.
+ *
+ * @param config Optional configuration for the agent.
+ * @param config.model The language model to use for generating responses.
+ * @param config.temperature The temperature to use for the model.
+ * @param config.maxTokens The maximum number of tokens to generate.
+ *
+ * @example
+ * ```typescript
+ * import { run, userSimulatorAgent, AgentRole, user, agent, AgentAdapter } from '@langwatch/scenario';
+ *
+ * const myAgent: AgentAdapter = {
+ *   role: AgentRole.AGENT,
+ *   async call(input) {
+ *     return `The user said: ${input.messages.at(-1)?.content}`;
+ *   }
+ * };
+ *
+ * async function main() {
+ *   const result = await run({
+ *     name: "User Simulator Test",
+ *     description: "A simple test to see if the user simulator works.",
+ *     agents: [myAgent, userSimulatorAgent()],
+ *     script: [
+ *       user(),
+ *       agent(),
+ *     ],
+ *   });
+ * }
+ * main();
+ * ```
+ */
+declare const userSimulatorAgent: (config?: TestingAgentConfig) => {
+    role: AgentRole.USER;
+    call: (input: AgentInput) => Promise<{
+        role: "user";
+        content: string;
+    }>;
+};
+
+export { AgentAdapter, type AgentInput, type AgentReturnTypes, AgentRole, type FinishTestArgs, JudgeAgentAdapter, type ScenarioConfig, type ScenarioConfigFinal, ScenarioExecution, type ScenarioExecutionLike, ScenarioExecutionState, type ScenarioExecutionStateLike, type ScenarioProjectConfig, type ScenarioResult, type ScriptStep, type TestingAgentConfig, type TestingAgentInferenceConfig, UserSimulatorAgentAdapter, agent, allAgentRoles, defineConfig, fail, judge, judgeAgent, message, proceed, run, scenarioProjectConfigSchema, succeed, user, userSimulatorAgent };