@langwatch/scenario 0.2.12 → 0.3.0

package/dist/index.d.mts

@@ -1,7 +1,8 @@
 import * as ai from 'ai';
-import { CoreMessage, CoreUserMessage, CoreAssistantMessage, CoreToolMessage, LanguageModel } from 'ai';
-import { z } from 'zod';
+import { CoreMessage, CoreUserMessage, CoreAssistantMessage, CoreToolMessage, LanguageModel, ModelMessage } from 'ai';
+import { z } from 'zod/v4';
 import { Observable } from 'rxjs';
+import { z as z$1 } from 'zod';
 
 declare enum AgentRole {
     USER = "User",
@@ -352,31 +353,12 @@ interface ScenarioExecutionStateLike {
 declare const DEFAULT_TEMPERATURE = 0;
 declare const scenarioProjectConfigSchema: z.ZodObject<{
     defaultModel: z.ZodOptional<z.ZodObject<{
-        model: z.ZodType<ai.LanguageModelV1, z.ZodTypeDef, ai.LanguageModelV1>;
+        model: z.ZodCustom<LanguageModel, LanguageModel>;
         temperature: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
-    }, "strip", z.ZodTypeAny, {
-        model: ai.LanguageModelV1;
-        temperature: number;
-        maxTokens?: number | undefined;
-    }, {
-        model: ai.LanguageModelV1;
-        temperature?: number | undefined;
-        maxTokens?: number | undefined;
-    }>>;
-}, "strict", z.ZodTypeAny, {
-    defaultModel?: {
-        model: ai.LanguageModelV1;
-        temperature: number;
-        maxTokens?: number | undefined;
-    } | undefined;
-}, {
-    defaultModel?: {
-        model: ai.LanguageModelV1;
-        temperature?: number | undefined;
-        maxTokens?: number | undefined;
-    } | undefined;
-}>;
+    }, z.core.$strip>>;
+    headless: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
+}, z.core.$strict>;
 type ScenarioProjectConfig = z.infer<typeof scenarioProjectConfigSchema>;
 declare function defineConfig(config: ScenarioProjectConfig): ScenarioProjectConfig;
 
@@ -491,7 +473,7 @@ declare class JudgeAgent extends JudgeAgentAdapter {
     constructor(cfg: JudgeAgentConfig);
     call(input: AgentInput): Promise<never[] | {
         success: boolean;
-        messages: CoreMessage[];
+        messages: ai.ModelMessage[];
         reasoning: string;
         metCriteria: string[];
         unmetCriteria: string[];
@@ -687,26 +669,26 @@ declare enum ScenarioRunStatus {
  * Discriminated union of all possible scenario event types.
  * Enables type-safe handling of different event types based on the 'type' field.
  */
-declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
-    timestamp: z.ZodNumber;
-    rawEvent: z.ZodOptional<z.ZodAny>;
-    batchRunId: z.ZodString;
-    scenarioId: z.ZodString;
-    scenarioRunId: z.ZodString;
-    scenarioSetId: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+declare const scenarioEventSchema: z$1.ZodDiscriminatedUnion<"type", [z$1.ZodObject<{
+    timestamp: z$1.ZodNumber;
+    rawEvent: z$1.ZodOptional<z$1.ZodAny>;
+    batchRunId: z$1.ZodString;
+    scenarioId: z$1.ZodString;
+    scenarioRunId: z$1.ZodString;
+    scenarioSetId: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodString>>;
 } & {
-    type: z.ZodLiteral<ScenarioEventType.RUN_STARTED>;
-    metadata: z.ZodObject<{
-        name: z.ZodOptional<z.ZodString>;
-        description: z.ZodOptional<z.ZodString>;
-    }, "strip", z.ZodTypeAny, {
+    type: z$1.ZodLiteral<ScenarioEventType.RUN_STARTED>;
+    metadata: z$1.ZodObject<{
+        name: z$1.ZodOptional<z$1.ZodString>;
+        description: z$1.ZodOptional<z$1.ZodString>;
+    }, "strip", z$1.ZodTypeAny, {
         name?: string | undefined;
         description?: string | undefined;
     }, {
         name?: string | undefined;
         description?: string | undefined;
     }>;
-}, "strip", z.ZodTypeAny, {
+}, "strip", z$1.ZodTypeAny, {
     type: ScenarioEventType.RUN_STARTED;
     timestamp: number;
     batchRunId: string;
@@ -730,36 +712,36 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
     };
     rawEvent?: any;
     scenarioSetId?: string | undefined;
-}>, z.ZodObject<{
-    timestamp: z.ZodNumber;
-    rawEvent: z.ZodOptional<z.ZodAny>;
-    batchRunId: z.ZodString;
-    scenarioId: z.ZodString;
-    scenarioRunId: z.ZodString;
-    scenarioSetId: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+}>, z$1.ZodObject<{
+    timestamp: z$1.ZodNumber;
+    rawEvent: z$1.ZodOptional<z$1.ZodAny>;
+    batchRunId: z$1.ZodString;
+    scenarioId: z$1.ZodString;
+    scenarioRunId: z$1.ZodString;
+    scenarioSetId: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodString>>;
 } & {
-    type: z.ZodLiteral<ScenarioEventType.RUN_FINISHED>;
-    status: z.ZodNativeEnum<typeof ScenarioRunStatus>;
-    results: z.ZodNullable<z.ZodOptional<z.ZodObject<{
-        verdict: z.ZodNativeEnum<typeof Verdict>;
-        reasoning: z.ZodOptional<z.ZodString>;
-        metCriteria: z.ZodArray<z.ZodString, "many">;
-        unmetCriteria: z.ZodArray<z.ZodString, "many">;
-        error: z.ZodOptional<z.ZodString>;
-    }, "strip", z.ZodTypeAny, {
+    type: z$1.ZodLiteral<ScenarioEventType.RUN_FINISHED>;
+    status: z$1.ZodNativeEnum<typeof ScenarioRunStatus>;
+    results: z$1.ZodNullable<z$1.ZodOptional<z$1.ZodObject<{
+        verdict: z$1.ZodNativeEnum<typeof Verdict>;
+        reasoning: z$1.ZodOptional<z$1.ZodString>;
+        metCriteria: z$1.ZodArray<z$1.ZodString, "many">;
+        unmetCriteria: z$1.ZodArray<z$1.ZodString, "many">;
+        error: z$1.ZodOptional<z$1.ZodString>;
+    }, "strip", z$1.ZodTypeAny, {
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     }, {
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     }>>>;
-}, "strip", z.ZodTypeAny, {
+}, "strip", z$1.ZodTypeAny, {
     type: ScenarioEventType.RUN_FINISHED;
     status: ScenarioRunStatus;
     timestamp: number;
@@ -772,8 +754,8 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     } | null | undefined;
 }, {
     type: ScenarioEventType.RUN_FINISHED;
@@ -788,19 +770,19 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     } | null | undefined;
-}>, z.ZodObject<{
-    messages: z.ZodArray<z.ZodDiscriminatedUnion<"role", [z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+}>, z$1.ZodObject<{
+    messages: z$1.ZodArray<z$1.ZodDiscriminatedUnion<"role", [z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"developer">;
-        content: z.ZodString;
-    }>, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"developer">;
+        content: z$1.ZodString;
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "developer";
         content: string;
@@ -810,15 +792,15 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         role: "developer";
         content: string;
         name?: string | undefined;
-    }>, z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+    }>, z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"system">;
-        content: z.ZodString;
-    }>, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"system">;
+        content: z$1.ZodString;
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "system";
         content: string;
@@ -828,28 +810,28 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         role: "system";
         content: string;
         name?: string | undefined;
-    }>, z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+    }>, z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"assistant">;
-        content: z.ZodOptional<z.ZodString>;
-        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
-            id: z.ZodString;
-            type: z.ZodLiteral<"function">;
-            function: z.ZodObject<{
-                name: z.ZodString;
-                arguments: z.ZodString;
-            }, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"assistant">;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        toolCalls: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
+            id: z$1.ZodString;
+            type: z$1.ZodLiteral<"function">;
+            function: z$1.ZodObject<{
+                name: z$1.ZodString;
+                arguments: z$1.ZodString;
+            }, "strip", z$1.ZodTypeAny, {
                 name: string;
                 arguments: string;
             }, {
                 name: string;
                 arguments: string;
             }>;
-        }, "strip", z.ZodTypeAny, {
+        }, "strip", z$1.ZodTypeAny, {
             function: {
                 name: string;
                 arguments: string;
@@ -864,7 +846,7 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
             type: "function";
             id: string;
         }>, "many">>;
-    }>, "strip", z.ZodTypeAny, {
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "assistant";
         name?: string | undefined;
@@ -890,15 +872,15 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
             type: "function";
             id: string;
         }[] | undefined;
-    }>, z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+    }>, z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"user">;
-        content: z.ZodString;
-    }>, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"user">;
+        content: z$1.ZodString;
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "user";
         content: string;
@@ -908,12 +890,12 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         role: "user";
         content: string;
         name?: string | undefined;
-    }>, z.ZodObject<{
-        id: z.ZodString;
-        content: z.ZodString;
-        role: z.ZodLiteral<"tool">;
-        toolCallId: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
+    }>, z$1.ZodObject<{
+        id: z$1.ZodString;
+        content: z$1.ZodString;
+        role: z$1.ZodLiteral<"tool">;
+        toolCallId: z$1.ZodString;
+    }, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "tool";
         content: string;
@@ -925,14 +907,14 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         toolCallId: string;
     }>]>, "many">;
 } & {
-    timestamp: z.ZodNumber;
-    rawEvent: z.ZodOptional<z.ZodAny>;
-    batchRunId: z.ZodString;
-    scenarioId: z.ZodString;
-    scenarioRunId: z.ZodString;
-    scenarioSetId: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-    type: z.ZodLiteral<ScenarioEventType.MESSAGE_SNAPSHOT>;
-}, "strip", z.ZodTypeAny, {
+    timestamp: z$1.ZodNumber;
+    rawEvent: z$1.ZodOptional<z$1.ZodAny>;
+    batchRunId: z$1.ZodString;
+    scenarioId: z$1.ZodString;
+    scenarioRunId: z$1.ZodString;
+    scenarioSetId: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodString>>;
+    type: z$1.ZodLiteral<ScenarioEventType.MESSAGE_SNAPSHOT>;
+}, "strip", z$1.ZodTypeAny, {
     type: ScenarioEventType.MESSAGE_SNAPSHOT;
     messages: ({
         id: string;
@@ -1017,7 +999,7 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
     rawEvent?: any;
     scenarioSetId?: string | undefined;
 }>]>;
-type ScenarioEvent = z.infer<typeof scenarioEventSchema>;
+type ScenarioEvent = z$1.infer<typeof scenarioEventSchema>;
 
 /**
  * Manages the execution of a single scenario test.
@@ -1152,9 +1134,9 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
     /**
      * Gets the complete conversation history as an array of messages.
      *
-     * @returns Array of CoreMessage objects representing the full conversation
+     * @returns Array of ModelMessage objects representing the full conversation
      */
-    get messages(): CoreMessage[];
+    get messages(): ModelMessage[];
     /**
      * Gets the unique identifier for the conversation thread.
      * This ID is used to maintain conversation context across multiple runs.
@@ -1227,7 +1209,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * }
      * ```
      */
-    step(): Promise<CoreMessage[] | ScenarioResult>;
+    step(): Promise<ModelMessage[] | ScenarioResult>;
     private _step;
     /**
      * Calls a specific agent to generate a response or make a decision.
@@ -1267,7 +1249,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * - "assistant" messages are routed to AGENT role agents
      * - Other message types are added directly to the conversation
      *
-     * @param message - The CoreMessage to add to the conversation
+     * @param message - The ModelMessage to add to the conversation
      *
      * @example
      * ```typescript
@@ -1277,7 +1259,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * });
      * ```
      */
-    message(message: CoreMessage): Promise<void>;
+    message(message: ModelMessage): Promise<void>;
     /**
      * Executes a user turn in the conversation.
      *
@@ -1287,7 +1269,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      *
      * This method is part of the ScenarioExecutionLike interface used by script steps.
      *
-     * @param content - Optional content for the user's message. Can be a string or CoreMessage.
+     * @param content - Optional content for the user's message. Can be a string or ModelMessage.
      *                 If not provided, the user simulator agent will generate the content.
      *
      * @example
@@ -1298,14 +1280,14 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * // Let user simulator generate content
      * await execution.user();
      *
-     * // Use a CoreMessage object
+     * // Use a ModelMessage object
      * await execution.user({
      *   role: "user",
      *   content: "Tell me a joke"
      * });
      * ```
      */
-    user(content?: string | CoreMessage): Promise<void>;
+    user(content?: string | ModelMessage): Promise<void>;
     /**
      * Executes an agent turn in the conversation.
      *
@@ -1315,7 +1297,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      *
      * This method is part of the ScenarioExecutionLike interface used by script steps.
      *
-     * @param content - Optional content for the agent's response. Can be a string or CoreMessage.
+     * @param content - Optional content for the agent's response. Can be a string or ModelMessage.
      *                 If not provided, the agent under test will generate the response.
      *
      * @example
@@ -1326,14 +1308,14 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * // Use provided content
      * await execution.agent("The weather is sunny today!");
      *
-     * // Use a CoreMessage object
+     * // Use a ModelMessage object
      * await execution.agent({
      *   role: "assistant",
      *   content: "I'm here to help you with weather information."
      * });
      * ```
      */
-    agent(content?: string | CoreMessage): Promise<void>;
+    agent(content?: string | ModelMessage): Promise<void>;
     /**
      * Invokes the judge agent to evaluate the current state of the conversation.
      *
@@ -1361,7 +1343,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * const result = await execution.judge("Please consider the user's satisfaction level");
      * ```
      */
-    judge(content?: string | CoreMessage): Promise<ScenarioResult | null>;
+    judge(content?: string | ModelMessage): Promise<ScenarioResult | null>;
     /**
      * Lets the scenario proceed automatically for a specified number of turns.
      *