@langwatch/scenario 0.4.0 → 0.4.1

package/dist/index.d.mts

@@ -1,5 +1,5 @@
 import * as ai from 'ai';
-import { CoreMessage, CoreUserMessage, CoreAssistantMessage, CoreToolMessage, LanguageModel, generateText, ModelMessage } from 'ai';
+import { ModelMessage, UserModelMessage, AssistantModelMessage, ToolModelMessage, LanguageModel, generateText } from 'ai';
 import { z } from 'zod/v4';
 import { SpanProcessor, ReadableSpan } from '@opentelemetry/sdk-trace-base';
 import { RealtimeSession } from '@openai/agents/realtime';
@@ -8,11 +8,11 @@ import { z as z$1 } from 'zod';
 
 /**
  * The possible return types from an agent's `call` method.
- * - string | CoreMessage | CoreMessage[]: Agent generated response
+ * - string | ModelMessage | ModelMessage[]: Agent generated response
  * - JudgeResult: Judge made a final decision
  * - null: Judge wants to continue observing (no decision yet)
  */
-type AgentReturnTypes = string | CoreMessage | CoreMessage[] | JudgeResult | null;
+type AgentReturnTypes = string | ModelMessage | ModelMessage[] | JudgeResult | null;
 
 declare enum AgentRole {
     USER = "User",
@@ -31,11 +31,11 @@ interface AgentInput {
     /**
      * The full history of messages in the conversation.
      */
-    messages: CoreMessage[];
+    messages: ModelMessage[];
     /**
      * New messages added since the last time this agent was called.
      */
-    newMessages: CoreMessage[];
+    newMessages: ModelMessage[];
     /**
      * The role the agent is being asked to play in this turn.
      */
@@ -191,7 +191,7 @@ interface ScenarioExecutionLike {
     /**
      * The history of messages in the conversation.
      */
-    readonly messages: CoreMessage[];
+    readonly messages: ModelMessage[];
     /**
      * The ID of the conversation thread.
      */
@@ -200,25 +200,25 @@ interface ScenarioExecutionLike {
      * Adds a message to the conversation.
      * @param message The message to add.
      */
-    message(message: CoreMessage): Promise<void>;
+    message(message: ModelMessage): Promise<void>;
     /**
      * Adds a user message to the conversation.
      * If no content is provided, the user simulator will generate a message.
      * @param content The content of the user message.
      */
-    user(content?: string | CoreMessage): Promise<void>;
+    user(content?: string | ModelMessage): Promise<void>;
     /**
      * Adds an agent message to the conversation.
      * If no content is provided, the agent under test will generate a message.
      * @param content The content of the agent message.
      */
-    agent(content?: string | CoreMessage): Promise<void>;
+    agent(content?: string | ModelMessage): Promise<void>;
     /**
      * Invokes the judge agent to evaluate the current state.
      * @param content Optional message to the judge.
      * @returns The result of the scenario if the judge makes a final decision.
      */
-    judge(content?: string | CoreMessage): Promise<ScenarioResult | null>;
+    judge(content?: string | ModelMessage): Promise<ScenarioResult | null>;
     /**
      * Proceeds with the scenario automatically for a number of turns.
      * @param turns The number of turns to proceed. Defaults to running until the scenario ends.
@@ -258,7 +258,7 @@ interface ScenarioResult {
     /**
      * The sequence of messages exchanged during the scenario.
      */
-    messages: CoreMessage[];
+    messages: ModelMessage[];
     /**
      * The reasoning behind the scenario's outcome.
      */
@@ -299,7 +299,7 @@ interface ScenarioExecutionStateLike {
     /**
      * The sequence of messages exchanged during the scenario.
      */
-    get messages(): CoreMessage[];
+    get messages(): ModelMessage[];
     /**
      * The unique identifier for the execution thread.
      */
@@ -313,28 +313,28 @@ interface ScenarioExecutionStateLike {
      *
      * @param message - The core message to add.
      */
-    addMessage(message: CoreMessage): void;
+    addMessage(message: ModelMessage): void;
     /**
      * Retrieves the last message from the execution state.
      * @returns The last message.
      */
-    lastMessage(): CoreMessage;
+    lastMessage(): ModelMessage;
     /**
      * Retrieves the last user message from the execution state.
      * @returns The last user message.
      */
-    lastUserMessage(): CoreUserMessage;
+    lastUserMessage(): UserModelMessage;
     /**
      * Retrieves the last agent message from the execution state.
      * @returns The last agent message.
      */
-    lastAgentMessage(): CoreAssistantMessage;
+    lastAgentMessage(): AssistantModelMessage;
     /**
      * Retrieves the last tool call message for a specific tool.
      * @param toolName - The name of the tool.
      * @returns The last tool call message.
      */
-    lastToolCall(toolName: string): CoreToolMessage;
+    lastToolCall(toolName: string): ToolModelMessage;
     /**
      * Checks if a tool call for a specific tool exists in the execution state.
      * @param toolName - The name of the tool.
@@ -1847,7 +1847,7 @@ declare class ScenarioExecutionState implements ScenarioExecutionStateLike {
     description: string;
     config: ScenarioConfig;
     constructor(config: ScenarioConfig);
-    get messages(): CoreMessage[];
+    get messages(): ModelMessage[];
     get currentTurn(): number;
     set currentTurn(turn: number);
     get threadId(): string;
@@ -1858,10 +1858,10 @@ declare class ScenarioExecutionState implements ScenarioExecutionStateLike {
      * @param message - The message to add.
      * @param traceId - Optional trace ID to associate with the message.
      */
-    addMessage(message: CoreMessage & {
+    addMessage(message: ModelMessage & {
         traceId?: string;
     }): void;
-    lastMessage(): ai.ModelMessage & {
+    lastMessage(): ModelMessage & {
         id: string;
         traceId?: string;
     };
@@ -1869,10 +1869,10 @@ declare class ScenarioExecutionState implements ScenarioExecutionStateLike {
         id: string;
         traceId?: string;
     };
-    lastAgentMessage(): CoreAssistantMessage & {
+    lastAgentMessage(): AssistantModelMessage & {
         traceId?: string;
     };
-    lastToolCall(toolName: string): CoreToolMessage & {
+    lastToolCall(toolName: string): ToolModelMessage & {
         traceId?: string;
     };
     hasToolCall(toolName: string): boolean;
@@ -1957,14 +1957,14 @@ declare namespace runner {
 /**
  * Add a specific message to the conversation.
  *
- * This function allows you to inject any CoreMessage compatible message directly
+ * This function allows you to inject any ModelMessage compatible message directly
  * into the conversation at a specific point in the script. Useful for
  * simulating tool responses, system messages, or specific conversational states.
  *
  * @param message The message to add to the conversation.
  * @returns A ScriptStep function that can be used in scenario scripts.
  */
-declare const message: (message: CoreMessage) => ScriptStep;
+declare const message: (message: ModelMessage) => ScriptStep;
 /**
  * Generate or specify an agent response in the conversation.
  *
@@ -1976,7 +1976,7 @@ declare const message: (message: CoreMessage) => ScriptStep;
  *                If undefined, the agent under test will generate content automatically.
  * @returns A ScriptStep function that can be used in scenario scripts.
  */
-declare const agent: (content?: string | CoreMessage) => ScriptStep;
+declare const agent: (content?: string | ModelMessage) => ScriptStep;
 /**
  * Invoke the judge agent to evaluate the current conversation state.
  *
@@ -1988,7 +1988,7 @@ declare const agent: (content?: string | CoreMessage) => ScriptStep;
  *                the judge evaluate based on its criteria.
  * @returns A ScriptStep function that can be used in scenario scripts.
  */
-declare const judge: (content?: string | CoreMessage) => ScriptStep;
+declare const judge: (content?: string | ModelMessage) => ScriptStep;
 /**
  * Generate or specify a user message in the conversation.
  *
@@ -2000,7 +2000,7 @@ declare const judge: (content?: string | CoreMessage) => ScriptStep;
  *                If undefined, the user simulator will generate content automatically.
  * @returns A ScriptStep function that can be used in scenario scripts.
  */
-declare const user: (content?: string | CoreMessage) => ScriptStep;
+declare const user: (content?: string | ModelMessage) => ScriptStep;
 /**
  * Let the scenario proceed automatically for a specified number of turns.
  *

package/dist/index.d.ts

@@ -1,5 +1,5 @@
 import * as ai from 'ai';
-import { CoreMessage, CoreUserMessage, CoreAssistantMessage, CoreToolMessage, LanguageModel, generateText, ModelMessage } from 'ai';
+import { ModelMessage, UserModelMessage, AssistantModelMessage, ToolModelMessage, LanguageModel, generateText } from 'ai';
 import { z } from 'zod/v4';
 import { SpanProcessor, ReadableSpan } from '@opentelemetry/sdk-trace-base';
 import { RealtimeSession } from '@openai/agents/realtime';
@@ -8,11 +8,11 @@ import { z as z$1 } from 'zod';
 
 /**
  * The possible return types from an agent's `call` method.
- * - string | CoreMessage | CoreMessage[]: Agent generated response
+ * - string | ModelMessage | ModelMessage[]: Agent generated response
  * - JudgeResult: Judge made a final decision
  * - null: Judge wants to continue observing (no decision yet)
  */
-type AgentReturnTypes = string | CoreMessage | CoreMessage[] | JudgeResult | null;
+type AgentReturnTypes = string | ModelMessage | ModelMessage[] | JudgeResult | null;
 
 declare enum AgentRole {
     USER = "User",
@@ -31,11 +31,11 @@ interface AgentInput {
     /**
      * The full history of messages in the conversation.
      */
-    messages: CoreMessage[];
+    messages: ModelMessage[];
     /**
      * New messages added since the last time this agent was called.
      */
-    newMessages: CoreMessage[];
+    newMessages: ModelMessage[];
     /**
      * The role the agent is being asked to play in this turn.
      */
@@ -191,7 +191,7 @@ interface ScenarioExecutionLike {
     /**
      * The history of messages in the conversation.
      */
-    readonly messages: CoreMessage[];
+    readonly messages: ModelMessage[];
     /**
      * The ID of the conversation thread.
      */
@@ -200,25 +200,25 @@ interface ScenarioExecutionLike {
      * Adds a message to the conversation.
      * @param message The message to add.
      */
-    message(message: CoreMessage): Promise<void>;
+    message(message: ModelMessage): Promise<void>;
     /**
      * Adds a user message to the conversation.
      * If no content is provided, the user simulator will generate a message.
      * @param content The content of the user message.
      */
-    user(content?: string | CoreMessage): Promise<void>;
+    user(content?: string | ModelMessage): Promise<void>;
     /**
      * Adds an agent message to the conversation.
      * If no content is provided, the agent under test will generate a message.
      * @param content The content of the agent message.
      */
-    agent(content?: string | CoreMessage): Promise<void>;
+    agent(content?: string | ModelMessage): Promise<void>;
     /**
      * Invokes the judge agent to evaluate the current state.
      * @param content Optional message to the judge.
      * @returns The result of the scenario if the judge makes a final decision.
      */
-    judge(content?: string | CoreMessage): Promise<ScenarioResult | null>;
+    judge(content?: string | ModelMessage): Promise<ScenarioResult | null>;
     /**
      * Proceeds with the scenario automatically for a number of turns.
      * @param turns The number of turns to proceed. Defaults to running until the scenario ends.
@@ -258,7 +258,7 @@ interface ScenarioResult {
     /**
      * The sequence of messages exchanged during the scenario.
      */
-    messages: CoreMessage[];
+    messages: ModelMessage[];
     /**
      * The reasoning behind the scenario's outcome.
      */
@@ -299,7 +299,7 @@ interface ScenarioExecutionStateLike {
     /**
      * The sequence of messages exchanged during the scenario.
      */
-    get messages(): CoreMessage[];
+    get messages(): ModelMessage[];
     /**
      * The unique identifier for the execution thread.
      */
@@ -313,28 +313,28 @@ interface ScenarioExecutionStateLike {
      *
      * @param message - The core message to add.
      */
-    addMessage(message: CoreMessage): void;
+    addMessage(message: ModelMessage): void;
     /**
      * Retrieves the last message from the execution state.
      * @returns The last message.
      */
-    lastMessage(): CoreMessage;
+    lastMessage(): ModelMessage;
     /**
      * Retrieves the last user message from the execution state.
      * @returns The last user message.
      */
-    lastUserMessage(): CoreUserMessage;
+    lastUserMessage(): UserModelMessage;
     /**
      * Retrieves the last agent message from the execution state.
      * @returns The last agent message.
      */
-    lastAgentMessage(): CoreAssistantMessage;
+    lastAgentMessage(): AssistantModelMessage;
     /**
      * Retrieves the last tool call message for a specific tool.
      * @param toolName - The name of the tool.
      * @returns The last tool call message.
      */
-    lastToolCall(toolName: string): CoreToolMessage;
+    lastToolCall(toolName: string): ToolModelMessage;
     /**
      * Checks if a tool call for a specific tool exists in the execution state.
      * @param toolName - The name of the tool.
@@ -1847,7 +1847,7 @@ declare class ScenarioExecutionState implements ScenarioExecutionStateLike {
     description: string;
     config: ScenarioConfig;
     constructor(config: ScenarioConfig);
-    get messages(): CoreMessage[];
+    get messages(): ModelMessage[];
     get currentTurn(): number;
     set currentTurn(turn: number);
     get threadId(): string;
@@ -1858,10 +1858,10 @@ declare class ScenarioExecutionState implements ScenarioExecutionStateLike {
      * @param message - The message to add.
      * @param traceId - Optional trace ID to associate with the message.
      */
-    addMessage(message: CoreMessage & {
+    addMessage(message: ModelMessage & {
         traceId?: string;
     }): void;
-    lastMessage(): ai.ModelMessage & {
+    lastMessage(): ModelMessage & {
         id: string;
         traceId?: string;
     };
@@ -1869,10 +1869,10 @@ declare class ScenarioExecutionState implements ScenarioExecutionStateLike {
         id: string;
         traceId?: string;
     };
-    lastAgentMessage(): CoreAssistantMessage & {
+    lastAgentMessage(): AssistantModelMessage & {
         traceId?: string;
     };
-    lastToolCall(toolName: string): CoreToolMessage & {
+    lastToolCall(toolName: string): ToolModelMessage & {
         traceId?: string;
     };
     hasToolCall(toolName: string): boolean;
@@ -1957,14 +1957,14 @@ declare namespace runner {
 /**
  * Add a specific message to the conversation.
  *
- * This function allows you to inject any CoreMessage compatible message directly
+ * This function allows you to inject any ModelMessage compatible message directly
  * into the conversation at a specific point in the script. Useful for
  * simulating tool responses, system messages, or specific conversational states.
  *
  * @param message The message to add to the conversation.
  * @returns A ScriptStep function that can be used in scenario scripts.
  */
-declare const message: (message: CoreMessage) => ScriptStep;
+declare const message: (message: ModelMessage) => ScriptStep;
 /**
  * Generate or specify an agent response in the conversation.
  *
@@ -1976,7 +1976,7 @@ declare const message: (message: CoreMessage) => ScriptStep;
  *                If undefined, the agent under test will generate content automatically.
  * @returns A ScriptStep function that can be used in scenario scripts.
  */
-declare const agent: (content?: string | CoreMessage) => ScriptStep;
+declare const agent: (content?: string | ModelMessage) => ScriptStep;
 /**
  * Invoke the judge agent to evaluate the current conversation state.
  *
@@ -1988,7 +1988,7 @@ declare const agent: (content?: string | CoreMessage) => ScriptStep;
  *                the judge evaluate based on its criteria.
  * @returns A ScriptStep function that can be used in scenario scripts.
  */
-declare const judge: (content?: string | CoreMessage) => ScriptStep;
+declare const judge: (content?: string | ModelMessage) => ScriptStep;
 /**
  * Generate or specify a user message in the conversation.
  *
@@ -2000,7 +2000,7 @@ declare const judge: (content?: string | CoreMessage) => ScriptStep;
  *                If undefined, the user simulator will generate content automatically.
  * @returns A ScriptStep function that can be used in scenario scripts.
  */
-declare const user: (content?: string | CoreMessage) => ScriptStep;
+declare const user: (content?: string | ModelMessage) => ScriptStep;
 /**
  * Let the scenario proceed automatically for a specified number of turns.
  *

package/dist/index.js

@@ -189,7 +189,7 @@ var DEFAULT_TEMPERATURE = 0;
 var modelSchema = import_v42.z.object({
   model: import_v42.z.custom((val) => Boolean(val), {
     message: "A model is required. Configure it in scenario.config.js defaultModel or pass directly to the agent."
-  }).describe("The OpenAI Language Model to use for generating responses."),
+  }).describe("Language model that is used by the AI SDK Core functions."),
   temperature: import_v42.z.number().min(0).max(1).optional().describe("The temperature for the language model.").default(DEFAULT_TEMPERATURE),
   maxTokens: import_v42.z.number().optional().describe("The maximum number of tokens to generate.")
 });
@@ -455,7 +455,7 @@ var JudgeUtils = {
   /**
    * Builds a minimal transcript from messages for judge evaluation.
    * Truncates base64 media to reduce token usage.
-   * @param messages - Array of CoreMessage from conversation
+   * @param messages - Array of ModelMessage from conversation
    * @returns Plain text transcript with one message per line
    */
   buildTranscriptFromMessages(messages) {
@@ -2466,13 +2466,15 @@ function convertModelMessagesToAguiMessages(modelMessages) {
       }
       case msg.role === "tool":
         msg.content.map((p, i) => {
-          var _a;
+          if ("type" in p && p.type !== "tool-result") return;
           aguiMessages.push({
             trace_id: msg.traceId,
             id: `${id}-${i}`,
             role: "tool",
             toolCallId: p.toolCallId,
-            content: JSON.stringify((_a = p.output) == null ? void 0 : _a.value)
+            content: JSON.stringify(
+              p.output && "value" in p.output ? p.output.value : p.output
+            )
           });
         });
         break;
@@ -3962,7 +3964,6 @@ function formatPart(part) {
     case "reasoning":
       return `(reasoning): ${part.text}`;
     default:
-      part;
       return `Unknown content: ${JSON.stringify(part)}`;
   }
 }

package/dist/index.mjs

@@ -131,7 +131,7 @@ var DEFAULT_TEMPERATURE = 0;
 var modelSchema = z2.object({
   model: z2.custom((val) => Boolean(val), {
     message: "A model is required. Configure it in scenario.config.js defaultModel or pass directly to the agent."
-  }).describe("The OpenAI Language Model to use for generating responses."),
+  }).describe("Language model that is used by the AI SDK Core functions."),
   temperature: z2.number().min(0).max(1).optional().describe("The temperature for the language model.").default(DEFAULT_TEMPERATURE),
   maxTokens: z2.number().optional().describe("The maximum number of tokens to generate.")
 });
@@ -397,7 +397,7 @@ var JudgeUtils = {
   /**
    * Builds a minimal transcript from messages for judge evaluation.
    * Truncates base64 media to reduce token usage.
-   * @param messages - Array of CoreMessage from conversation
+   * @param messages - Array of ModelMessage from conversation
    * @returns Plain text transcript with one message per line
    */
   buildTranscriptFromMessages(messages) {
@@ -2408,13 +2408,15 @@ function convertModelMessagesToAguiMessages(modelMessages) {
       }
       case msg.role === "tool":
         msg.content.map((p, i) => {
-          var _a;
+          if ("type" in p && p.type !== "tool-result") return;
           aguiMessages.push({
             trace_id: msg.traceId,
             id: `${id}-${i}`,
             role: "tool",
             toolCallId: p.toolCallId,
-            content: JSON.stringify((_a = p.output) == null ? void 0 : _a.value)
+            content: JSON.stringify(
+              p.output && "value" in p.output ? p.output.value : p.output
+            )
           });
         });
         break;
@@ -3911,7 +3913,6 @@ function formatPart(part) {
     case "reasoning":
       return `(reasoning): ${part.text}`;
     default:
-      part;
       return `Unknown content: ${JSON.stringify(part)}`;
   }
 }

package/dist/integrations/vitest/setup.js

@@ -104,7 +104,7 @@ var DEFAULT_TEMPERATURE = 0;
 var modelSchema = import_v42.z.object({
   model: import_v42.z.custom((val) => Boolean(val), {
     message: "A model is required. Configure it in scenario.config.js defaultModel or pass directly to the agent."
-  }).describe("The OpenAI Language Model to use for generating responses."),
+  }).describe("Language model that is used by the AI SDK Core functions."),
   temperature: import_v42.z.number().min(0).max(1).optional().describe("The temperature for the language model.").default(DEFAULT_TEMPERATURE),
   maxTokens: import_v42.z.number().optional().describe("The maximum number of tokens to generate.")
 });

package/dist/integrations/vitest/setup.mjs

@@ -87,7 +87,7 @@ var DEFAULT_TEMPERATURE = 0;
 var modelSchema = z2.object({
   model: z2.custom((val) => Boolean(val), {
     message: "A model is required. Configure it in scenario.config.js defaultModel or pass directly to the agent."
-  }).describe("The OpenAI Language Model to use for generating responses."),
+  }).describe("Language model that is used by the AI SDK Core functions."),
   temperature: z2.number().min(0).max(1).optional().describe("The temperature for the language model.").default(DEFAULT_TEMPERATURE),
   maxTokens: z2.number().optional().describe("The maximum number of tokens to generate.")
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langwatch/scenario",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "A TypeScript library for testing AI agents using scenarios",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -31,7 +31,7 @@
     "@ag-ui/core": "^0.0.28",
     "@ai-sdk/openai": "^2.0.74",
     "@openai/agents": "^0.3.3",
-    "ai": "5.0.104",
+    "ai": "^6.0.0",
     "chalk": "^5.6.2",
     "langwatch": "0.9.0",
     "open": "11.0.0",
@@ -88,7 +88,7 @@
     }
   },
   "peerDependencies": {
-    "ai": ">=5.0.0",
+    "ai": ">=6.0.0",
     "vitest": ">=3.2.4"
   },
   "scripts": {