@langwatch/scenario 0.2.13 → 0.3.0

package/README.md

@@ -78,7 +78,7 @@ import { describe, it, expect } from "vitest";
 import { openai } from "@ai-sdk/openai";
 import scenario, { type AgentAdapter, AgentRole } from "@langwatch/scenario";
 import { generateText, tool } from "ai";
-import { z } from "zod";
+import { z } from "zod/v4";
 
 describe("Weather Agent", () => {
   it("should get the weather for a city", async () => {
@@ -103,13 +103,40 @@ describe("Weather Agent", () => {
           tools: { get_current_weather: getCurrentWeather },
         });
 
-        if (response.toolCalls?.length) {
-          // For simplicity, we'll just return the arguments of the first tool call
-          const { toolName, args } = response.toolCalls[0];
-          return {
-            role: "tool",
-            content: [{ type: "tool-result", toolName, result: args }],
-          };
+        if (response.toolCalls && response.toolCalls.length > 0) {
+          const toolCall = response.toolCalls[0];
+          // Agent executes the tool directly and returns both messages
+          const toolResult = await getCurrentWeather.execute(
+            toolCall.input as { city: string },
+            {
+              toolCallId: toolCall.toolCallId,
+              messages: input.messages,
+            }
+          );
+          return [
+            {
+              role: "assistant",
+              content: [
+                {
+                  type: "tool-call",
+                  toolName: toolCall.toolName,
+                  toolCallId: toolCall.toolCallId,
+                  input: toolCall.input,
+                },
+              ],
+            },
+            {
+              role: "tool",
+              content: [
+                {
+                  type: "tool-result",
+                  toolName: toolCall.toolName,
+                  toolCallId: toolCall.toolCallId,
+                  output: { type: "text", value: toolResult as string },
+                },
+              ],
+            },
+          ];
         }
 
         return response.text;

package/dist/{chunk-6SKQWXT7.mjs → chunk-3Z7E24UI.mjs}

@@ -1,13 +1,13 @@
 import {
   Logger,
   getEnv
-} from "./chunk-OL4RFXV4.mjs";
+} from "./chunk-RHTLQKEJ.mjs";
 import {
   __export
 } from "./chunk-7P6ASYW6.mjs";
 
 // src/domain/core/config.ts
-import { z } from "zod";
+import { z } from "zod/v4";
 var DEFAULT_TEMPERATURE = 0;
 var scenarioProjectConfigSchema = z.object({
   defaultModel: z.object({
@@ -75,6 +75,9 @@ import {
 } from "rxjs";
 
 // src/events/event-alert-message-logger.ts
+import * as fs2 from "fs";
+import * as os from "os";
+import * as path2 from "path";
 import open from "open";
 
 // src/config/load.ts
@@ -184,8 +187,23 @@ function generateMessageId() {
 }
 
 // src/events/event-alert-message-logger.ts
-var EventAlertMessageLogger = class _EventAlertMessageLogger {
-  static shownBatchIds = /* @__PURE__ */ new Set();
+var EventAlertMessageLogger = class {
+  /**
+   * Creates a coordination file to prevent duplicate messages across processes.
+   * Returns true if this process should show the message (first one to create the file).
+   */
+  createCoordinationFile(type) {
+    try {
+      const batchId = getBatchRunId();
+      const tmpDir = os.tmpdir();
+      const fileName = `scenario-${type}-${batchId}`;
+      const filePath = path2.join(tmpDir, fileName);
+      fs2.writeFileSync(filePath, process.pid.toString(), { flag: "wx" });
+      return true;
+    } catch {
+      return false;
+    }
+  }
   /**
    * Shows a fancy greeting message about simulation reporting status.
    * Only shows once per batch run to avoid spam.
@@ -194,10 +212,9 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
     if (this.isGreetingDisabled()) {
       return;
     }
-    if (_EventAlertMessageLogger.shownBatchIds.has(getBatchRunId())) {
+    if (!this.createCoordinationFile("greeting")) {
       return;
     }
-    _EventAlertMessageLogger.shownBatchIds.add(getBatchRunId());
     this.displayGreeting();
   }
   /**
@@ -208,6 +225,9 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
     if (this.isGreetingDisabled()) {
       return;
     }
+    if (!this.createCoordinationFile(`watch-${params.scenarioSetId}`)) {
+      return;
+    }
     await this.displayWatchMessage(params);
   }
   isGreetingDisabled() {

package/dist/{chunk-OL4RFXV4.mjs → chunk-RHTLQKEJ.mjs}

@@ -1,5 +1,5 @@
 // src/config/env.ts
-import { z } from "zod";
+import { z } from "zod/v4";
 
 // src/config/log-levels.ts
 var LogLevel = /* @__PURE__ */ ((LogLevel2) => {

package/dist/index.d.mts

@@ -1,7 +1,8 @@
 import * as ai from 'ai';
-import { CoreMessage, CoreUserMessage, CoreAssistantMessage, CoreToolMessage, LanguageModel } from 'ai';
-import { z } from 'zod';
+import { CoreMessage, CoreUserMessage, CoreAssistantMessage, CoreToolMessage, LanguageModel, ModelMessage } from 'ai';
+import { z } from 'zod/v4';
 import { Observable } from 'rxjs';
+import { z as z$1 } from 'zod';
 
 declare enum AgentRole {
     USER = "User",
@@ -352,34 +353,12 @@ interface ScenarioExecutionStateLike {
 declare const DEFAULT_TEMPERATURE = 0;
 declare const scenarioProjectConfigSchema: z.ZodObject<{
     defaultModel: z.ZodOptional<z.ZodObject<{
-        model: z.ZodType<ai.LanguageModelV1, z.ZodTypeDef, ai.LanguageModelV1>;
+        model: z.ZodCustom<LanguageModel, LanguageModel>;
         temperature: z.ZodDefault<z.ZodOptional<z.ZodNumber>>;
         maxTokens: z.ZodOptional<z.ZodNumber>;
-    }, "strip", z.ZodTypeAny, {
-        model: ai.LanguageModelV1;
-        temperature: number;
-        maxTokens?: number | undefined;
-    }, {
-        model: ai.LanguageModelV1;
-        temperature?: number | undefined;
-        maxTokens?: number | undefined;
-    }>>;
+    }, z.core.$strip>>;
     headless: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
-}, "strict", z.ZodTypeAny, {
-    headless: boolean;
-    defaultModel?: {
-        model: ai.LanguageModelV1;
-        temperature: number;
-        maxTokens?: number | undefined;
-    } | undefined;
-}, {
-    defaultModel?: {
-        model: ai.LanguageModelV1;
-        temperature?: number | undefined;
-        maxTokens?: number | undefined;
-    } | undefined;
-    headless?: boolean | undefined;
-}>;
+}, z.core.$strict>;
 type ScenarioProjectConfig = z.infer<typeof scenarioProjectConfigSchema>;
 declare function defineConfig(config: ScenarioProjectConfig): ScenarioProjectConfig;
 
@@ -494,7 +473,7 @@ declare class JudgeAgent extends JudgeAgentAdapter {
     constructor(cfg: JudgeAgentConfig);
     call(input: AgentInput): Promise<never[] | {
         success: boolean;
-        messages: CoreMessage[];
+        messages: ai.ModelMessage[];
         reasoning: string;
         metCriteria: string[];
         unmetCriteria: string[];
@@ -690,26 +669,26 @@ declare enum ScenarioRunStatus {
  * Discriminated union of all possible scenario event types.
  * Enables type-safe handling of different event types based on the 'type' field.
  */
-declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
-    timestamp: z.ZodNumber;
-    rawEvent: z.ZodOptional<z.ZodAny>;
-    batchRunId: z.ZodString;
-    scenarioId: z.ZodString;
-    scenarioRunId: z.ZodString;
-    scenarioSetId: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+declare const scenarioEventSchema: z$1.ZodDiscriminatedUnion<"type", [z$1.ZodObject<{
+    timestamp: z$1.ZodNumber;
+    rawEvent: z$1.ZodOptional<z$1.ZodAny>;
+    batchRunId: z$1.ZodString;
+    scenarioId: z$1.ZodString;
+    scenarioRunId: z$1.ZodString;
+    scenarioSetId: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodString>>;
 } & {
-    type: z.ZodLiteral<ScenarioEventType.RUN_STARTED>;
-    metadata: z.ZodObject<{
-        name: z.ZodOptional<z.ZodString>;
-        description: z.ZodOptional<z.ZodString>;
-    }, "strip", z.ZodTypeAny, {
+    type: z$1.ZodLiteral<ScenarioEventType.RUN_STARTED>;
+    metadata: z$1.ZodObject<{
+        name: z$1.ZodOptional<z$1.ZodString>;
+        description: z$1.ZodOptional<z$1.ZodString>;
+    }, "strip", z$1.ZodTypeAny, {
         name?: string | undefined;
         description?: string | undefined;
     }, {
         name?: string | undefined;
         description?: string | undefined;
     }>;
-}, "strip", z.ZodTypeAny, {
+}, "strip", z$1.ZodTypeAny, {
     type: ScenarioEventType.RUN_STARTED;
     timestamp: number;
     batchRunId: string;
@@ -733,36 +712,36 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
     };
     rawEvent?: any;
     scenarioSetId?: string | undefined;
-}>, z.ZodObject<{
-    timestamp: z.ZodNumber;
-    rawEvent: z.ZodOptional<z.ZodAny>;
-    batchRunId: z.ZodString;
-    scenarioId: z.ZodString;
-    scenarioRunId: z.ZodString;
-    scenarioSetId: z.ZodDefault<z.ZodOptional<z.ZodString>>;
+}>, z$1.ZodObject<{
+    timestamp: z$1.ZodNumber;
+    rawEvent: z$1.ZodOptional<z$1.ZodAny>;
+    batchRunId: z$1.ZodString;
+    scenarioId: z$1.ZodString;
+    scenarioRunId: z$1.ZodString;
+    scenarioSetId: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodString>>;
 } & {
-    type: z.ZodLiteral<ScenarioEventType.RUN_FINISHED>;
-    status: z.ZodNativeEnum<typeof ScenarioRunStatus>;
-    results: z.ZodNullable<z.ZodOptional<z.ZodObject<{
-        verdict: z.ZodNativeEnum<typeof Verdict>;
-        reasoning: z.ZodOptional<z.ZodString>;
-        metCriteria: z.ZodArray<z.ZodString, "many">;
-        unmetCriteria: z.ZodArray<z.ZodString, "many">;
-        error: z.ZodOptional<z.ZodString>;
-    }, "strip", z.ZodTypeAny, {
+    type: z$1.ZodLiteral<ScenarioEventType.RUN_FINISHED>;
+    status: z$1.ZodNativeEnum<typeof ScenarioRunStatus>;
+    results: z$1.ZodNullable<z$1.ZodOptional<z$1.ZodObject<{
+        verdict: z$1.ZodNativeEnum<typeof Verdict>;
+        reasoning: z$1.ZodOptional<z$1.ZodString>;
+        metCriteria: z$1.ZodArray<z$1.ZodString, "many">;
+        unmetCriteria: z$1.ZodArray<z$1.ZodString, "many">;
+        error: z$1.ZodOptional<z$1.ZodString>;
+    }, "strip", z$1.ZodTypeAny, {
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     }, {
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     }>>>;
-}, "strip", z.ZodTypeAny, {
+}, "strip", z$1.ZodTypeAny, {
     type: ScenarioEventType.RUN_FINISHED;
     status: ScenarioRunStatus;
     timestamp: number;
@@ -775,8 +754,8 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     } | null | undefined;
 }, {
     type: ScenarioEventType.RUN_FINISHED;
@@ -791,19 +770,19 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         verdict: Verdict;
         metCriteria: string[];
         unmetCriteria: string[];
-        reasoning?: string | undefined;
         error?: string | undefined;
+        reasoning?: string | undefined;
     } | null | undefined;
-}>, z.ZodObject<{
-    messages: z.ZodArray<z.ZodDiscriminatedUnion<"role", [z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+}>, z$1.ZodObject<{
+    messages: z$1.ZodArray<z$1.ZodDiscriminatedUnion<"role", [z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"developer">;
-        content: z.ZodString;
-    }>, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"developer">;
+        content: z$1.ZodString;
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "developer";
         content: string;
@@ -813,15 +792,15 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         role: "developer";
         content: string;
         name?: string | undefined;
-    }>, z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+    }>, z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"system">;
-        content: z.ZodString;
-    }>, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"system">;
+        content: z$1.ZodString;
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "system";
         content: string;
@@ -831,28 +810,28 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         role: "system";
         content: string;
         name?: string | undefined;
-    }>, z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+    }>, z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"assistant">;
-        content: z.ZodOptional<z.ZodString>;
-        toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
-            id: z.ZodString;
-            type: z.ZodLiteral<"function">;
-            function: z.ZodObject<{
-                name: z.ZodString;
-                arguments: z.ZodString;
-            }, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"assistant">;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        toolCalls: z$1.ZodOptional<z$1.ZodArray<z$1.ZodObject<{
+            id: z$1.ZodString;
+            type: z$1.ZodLiteral<"function">;
+            function: z$1.ZodObject<{
+                name: z$1.ZodString;
+                arguments: z$1.ZodString;
+            }, "strip", z$1.ZodTypeAny, {
                 name: string;
                 arguments: string;
             }, {
                 name: string;
                 arguments: string;
             }>;
-        }, "strip", z.ZodTypeAny, {
+        }, "strip", z$1.ZodTypeAny, {
             function: {
                 name: string;
                 arguments: string;
@@ -867,7 +846,7 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
             type: "function";
             id: string;
         }>, "many">>;
-    }>, "strip", z.ZodTypeAny, {
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "assistant";
         name?: string | undefined;
@@ -893,15 +872,15 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
             type: "function";
             id: string;
         }[] | undefined;
-    }>, z.ZodObject<z.objectUtil.extendShape<{
-        id: z.ZodString;
-        role: z.ZodString;
-        content: z.ZodOptional<z.ZodString>;
-        name: z.ZodOptional<z.ZodString>;
+    }>, z$1.ZodObject<z$1.objectUtil.extendShape<{
+        id: z$1.ZodString;
+        role: z$1.ZodString;
+        content: z$1.ZodOptional<z$1.ZodString>;
+        name: z$1.ZodOptional<z$1.ZodString>;
     }, {
-        role: z.ZodLiteral<"user">;
-        content: z.ZodString;
-    }>, "strip", z.ZodTypeAny, {
+        role: z$1.ZodLiteral<"user">;
+        content: z$1.ZodString;
+    }>, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "user";
         content: string;
@@ -911,12 +890,12 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         role: "user";
         content: string;
         name?: string | undefined;
-    }>, z.ZodObject<{
-        id: z.ZodString;
-        content: z.ZodString;
-        role: z.ZodLiteral<"tool">;
-        toolCallId: z.ZodString;
-    }, "strip", z.ZodTypeAny, {
+    }>, z$1.ZodObject<{
+        id: z$1.ZodString;
+        content: z$1.ZodString;
+        role: z$1.ZodLiteral<"tool">;
+        toolCallId: z$1.ZodString;
+    }, "strip", z$1.ZodTypeAny, {
         id: string;
         role: "tool";
         content: string;
@@ -928,14 +907,14 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
         toolCallId: string;
     }>]>, "many">;
 } & {
-    timestamp: z.ZodNumber;
-    rawEvent: z.ZodOptional<z.ZodAny>;
-    batchRunId: z.ZodString;
-    scenarioId: z.ZodString;
-    scenarioRunId: z.ZodString;
-    scenarioSetId: z.ZodDefault<z.ZodOptional<z.ZodString>>;
-    type: z.ZodLiteral<ScenarioEventType.MESSAGE_SNAPSHOT>;
-}, "strip", z.ZodTypeAny, {
+    timestamp: z$1.ZodNumber;
+    rawEvent: z$1.ZodOptional<z$1.ZodAny>;
+    batchRunId: z$1.ZodString;
+    scenarioId: z$1.ZodString;
+    scenarioRunId: z$1.ZodString;
+    scenarioSetId: z$1.ZodDefault<z$1.ZodOptional<z$1.ZodString>>;
+    type: z$1.ZodLiteral<ScenarioEventType.MESSAGE_SNAPSHOT>;
+}, "strip", z$1.ZodTypeAny, {
     type: ScenarioEventType.MESSAGE_SNAPSHOT;
     messages: ({
         id: string;
@@ -1020,7 +999,7 @@ declare const scenarioEventSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<
     rawEvent?: any;
     scenarioSetId?: string | undefined;
 }>]>;
-type ScenarioEvent = z.infer<typeof scenarioEventSchema>;
+type ScenarioEvent = z$1.infer<typeof scenarioEventSchema>;
 
 /**
  * Manages the execution of a single scenario test.
@@ -1155,9 +1134,9 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
     /**
      * Gets the complete conversation history as an array of messages.
      *
-     * @returns Array of CoreMessage objects representing the full conversation
+     * @returns Array of ModelMessage objects representing the full conversation
      */
-    get messages(): CoreMessage[];
+    get messages(): ModelMessage[];
     /**
      * Gets the unique identifier for the conversation thread.
      * This ID is used to maintain conversation context across multiple runs.
@@ -1230,7 +1209,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * }
      * ```
      */
-    step(): Promise<CoreMessage[] | ScenarioResult>;
+    step(): Promise<ModelMessage[] | ScenarioResult>;
     private _step;
     /**
      * Calls a specific agent to generate a response or make a decision.
@@ -1270,7 +1249,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * - "assistant" messages are routed to AGENT role agents
      * - Other message types are added directly to the conversation
      *
-     * @param message - The CoreMessage to add to the conversation
+     * @param message - The ModelMessage to add to the conversation
      *
      * @example
      * ```typescript
@@ -1280,7 +1259,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * });
      * ```
      */
-    message(message: CoreMessage): Promise<void>;
+    message(message: ModelMessage): Promise<void>;
     /**
      * Executes a user turn in the conversation.
      *
@@ -1290,7 +1269,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      *
      * This method is part of the ScenarioExecutionLike interface used by script steps.
      *
-     * @param content - Optional content for the user's message. Can be a string or CoreMessage.
+     * @param content - Optional content for the user's message. Can be a string or ModelMessage.
      *                 If not provided, the user simulator agent will generate the content.
      *
      * @example
@@ -1301,14 +1280,14 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * // Let user simulator generate content
      * await execution.user();
      *
-     * // Use a CoreMessage object
+     * // Use a ModelMessage object
      * await execution.user({
      *   role: "user",
      *   content: "Tell me a joke"
      * });
      * ```
      */
-    user(content?: string | CoreMessage): Promise<void>;
+    user(content?: string | ModelMessage): Promise<void>;
     /**
      * Executes an agent turn in the conversation.
      *
@@ -1318,7 +1297,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      *
      * This method is part of the ScenarioExecutionLike interface used by script steps.
      *
-     * @param content - Optional content for the agent's response. Can be a string or CoreMessage.
+     * @param content - Optional content for the agent's response. Can be a string or ModelMessage.
      *                 If not provided, the agent under test will generate the response.
      *
      * @example
@@ -1329,14 +1308,14 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * // Use provided content
      * await execution.agent("The weather is sunny today!");
      *
-     * // Use a CoreMessage object
+     * // Use a ModelMessage object
      * await execution.agent({
      *   role: "assistant",
      *   content: "I'm here to help you with weather information."
      * });
      * ```
      */
-    agent(content?: string | CoreMessage): Promise<void>;
+    agent(content?: string | ModelMessage): Promise<void>;
     /**
      * Invokes the judge agent to evaluate the current state of the conversation.
      *
@@ -1364,7 +1343,7 @@ declare class ScenarioExecution implements ScenarioExecutionLike {
      * const result = await execution.judge("Please consider the user's satisfaction level");
      * ```
      */
-    judge(content?: string | CoreMessage): Promise<ScenarioResult | null>;
+    judge(content?: string | ModelMessage): Promise<ScenarioResult | null>;
     /**
      * Lets the scenario proceed automatically for a specified number of turns.
      *