@langwatch/scenario 0.2.0 → 0.2.1

package/dist/index.js

@@ -63,7 +63,7 @@ __export(agents_exports, {
 
 // src/agents/judge-agent.ts
 var import_ai = require("ai");
-var import_zod2 = require("zod");
+var import_zod3 = require("zod");
 
 // src/domain/index.ts
 var domain_exports = {};
@@ -218,21 +218,25 @@ var Logger = class _Logger {
   static create(context) {
     return new _Logger(context);
   }
+  getLogLevel() {
+    return env.SCENARIO_LOG_LEVEL ?? "INFO" /* INFO */;
+  }
+  getLogLevelIndex(level) {
+    return Object.values(LogLevel).indexOf(level);
+  }
   /**
    * Checks if logging should occur based on LOG_LEVEL env var
    */
   shouldLog(level) {
-    const logLevel = (process.env.SCENARIO_LOG_LEVEL || "").toLowerCase();
-    const levels = ["error", "warn", "info", "debug"];
-    const currentLevelIndex = levels.indexOf(logLevel);
-    const requestedLevelIndex = levels.indexOf(level);
+    const currentLevelIndex = this.getLogLevelIndex(this.getLogLevel());
+    const requestedLevelIndex = this.getLogLevelIndex(level);
     return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
   }
   formatMessage(message2) {
     return this.context ? `[${this.context}] ${message2}` : message2;
   }
   error(message2, data) {
-    if (this.shouldLog("error")) {
+    if (this.shouldLog("ERROR" /* ERROR */)) {
       const formattedMessage = this.formatMessage(message2);
       if (data) {
         console.error(formattedMessage, data);
@@ -242,7 +246,7 @@ var Logger = class _Logger {
     }
   }
   warn(message2, data) {
-    if (this.shouldLog("warn")) {
+    if (this.shouldLog("WARN" /* WARN */)) {
       const formattedMessage = this.formatMessage(message2);
       if (data) {
         console.warn(formattedMessage, data);
@@ -252,7 +256,7 @@ var Logger = class _Logger {
     }
   }
   info(message2, data) {
-    if (this.shouldLog("info")) {
+    if (this.shouldLog("INFO" /* INFO */)) {
       const formattedMessage = this.formatMessage(message2);
       if (data) {
         console.info(formattedMessage, data);
@@ -262,7 +266,7 @@ var Logger = class _Logger {
     }
   }
   debug(message2, data) {
-    if (this.shouldLog("debug")) {
+    if (this.shouldLog("DEBUG" /* DEBUG */)) {
       const formattedMessage = this.formatMessage(message2);
       if (data) {
         console.log(formattedMessage, data);
@@ -273,6 +277,53 @@ var Logger = class _Logger {
   }
 };
 
+// src/config/env.ts
+var import_zod2 = require("zod");
+
+// src/config/log-levels.ts
+var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
+  LogLevel2["ERROR"] = "ERROR";
+  LogLevel2["WARN"] = "WARN";
+  LogLevel2["INFO"] = "INFO";
+  LogLevel2["DEBUG"] = "DEBUG";
+  return LogLevel2;
+})(LogLevel || {});
+
+// src/config/env.ts
+var envSchema = import_zod2.z.object({
+  /**
+   * LangWatch API key for event reporting.
+   * If not provided, events will not be sent to LangWatch.
+   */
+  LANGWATCH_API_KEY: import_zod2.z.string().optional(),
+  /**
+   * LangWatch endpoint URL for event reporting.
+   * Defaults to the production LangWatch endpoint.
+   */
+  LANGWATCH_ENDPOINT: import_zod2.z.string().url().default("https://app.langwatch.ai"),
+  /**
+   * Disables simulation report info messages when set to any truthy value.
+   * Useful for CI/CD environments or when you want cleaner output.
+   */
+  SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod2.z.string().optional().transform((val) => Boolean(val)),
+  /**
+   * Node environment - affects logging and behavior.
+   * Defaults to 'development' if not specified.
+   */
+  NODE_ENV: import_zod2.z.enum(["development", "production", "test"]).default("development"),
+  /**
+   * Log level for the scenario package.
+   * Defaults to 'info' if not specified.
+   */
+  SCENARIO_LOG_LEVEL: import_zod2.z.nativeEnum(LogLevel).optional(),
+  /**
+   * Scenario batch run ID.
+   * If not provided, a random ID will be generated.
+   */
+  SCENARIO_BATCH_RUN_ID: import_zod2.z.string().optional()
+});
+var env = envSchema.parse(process.env);
+
 // src/config/index.ts
 var logger = new Logger("scenario.config");
 var configLoaded = false;
@@ -352,24 +403,24 @@ ${criteriaList}
 function buildContinueTestTool() {
   return (0, import_ai.tool)({
     description: "Continue the test with the next step",
-    parameters: import_zod2.z.object({})
+    parameters: import_zod3.z.object({})
   });
 }
 function buildFinishTestTool(criteria) {
   const criteriaNames = criteria.map(criterionToParamName);
   return (0, import_ai.tool)({
     description: "Complete the test with a final verdict",
-    parameters: import_zod2.z.object({
-      criteria: import_zod2.z.object(
+    parameters: import_zod3.z.object({
+      criteria: import_zod3.z.object(
         Object.fromEntries(
           criteriaNames.map((name, idx) => [
             name,
-            import_zod2.z.enum(["true", "false", "inconclusive"]).describe(criteria[idx])
+            import_zod3.z.enum(["true", "false", "inconclusive"]).describe(criteria[idx])
           ])
         )
       ).strict().describe("Strict verdict for each criterion"),
-      reasoning: import_zod2.z.string().describe("Explanation of what the final verdict should be"),
-      verdict: import_zod2.z.enum(["success", "failure", "inconclusive"]).describe("The final verdict of the test")
+      reasoning: import_zod3.z.string().describe("Explanation of what the final verdict should be"),
+      verdict: import_zod3.z.enum(["success", "failure", "inconclusive"]).describe("The final verdict of the test")
     })
   });
 }
@@ -522,7 +573,6 @@ var import_rxjs = require("rxjs");
 
 // src/utils/ids.ts
 var import_xksuid = require("xksuid");
-var batchRunId = null;
 function generateThreadId() {
   return `thread_${(0, import_xksuid.generate)()}`;
 }
@@ -533,10 +583,10 @@ function generateScenarioId() {
   return `scenario_${(0, import_xksuid.generate)()}`;
 }
 function getBatchRunId() {
-  if (!batchRunId) {
-    batchRunId = process.env.SCENARIO_BATCH_RUN_ID ?? `scenariobatchrun_${(0, import_xksuid.generate)()}`;
+  if (!env.SCENARIO_BATCH_RUN_ID) {
+    env.SCENARIO_BATCH_RUN_ID = `scenariobatchrun_${(0, import_xksuid.generate)()}`;
   }
-  return batchRunId;
+  return env.SCENARIO_BATCH_RUN_ID;
 }
 function generateMessageId() {
   return `scenariomsg_${(0, import_xksuid.generate)()}`;
@@ -615,7 +665,7 @@ var ScenarioExecutionState = class {
 
 // src/events/schema.ts
 var import_core = require("@ag-ui/core");
-var import_zod3 = require("zod");
+var import_zod4 = require("zod");
 var Verdict = /* @__PURE__ */ ((Verdict2) => {
   Verdict2["SUCCESS"] = "success";
   Verdict2["FAILURE"] = "failure";
@@ -631,62 +681,62 @@ var ScenarioRunStatus = /* @__PURE__ */ ((ScenarioRunStatus2) => {
   ScenarioRunStatus2["FAILED"] = "FAILED";
   return ScenarioRunStatus2;
 })(ScenarioRunStatus || {});
-var baseEventSchema = import_zod3.z.object({
-  type: import_zod3.z.nativeEnum(import_core.EventType),
-  timestamp: import_zod3.z.number(),
-  rawEvent: import_zod3.z.any().optional()
+var baseEventSchema = import_zod4.z.object({
+  type: import_zod4.z.nativeEnum(import_core.EventType),
+  timestamp: import_zod4.z.number(),
+  rawEvent: import_zod4.z.any().optional()
 });
-var batchRunIdSchema = import_zod3.z.string();
-var scenarioRunIdSchema = import_zod3.z.string();
-var scenarioIdSchema = import_zod3.z.string();
+var batchRunIdSchema = import_zod4.z.string();
+var scenarioRunIdSchema = import_zod4.z.string();
+var scenarioIdSchema = import_zod4.z.string();
 var baseScenarioEventSchema = baseEventSchema.extend({
   batchRunId: batchRunIdSchema,
   scenarioId: scenarioIdSchema,
   scenarioRunId: scenarioRunIdSchema,
-  scenarioSetId: import_zod3.z.string().optional().default("default")
+  scenarioSetId: import_zod4.z.string().optional().default("default")
 });
 var scenarioRunStartedSchema = baseScenarioEventSchema.extend({
-  type: import_zod3.z.literal("SCENARIO_RUN_STARTED" /* RUN_STARTED */),
-  metadata: import_zod3.z.object({
-    name: import_zod3.z.string().optional(),
-    description: import_zod3.z.string().optional()
+  type: import_zod4.z.literal("SCENARIO_RUN_STARTED" /* RUN_STARTED */),
+  metadata: import_zod4.z.object({
+    name: import_zod4.z.string().optional(),
+    description: import_zod4.z.string().optional()
   })
 });
-var scenarioResultsSchema = import_zod3.z.object({
-  verdict: import_zod3.z.nativeEnum(Verdict),
-  reasoning: import_zod3.z.string().optional(),
-  metCriteria: import_zod3.z.array(import_zod3.z.string()),
-  unmetCriteria: import_zod3.z.array(import_zod3.z.string()),
-  error: import_zod3.z.string().optional()
+var scenarioResultsSchema = import_zod4.z.object({
+  verdict: import_zod4.z.nativeEnum(Verdict),
+  reasoning: import_zod4.z.string().optional(),
+  metCriteria: import_zod4.z.array(import_zod4.z.string()),
+  unmetCriteria: import_zod4.z.array(import_zod4.z.string()),
+  error: import_zod4.z.string().optional()
 });
 var scenarioRunFinishedSchema = baseScenarioEventSchema.extend({
-  type: import_zod3.z.literal("SCENARIO_RUN_FINISHED" /* RUN_FINISHED */),
-  status: import_zod3.z.nativeEnum(ScenarioRunStatus),
+  type: import_zod4.z.literal("SCENARIO_RUN_FINISHED" /* RUN_FINISHED */),
+  status: import_zod4.z.nativeEnum(ScenarioRunStatus),
   results: scenarioResultsSchema.optional().nullable()
 });
 var scenarioMessageSnapshotSchema = import_core.MessagesSnapshotEventSchema.merge(
   baseScenarioEventSchema.extend({
-    type: import_zod3.z.literal("SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */)
+    type: import_zod4.z.literal("SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */)
   })
 );
-var scenarioEventSchema = import_zod3.z.discriminatedUnion("type", [
+var scenarioEventSchema = import_zod4.z.discriminatedUnion("type", [
   scenarioRunStartedSchema,
   scenarioRunFinishedSchema,
   scenarioMessageSnapshotSchema
 ]);
-var successSchema = import_zod3.z.object({ success: import_zod3.z.boolean() });
-var errorSchema = import_zod3.z.object({ error: import_zod3.z.string() });
-var stateSchema = import_zod3.z.object({
-  state: import_zod3.z.object({
-    messages: import_zod3.z.array(import_zod3.z.any()),
-    status: import_zod3.z.string()
+var successSchema = import_zod4.z.object({ success: import_zod4.z.boolean() });
+var errorSchema = import_zod4.z.object({ error: import_zod4.z.string() });
+var stateSchema = import_zod4.z.object({
+  state: import_zod4.z.object({
+    messages: import_zod4.z.array(import_zod4.z.any()),
+    status: import_zod4.z.string()
   })
 });
-var runsSchema = import_zod3.z.object({ runs: import_zod3.z.array(import_zod3.z.string()) });
-var eventsSchema = import_zod3.z.object({ events: import_zod3.z.array(scenarioEventSchema) });
+var runsSchema = import_zod4.z.object({ runs: import_zod4.z.array(import_zod4.z.string()) });
+var eventsSchema = import_zod4.z.object({ events: import_zod4.z.array(scenarioEventSchema) });
 
 // src/execution/scenario-execution.ts
-var batchRunId2 = getBatchRunId();
+var batchRunId = getBatchRunId();
 var ScenarioExecution = class {
   state;
   eventSubject = new import_rxjs.Subject();
@@ -1098,7 +1148,7 @@ var ScenarioExecution = class {
       type: "placeholder",
       // This will be replaced by the specific event type
       timestamp: Date.now(),
-      batchRunId: batchRunId2,
+      batchRunId,
       scenarioId: this.config.id,
       scenarioRunId,
       scenarioSetId: this.config.setId
@@ -1187,45 +1237,111 @@ __export(runner_exports, {
 // src/events/event-bus.ts
 var import_rxjs2 = require("rxjs");
 
+// src/events/event-alert-message-logger.ts
+var EventAlertMessageLogger = class _EventAlertMessageLogger {
+  static shownBatchIds = /* @__PURE__ */ new Set();
+  /**
+   * Shows a fancy greeting message about simulation reporting status.
+   * Only shows once per batch run to avoid spam.
+   */
+  handleGreeting() {
+    if (this.isGreetingDisabled()) {
+      return;
+    }
+    const batchRunId2 = getBatchRunId();
+    if (_EventAlertMessageLogger.shownBatchIds.has(batchRunId2)) {
+      return;
+    }
+    _EventAlertMessageLogger.shownBatchIds.add(batchRunId2);
+    this.displayGreeting(batchRunId2);
+  }
+  /**
+   * Shows a fancy message about how to watch the simulation.
+   * Called when a run started event is received with a session ID.
+   */
+  handleWatchMessage(params) {
+    if (this.isGreetingDisabled()) {
+      return;
+    }
+    this.displayWatchMessage(params);
+  }
+  isGreetingDisabled() {
+    return env.SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
+  }
+  displayGreeting(batchRunId2) {
+    const separator = "\u2500".repeat(60);
+    if (!env.LANGWATCH_API_KEY) {
+      console.log(`
+${separator}`);
+      console.log("\u{1F680}  LangWatch Simulation Reporting");
+      console.log(`${separator}`);
+      console.log("\u27A1\uFE0F  API key not configured");
+      console.log("   Simulations will only output final results");
+      console.log("");
+      console.log("\u{1F4A1} To visualize conversations in real time:");
+      console.log("   \u2022 Set LANGWATCH_API_KEY environment variable");
+      console.log("   \u2022 Or configure apiKey in scenario.config.js");
+      console.log("");
+      console.log(`\u{1F4E6} Batch Run ID: ${batchRunId2}`);
+      console.log(`${separator}
+`);
+    } else {
+      console.log(`
+${separator}`);
+      console.log("\u{1F680}  LangWatch Simulation Reporting");
+      console.log(`${separator}`);
+      console.log("\u2705 Simulation reporting enabled");
+      console.log(`   Endpoint: ${env.LANGWATCH_ENDPOINT}`);
+      console.log(
+        `   API Key: ${env.LANGWATCH_API_KEY.length > 0 ? "Configured" : "Not configured"}`
+      );
+      console.log("");
+      console.log(`\u{1F4E6} Batch Run ID: ${batchRunId2}`);
+      console.log(`${separator}
+`);
+    }
+  }
+  displayWatchMessage(params) {
+    const separator = "\u2500".repeat(60);
+    const setUrl = params.setUrl;
+    const batchUrl = `${setUrl}/${getBatchRunId()}`;
+    console.log(`
+${separator}`);
+    console.log("\u{1F440} Watch Your Simulation Live");
+    console.log(`${separator}`);
+    console.log("\u{1F310} Open in your browser:");
+    console.log(`   Scenario Set: ${setUrl}`);
+    console.log(`   Batch Run: ${batchUrl}`);
+    console.log("");
+    console.log(`${separator}
+`);
+  }
+};
+
 // src/events/event-reporter.ts
 var EventReporter = class {
-  eventsEndpoint;
   apiKey;
+  eventsEndpoint;
+  eventAlertMessageLogger;
   logger = new Logger("scenario.events.EventReporter");
   constructor(config2) {
-    this.eventsEndpoint = new URL("/api/scenario-events", config2.endpoint);
     this.apiKey = config2.apiKey ?? "";
-    if (!process.env.SCENARIO_DISABLE_SIMULATION_REPORT_INFO) {
-      if (!this.apiKey) {
-        console.log(
-          "\u27A1\uFE0F  LangWatch API key not configured, simulations will only output the final result"
-        );
-        console.log(
-          "To visualize the conversations in real time, configure your LangWatch API key (via LANGWATCH_API_KEY, or scenario.config.js)"
-        );
-      } else {
-        console.log(`simulation reporting is enabled, endpoint:(${this.eventsEndpoint}) api_key_configured:(${this.apiKey.length > 0 ? "true" : "false"})`);
-      }
-    }
+    this.eventsEndpoint = new URL("/api/scenario-events", config2.endpoint);
+    this.eventAlertMessageLogger = new EventAlertMessageLogger();
+    this.eventAlertMessageLogger.handleGreeting();
   }
   /**
    * Posts an event to the configured endpoint.
    * Logs success/failure but doesn't throw - event posting shouldn't break scenario execution.
    */
   async postEvent(event) {
-    this.logger.debug(`[${event.type}] Posting event`, {
-      event
-    });
-    if (!this.eventsEndpoint) {
-      this.logger.warn(
-        "No LANGWATCH_ENDPOINT configured, skipping event posting"
-      );
-      return;
-    }
+    const result = {};
+    this.logger.debug(`[${event.type}] Posting event`, { event });
+    const processedEvent = this.processEventForApi(event);
     try {
       const response = await fetch(this.eventsEndpoint.href, {
         method: "POST",
-        body: JSON.stringify(event),
+        body: JSON.stringify(processedEvent),
         headers: {
           "Content-Type": "application/json",
           "X-Auth-Token": this.apiKey
@@ -1237,22 +1353,40 @@ var EventReporter = class {
       if (response.ok) {
         const data = await response.json();
         this.logger.debug(`[${event.type}] Event POST response:`, data);
+        result.setUrl = data.url;
       } else {
         const errorText = await response.text();
         this.logger.error(`[${event.type}] Event POST failed:`, {
           status: response.status,
           statusText: response.statusText,
           error: errorText,
-          event
+          event: JSON.stringify(processedEvent)
         });
       }
     } catch (error) {
       this.logger.error(`[${event.type}] Event POST error:`, {
         error,
-        event,
-        endpoint: this.eventsEndpoint
+        event: JSON.stringify(processedEvent),
+        endpoint: this.eventsEndpoint.href
       });
     }
+    return result;
+  }
+  /**
+   * Processes event data to ensure API compatibility.
+   * Converts message content objects to strings when needed.
+   */
+  processEventForApi(event) {
+    if (event.type === "SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */) {
+      return {
+        ...event,
+        messages: event.messages.map((message2) => ({
+          ...message2,
+          content: typeof message2.content !== "string" ? JSON.stringify(message2.content) : message2.content
+        }))
+      };
+    }
+    return event;
   }
 };
 
@@ -1261,11 +1395,13 @@ var EventBus = class _EventBus {
   static registry = /* @__PURE__ */ new Set();
   events$ = new import_rxjs2.Subject();
   eventReporter;
+  eventAlertMessageLogger;
   processingPromise = null;
   logger = new Logger("scenario.events.EventBus");
   static globalListeners = [];
   constructor(config2) {
     this.eventReporter = new EventReporter(config2);
+    this.eventAlertMessageLogger = new EventAlertMessageLogger();
     _EventBus.registry.add(this);
     for (const listener of _EventBus.globalListeners) {
       listener(this);
@@ -1297,22 +1433,31 @@ var EventBus = class _EventBus {
     }
     this.processingPromise = new Promise((resolve, reject) => {
       this.events$.pipe(
+        // Post events and get results
         (0, import_rxjs2.concatMap)(async (event) => {
-          this.logger.debug(`[${event.type}] Processing event`, {
-            event
-          });
-          await this.eventReporter.postEvent(event);
-          return event;
+          this.logger.debug(`[${event.type}] Processing event`, { event });
+          const result = await this.eventReporter.postEvent(event);
+          return { event, result };
+        }),
+        // Handle watch messages reactively
+        (0, import_rxjs2.tap)(({ event, result }) => {
+          if (event.type === "SCENARIO_RUN_STARTED" /* RUN_STARTED */ && result.setUrl) {
+            this.eventAlertMessageLogger.handleWatchMessage({
+              scenarioSetId: event.scenarioSetId,
+              scenarioRunId: event.scenarioRunId,
+              setUrl: result.setUrl
+            });
+          }
         }),
+        // Extract just the event for downstream processing
+        (0, import_rxjs2.map)(({ event }) => event),
         (0, import_rxjs2.catchError)((error) => {
           this.logger.error("Error in event stream:", error);
           return import_rxjs2.EMPTY;
         })
       ).subscribe({
         next: (event) => {
-          this.logger.debug(`[${event.type}] Event processed`, {
-            event
-          });
+          this.logger.debug(`[${event.type}] Event processed`, { event });
           if (event.type === "SCENARIO_RUN_FINISHED" /* RUN_FINISHED */) {
             resolve();
           }