@langwatch/scenario 0.2.0 → 0.2.2

package/dist/integrations/vitest/setup.js

@@ -30,6 +30,22 @@ var import_vitest = require("vitest");
 // src/events/event-bus.ts
 var import_rxjs = require("rxjs");
 
+// src/config/load.ts
+var import_promises = __toESM(require("fs/promises"));
+var import_node_path = __toESM(require("path"));
+var import_node_url = require("url");
+
+// src/domain/core/config.ts
+var import_zod = require("zod");
+var DEFAULT_TEMPERATURE = 0;
+var scenarioProjectConfigSchema = import_zod.z.object({
+  defaultModel: import_zod.z.object({
+    model: import_zod.z.custom(),
+    temperature: import_zod.z.number().min(0).max(1).optional().default(DEFAULT_TEMPERATURE),
+    maxTokens: import_zod.z.number().optional()
+  }).optional()
+}).strict();
+
 // src/utils/logger.ts
 var Logger = class _Logger {
   constructor(context) {
@@ -41,21 +57,25 @@ var Logger = class _Logger {
   static create(context) {
     return new _Logger(context);
   }
+  getLogLevel() {
+    return env.SCENARIO_LOG_LEVEL ?? "INFO" /* INFO */;
+  }
+  getLogLevelIndex(level) {
+    return Object.values(LogLevel).indexOf(level);
+  }
   /**
    * Checks if logging should occur based on LOG_LEVEL env var
    */
   shouldLog(level) {
-    const logLevel = (process.env.SCENARIO_LOG_LEVEL || "").toLowerCase();
-    const levels = ["error", "warn", "info", "debug"];
-    const currentLevelIndex = levels.indexOf(logLevel);
-    const requestedLevelIndex = levels.indexOf(level);
+    const currentLevelIndex = this.getLogLevelIndex(this.getLogLevel());
+    const requestedLevelIndex = this.getLogLevelIndex(level);
     return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
   }
   formatMessage(message) {
     return this.context ? `[${this.context}] ${message}` : message;
   }
   error(message, data) {
-    if (this.shouldLog("error")) {
+    if (this.shouldLog("ERROR" /* ERROR */)) {
       const formattedMessage = this.formatMessage(message);
       if (data) {
         console.error(formattedMessage, data);
@@ -65,7 +85,7 @@ var Logger = class _Logger {
     }
   }
   warn(message, data) {
-    if (this.shouldLog("warn")) {
+    if (this.shouldLog("WARN" /* WARN */)) {
       const formattedMessage = this.formatMessage(message);
       if (data) {
         console.warn(formattedMessage, data);
@@ -75,7 +95,7 @@ var Logger = class _Logger {
     }
   }
   info(message, data) {
-    if (this.shouldLog("info")) {
+    if (this.shouldLog("INFO" /* INFO */)) {
       const formattedMessage = this.formatMessage(message);
       if (data) {
         console.info(formattedMessage, data);
@@ -85,7 +105,7 @@ var Logger = class _Logger {
     }
   }
   debug(message, data) {
-    if (this.shouldLog("debug")) {
+    if (this.shouldLog("DEBUG" /* DEBUG */)) {
       const formattedMessage = this.formatMessage(message);
       if (data) {
         console.log(formattedMessage, data);
@@ -96,78 +116,149 @@ var Logger = class _Logger {
   }
 };
 
-// src/events/event-reporter.ts
-var EventReporter = class {
-  eventsEndpoint;
-  apiKey;
-  logger = new Logger("scenario.events.EventReporter");
-  constructor(config) {
-    this.eventsEndpoint = new URL("/api/scenario-events", config.endpoint);
-    this.apiKey = config.apiKey ?? "";
-    if (!process.env.SCENARIO_DISABLE_SIMULATION_REPORT_INFO) {
-      if (!this.apiKey) {
-        console.log(
-          "\u27A1\uFE0F  LangWatch API key not configured, simulations will only output the final result"
-        );
-        console.log(
-          "To visualize the conversations in real time, configure your LangWatch API key (via LANGWATCH_API_KEY, or scenario.config.js)"
-        );
-      } else {
-        console.log(`simulation reporting is enabled, endpoint:(${this.eventsEndpoint}) api_key_configured:(${this.apiKey.length > 0 ? "true" : "false"})`);
-      }
+// src/config/env.ts
+var import_zod2 = require("zod");
+
+// src/config/log-levels.ts
+var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
+  LogLevel2["ERROR"] = "ERROR";
+  LogLevel2["WARN"] = "WARN";
+  LogLevel2["INFO"] = "INFO";
+  LogLevel2["DEBUG"] = "DEBUG";
+  return LogLevel2;
+})(LogLevel || {});
+
+// src/config/env.ts
+var envSchema = import_zod2.z.object({
+  /**
+   * LangWatch API key for event reporting.
+   * If not provided, events will not be sent to LangWatch.
+   */
+  LANGWATCH_API_KEY: import_zod2.z.string().optional(),
+  /**
+   * LangWatch endpoint URL for event reporting.
+   * Defaults to the production LangWatch endpoint.
+   */
+  LANGWATCH_ENDPOINT: import_zod2.z.string().url().default("https://app.langwatch.ai"),
+  /**
+   * Disables simulation report info messages when set to any truthy value.
+   * Useful for CI/CD environments or when you want cleaner output.
+   */
+  SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod2.z.string().optional().transform((val) => Boolean(val)),
+  /**
+   * Node environment - affects logging and behavior.
+   * Defaults to 'development' if not specified.
+   */
+  NODE_ENV: import_zod2.z.enum(["development", "production", "test"]).default("development"),
+  /**
+   * Log level for the scenario package.
+   * Defaults to 'info' if not specified.
+   */
+  SCENARIO_LOG_LEVEL: import_zod2.z.nativeEnum(LogLevel).optional(),
+  /**
+   * Scenario batch run ID.
+   * If not provided, a random ID will be generated.
+   */
+  SCENARIO_BATCH_RUN_ID: import_zod2.z.string().optional()
+});
+var env = envSchema.parse(process.env);
+
+// src/config/index.ts
+var logger = new Logger("scenario.config");
+
+// src/utils/ids.ts
+var import_xksuid = require("xksuid");
+function getBatchRunId() {
+  if (!env.SCENARIO_BATCH_RUN_ID) {
+    env.SCENARIO_BATCH_RUN_ID = `scenariobatchrun_${(0, import_xksuid.generate)()}`;
+  }
+  return env.SCENARIO_BATCH_RUN_ID;
+}
+
+// src/events/event-alert-message-logger.ts
+var EventAlertMessageLogger = class _EventAlertMessageLogger {
+  static shownBatchIds = /* @__PURE__ */ new Set();
+  /**
+   * Shows a fancy greeting message about simulation reporting status.
+   * Only shows once per batch run to avoid spam.
+   */
+  handleGreeting() {
+    if (this.isGreetingDisabled()) {
+      return;
+    }
+    const batchRunId = getBatchRunId();
+    if (_EventAlertMessageLogger.shownBatchIds.has(batchRunId)) {
+      return;
     }
+    _EventAlertMessageLogger.shownBatchIds.add(batchRunId);
+    this.displayGreeting(batchRunId);
   }
   /**
-   * Posts an event to the configured endpoint.
-   * Logs success/failure but doesn't throw - event posting shouldn't break scenario execution.
+   * Shows a fancy message about how to watch the simulation.
+   * Called when a run started event is received with a session ID.
    */
-  async postEvent(event) {
-    this.logger.debug(`[${event.type}] Posting event`, {
-      event
-    });
-    if (!this.eventsEndpoint) {
-      this.logger.warn(
-        "No LANGWATCH_ENDPOINT configured, skipping event posting"
-      );
+  handleWatchMessage(params) {
+    if (this.isGreetingDisabled()) {
       return;
     }
-    try {
-      const response = await fetch(this.eventsEndpoint.href, {
-        method: "POST",
-        body: JSON.stringify(event),
-        headers: {
-          "Content-Type": "application/json",
-          "X-Auth-Token": this.apiKey
-        }
-      });
-      this.logger.debug(
-        `[${event.type}] Event POST response status: ${response.status}`
+    this.displayWatchMessage(params);
+  }
+  isGreetingDisabled() {
+    return env.SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
+  }
+  displayGreeting(batchRunId) {
+    const separator = "\u2500".repeat(60);
+    if (!env.LANGWATCH_API_KEY) {
+      console.log(`
+${separator}`);
+      console.log("\u{1F680}  LangWatch Simulation Reporting");
+      console.log(`${separator}`);
+      console.log("\u27A1\uFE0F  API key not configured");
+      console.log("   Simulations will only output final results");
+      console.log("");
+      console.log("\u{1F4A1} To visualize conversations in real time:");
+      console.log("   \u2022 Set LANGWATCH_API_KEY environment variable");
+      console.log("   \u2022 Or configure apiKey in scenario.config.js");
+      console.log("");
+      console.log(`\u{1F4E6} Batch Run ID: ${batchRunId}`);
+      console.log(`${separator}
+`);
+    } else {
+      console.log(`
+${separator}`);
+      console.log("\u{1F680}  LangWatch Simulation Reporting");
+      console.log(`${separator}`);
+      console.log("\u2705 Simulation reporting enabled");
+      console.log(`   Endpoint: ${env.LANGWATCH_ENDPOINT}`);
+      console.log(
+        `   API Key: ${env.LANGWATCH_API_KEY.length > 0 ? "Configured" : "Not configured"}`
       );
-      if (response.ok) {
-        const data = await response.json();
-        this.logger.debug(`[${event.type}] Event POST response:`, data);
-      } else {
-        const errorText = await response.text();
-        this.logger.error(`[${event.type}] Event POST failed:`, {
-          status: response.status,
-          statusText: response.statusText,
-          error: errorText,
-          event
-        });
-      }
-    } catch (error) {
-      this.logger.error(`[${event.type}] Event POST error:`, {
-        error,
-        event,
-        endpoint: this.eventsEndpoint
-      });
+      console.log("");
+      console.log(`\u{1F4E6} Batch Run ID: ${batchRunId}`);
+      console.log(`${separator}
+`);
     }
   }
+  displayWatchMessage(params) {
+    const separator = "\u2500".repeat(60);
+    const setUrl = params.setUrl;
+    const batchUrl = `${setUrl}/${getBatchRunId()}`;
+    console.log(`
+${separator}`);
+    console.log("\u{1F440} Watch Your Simulation Live");
+    console.log(`${separator}`);
+    console.log("\u{1F310} Open in your browser:");
+    console.log(`   Scenario Set: ${setUrl}`);
+    console.log(`   Batch Run: ${batchUrl}`);
+    console.log("");
+    console.log(`${separator}
+`);
+  }
 };
 
 // src/events/schema.ts
 var import_core = require("@ag-ui/core");
-var import_zod = require("zod");
+var import_zod3 = require("zod");
 var Verdict = /* @__PURE__ */ ((Verdict2) => {
   Verdict2["SUCCESS"] = "success";
   Verdict2["FAILURE"] = "failure";
@@ -183,70 +274,147 @@ var ScenarioRunStatus = /* @__PURE__ */ ((ScenarioRunStatus2) => {
   ScenarioRunStatus2["FAILED"] = "FAILED";
   return ScenarioRunStatus2;
 })(ScenarioRunStatus || {});
-var baseEventSchema = import_zod.z.object({
-  type: import_zod.z.nativeEnum(import_core.EventType),
-  timestamp: import_zod.z.number(),
-  rawEvent: import_zod.z.any().optional()
+var baseEventSchema = import_zod3.z.object({
+  type: import_zod3.z.nativeEnum(import_core.EventType),
+  timestamp: import_zod3.z.number(),
+  rawEvent: import_zod3.z.any().optional()
 });
-var batchRunIdSchema = import_zod.z.string();
-var scenarioRunIdSchema = import_zod.z.string();
-var scenarioIdSchema = import_zod.z.string();
+var batchRunIdSchema = import_zod3.z.string();
+var scenarioRunIdSchema = import_zod3.z.string();
+var scenarioIdSchema = import_zod3.z.string();
 var baseScenarioEventSchema = baseEventSchema.extend({
   batchRunId: batchRunIdSchema,
   scenarioId: scenarioIdSchema,
   scenarioRunId: scenarioRunIdSchema,
-  scenarioSetId: import_zod.z.string().optional().default("default")
+  scenarioSetId: import_zod3.z.string().optional().default("default")
 });
 var scenarioRunStartedSchema = baseScenarioEventSchema.extend({
-  type: import_zod.z.literal("SCENARIO_RUN_STARTED" /* RUN_STARTED */),
-  metadata: import_zod.z.object({
-    name: import_zod.z.string().optional(),
-    description: import_zod.z.string().optional()
+  type: import_zod3.z.literal("SCENARIO_RUN_STARTED" /* RUN_STARTED */),
+  metadata: import_zod3.z.object({
+    name: import_zod3.z.string().optional(),
+    description: import_zod3.z.string().optional()
   })
 });
-var scenarioResultsSchema = import_zod.z.object({
-  verdict: import_zod.z.nativeEnum(Verdict),
-  reasoning: import_zod.z.string().optional(),
-  metCriteria: import_zod.z.array(import_zod.z.string()),
-  unmetCriteria: import_zod.z.array(import_zod.z.string()),
-  error: import_zod.z.string().optional()
+var scenarioResultsSchema = import_zod3.z.object({
+  verdict: import_zod3.z.nativeEnum(Verdict),
+  reasoning: import_zod3.z.string().optional(),
+  metCriteria: import_zod3.z.array(import_zod3.z.string()),
+  unmetCriteria: import_zod3.z.array(import_zod3.z.string()),
+  error: import_zod3.z.string().optional()
 });
 var scenarioRunFinishedSchema = baseScenarioEventSchema.extend({
-  type: import_zod.z.literal("SCENARIO_RUN_FINISHED" /* RUN_FINISHED */),
-  status: import_zod.z.nativeEnum(ScenarioRunStatus),
+  type: import_zod3.z.literal("SCENARIO_RUN_FINISHED" /* RUN_FINISHED */),
+  status: import_zod3.z.nativeEnum(ScenarioRunStatus),
   results: scenarioResultsSchema.optional().nullable()
 });
 var scenarioMessageSnapshotSchema = import_core.MessagesSnapshotEventSchema.merge(
   baseScenarioEventSchema.extend({
-    type: import_zod.z.literal("SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */)
+    type: import_zod3.z.literal("SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */)
   })
 );
-var scenarioEventSchema = import_zod.z.discriminatedUnion("type", [
+var scenarioEventSchema = import_zod3.z.discriminatedUnion("type", [
   scenarioRunStartedSchema,
   scenarioRunFinishedSchema,
   scenarioMessageSnapshotSchema
 ]);
-var successSchema = import_zod.z.object({ success: import_zod.z.boolean() });
-var errorSchema = import_zod.z.object({ error: import_zod.z.string() });
-var stateSchema = import_zod.z.object({
-  state: import_zod.z.object({
-    messages: import_zod.z.array(import_zod.z.any()),
-    status: import_zod.z.string()
+var successSchema = import_zod3.z.object({ success: import_zod3.z.boolean() });
+var errorSchema = import_zod3.z.object({ error: import_zod3.z.string() });
+var stateSchema = import_zod3.z.object({
+  state: import_zod3.z.object({
+    messages: import_zod3.z.array(import_zod3.z.any()),
+    status: import_zod3.z.string()
   })
 });
-var runsSchema = import_zod.z.object({ runs: import_zod.z.array(import_zod.z.string()) });
-var eventsSchema = import_zod.z.object({ events: import_zod.z.array(scenarioEventSchema) });
+var runsSchema = import_zod3.z.object({ runs: import_zod3.z.array(import_zod3.z.string()) });
+var eventsSchema = import_zod3.z.object({ events: import_zod3.z.array(scenarioEventSchema) });
+
+// src/events/event-reporter.ts
+var EventReporter = class {
+  apiKey;
+  eventsEndpoint;
+  eventAlertMessageLogger;
+  logger = new Logger("scenario.events.EventReporter");
+  isEnabled;
+  constructor(config) {
+    this.apiKey = config.apiKey ?? "";
+    this.eventsEndpoint = new URL("/api/scenario-events", config.endpoint);
+    this.eventAlertMessageLogger = new EventAlertMessageLogger();
+    this.eventAlertMessageLogger.handleGreeting();
+    this.isEnabled = this.apiKey.length > 0 && this.eventsEndpoint.href.length > 0;
+  }
+  /**
+   * Posts an event to the configured endpoint.
+   * Logs success/failure but doesn't throw - event posting shouldn't break scenario execution.
+   */
+  async postEvent(event) {
+    if (!this.isEnabled) return {};
+    const result = {};
+    this.logger.debug(`[${event.type}] Posting event`, { event });
+    const processedEvent = this.processEventForApi(event);
+    try {
+      const response = await fetch(this.eventsEndpoint.href, {
+        method: "POST",
+        body: JSON.stringify(processedEvent),
+        headers: {
+          "Content-Type": "application/json",
+          "X-Auth-Token": this.apiKey
+        }
+      });
+      this.logger.debug(
+        `[${event.type}] Event POST response status: ${response.status}`
+      );
+      if (response.ok) {
+        const data = await response.json();
+        this.logger.debug(`[${event.type}] Event POST response:`, data);
+        result.setUrl = data.url;
+      } else {
+        const errorText = await response.text();
+        this.logger.error(`[${event.type}] Event POST failed:`, {
+          status: response.status,
+          statusText: response.statusText,
+          error: errorText,
+          event: JSON.stringify(processedEvent)
+        });
+      }
+    } catch (error) {
+      this.logger.error(`[${event.type}] Event POST error:`, {
+        error,
+        event: JSON.stringify(processedEvent),
+        endpoint: this.eventsEndpoint.href
+      });
+    }
+    return result;
+  }
+  /**
+   * Processes event data to ensure API compatibility.
+   * Converts message content objects to strings when needed.
+   */
+  processEventForApi(event) {
+    if (event.type === "SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */) {
+      return {
+        ...event,
+        messages: event.messages.map((message) => ({
+          ...message,
+          content: typeof message.content !== "string" ? JSON.stringify(message.content) : message.content
+        }))
+      };
+    }
+    return event;
+  }
+};
 
 // src/events/event-bus.ts
 var EventBus = class _EventBus {
   static registry = /* @__PURE__ */ new Set();
   events$ = new import_rxjs.Subject();
   eventReporter;
+  eventAlertMessageLogger;
   processingPromise = null;
   logger = new Logger("scenario.events.EventBus");
   static globalListeners = [];
   constructor(config) {
     this.eventReporter = new EventReporter(config);
+    this.eventAlertMessageLogger = new EventAlertMessageLogger();
     _EventBus.registry.add(this);
     for (const listener of _EventBus.globalListeners) {
       listener(this);
@@ -278,22 +446,31 @@ var EventBus = class _EventBus {
     }
     this.processingPromise = new Promise((resolve, reject) => {
       this.events$.pipe(
+        // Post events and get results
         (0, import_rxjs.concatMap)(async (event) => {
-          this.logger.debug(`[${event.type}] Processing event`, {
-            event
-          });
-          await this.eventReporter.postEvent(event);
-          return event;
+          this.logger.debug(`[${event.type}] Processing event`, { event });
+          const result = await this.eventReporter.postEvent(event);
+          return { event, result };
+        }),
+        // Handle watch messages reactively
+        (0, import_rxjs.tap)(({ event, result }) => {
+          if (event.type === "SCENARIO_RUN_STARTED" /* RUN_STARTED */ && result.setUrl) {
+            this.eventAlertMessageLogger.handleWatchMessage({
+              scenarioSetId: event.scenarioSetId,
+              scenarioRunId: event.scenarioRunId,
+              setUrl: result.setUrl
+            });
+          }
         }),
+        // Extract just the event for downstream processing
+        (0, import_rxjs.map)(({ event }) => event),
         (0, import_rxjs.catchError)((error) => {
           this.logger.error("Error in event stream:", error);
           return import_rxjs.EMPTY;
         })
       ).subscribe({
         next: (event) => {
-          this.logger.debug(`[${event.type}] Event processed`, {
-            event
-          });
+          this.logger.debug(`[${event.type}] Event processed`, { event });
           if (event.type === "SCENARIO_RUN_FINISHED" /* RUN_FINISHED */) {
             resolve();
           }

package/dist/integrations/vitest/setup.mjs

@@ -1,6 +1,6 @@
 import {
   EventBus
-} from "../../chunk-ORWSJC5F.mjs";
+} from "../../chunk-NUZZAQV2.mjs";
 import "../../chunk-7P6ASYW6.mjs";
 
 // src/integrations/vitest/setup.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langwatch/scenario",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "description": "A TypeScript library for testing AI agents using scenarios",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -84,7 +84,8 @@
     "test": "vitest",
     "test:ci": "vitest run",
     "lint": "eslint .",
-    "examples:vitest:run": "export SCENARIO_BATCH_ID=scenariobatch_$(uuidgen) && pnpm run buildpack && (cd examples/vitest && pnpm install) && pnpm -F vitest run test",
-    "generate:api-reference": "npx typedoc src --out api-reference-docs"
+    "examples:vitest:run": "export SCENARIO_BATCH_ID=scenariobatch_$(uuidgen) && pnpm run buildpack && (cd examples/vitest && pnpm install) && pnpm -F vitest-example run test",
+    "hash-source": "find src -name \"*.ts\" -type f | sort | xargs cat | sha256sum | cut -d' ' -f1",
+    "generate:api-reference": "npx typedoc src --out api-reference-docs && rm -rf ../docs/docs/public/reference/javascript/scenario && mv api-reference-docs ../docs/docs/public/reference/javascript/scenario && pnpm run hash-source > ../docs/docs/public/reference/javascript/.docs-source-hash"
   }
 }