@langwatch/scenario 0.2.13 → 0.4.0

package/dist/integrations/vitest/setup.mjs

@@ -1,37 +1,638 @@
+// src/integrations/vitest/setup.ts
+import fs3 from "fs";
+import path3 from "path";
+import { beforeEach, afterEach } from "vitest";
+
+// src/events/event-bus.ts
 import {
-  EventBus
-} from "../../chunk-6SKQWXT7.mjs";
-import {
-  Logger
-} from "../../chunk-OL4RFXV4.mjs";
-import "../../chunk-7P6ASYW6.mjs";
+  concatMap,
+  EMPTY,
+  catchError,
+  Subject,
+  tap,
+  map
+} from "rxjs";
 
-// src/integrations/vitest/setup.ts
-import fs from "fs";
+// src/events/event-alert-message-logger.ts
+import * as fs2 from "fs";
+import * as os from "os";
+import * as path2 from "path";
+import open from "open";
+
+// src/config/env.ts
+import { z } from "zod/v4";
+
+// src/config/log-levels.ts
+var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
+  LogLevel2["ERROR"] = "ERROR";
+  LogLevel2["WARN"] = "WARN";
+  LogLevel2["INFO"] = "INFO";
+  LogLevel2["DEBUG"] = "DEBUG";
+  return LogLevel2;
+})(LogLevel || {});
+var LOG_LEVELS = Object.values(LogLevel);
+
+// src/config/env.ts
+var envSchema = z.object({
+  /**
+   * LangWatch API key for event reporting.
+   * If not provided, events will not be sent to LangWatch.
+   */
+  LANGWATCH_API_KEY: z.string().optional(),
+  /**
+   * LangWatch endpoint URL for event reporting.
+   * Defaults to the production LangWatch endpoint.
+   */
+  LANGWATCH_ENDPOINT: z.string().url().optional().default("https://app.langwatch.ai"),
+  /**
+   * Disables simulation report info messages when set to any truthy value.
+   * Useful for CI/CD environments or when you want cleaner output.
+   */
+  SCENARIO_DISABLE_SIMULATION_REPORT_INFO: z.string().optional().transform((val) => Boolean(val)),
+  /**
+   * Node environment - affects logging and behavior.
+   * Defaults to 'development' if not specified.
+   */
+  NODE_ENV: z.enum(["development", "production", "test"]).default("development"),
+  /**
+   * Case-insensitive log level for the scenario package.
+   * Defaults to 'info' if not specified.
+   */
+  LOG_LEVEL: z.string().toUpperCase().pipe(z.nativeEnum(LogLevel)).optional().default("INFO" /* INFO */),
+  /**
+   * Scenario batch run ID.
+   * If not provided, a random ID will be generated.
+   */
+  SCENARIO_BATCH_RUN_ID: z.string().optional()
+});
+function getEnv() {
+  return envSchema.parse(process.env);
+}
+
+// src/config/load.ts
+import fs from "fs/promises";
 import path from "path";
-import { beforeEach, afterEach } from "vitest";
-var logger = Logger.create("integrations:vitest:setup");
+import { pathToFileURL } from "url";
+
+// src/domain/core/config.ts
+import { z as z3 } from "zod/v4";
+
+// src/domain/core/schemas/model.schema.ts
+import { z as z2 } from "zod/v4";
+
+// src/domain/core/constants.ts
+var DEFAULT_TEMPERATURE = 0;
+
+// src/domain/core/schemas/model.schema.ts
+var modelSchema = z2.object({
+  model: z2.custom((val) => Boolean(val), {
+    message: "A model is required. Configure it in scenario.config.js defaultModel or pass directly to the agent."
+  }).describe("The OpenAI Language Model to use for generating responses."),
+  temperature: z2.number().min(0).max(1).optional().describe("The temperature for the language model.").default(DEFAULT_TEMPERATURE),
+  maxTokens: z2.number().optional().describe("The maximum number of tokens to generate.")
+});
+
+// src/domain/core/config.ts
+var headless = typeof process !== "undefined" ? process.env.SCENARIO_HEADLESS === "true" : false;
+var scenarioProjectConfigSchema = z3.object({
+  defaultModel: modelSchema.optional(),
+  headless: z3.boolean().optional().default(headless)
+}).strict();
+
+// src/config/load.ts
+async function loadScenarioProjectConfig() {
+  const cwd = process.cwd();
+  const configNames = [
+    "scenario.config.js",
+    "scenario.config.mjs"
+  ];
+  for (const name of configNames) {
+    const fullPath = path.join(cwd, name);
+    try {
+      await fs.access(fullPath);
+      const configModule = await import(pathToFileURL(fullPath).href);
+      const config2 = configModule.default || configModule;
+      const parsed = scenarioProjectConfigSchema.safeParse(config2);
+      if (!parsed.success) {
+        throw new Error(
+          `Invalid config file ${name}: ${JSON.stringify(parsed.error.format(), null, 2)}`
+        );
+      }
+      return parsed.data;
+    } catch (error) {
+      if (error instanceof Error && "code" in error && error.code === "ENOENT") {
+        continue;
+      }
+      throw error;
+    }
+  }
+  return await scenarioProjectConfigSchema.parseAsync({});
+}
+
+// src/utils/logger.ts
+var Logger = class _Logger {
+  constructor(context) {
+    this.context = context;
+  }
+  /**
+   * Creates a logger with context (e.g., class name)
+   */
+  static create(context) {
+    return new _Logger(context);
+  }
+  /**
+   * Returns the current log level from environment.
+   * Uses a getter for clarity and idiomatic usage.
+   */
+  get LOG_LEVEL() {
+    return getEnv().LOG_LEVEL;
+  }
+  /**
+   * Returns the index of the given log level in the LOG_LEVELS array.
+   * @param level - The log level to get the index for.
+   * @returns The index of the log level in the LOG_LEVELS array.
+   */
+  getLogLevelIndexFor(level) {
+    return LOG_LEVELS.indexOf(level);
+  }
+  /**
+   * Checks if logging should occur based on LOG_LEVEL env var
+   */
+  shouldLog(level) {
+    const currentLevelIndex = this.getLogLevelIndexFor(this.LOG_LEVEL);
+    const requestedLevelIndex = this.getLogLevelIndexFor(level);
+    return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
+  }
+  formatMessage(message) {
+    return this.context ? `[${this.context}] ${message}` : message;
+  }
+  error(message, data) {
+    if (this.shouldLog("ERROR" /* ERROR */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.error(formattedMessage, data);
+      } else {
+        console.error(formattedMessage);
+      }
+    }
+  }
+  warn(message, data) {
+    if (this.shouldLog("WARN" /* WARN */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.warn(formattedMessage, data);
+      } else {
+        console.warn(formattedMessage);
+      }
+    }
+  }
+  info(message, data) {
+    if (this.shouldLog("INFO" /* INFO */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.info(formattedMessage, data);
+      } else {
+        console.info(formattedMessage);
+      }
+    }
+  }
+  debug(message, data) {
+    if (this.shouldLog("DEBUG" /* DEBUG */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.log(formattedMessage, data);
+      } else {
+        console.log(formattedMessage);
+      }
+    }
+  }
+};
+
+// src/config/get-project-config.ts
+var logger = new Logger("scenario.config");
+var configLoaded = false;
+var config = null;
+var configLoadPromise = null;
+async function loadProjectConfig() {
+  if (configLoaded) {
+    return;
+  }
+  if (configLoadPromise) {
+    return configLoadPromise;
+  }
+  configLoadPromise = (async () => {
+    try {
+      config = await loadScenarioProjectConfig();
+      logger.debug("loaded scenario project config", { config });
+    } catch (error) {
+      logger.error("error loading scenario project config", { error });
+    } finally {
+      configLoaded = true;
+    }
+  })();
+  return configLoadPromise;
+}
+async function getProjectConfig() {
+  await loadProjectConfig();
+  return config;
+}
+
+// src/utils/ids.ts
+import crypto from "crypto";
+import process2 from "process";
+import { generate, parse } from "xksuid";
+var batchRunId;
+function getBatchRunId() {
+  if (batchRunId) {
+    return batchRunId;
+  }
+  if (process2.env.SCENARIO_BATCH_RUN_ID) {
+    return batchRunId = process2.env.SCENARIO_BATCH_RUN_ID;
+  }
+  if (process2.env.VITEST_WORKER_ID || process2.env.JEST_WORKER_ID) {
+    const parentProcessId = process2.ppid;
+    const now = /* @__PURE__ */ new Date();
+    const year = now.getUTCFullYear();
+    const week = String(getISOWeekNumber(now)).padStart(2, "0");
+    const raw = `${parentProcessId}_${year}_w${week}`;
+    const hash = crypto.createHash("sha256").update(raw).digest("hex").slice(0, 12);
+    return batchRunId = `scenariobatch_${hash}`;
+  }
+  return batchRunId = `scenariobatch_${generate()}`;
+}
+function getISOWeekNumber(date) {
+  const tmp = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
+  const dayNum = tmp.getUTCDay() || 7;
+  tmp.setUTCDate(tmp.getUTCDate() + 4 - dayNum);
+  const yearStart = new Date(Date.UTC(tmp.getUTCFullYear(), 0, 1));
+  const weekNo = Math.ceil(((tmp.getTime() - yearStart.getTime()) / 864e5 + 1) / 7);
+  return weekNo;
+}
+
+// src/events/event-alert-message-logger.ts
+var EventAlertMessageLogger = class {
+  /**
+   * Creates a coordination file to prevent duplicate messages across processes.
+   * Returns true if this process should show the message (first one to create the file).
+   */
+  createCoordinationFile(type) {
+    try {
+      const batchId = getBatchRunId();
+      const tmpDir = os.tmpdir();
+      const fileName = `scenario-${type}-${batchId}`;
+      const filePath = path2.join(tmpDir, fileName);
+      fs2.writeFileSync(filePath, process.pid.toString(), { flag: "wx" });
+      return true;
+    } catch {
+      return false;
+    }
+  }
+  /**
+   * Shows a fancy greeting message about simulation reporting status.
+   * Only shows once per batch run to avoid spam.
+   */
+  handleGreeting() {
+    if (this.isGreetingDisabled()) {
+      return;
+    }
+    if (!this.createCoordinationFile("greeting")) {
+      return;
+    }
+    this.displayGreeting();
+  }
+  /**
+   * Shows a fancy message about how to watch the simulation.
+   * Called when a run started event is received with a session ID.
+   */
+  async handleWatchMessage(params) {
+    if (this.isGreetingDisabled()) {
+      return;
+    }
+    if (!this.createCoordinationFile(`watch-${params.scenarioSetId}`)) {
+      return;
+    }
+    await this.displayWatchMessage(params);
+  }
+  isGreetingDisabled() {
+    return getEnv().SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
+  }
+  displayGreeting() {
+    const separator = "\u2500".repeat(60);
+    const env = getEnv();
+    if (!env.LANGWATCH_API_KEY) {
+      console.log(`
+${separator}`);
+      console.log("\u{1F3AD}  Running Scenario Tests");
+      console.log(`${separator}`);
+      console.log("\u27A1\uFE0F  LangWatch API key not configured");
+      console.log("   Simulations will only output final results");
+      console.log("");
+      console.log("\u{1F4A1} To visualize conversations in real time:");
+      console.log("   \u2022 Set LANGWATCH_API_KEY environment variable");
+      console.log("   \u2022 Or configure apiKey in scenario.config.js");
+      console.log("");
+      console.log(`${separator}
+`);
+    }
+  }
+  async displayWatchMessage(params) {
+    const separator = "\u2500".repeat(60);
+    const setUrl = params.setUrl;
+    const batchUrl = `${setUrl}/${getBatchRunId()}`;
+    console.log(`
+${separator}`);
+    console.log("\u{1F3AD}  Running Scenario Tests");
+    console.log(`${separator}`);
+    console.log(`Follow it live: ${batchUrl}`);
+    console.log(`${separator}
+`);
+    const projectConfig = await getProjectConfig();
+    if (!(projectConfig == null ? void 0 : projectConfig.headless)) {
+      try {
+        open(batchUrl);
+      } catch (_) {
+      }
+    }
+  }
+};
+
+// src/events/schema.ts
+import { EventType, MessagesSnapshotEventSchema } from "@ag-ui/core";
+import { z as z4 } from "zod";
+var Verdict = /* @__PURE__ */ ((Verdict2) => {
+  Verdict2["SUCCESS"] = "success";
+  Verdict2["FAILURE"] = "failure";
+  Verdict2["INCONCLUSIVE"] = "inconclusive";
+  return Verdict2;
+})(Verdict || {});
+var ScenarioRunStatus = /* @__PURE__ */ ((ScenarioRunStatus2) => {
+  ScenarioRunStatus2["SUCCESS"] = "SUCCESS";
+  ScenarioRunStatus2["ERROR"] = "ERROR";
+  ScenarioRunStatus2["CANCELLED"] = "CANCELLED";
+  ScenarioRunStatus2["IN_PROGRESS"] = "IN_PROGRESS";
+  ScenarioRunStatus2["PENDING"] = "PENDING";
+  ScenarioRunStatus2["FAILED"] = "FAILED";
+  return ScenarioRunStatus2;
+})(ScenarioRunStatus || {});
+var baseEventSchema = z4.object({
+  type: z4.nativeEnum(EventType),
+  timestamp: z4.number(),
+  rawEvent: z4.any().optional()
+});
+var batchRunIdSchema = z4.string();
+var scenarioRunIdSchema = z4.string();
+var scenarioIdSchema = z4.string();
+var baseScenarioEventSchema = baseEventSchema.extend({
+  batchRunId: batchRunIdSchema,
+  scenarioId: scenarioIdSchema,
+  scenarioRunId: scenarioRunIdSchema,
+  scenarioSetId: z4.string().optional().default("default")
+});
+var scenarioRunStartedSchema = baseScenarioEventSchema.extend({
+  type: z4.literal("SCENARIO_RUN_STARTED" /* RUN_STARTED */),
+  metadata: z4.object({
+    name: z4.string().optional(),
+    description: z4.string().optional()
+  })
+});
+var scenarioResultsSchema = z4.object({
+  verdict: z4.nativeEnum(Verdict),
+  reasoning: z4.string().optional(),
+  metCriteria: z4.array(z4.string()),
+  unmetCriteria: z4.array(z4.string()),
+  error: z4.string().optional()
+});
+var scenarioRunFinishedSchema = baseScenarioEventSchema.extend({
+  type: z4.literal("SCENARIO_RUN_FINISHED" /* RUN_FINISHED */),
+  status: z4.nativeEnum(ScenarioRunStatus),
+  results: scenarioResultsSchema.optional().nullable()
+});
+var scenarioMessageSnapshotSchema = MessagesSnapshotEventSchema.merge(
+  baseScenarioEventSchema.extend({
+    type: z4.literal("SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */)
+  })
+);
+var scenarioEventSchema = z4.discriminatedUnion("type", [
+  scenarioRunStartedSchema,
+  scenarioRunFinishedSchema,
+  scenarioMessageSnapshotSchema
+]);
+var successSchema = z4.object({ success: z4.boolean() });
+var errorSchema = z4.object({ error: z4.string() });
+var stateSchema = z4.object({
+  state: z4.object({
+    messages: z4.array(z4.any()),
+    status: z4.string()
+  })
+});
+var runsSchema = z4.object({ runs: z4.array(z4.string()) });
+var eventsSchema = z4.object({ events: z4.array(scenarioEventSchema) });
+
+// src/events/event-reporter.ts
+var EventReporter = class {
+  apiKey;
+  eventsEndpoint;
+  eventAlertMessageLogger;
+  logger = new Logger("scenario.events.EventReporter");
+  isEnabled;
+  constructor(config2) {
+    this.apiKey = config2.apiKey ?? "";
+    this.eventsEndpoint = new URL("/api/scenario-events", config2.endpoint);
+    this.eventAlertMessageLogger = new EventAlertMessageLogger();
+    this.eventAlertMessageLogger.handleGreeting();
+    this.isEnabled = this.apiKey.length > 0 && this.eventsEndpoint.href.length > 0;
+  }
+  /**
+   * Posts an event to the configured endpoint.
+   * Logs success/failure but doesn't throw - event posting shouldn't break scenario execution.
+   */
+  async postEvent(event) {
+    if (!this.isEnabled) return {};
+    const result = {};
+    this.logger.debug(`[${event.type}] Posting event`, { event });
+    const processedEvent = this.processEventForApi(event);
+    try {
+      const response = await fetch(this.eventsEndpoint.href, {
+        method: "POST",
+        body: JSON.stringify(processedEvent),
+        headers: {
+          "Content-Type": "application/json",
+          "X-Auth-Token": this.apiKey
+        }
+      });
+      this.logger.debug(
+        `[${event.type}] Event POST response status: ${response.status}`
+      );
+      if (response.ok) {
+        const data = await response.json();
+        this.logger.debug(`[${event.type}] Event POST response:`, data);
+        result.setUrl = data.url;
+      } else {
+        const errorText = await response.text();
+        this.logger.error(`[${event.type}] Event POST failed:`, {
+          endpoint: this.eventsEndpoint.href,
+          status: response.status,
+          statusText: response.statusText,
+          error: errorText,
+          event: JSON.stringify(processedEvent)
+        });
+      }
+    } catch (error) {
+      this.logger.error(`[${event.type}] Event POST error:`, {
+        error,
+        event: JSON.stringify(processedEvent),
+        endpoint: this.eventsEndpoint.href
+      });
+    }
+    return result;
+  }
+  /**
+   * Processes event data to ensure API compatibility.
+   * Converts message content objects to strings when needed.
+   */
+  processEventForApi(event) {
+    if (event.type === "SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */) {
+      return {
+        ...event,
+        messages: event.messages.map((message) => ({
+          ...message,
+          content: typeof message.content !== "string" ? JSON.stringify(message.content) : message.content
+        }))
+      };
+    }
+    return event;
+  }
+};
+
+// src/events/event-bus.ts
+var EventBus = class _EventBus {
+  static registry = /* @__PURE__ */ new Set();
+  events$ = new Subject();
+  eventReporter;
+  eventAlertMessageLogger;
+  processingPromise = null;
+  logger = new Logger("scenario.events.EventBus");
+  static globalListeners = [];
+  constructor(config2) {
+    this.eventReporter = new EventReporter(config2);
+    this.eventAlertMessageLogger = new EventAlertMessageLogger();
+    _EventBus.registry.add(this);
+    for (const listener of _EventBus.globalListeners) {
+      listener(this);
+    }
+  }
+  static getAllBuses() {
+    return _EventBus.registry;
+  }
+  static addGlobalListener(listener) {
+    _EventBus.globalListeners.push(listener);
+  }
+  /**
+   * Publishes an event into the processing pipeline.
+   */
+  publish(event) {
+    this.logger.debug(`[${event.type}] Publishing event`, {
+      event
+    });
+    this.events$.next(event);
+  }
+  /**
+   * Begins listening for and processing events.
+   * Returns a promise that resolves when a RUN_FINISHED event is fully processed.
+   */
+  listen() {
+    this.logger.debug("Listening for events");
+    if (this.processingPromise) {
+      return this.processingPromise;
+    }
+    this.processingPromise = new Promise((resolve, reject) => {
+      this.events$.pipe(
+        // Post events and get results
+        concatMap(async (event) => {
+          this.logger.debug(`[${event.type}] Processing event`, { event });
+          const result = await this.eventReporter.postEvent(event);
+          return { event, result };
+        }),
+        // Handle watch messages reactively
+        tap(async ({ event, result }) => {
+          if (event.type === "SCENARIO_RUN_STARTED" /* RUN_STARTED */ && result.setUrl) {
+            await this.eventAlertMessageLogger.handleWatchMessage({
+              scenarioSetId: event.scenarioSetId,
+              scenarioRunId: event.scenarioRunId,
+              setUrl: result.setUrl
+            });
+          }
+        }),
+        // Extract just the event for downstream processing
+        map(({ event }) => event),
+        catchError((error) => {
+          this.logger.error("Error in event stream:", error);
+          return EMPTY;
+        })
+      ).subscribe({
+        next: (event) => {
+          this.logger.debug(`[${event.type}] Event processed`, { event });
+          if (event.type === "SCENARIO_RUN_FINISHED" /* RUN_FINISHED */) {
+            resolve();
+          }
+        },
+        error: (error) => {
+          this.logger.error("Error in event stream:", error);
+          reject(error);
+        }
+      });
+    });
+    return this.processingPromise;
+  }
+  /**
+   * Stops accepting new events and drains the processing queue.
+   */
+  async drain() {
+    this.logger.debug("Draining event stream");
+    this.events$.complete();
+    if (this.processingPromise) {
+      await this.processingPromise;
+    }
+  }
+  /**
+   * Subscribes to an event stream.
+   * @param source$ - The event stream to subscribe to.
+   */
+  subscribeTo(source$) {
+    this.logger.debug("Subscribing to event stream");
+    return source$.subscribe(this.events$);
+  }
+  /**
+   * Expose the events$ observable for external subscription (read-only).
+   */
+  get eventsObservable() {
+    return this.events$.asObservable();
+  }
+};
+
+// src/integrations/vitest/setup.ts
+var logger2 = Logger.create("integrations:vitest:setup");
 function getProjectRoot() {
   return process.cwd();
 }
 var projectRoot = getProjectRoot();
-var logDir = path.join(projectRoot, ".scenario");
-if (!fs.existsSync(logDir)) fs.mkdirSync(logDir, { recursive: true });
+var logDir = path3.join(projectRoot, ".scenario");
+if (!fs3.existsSync(logDir)) fs3.mkdirSync(logDir, { recursive: true });
 function getLogFilePath(testName) {
-  return path.join(logDir, `${testName.replace(/[^a-z0-9]/gi, "_")}.log`);
+  return path3.join(logDir, `${testName.replace(/[^a-z0-9]/gi, "_")}.log`);
 }
 var currentTestName = "";
 var subs = [];
 beforeEach((ctx) => {
   currentTestName = ctx.task.id;
   const filePath = getLogFilePath(currentTestName);
-  if (fs.existsSync(filePath)) fs.unlinkSync(filePath);
+  if (fs3.existsSync(filePath)) fs3.unlinkSync(filePath);
   subs = Array.from(EventBus.getAllBuses()).map(
     (bus) => bus.eventsObservable.subscribe((event) => {
       try {
-        fs.appendFileSync(filePath, JSON.stringify(event) + "\n");
+        fs3.appendFileSync(filePath, JSON.stringify(event) + "\n");
       } catch (error) {
-        logger.error("Error writing to log file:", error);
+        logger2.error("Error writing to log file:", error);
       }
     })
   );
@@ -41,9 +642,9 @@ EventBus.addGlobalListener((bus) => {
     bus.eventsObservable.subscribe((event) => {
       const filePath = getLogFilePath(currentTestName);
       try {
-        fs.appendFileSync(filePath, JSON.stringify(event) + "\n");
+        fs3.appendFileSync(filePath, JSON.stringify(event) + "\n");
       } catch (error) {
-        logger.error("Error writing to log file:", error);
+        logger2.error("Error writing to log file:", error);
       }
     })
   );