@langwatch/scenario 0.2.0 → 0.2.1

package/dist/chunk-ZMHTHRDR.mjs

@@ -0,0 +1,618 @@
+import {
+  __export
+} from "./chunk-7P6ASYW6.mjs";
+
+// src/domain/core/config.ts
+import { z } from "zod";
+var scenarioProjectConfigSchema = z.object({
+  defaultModel: z.object({
+    model: z.custom(),
+    temperature: z.number().min(0).max(1).optional().default(0),
+    maxTokens: z.number().optional()
+  }).optional(),
+  langwatchEndpoint: z.string().optional(),
+  langwatchApiKey: z.string().optional()
+}).strict();
+function defineConfig(config2) {
+  return config2;
+}
+
+// src/domain/agents/index.ts
+var AgentRole = /* @__PURE__ */ ((AgentRole2) => {
+  AgentRole2["USER"] = "User";
+  AgentRole2["AGENT"] = "Agent";
+  AgentRole2["JUDGE"] = "Judge";
+  return AgentRole2;
+})(AgentRole || {});
+var allAgentRoles = ["User" /* USER */, "Agent" /* AGENT */, "Judge" /* JUDGE */];
+var AgentAdapter = class {
+  role = "Agent" /* AGENT */;
+  constructor(input) {
+    void input;
+  }
+};
+var UserSimulatorAgentAdapter = class {
+  role = "User" /* USER */;
+  constructor(input) {
+    void input;
+  }
+};
+var JudgeAgentAdapter = class {
+  role = "Judge" /* JUDGE */;
+  constructor(input) {
+    void input;
+  }
+};
+
+// src/domain/index.ts
+var domain_exports = {};
+__export(domain_exports, {
+  AgentAdapter: () => AgentAdapter,
+  AgentRole: () => AgentRole,
+  JudgeAgentAdapter: () => JudgeAgentAdapter,
+  UserSimulatorAgentAdapter: () => UserSimulatorAgentAdapter,
+  allAgentRoles: () => allAgentRoles,
+  defineConfig: () => defineConfig,
+  scenarioProjectConfigSchema: () => scenarioProjectConfigSchema
+});
+
+// src/events/event-bus.ts
+import {
+  concatMap,
+  EMPTY,
+  catchError,
+  Subject,
+  tap,
+  map
+} from "rxjs";
+
+// src/config/load.ts
+import fs from "node:fs/promises";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+async function loadScenarioProjectConfig() {
+  const cwd = process.cwd();
+  const configNames = [
+    "scenario.config.js",
+    "scenario.config.mjs"
+  ];
+  for (const name of configNames) {
+    const fullPath = path.join(cwd, name);
+    try {
+      await fs.access(fullPath);
+      const configModule = await import(pathToFileURL(fullPath).href);
+      const config2 = configModule.default || configModule;
+      const parsed = scenarioProjectConfigSchema.safeParse(config2);
+      if (!parsed.success) {
+        throw new Error(
+          `Invalid config file ${name}: ${JSON.stringify(parsed.error.format(), null, 2)}`
+        );
+      }
+      return parsed.data;
+    } catch (error) {
+      if (error instanceof Error && "code" in error && error.code === "ENOENT") {
+        continue;
+      }
+      throw error;
+    }
+  }
+  return await scenarioProjectConfigSchema.parseAsync({});
+}
+
+// src/utils/logger.ts
+var Logger = class _Logger {
+  constructor(context) {
+    this.context = context;
+  }
+  /**
+   * Creates a logger with context (e.g., class name)
+   */
+  static create(context) {
+    return new _Logger(context);
+  }
+  getLogLevel() {
+    return env.SCENARIO_LOG_LEVEL ?? "INFO" /* INFO */;
+  }
+  getLogLevelIndex(level) {
+    return Object.values(LogLevel).indexOf(level);
+  }
+  /**
+   * Checks if logging should occur based on LOG_LEVEL env var
+   */
+  shouldLog(level) {
+    const currentLevelIndex = this.getLogLevelIndex(this.getLogLevel());
+    const requestedLevelIndex = this.getLogLevelIndex(level);
+    return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
+  }
+  formatMessage(message) {
+    return this.context ? `[${this.context}] ${message}` : message;
+  }
+  error(message, data) {
+    if (this.shouldLog("ERROR" /* ERROR */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.error(formattedMessage, data);
+      } else {
+        console.error(formattedMessage);
+      }
+    }
+  }
+  warn(message, data) {
+    if (this.shouldLog("WARN" /* WARN */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.warn(formattedMessage, data);
+      } else {
+        console.warn(formattedMessage);
+      }
+    }
+  }
+  info(message, data) {
+    if (this.shouldLog("INFO" /* INFO */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.info(formattedMessage, data);
+      } else {
+        console.info(formattedMessage);
+      }
+    }
+  }
+  debug(message, data) {
+    if (this.shouldLog("DEBUG" /* DEBUG */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.log(formattedMessage, data);
+      } else {
+        console.log(formattedMessage);
+      }
+    }
+  }
+};
+
+// src/config/env.ts
+import { z as z2 } from "zod";
+
+// src/config/log-levels.ts
+var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
+  LogLevel2["ERROR"] = "ERROR";
+  LogLevel2["WARN"] = "WARN";
+  LogLevel2["INFO"] = "INFO";
+  LogLevel2["DEBUG"] = "DEBUG";
+  return LogLevel2;
+})(LogLevel || {});
+
+// src/config/env.ts
+var envSchema = z2.object({
+  /**
+   * LangWatch API key for event reporting.
+   * If not provided, events will not be sent to LangWatch.
+   */
+  LANGWATCH_API_KEY: z2.string().optional(),
+  /**
+   * LangWatch endpoint URL for event reporting.
+   * Defaults to the production LangWatch endpoint.
+   */
+  LANGWATCH_ENDPOINT: z2.string().url().default("https://app.langwatch.ai"),
+  /**
+   * Disables simulation report info messages when set to any truthy value.
+   * Useful for CI/CD environments or when you want cleaner output.
+   */
+  SCENARIO_DISABLE_SIMULATION_REPORT_INFO: z2.string().optional().transform((val) => Boolean(val)),
+  /**
+   * Node environment - affects logging and behavior.
+   * Defaults to 'development' if not specified.
+   */
+  NODE_ENV: z2.enum(["development", "production", "test"]).default("development"),
+  /**
+   * Log level for the scenario package.
+   * Defaults to 'info' if not specified.
+   */
+  SCENARIO_LOG_LEVEL: z2.nativeEnum(LogLevel).optional(),
+  /**
+   * Scenario batch run ID.
+   * If not provided, a random ID will be generated.
+   */
+  SCENARIO_BATCH_RUN_ID: z2.string().optional()
+});
+var env = envSchema.parse(process.env);
+
+// src/config/index.ts
+var logger = new Logger("scenario.config");
+var configLoaded = false;
+var config = null;
+var configLoadPromise = null;
+async function loadProjectConfig() {
+  if (configLoaded) {
+    return;
+  }
+  if (configLoadPromise) {
+    return configLoadPromise;
+  }
+  configLoadPromise = (async () => {
+    try {
+      config = await loadScenarioProjectConfig();
+      logger.info("loaded scenario project config", { config });
+    } catch (error) {
+      logger.error("error loading scenario project config", { error });
+    } finally {
+      configLoaded = true;
+    }
+  })();
+  return configLoadPromise;
+}
+async function getProjectConfig() {
+  await loadProjectConfig();
+  return config;
+}
+
+// src/utils/ids.ts
+import { generate, parse } from "xksuid";
+function generateThreadId() {
+  return `thread_${generate()}`;
+}
+function generateScenarioRunId() {
+  return `scenariorun_${generate()}`;
+}
+function generateScenarioId() {
+  return `scenario_${generate()}`;
+}
+function getBatchRunId() {
+  if (!env.SCENARIO_BATCH_RUN_ID) {
+    env.SCENARIO_BATCH_RUN_ID = `scenariobatchrun_${generate()}`;
+  }
+  return env.SCENARIO_BATCH_RUN_ID;
+}
+function generateMessageId() {
+  return `scenariomsg_${generate()}`;
+}
+
+// src/events/event-alert-message-logger.ts
+var EventAlertMessageLogger = class _EventAlertMessageLogger {
+  static shownBatchIds = /* @__PURE__ */ new Set();
+  /**
+   * Shows a fancy greeting message about simulation reporting status.
+   * Only shows once per batch run to avoid spam.
+   */
+  handleGreeting() {
+    if (this.isGreetingDisabled()) {
+      return;
+    }
+    const batchRunId = getBatchRunId();
+    if (_EventAlertMessageLogger.shownBatchIds.has(batchRunId)) {
+      return;
+    }
+    _EventAlertMessageLogger.shownBatchIds.add(batchRunId);
+    this.displayGreeting(batchRunId);
+  }
+  /**
+   * Shows a fancy message about how to watch the simulation.
+   * Called when a run started event is received with a session ID.
+   */
+  handleWatchMessage(params) {
+    if (this.isGreetingDisabled()) {
+      return;
+    }
+    this.displayWatchMessage(params);
+  }
+  isGreetingDisabled() {
+    return env.SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
+  }
+  displayGreeting(batchRunId) {
+    const separator = "\u2500".repeat(60);
+    if (!env.LANGWATCH_API_KEY) {
+      console.log(`
+${separator}`);
+      console.log("\u{1F680}  LangWatch Simulation Reporting");
+      console.log(`${separator}`);
+      console.log("\u27A1\uFE0F  API key not configured");
+      console.log("   Simulations will only output final results");
+      console.log("");
+      console.log("\u{1F4A1} To visualize conversations in real time:");
+      console.log("   \u2022 Set LANGWATCH_API_KEY environment variable");
+      console.log("   \u2022 Or configure apiKey in scenario.config.js");
+      console.log("");
+      console.log(`\u{1F4E6} Batch Run ID: ${batchRunId}`);
+      console.log(`${separator}
+`);
+    } else {
+      console.log(`
+${separator}`);
+      console.log("\u{1F680}  LangWatch Simulation Reporting");
+      console.log(`${separator}`);
+      console.log("\u2705 Simulation reporting enabled");
+      console.log(`   Endpoint: ${env.LANGWATCH_ENDPOINT}`);
+      console.log(
+        `   API Key: ${env.LANGWATCH_API_KEY.length > 0 ? "Configured" : "Not configured"}`
+      );
+      console.log("");
+      console.log(`\u{1F4E6} Batch Run ID: ${batchRunId}`);
+      console.log(`${separator}
+`);
+    }
+  }
+  displayWatchMessage(params) {
+    const separator = "\u2500".repeat(60);
+    const setUrl = params.setUrl;
+    const batchUrl = `${setUrl}/${getBatchRunId()}`;
+    console.log(`
+${separator}`);
+    console.log("\u{1F440} Watch Your Simulation Live");
+    console.log(`${separator}`);
+    console.log("\u{1F310} Open in your browser:");
+    console.log(`   Scenario Set: ${setUrl}`);
+    console.log(`   Batch Run: ${batchUrl}`);
+    console.log("");
+    console.log(`${separator}
+`);
+  }
+};
+
+// src/events/schema.ts
+import { EventType, MessagesSnapshotEventSchema } from "@ag-ui/core";
+import { z as z3 } from "zod";
+var Verdict = /* @__PURE__ */ ((Verdict2) => {
+  Verdict2["SUCCESS"] = "success";
+  Verdict2["FAILURE"] = "failure";
+  Verdict2["INCONCLUSIVE"] = "inconclusive";
+  return Verdict2;
+})(Verdict || {});
+var ScenarioRunStatus = /* @__PURE__ */ ((ScenarioRunStatus2) => {
+  ScenarioRunStatus2["SUCCESS"] = "SUCCESS";
+  ScenarioRunStatus2["ERROR"] = "ERROR";
+  ScenarioRunStatus2["CANCELLED"] = "CANCELLED";
+  ScenarioRunStatus2["IN_PROGRESS"] = "IN_PROGRESS";
+  ScenarioRunStatus2["PENDING"] = "PENDING";
+  ScenarioRunStatus2["FAILED"] = "FAILED";
+  return ScenarioRunStatus2;
+})(ScenarioRunStatus || {});
+var baseEventSchema = z3.object({
+  type: z3.nativeEnum(EventType),
+  timestamp: z3.number(),
+  rawEvent: z3.any().optional()
+});
+var batchRunIdSchema = z3.string();
+var scenarioRunIdSchema = z3.string();
+var scenarioIdSchema = z3.string();
+var baseScenarioEventSchema = baseEventSchema.extend({
+  batchRunId: batchRunIdSchema,
+  scenarioId: scenarioIdSchema,
+  scenarioRunId: scenarioRunIdSchema,
+  scenarioSetId: z3.string().optional().default("default")
+});
+var scenarioRunStartedSchema = baseScenarioEventSchema.extend({
+  type: z3.literal("SCENARIO_RUN_STARTED" /* RUN_STARTED */),
+  metadata: z3.object({
+    name: z3.string().optional(),
+    description: z3.string().optional()
+  })
+});
+var scenarioResultsSchema = z3.object({
+  verdict: z3.nativeEnum(Verdict),
+  reasoning: z3.string().optional(),
+  metCriteria: z3.array(z3.string()),
+  unmetCriteria: z3.array(z3.string()),
+  error: z3.string().optional()
+});
+var scenarioRunFinishedSchema = baseScenarioEventSchema.extend({
+  type: z3.literal("SCENARIO_RUN_FINISHED" /* RUN_FINISHED */),
+  status: z3.nativeEnum(ScenarioRunStatus),
+  results: scenarioResultsSchema.optional().nullable()
+});
+var scenarioMessageSnapshotSchema = MessagesSnapshotEventSchema.merge(
+  baseScenarioEventSchema.extend({
+    type: z3.literal("SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */)
+  })
+);
+var scenarioEventSchema = z3.discriminatedUnion("type", [
+  scenarioRunStartedSchema,
+  scenarioRunFinishedSchema,
+  scenarioMessageSnapshotSchema
+]);
+var successSchema = z3.object({ success: z3.boolean() });
+var errorSchema = z3.object({ error: z3.string() });
+var stateSchema = z3.object({
+  state: z3.object({
+    messages: z3.array(z3.any()),
+    status: z3.string()
+  })
+});
+var runsSchema = z3.object({ runs: z3.array(z3.string()) });
+var eventsSchema = z3.object({ events: z3.array(scenarioEventSchema) });
+
+// src/events/event-reporter.ts
+var EventReporter = class {
+  apiKey;
+  eventsEndpoint;
+  eventAlertMessageLogger;
+  logger = new Logger("scenario.events.EventReporter");
+  constructor(config2) {
+    this.apiKey = config2.apiKey ?? "";
+    this.eventsEndpoint = new URL("/api/scenario-events", config2.endpoint);
+    this.eventAlertMessageLogger = new EventAlertMessageLogger();
+    this.eventAlertMessageLogger.handleGreeting();
+  }
+  /**
+   * Posts an event to the configured endpoint.
+   * Logs success/failure but doesn't throw - event posting shouldn't break scenario execution.
+   */
+  async postEvent(event) {
+    const result = {};
+    this.logger.debug(`[${event.type}] Posting event`, { event });
+    const processedEvent = this.processEventForApi(event);
+    try {
+      const response = await fetch(this.eventsEndpoint.href, {
+        method: "POST",
+        body: JSON.stringify(processedEvent),
+        headers: {
+          "Content-Type": "application/json",
+          "X-Auth-Token": this.apiKey
+        }
+      });
+      this.logger.debug(
+        `[${event.type}] Event POST response status: ${response.status}`
+      );
+      if (response.ok) {
+        const data = await response.json();
+        this.logger.debug(`[${event.type}] Event POST response:`, data);
+        result.setUrl = data.url;
+      } else {
+        const errorText = await response.text();
+        this.logger.error(`[${event.type}] Event POST failed:`, {
+          status: response.status,
+          statusText: response.statusText,
+          error: errorText,
+          event: JSON.stringify(processedEvent)
+        });
+      }
+    } catch (error) {
+      this.logger.error(`[${event.type}] Event POST error:`, {
+        error,
+        event: JSON.stringify(processedEvent),
+        endpoint: this.eventsEndpoint.href
+      });
+    }
+    return result;
+  }
+  /**
+   * Processes event data to ensure API compatibility.
+   * Converts message content objects to strings when needed.
+   */
+  processEventForApi(event) {
+    if (event.type === "SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */) {
+      return {
+        ...event,
+        messages: event.messages.map((message) => ({
+          ...message,
+          content: typeof message.content !== "string" ? JSON.stringify(message.content) : message.content
+        }))
+      };
+    }
+    return event;
+  }
+};
+
+// src/events/event-bus.ts
+var EventBus = class _EventBus {
+  static registry = /* @__PURE__ */ new Set();
+  events$ = new Subject();
+  eventReporter;
+  eventAlertMessageLogger;
+  processingPromise = null;
+  logger = new Logger("scenario.events.EventBus");
+  static globalListeners = [];
+  constructor(config2) {
+    this.eventReporter = new EventReporter(config2);
+    this.eventAlertMessageLogger = new EventAlertMessageLogger();
+    _EventBus.registry.add(this);
+    for (const listener of _EventBus.globalListeners) {
+      listener(this);
+    }
+  }
+  static getAllBuses() {
+    return _EventBus.registry;
+  }
+  static addGlobalListener(listener) {
+    _EventBus.globalListeners.push(listener);
+  }
+  /**
+   * Publishes an event into the processing pipeline.
+   */
+  publish(event) {
+    this.logger.debug(`[${event.type}] Publishing event`, {
+      event
+    });
+    this.events$.next(event);
+  }
+  /**
+   * Begins listening for and processing events.
+   * Returns a promise that resolves when a RUN_FINISHED event is fully processed.
+   */
+  listen() {
+    this.logger.debug("Listening for events");
+    if (this.processingPromise) {
+      return this.processingPromise;
+    }
+    this.processingPromise = new Promise((resolve, reject) => {
+      this.events$.pipe(
+        // Post events and get results
+        concatMap(async (event) => {
+          this.logger.debug(`[${event.type}] Processing event`, { event });
+          const result = await this.eventReporter.postEvent(event);
+          return { event, result };
+        }),
+        // Handle watch messages reactively
+        tap(({ event, result }) => {
+          if (event.type === "SCENARIO_RUN_STARTED" /* RUN_STARTED */ && result.setUrl) {
+            this.eventAlertMessageLogger.handleWatchMessage({
+              scenarioSetId: event.scenarioSetId,
+              scenarioRunId: event.scenarioRunId,
+              setUrl: result.setUrl
+            });
+          }
+        }),
+        // Extract just the event for downstream processing
+        map(({ event }) => event),
+        catchError((error) => {
+          this.logger.error("Error in event stream:", error);
+          return EMPTY;
+        })
+      ).subscribe({
+        next: (event) => {
+          this.logger.debug(`[${event.type}] Event processed`, { event });
+          if (event.type === "SCENARIO_RUN_FINISHED" /* RUN_FINISHED */) {
+            resolve();
+          }
+        },
+        error: (error) => {
+          this.logger.error("Error in event stream:", error);
+          reject(error);
+        }
+      });
+    });
+    return this.processingPromise;
+  }
+  /**
+   * Stops accepting new events and drains the processing queue.
+   */
+  async drain() {
+    this.logger.debug("Draining event stream");
+    this.events$.complete();
+    if (this.processingPromise) {
+      await this.processingPromise;
+    }
+  }
+  /**
+   * Subscribes to an event stream.
+   * @param source$ - The event stream to subscribe to.
+   */
+  subscribeTo(source$) {
+    this.logger.debug("Subscribing to event stream");
+    return source$.subscribe(this.events$);
+  }
+  /**
+   * Expose the events$ observable for external subscription (read-only).
+   */
+  get eventsObservable() {
+    return this.events$.asObservable();
+  }
+};
+
+export {
+  scenarioProjectConfigSchema,
+  defineConfig,
+  AgentRole,
+  allAgentRoles,
+  AgentAdapter,
+  UserSimulatorAgentAdapter,
+  JudgeAgentAdapter,
+  domain_exports,
+  loadScenarioProjectConfig,
+  Logger,
+  getProjectConfig,
+  generateThreadId,
+  generateScenarioRunId,
+  generateScenarioId,
+  getBatchRunId,
+  generateMessageId,
+  EventBus
+};