@langwatch/scenario 0.2.13 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langwatch/scenario",
-  "version": "0.2.13",
+  "version": "0.4.0",
   "description": "A TypeScript library for testing AI agents using scenarios",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -29,33 +29,36 @@
   },
   "dependencies": {
     "@ag-ui/core": "^0.0.28",
-    "@ai-sdk/openai": "^1.3.22",
-    "ai": ">=4.0.0",
-    "chalk": "^5.4.1",
-    "open": "10.2.0",
+    "@ai-sdk/openai": "^2.0.74",
+    "@openai/agents": "^0.3.3",
+    "ai": "5.0.104",
+    "chalk": "^5.6.2",
+    "langwatch": "0.9.0",
+    "open": "11.0.0",
     "rxjs": "^7.8.2",
     "xksuid": "^0.0.4",
-    "zod": "^3.24.4"
+    "zod": "^3.25.76"
   },
   "devDependencies": {
-    "@eslint/js": "^9.26.0",
-    "@types/jest": "^29.0.0",
-    "@types/node": "^22.15.15",
-    "@typescript-eslint/parser": "^8.32.0",
-    "@typescript/native-preview": "^7.0.0-dev.20250617.1",
-    "dotenv": "^16.5.0",
-    "eslint": "^9.26.0",
-    "eslint-import-resolver-typescript": "^4.3.4",
-    "eslint-plugin-import": "^2.31.0",
-    "eslint-plugin-unused-imports": "^4.1.4",
-    "globals": "^16.1.0",
-    "jest": "^29.0.0",
-    "ts-jest": "^29.0.0",
-    "tsup": "^8.4.0",
-    "tsx": "^4.19.4",
-    "typescript": "^5.0.0",
-    "typescript-eslint": "^8.32.0",
-    "vitest": "^3.2.4"
+    "@eslint/js": "^9.39.1",
+    "@types/jest": "^30.0.0",
+    "@types/node": "^24.10.1",
+    "@typescript-eslint/parser": "^8.48.0",
+    "@typescript/native-preview": "7.0.0-dev.20251128.1",
+    "@vitest/coverage-v8": "4.0.14",
+    "dotenv": "^17.2.3",
+    "eslint": "^9.39.1",
+    "eslint-import-resolver-typescript": "^4.4.4",
+    "eslint-plugin-import": "^2.32.0",
+    "eslint-plugin-unused-imports": "^4.3.0",
+    "globals": "^16.5.0",
+    "jest": "^30.2.0",
+    "ts-jest": "^29.4.5",
+    "tsup": "^8.5.1",
+    "tsx": "^4.20.6",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.48.0",
+    "vitest": "^4.0.14"
   },
   "exports": {
     ".": {
@@ -85,16 +88,29 @@
     }
   },
   "peerDependencies": {
+    "ai": ">=5.0.0",
     "vitest": ">=3.2.4"
   },
   "scripts": {
-    "build": "tsup src/index.ts src/integrations/vitest/*.ts --format cjs,esm --dts --clean --external vitest",
+    "clean": "rm -rf dist *.tgz",
+    "build": "tsup",
     "buildpack": "pnpm run build && pnpm pack",
-    "watch": "pnpm run build -- --watch src",
-    "test": "vitest",
-    "test:ci": "vitest run",
+    "watch": "pnpm run build -- --watch",
+    "typecheck": "tsc --noEmit",
     "lint": "eslint .",
-    "examples:vitest:run": "(cd examples/vitest && pnpm install) && pnpm -F vitest-example run test",
-    "generate:api-reference": "npx typedoc src --out api-reference-docs && rm -rf ../docs/docs/public/reference/javascript/scenario && mv api-reference-docs ../docs/docs/public/reference/javascript/scenario"
+    "format": "eslint . --fix",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:ci": "vitest run --coverage",
+    "generate:api-reference": "npx typedoc src --out api-reference-docs && rm -rf ../docs/docs/public/reference/javascript/scenario && mv api-reference-docs ../docs/docs/public/reference/javascript/scenario",
+    "clean:all": "pnpm -r --parallel exec rm -rf dist *.tgz .cache",
+    "build:all": "pnpm run build",
+    "typecheck:all": "pnpm -r --parallel run typecheck",
+    "lint:all": "pnpm -r --parallel run lint",
+    "format:all": "pnpm -r --parallel run format",
+    "test:all": "pnpm -r --parallel run test",
+    "vitest-examples": "pnpm -F vitest-examples",
+    "openai-realtime-demo": "pnpm -F openai-realtime-demo",
+    "realtime-client": "pnpm -F realtime-client"
   }
 }

package/dist/chunk-6SKQWXT7.mjs

@@ -1,528 +0,0 @@
-import {
-  Logger,
-  getEnv
-} from "./chunk-OL4RFXV4.mjs";
-import {
-  __export
-} from "./chunk-7P6ASYW6.mjs";
-
-// src/domain/core/config.ts
-import { z } from "zod";
-var DEFAULT_TEMPERATURE = 0;
-var scenarioProjectConfigSchema = z.object({
-  defaultModel: z.object({
-    model: z.custom(),
-    temperature: z.number().min(0).max(1).optional().default(DEFAULT_TEMPERATURE),
-    maxTokens: z.number().optional()
-  }).optional(),
-  headless: z.boolean().optional().default(
-    typeof process !== "undefined" ? !["false", "0"].includes(process.env.SCENARIO_HEADLESS || "false") : false
-  )
-}).strict();
-function defineConfig(config2) {
-  return config2;
-}
-
-// src/domain/agents/index.ts
-var AgentRole = /* @__PURE__ */ ((AgentRole2) => {
-  AgentRole2["USER"] = "User";
-  AgentRole2["AGENT"] = "Agent";
-  AgentRole2["JUDGE"] = "Judge";
-  return AgentRole2;
-})(AgentRole || {});
-var allAgentRoles = [
-  "User" /* USER */,
-  "Agent" /* AGENT */,
-  "Judge" /* JUDGE */
-];
-var AgentAdapter = class {
-  role = "Agent" /* AGENT */;
-};
-var UserSimulatorAgentAdapter = class {
-  role = "User" /* USER */;
-};
-var JudgeAgentAdapter = class {
-  role = "Judge" /* JUDGE */;
-};
-
-// src/domain/scenarios/index.ts
-var DEFAULT_MAX_TURNS = 10;
-var DEFAULT_VERBOSE = false;
-
-// src/domain/index.ts
-var domain_exports = {};
-__export(domain_exports, {
-  AgentAdapter: () => AgentAdapter,
-  AgentRole: () => AgentRole,
-  DEFAULT_MAX_TURNS: () => DEFAULT_MAX_TURNS,
-  DEFAULT_TEMPERATURE: () => DEFAULT_TEMPERATURE,
-  DEFAULT_VERBOSE: () => DEFAULT_VERBOSE,
-  JudgeAgentAdapter: () => JudgeAgentAdapter,
-  UserSimulatorAgentAdapter: () => UserSimulatorAgentAdapter,
-  allAgentRoles: () => allAgentRoles,
-  defineConfig: () => defineConfig,
-  scenarioProjectConfigSchema: () => scenarioProjectConfigSchema
-});
-
-// src/events/event-bus.ts
-import {
-  concatMap,
-  EMPTY,
-  catchError,
-  Subject,
-  tap,
-  map
-} from "rxjs";
-
-// src/events/event-alert-message-logger.ts
-import open from "open";
-
-// src/config/load.ts
-import fs from "node:fs/promises";
-import path from "node:path";
-import { pathToFileURL } from "node:url";
-async function loadScenarioProjectConfig() {
-  const cwd = process.cwd();
-  const configNames = [
-    "scenario.config.js",
-    "scenario.config.mjs"
-  ];
-  for (const name of configNames) {
-    const fullPath = path.join(cwd, name);
-    try {
-      await fs.access(fullPath);
-      const configModule = await import(pathToFileURL(fullPath).href);
-      const config2 = configModule.default || configModule;
-      const parsed = scenarioProjectConfigSchema.safeParse(config2);
-      if (!parsed.success) {
-        throw new Error(
-          `Invalid config file ${name}: ${JSON.stringify(parsed.error.format(), null, 2)}`
-        );
-      }
-      return parsed.data;
-    } catch (error) {
-      if (error instanceof Error && "code" in error && error.code === "ENOENT") {
-        continue;
-      }
-      throw error;
-    }
-  }
-  return await scenarioProjectConfigSchema.parseAsync({});
-}
-
-// src/config/get-project-config.ts
-var logger = new Logger("scenario.config");
-var configLoaded = false;
-var config = null;
-var configLoadPromise = null;
-async function loadProjectConfig() {
-  if (configLoaded) {
-    return;
-  }
-  if (configLoadPromise) {
-    return configLoadPromise;
-  }
-  configLoadPromise = (async () => {
-    try {
-      config = await loadScenarioProjectConfig();
-      logger.debug("loaded scenario project config", { config });
-    } catch (error) {
-      logger.error("error loading scenario project config", { error });
-    } finally {
-      configLoaded = true;
-    }
-  })();
-  return configLoadPromise;
-}
-async function getProjectConfig() {
-  await loadProjectConfig();
-  return config;
-}
-
-// src/utils/ids.ts
-import crypto from "node:crypto";
-import process2 from "node:process";
-import { generate, parse } from "xksuid";
-var batchRunId;
-function generateThreadId() {
-  return `scenariothread_${generate()}`;
-}
-function generateScenarioRunId() {
-  return `scenariorun_${generate()}`;
-}
-function generateScenarioId() {
-  return `scenario_${generate()}`;
-}
-function getBatchRunId() {
-  if (batchRunId) {
-    return batchRunId;
-  }
-  if (process2.env.SCENARIO_BATCH_RUN_ID) {
-    return batchRunId = process2.env.SCENARIO_BATCH_RUN_ID;
-  }
-  if (process2.env.VITEST_WORKER_ID || process2.env.JEST_WORKER_ID) {
-    const parentProcessId = process2.ppid;
-    const now = /* @__PURE__ */ new Date();
-    const year = now.getUTCFullYear();
-    const week = String(getISOWeekNumber(now)).padStart(2, "0");
-    const raw = `${parentProcessId}_${year}_w${week}`;
-    const hash = crypto.createHash("sha256").update(raw).digest("hex").slice(0, 12);
-    return batchRunId = `scenariobatch_${hash}`;
-  }
-  return batchRunId = `scenariobatch_${generate()}`;
-}
-function getISOWeekNumber(date) {
-  const tmp = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
-  const dayNum = tmp.getUTCDay() || 7;
-  tmp.setUTCDate(tmp.getUTCDate() + 4 - dayNum);
-  const yearStart = new Date(Date.UTC(tmp.getUTCFullYear(), 0, 1));
-  const weekNo = Math.ceil(((tmp.getTime() - yearStart.getTime()) / 864e5 + 1) / 7);
-  return weekNo;
-}
-function generateMessageId() {
-  return `scenariomsg_${generate()}`;
-}
-
-// src/events/event-alert-message-logger.ts
-var EventAlertMessageLogger = class _EventAlertMessageLogger {
-  static shownBatchIds = /* @__PURE__ */ new Set();
-  /**
-   * Shows a fancy greeting message about simulation reporting status.
-   * Only shows once per batch run to avoid spam.
-   */
-  handleGreeting() {
-    if (this.isGreetingDisabled()) {
-      return;
-    }
-    if (_EventAlertMessageLogger.shownBatchIds.has(getBatchRunId())) {
-      return;
-    }
-    _EventAlertMessageLogger.shownBatchIds.add(getBatchRunId());
-    this.displayGreeting();
-  }
-  /**
-   * Shows a fancy message about how to watch the simulation.
-   * Called when a run started event is received with a session ID.
-   */
-  async handleWatchMessage(params) {
-    if (this.isGreetingDisabled()) {
-      return;
-    }
-    await this.displayWatchMessage(params);
-  }
-  isGreetingDisabled() {
-    return getEnv().SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
-  }
-  displayGreeting() {
-    const separator = "\u2500".repeat(60);
-    const env = getEnv();
-    if (!env.LANGWATCH_API_KEY) {
-      console.log(`
-${separator}`);
-      console.log("\u{1F3AD}  Running Scenario Tests");
-      console.log(`${separator}`);
-      console.log("\u27A1\uFE0F  LangWatch API key not configured");
-      console.log("   Simulations will only output final results");
-      console.log("");
-      console.log("\u{1F4A1} To visualize conversations in real time:");
-      console.log("   \u2022 Set LANGWATCH_API_KEY environment variable");
-      console.log("   \u2022 Or configure apiKey in scenario.config.js");
-      console.log("");
-      console.log(`${separator}
-`);
-    }
-  }
-  async displayWatchMessage(params) {
-    const separator = "\u2500".repeat(60);
-    const setUrl = params.setUrl;
-    const batchUrl = `${setUrl}/${getBatchRunId()}`;
-    console.log(`
-${separator}`);
-    console.log("\u{1F3AD}  Running Scenario Tests");
-    console.log(`${separator}`);
-    console.log(`Follow it live: ${batchUrl}`);
-    console.log(`${separator}
-`);
-    const projectConfig = await getProjectConfig();
-    if (!(projectConfig == null ? void 0 : projectConfig.headless)) {
-      try {
-        open(batchUrl);
-      } catch (_) {
-      }
-    }
-  }
-};
-
-// src/events/schema.ts
-import { EventType, MessagesSnapshotEventSchema } from "@ag-ui/core";
-import { z as z2 } from "zod";
-var Verdict = /* @__PURE__ */ ((Verdict2) => {
-  Verdict2["SUCCESS"] = "success";
-  Verdict2["FAILURE"] = "failure";
-  Verdict2["INCONCLUSIVE"] = "inconclusive";
-  return Verdict2;
-})(Verdict || {});
-var ScenarioRunStatus = /* @__PURE__ */ ((ScenarioRunStatus2) => {
-  ScenarioRunStatus2["SUCCESS"] = "SUCCESS";
-  ScenarioRunStatus2["ERROR"] = "ERROR";
-  ScenarioRunStatus2["CANCELLED"] = "CANCELLED";
-  ScenarioRunStatus2["IN_PROGRESS"] = "IN_PROGRESS";
-  ScenarioRunStatus2["PENDING"] = "PENDING";
-  ScenarioRunStatus2["FAILED"] = "FAILED";
-  return ScenarioRunStatus2;
-})(ScenarioRunStatus || {});
-var baseEventSchema = z2.object({
-  type: z2.nativeEnum(EventType),
-  timestamp: z2.number(),
-  rawEvent: z2.any().optional()
-});
-var batchRunIdSchema = z2.string();
-var scenarioRunIdSchema = z2.string();
-var scenarioIdSchema = z2.string();
-var baseScenarioEventSchema = baseEventSchema.extend({
-  batchRunId: batchRunIdSchema,
-  scenarioId: scenarioIdSchema,
-  scenarioRunId: scenarioRunIdSchema,
-  scenarioSetId: z2.string().optional().default("default")
-});
-var scenarioRunStartedSchema = baseScenarioEventSchema.extend({
-  type: z2.literal("SCENARIO_RUN_STARTED" /* RUN_STARTED */),
-  metadata: z2.object({
-    name: z2.string().optional(),
-    description: z2.string().optional()
-  })
-});
-var scenarioResultsSchema = z2.object({
-  verdict: z2.nativeEnum(Verdict),
-  reasoning: z2.string().optional(),
-  metCriteria: z2.array(z2.string()),
-  unmetCriteria: z2.array(z2.string()),
-  error: z2.string().optional()
-});
-var scenarioRunFinishedSchema = baseScenarioEventSchema.extend({
-  type: z2.literal("SCENARIO_RUN_FINISHED" /* RUN_FINISHED */),
-  status: z2.nativeEnum(ScenarioRunStatus),
-  results: scenarioResultsSchema.optional().nullable()
-});
-var scenarioMessageSnapshotSchema = MessagesSnapshotEventSchema.merge(
-  baseScenarioEventSchema.extend({
-    type: z2.literal("SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */)
-  })
-);
-var scenarioEventSchema = z2.discriminatedUnion("type", [
-  scenarioRunStartedSchema,
-  scenarioRunFinishedSchema,
-  scenarioMessageSnapshotSchema
-]);
-var successSchema = z2.object({ success: z2.boolean() });
-var errorSchema = z2.object({ error: z2.string() });
-var stateSchema = z2.object({
-  state: z2.object({
-    messages: z2.array(z2.any()),
-    status: z2.string()
-  })
-});
-var runsSchema = z2.object({ runs: z2.array(z2.string()) });
-var eventsSchema = z2.object({ events: z2.array(scenarioEventSchema) });
-
-// src/events/event-reporter.ts
-var EventReporter = class {
-  apiKey;
-  eventsEndpoint;
-  eventAlertMessageLogger;
-  logger = new Logger("scenario.events.EventReporter");
-  isEnabled;
-  constructor(config2) {
-    this.apiKey = config2.apiKey ?? "";
-    this.eventsEndpoint = new URL("/api/scenario-events", config2.endpoint);
-    this.eventAlertMessageLogger = new EventAlertMessageLogger();
-    this.eventAlertMessageLogger.handleGreeting();
-    this.isEnabled = this.apiKey.length > 0 && this.eventsEndpoint.href.length > 0;
-  }
-  /**
-   * Posts an event to the configured endpoint.
-   * Logs success/failure but doesn't throw - event posting shouldn't break scenario execution.
-   */
-  async postEvent(event) {
-    if (!this.isEnabled) return {};
-    const result = {};
-    this.logger.debug(`[${event.type}] Posting event`, { event });
-    const processedEvent = this.processEventForApi(event);
-    try {
-      const response = await fetch(this.eventsEndpoint.href, {
-        method: "POST",
-        body: JSON.stringify(processedEvent),
-        headers: {
-          "Content-Type": "application/json",
-          "X-Auth-Token": this.apiKey
-        }
-      });
-      this.logger.debug(
-        `[${event.type}] Event POST response status: ${response.status}`
-      );
-      if (response.ok) {
-        const data = await response.json();
-        this.logger.debug(`[${event.type}] Event POST response:`, data);
-        result.setUrl = data.url;
-      } else {
-        const errorText = await response.text();
-        this.logger.error(`[${event.type}] Event POST failed:`, {
-          status: response.status,
-          statusText: response.statusText,
-          error: errorText,
-          event: JSON.stringify(processedEvent)
-        });
-      }
-    } catch (error) {
-      this.logger.error(`[${event.type}] Event POST error:`, {
-        error,
-        event: JSON.stringify(processedEvent),
-        endpoint: this.eventsEndpoint.href
-      });
-    }
-    return result;
-  }
-  /**
-   * Processes event data to ensure API compatibility.
-   * Converts message content objects to strings when needed.
-   */
-  processEventForApi(event) {
-    if (event.type === "SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */) {
-      return {
-        ...event,
-        messages: event.messages.map((message) => ({
-          ...message,
-          content: typeof message.content !== "string" ? JSON.stringify(message.content) : message.content
-        }))
-      };
-    }
-    return event;
-  }
-};
-
-// src/events/event-bus.ts
-var EventBus = class _EventBus {
-  static registry = /* @__PURE__ */ new Set();
-  events$ = new Subject();
-  eventReporter;
-  eventAlertMessageLogger;
-  processingPromise = null;
-  logger = new Logger("scenario.events.EventBus");
-  static globalListeners = [];
-  constructor(config2) {
-    this.eventReporter = new EventReporter(config2);
-    this.eventAlertMessageLogger = new EventAlertMessageLogger();
-    _EventBus.registry.add(this);
-    for (const listener of _EventBus.globalListeners) {
-      listener(this);
-    }
-  }
-  static getAllBuses() {
-    return _EventBus.registry;
-  }
-  static addGlobalListener(listener) {
-    _EventBus.globalListeners.push(listener);
-  }
-  /**
-   * Publishes an event into the processing pipeline.
-   */
-  publish(event) {
-    this.logger.debug(`[${event.type}] Publishing event`, {
-      event
-    });
-    this.events$.next(event);
-  }
-  /**
-   * Begins listening for and processing events.
-   * Returns a promise that resolves when a RUN_FINISHED event is fully processed.
-   */
-  listen() {
-    this.logger.debug("Listening for events");
-    if (this.processingPromise) {
-      return this.processingPromise;
-    }
-    this.processingPromise = new Promise((resolve, reject) => {
-      this.events$.pipe(
-        // Post events and get results
-        concatMap(async (event) => {
-          this.logger.debug(`[${event.type}] Processing event`, { event });
-          const result = await this.eventReporter.postEvent(event);
-          return { event, result };
-        }),
-        // Handle watch messages reactively
-        tap(async ({ event, result }) => {
-          if (event.type === "SCENARIO_RUN_STARTED" /* RUN_STARTED */ && result.setUrl) {
-            await this.eventAlertMessageLogger.handleWatchMessage({
-              scenarioSetId: event.scenarioSetId,
-              scenarioRunId: event.scenarioRunId,
-              setUrl: result.setUrl
-            });
-          }
-        }),
-        // Extract just the event for downstream processing
-        map(({ event }) => event),
-        catchError((error) => {
-          this.logger.error("Error in event stream:", error);
-          return EMPTY;
-        })
-      ).subscribe({
-        next: (event) => {
-          this.logger.debug(`[${event.type}] Event processed`, { event });
-          if (event.type === "SCENARIO_RUN_FINISHED" /* RUN_FINISHED */) {
-            resolve();
-          }
-        },
-        error: (error) => {
-          this.logger.error("Error in event stream:", error);
-          reject(error);
-        }
-      });
-    });
-    return this.processingPromise;
-  }
-  /**
-   * Stops accepting new events and drains the processing queue.
-   */
-  async drain() {
-    this.logger.debug("Draining event stream");
-    this.events$.complete();
-    if (this.processingPromise) {
-      await this.processingPromise;
-    }
-  }
-  /**
-   * Subscribes to an event stream.
-   * @param source$ - The event stream to subscribe to.
-   */
-  subscribeTo(source$) {
-    this.logger.debug("Subscribing to event stream");
-    return source$.subscribe(this.events$);
-  }
-  /**
-   * Expose the events$ observable for external subscription (read-only).
-   */
-  get eventsObservable() {
-    return this.events$.asObservable();
-  }
-};
-
-export {
-  DEFAULT_TEMPERATURE,
-  scenarioProjectConfigSchema,
-  defineConfig,
-  AgentRole,
-  allAgentRoles,
-  AgentAdapter,
-  UserSimulatorAgentAdapter,
-  JudgeAgentAdapter,
-  DEFAULT_MAX_TURNS,
-  DEFAULT_VERBOSE,
-  domain_exports,
-  getProjectConfig,
-  generateThreadId,
-  generateScenarioRunId,
-  generateScenarioId,
-  getBatchRunId,
-  generateMessageId,
-  EventBus
-};

package/dist/chunk-7P6ASYW6.mjs

@@ -1,9 +0,0 @@
-var __defProp = Object.defineProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-
-export {
-  __export
-};