@langwatch/scenario 0.2.9 → 0.2.12

package/dist/integrations/vitest/reporter.mjs

@@ -1,8 +1,186 @@
 import {
-  VitestReporter
-} from "../../chunk-K7KLHTDI.mjs";
-import "../../chunk-YPJZSK4J.mjs";
+  Logger
+} from "../../chunk-OL4RFXV4.mjs";
 import "../../chunk-7P6ASYW6.mjs";
+
+// src/integrations/vitest/reporter.ts
+import fs from "fs";
+import path from "path";
+import chalk from "chalk";
+var logger = Logger.create("integrations:vitest:reporter");
+function getProjectRoot() {
+  return process.cwd();
+}
+var projectRoot = getProjectRoot();
+var logDir = path.join(projectRoot, ".scenario");
+if (!fs.existsSync(logDir)) fs.mkdirSync(logDir);
+function getLogFilePath(testId) {
+  return path.join(logDir, `${testId}.log`);
+}
+function getFullTestName(task) {
+  let name = task.name;
+  let parent = task.suite;
+  while (parent) {
+    name = `${parent.name} > ${name}`;
+    parent = parent.suite;
+  }
+  return name;
+}
+function indent(str, n = 2) {
+  return str.replace(/^/gm, " ".repeat(n));
+}
+var VitestReporter = class {
+  results = [];
+  async onTestCaseResult(test) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
+    const fullName = getFullTestName(test);
+    const filePath = getLogFilePath(test.id);
+    if (!fs.existsSync(filePath)) {
+      logger.warn(
+        `No log file found ${filePath} for test ${fullName}`,
+        test.id
+      );
+      return;
+    }
+    const lines = fs.readFileSync(filePath, "utf-8").split("\n").filter(Boolean);
+    const events = lines.map((line) => JSON.parse(line));
+    const runs = /* @__PURE__ */ new Map();
+    for (const event of events) {
+      const runId = event.scenarioRunId ?? "unknown";
+      if (!runs.has(runId)) runs.set(runId, []);
+      runs.get(runId).push(event);
+    }
+    for (const [runId, runEvents] of Array.from(runs.entries())) {
+      const started = runEvents.find(
+        (e) => e.type === "SCENARIO_RUN_STARTED"
+      );
+      const finished = runEvents.find(
+        (e) => e.type === "SCENARIO_RUN_FINISHED"
+      );
+      const messages = runEvents.filter(
+        (e) => e.type === "SCENARIO_MESSAGE_SNAPSHOT"
+      );
+      this.results.push({
+        name: ((_a = started == null ? void 0 : started.metadata) == null ? void 0 : _a.name) ?? fullName,
+        status: (finished == null ? void 0 : finished.status) ?? "UNKNOWN",
+        duration: started && finished ? finished.timestamp - started.timestamp : 0,
+        reasoning: (_b = finished == null ? void 0 : finished.results) == null ? void 0 : _b.reasoning,
+        criteria: (finished == null ? void 0 : finished.results) ? `Success Criteria: ${((_c = finished.results.metCriteria) == null ? void 0 : _c.length) ?? 0}/${(((_d = finished.results.metCriteria) == null ? void 0 : _d.length) ?? 0) + (((_e = finished.results.unmetCriteria) == null ? void 0 : _e.length) ?? 0)}` : void 0
+      });
+      console.log(
+        `
+--- Scenario Run: ${((_f = started == null ? void 0 : started.metadata) == null ? void 0 : _f.name) ?? runId} ---`
+      );
+      if (started) {
+        console.log(`Description: ${((_g = started.metadata) == null ? void 0 : _g.description) ?? ""}`);
+      }
+      if (messages.length) {
+        console.log("Chat log:\n");
+        let lastMessageCount = 0;
+        for (const msg of messages) {
+          const allMessages = msg.messages ?? [];
+          for (const m of allMessages.slice(lastMessageCount)) {
+            const role = m.role;
+            if (role.toLowerCase() === "assistant" && "toolCalls" in m && Array.isArray(m.toolCalls) && m.toolCalls.length > 0) {
+              for (const toolCall of m.toolCalls) {
+                const functionName = toolCall.function.name;
+                let parsedJson = "";
+                try {
+                  parsedJson = JSON.stringify(
+                    JSON.parse(toolCall.function.arguments),
+                    null,
+                    2
+                  );
+                } catch {
+                  parsedJson = toolCall.function.arguments;
+                }
+                const role2 = chalk.magenta(`ToolCall(${functionName}):`);
+                console.log(`${role2}:
+
+${indent(parsedJson)}
+`);
+              }
+              continue;
+            }
+            let roleLabel = role;
+            if (role.toLowerCase() === "user") roleLabel = chalk.green("User");
+            else if (role.toLowerCase() === "agent")
+              roleLabel = chalk.cyan("Agent");
+            else if (role.toLowerCase() === "assistant")
+              if (Array.isArray(m.content) && typeof m.content.at(0) === "object" && ((_h = m.content.at(0)) == null ? void 0 : _h.type) === "tool-call")
+                roleLabel = chalk.cyan("ToolCall");
+              else roleLabel = chalk.cyan("Assistant");
+            else if (role.toLowerCase() === "tool") {
+              roleLabel = chalk.magenta("ToolResult");
+              let parsedJson = "";
+              try {
+                parsedJson = JSON.stringify(JSON.parse(m.content), null, 2);
+              } catch {
+                parsedJson = m.content;
+              }
+              console.log(`${roleLabel}:
+
+${indent(parsedJson)}
+`);
+              continue;
+            } else roleLabel = chalk.yellow(role);
+            console.log(`${roleLabel}: ${m.content}`);
+          }
+          lastMessageCount = allMessages.length;
+        }
+      }
+      if (finished) {
+        console.log("--- Verdict ---");
+        console.log(`Status: ${finished.status}`);
+        if (finished.results) {
+          console.log(`Verdict: ${finished.results.verdict}`);
+          if (finished.results.reasoning)
+            console.log(`Reasoning: ${finished.results.reasoning}`);
+          if ((_i = finished.results.metCriteria) == null ? void 0 : _i.length)
+            console.log(
+              `Met criteria: ${finished.results.metCriteria.join(", ")}`
+            );
+          if ((_j = finished.results.unmetCriteria) == null ? void 0 : _j.length)
+            console.log(
+              `Unmet criteria: ${finished.results.unmetCriteria.join(", ")}`
+            );
+          if (finished.results.error)
+            console.log(`Error: ${finished.results.error}`);
+        }
+      }
+      console.log("-----------------------------\n");
+    }
+    fs.unlinkSync(filePath);
+  }
+  async onTestRunEnd() {
+    if (this.results.length === 0) return;
+    const total = this.results.length;
+    const passed = this.results.filter((r) => r.status === "SUCCESS").length;
+    const failed = this.results.filter((r) => r.status !== "SUCCESS").length;
+    const successRate = (passed / total * 100).toFixed(1);
+    console.log();
+    console.log(chalk.bold.cyan("=== Scenario Test Report ==="));
+    console.log(`Total Scenarios: ${total}`);
+    console.log(chalk.green(`Passed: ${passed}`));
+    console.log(chalk.red(`Failed: ${failed}`));
+    console.log(`Success Rate: ${chalk.bold(`${successRate}%`)}`);
+    this.results.forEach((r, i) => {
+      const statusColor = r.status === "SUCCESS" ? chalk.green : chalk.red;
+      console.log();
+      console.log(
+        `${i + 1}. ${r.name} - ${statusColor(r.status)} in ${(r.duration / 1e3).toFixed(2)}s`
+      );
+      if (r.reasoning) {
+        console.log(chalk.greenBright("  Reasoning: ") + r.reasoning);
+      }
+      if (r.criteria) {
+        console.log(chalk.bold("  " + r.criteria));
+      }
+    });
+    console.log();
+  }
+};
+var reporter_default = VitestReporter;
 export {
-  VitestReporter as default
+  reporter_default as default
 };

package/dist/integrations/vitest/setup.js

@@ -41,6 +41,7 @@ var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
   LogLevel2["DEBUG"] = "DEBUG";
   return LogLevel2;
 })(LogLevel || {});
+var LOG_LEVELS = Object.values(LogLevel);
 
 // src/config/env.ts
 var envSchema = import_zod.z.object({
@@ -53,7 +54,7 @@ var envSchema = import_zod.z.object({
    * LangWatch endpoint URL for event reporting.
    * Defaults to the production LangWatch endpoint.
    */
-  LANGWATCH_ENDPOINT: import_zod.z.string().url().default("https://app.langwatch.ai"),
+  LANGWATCH_ENDPOINT: import_zod.z.string().url().optional().default("https://app.langwatch.ai"),
   /**
    * Disables simulation report info messages when set to any truthy value.
    * Useful for CI/CD environments or when you want cleaner output.
@@ -65,17 +66,19 @@ var envSchema = import_zod.z.object({
    */
   NODE_ENV: import_zod.z.enum(["development", "production", "test"]).default("development"),
   /**
-   * Log level for the scenario package.
+   * Case-insensitive log level for the scenario package.
    * Defaults to 'info' if not specified.
    */
-  LOG_LEVEL: import_zod.z.nativeEnum(LogLevel).optional(),
+  LOG_LEVEL: import_zod.z.string().toUpperCase().pipe(import_zod.z.nativeEnum(LogLevel)).optional().default("INFO" /* INFO */),
   /**
    * Scenario batch run ID.
    * If not provided, a random ID will be generated.
    */
   SCENARIO_BATCH_RUN_ID: import_zod.z.string().optional()
 });
-var env = envSchema.parse(process.env);
+function getEnv() {
+  return envSchema.parse(process.env);
+}
 
 // src/config/load.ts
 var import_promises = __toESM(require("fs/promises"));
@@ -104,18 +107,27 @@ var Logger = class _Logger {
   static create(context) {
     return new _Logger(context);
   }
-  getLogLevel() {
-    return env.LOG_LEVEL ?? "INFO" /* INFO */;
+  /**
+   * Returns the current log level from environment.
+   * Uses a getter for clarity and idiomatic usage.
+   */
+  get LOG_LEVEL() {
+    return getEnv().LOG_LEVEL;
   }
-  getLogLevelIndex(level) {
-    return Object.values(LogLevel).indexOf(level);
+  /**
+   * Returns the index of the given log level in the LOG_LEVELS array.
+   * @param level - The log level to get the index for.
+   * @returns The index of the log level in the LOG_LEVELS array.
+   */
+  getLogLevelIndexFor(level) {
+    return LOG_LEVELS.indexOf(level);
   }
   /**
    * Checks if logging should occur based on LOG_LEVEL env var
    */
   shouldLog(level) {
-    const currentLevelIndex = this.getLogLevelIndex(this.getLogLevel());
-    const requestedLevelIndex = this.getLogLevelIndex(level);
+    const currentLevelIndex = this.getLogLevelIndexFor(this.LOG_LEVEL);
+    const requestedLevelIndex = this.getLogLevelIndexFor(level);
     return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
   }
   formatMessage(message) {
@@ -176,7 +188,6 @@ function getBatchRunId() {
     return batchRunId;
   }
   if (import_node_process.default.env.SCENARIO_BATCH_RUN_ID) {
-    console.log("process.env.SCENARIO_BATCH_RUN_ID", import_node_process.default.env.SCENARIO_BATCH_RUN_ID);
     return batchRunId = import_node_process.default.env.SCENARIO_BATCH_RUN_ID;
   }
   if (import_node_process.default.env.VITEST_WORKER_ID || import_node_process.default.env.JEST_WORKER_ID) {
@@ -227,10 +238,11 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
     this.displayWatchMessage(params);
   }
   isGreetingDisabled() {
-    return env.SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
+    return getEnv().SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
   }
   displayGreeting() {
     const separator = "\u2500".repeat(60);
+    const env = getEnv();
     if (!env.LANGWATCH_API_KEY) {
       console.log(`
 ${separator}`);

package/dist/integrations/vitest/setup.mjs

@@ -1,9 +1,9 @@
 import {
   EventBus
-} from "../../chunk-7H6OGEQ5.mjs";
+} from "../../chunk-7HLDX5EL.mjs";
 import {
   Logger
-} from "../../chunk-YPJZSK4J.mjs";
+} from "../../chunk-OL4RFXV4.mjs";
 import "../../chunk-7P6ASYW6.mjs";
 
 // src/integrations/vitest/setup.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langwatch/scenario",
-  "version": "0.2.9",
+  "version": "0.2.12",
   "description": "A TypeScript library for testing AI agents using scenarios",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -28,34 +28,33 @@
     "pnpm": ">=8"
   },
   "dependencies": {
-    "@ag-ui/core": "0.0.28",
-    "@ai-sdk/openai": "1.3.22",
+    "@ag-ui/core": "^0.0.28",
+    "@ai-sdk/openai": "^1.3.22",
     "ai": ">=4.0.0",
-    "chalk": "5.4.1",
-    "rxjs": "7.8.2",
-    "stringify": "5.2.0",
-    "xksuid": "0.0.4",
-    "zod": "3.24.4"
+    "chalk": "^5.4.1",
+    "rxjs": "^7.8.2",
+    "xksuid": "^0.0.4",
+    "zod": "^3.24.4"
   },
   "devDependencies": {
-    "@eslint/js": "9.26.0",
+    "@eslint/js": "^9.26.0",
     "@types/jest": "^29.0.0",
-    "@types/node": "22.15.15",
-    "@typescript-eslint/parser": "8.32.0",
-    "@typescript/native-preview": "7.0.0-dev.20250617.1",
-    "dotenv": "16.5.0",
-    "eslint": "9.26.0",
-    "eslint-import-resolver-typescript": "4.3.4",
-    "eslint-plugin-import": "2.31.0",
-    "eslint-plugin-unused-imports": "4.1.4",
-    "globals": "16.1.0",
+    "@types/node": "^22.15.15",
+    "@typescript-eslint/parser": "^8.32.0",
+    "@typescript/native-preview": "^7.0.0-dev.20250617.1",
+    "dotenv": "^16.5.0",
+    "eslint": "^9.26.0",
+    "eslint-import-resolver-typescript": "^4.3.4",
+    "eslint-plugin-import": "^2.31.0",
+    "eslint-plugin-unused-imports": "^4.1.4",
+    "globals": "^16.1.0",
     "jest": "^29.0.0",
     "ts-jest": "^29.0.0",
-    "tsup": "8.4.0",
-    "tsx": "4.19.4",
+    "tsup": "^8.4.0",
+    "tsx": "^4.19.4",
     "typescript": "^5.0.0",
-    "typescript-eslint": "8.32.0",
-    "vitest": "3.2.4"
+    "typescript-eslint": "^8.32.0",
+    "vitest": "^3.2.4"
   },
   "exports": {
     ".": {

package/dist/chunk-K7KLHTDI.mjs

@@ -1,146 +0,0 @@
-import {
-  Logger
-} from "./chunk-YPJZSK4J.mjs";
-
-// src/integrations/vitest/reporter.ts
-import fs from "fs";
-import path from "path";
-import chalk from "chalk";
-var logger = Logger.create("integrations:vitest:reporter");
-function getProjectRoot() {
-  return process.cwd();
-}
-var projectRoot = getProjectRoot();
-var logDir = path.join(projectRoot, ".scenario");
-if (!fs.existsSync(logDir)) fs.mkdirSync(logDir);
-function getLogFilePath(testId) {
-  return path.join(logDir, `${testId}.log`);
-}
-function getFullTestName(task) {
-  let name = task.name;
-  let parent = task.suite;
-  while (parent) {
-    name = `${parent.name} > ${name}`;
-    parent = parent.suite;
-  }
-  return name;
-}
-var VitestReporter = class {
-  results = [];
-  async onTestCaseResult(test) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _i;
-    const fullName = getFullTestName(test);
-    const filePath = getLogFilePath(test.id);
-    if (!fs.existsSync(filePath)) {
-      logger.warn(
-        `No log file found ${filePath} for test ${fullName}`,
-        test.id
-      );
-      return;
-    }
-    const lines = fs.readFileSync(filePath, "utf-8").split("\n").filter(Boolean);
-    const events = lines.map((line) => JSON.parse(line));
-    const runs = /* @__PURE__ */ new Map();
-    for (const event of events) {
-      const runId = event.scenarioRunId ?? "unknown";
-      if (!runs.has(runId)) runs.set(runId, []);
-      runs.get(runId).push(event);
-    }
-    for (const [runId, runEvents] of Array.from(runs.entries())) {
-      const started = runEvents.find(
-        (e) => e.type === "SCENARIO_RUN_STARTED"
-      );
-      const finished = runEvents.find(
-        (e) => e.type === "SCENARIO_RUN_FINISHED"
-      );
-      const messages = runEvents.filter(
-        (e) => e.type === "SCENARIO_MESSAGE_SNAPSHOT"
-      );
-      this.results.push({
-        name: ((_a = started == null ? void 0 : started.metadata) == null ? void 0 : _a.name) ?? fullName,
-        status: (finished == null ? void 0 : finished.status) ?? "UNKNOWN",
-        duration: started && finished ? finished.timestamp - started.timestamp : 0,
-        reasoning: (_b = finished == null ? void 0 : finished.results) == null ? void 0 : _b.reasoning,
-        criteria: (finished == null ? void 0 : finished.results) ? `Success Criteria: ${((_c = finished.results.metCriteria) == null ? void 0 : _c.length) ?? 0}/${(((_d = finished.results.metCriteria) == null ? void 0 : _d.length) ?? 0) + (((_e = finished.results.unmetCriteria) == null ? void 0 : _e.length) ?? 0)}` : void 0
-      });
-      console.log(
-        `
---- Scenario Run: ${((_f = started == null ? void 0 : started.metadata) == null ? void 0 : _f.name) ?? runId} ---`
-      );
-      if (started) {
-        console.log(`Description: ${((_g = started.metadata) == null ? void 0 : _g.description) ?? ""}`);
-      }
-      if (messages.length) {
-        console.log("Chat log:");
-        let lastMessageCount = 0;
-        for (const msg of messages) {
-          const allMessages = msg.messages ?? [];
-          for (const m of allMessages.slice(lastMessageCount)) {
-            const role = m.role;
-            let roleLabel = role;
-            if (role.toLowerCase() === "user") roleLabel = chalk.green("User");
-            else if (role.toLowerCase() === "agent")
-              roleLabel = chalk.cyan("Agent");
-            else if (role.toLowerCase() === "assistant")
-              roleLabel = chalk.cyan("Assistant");
-            else roleLabel = chalk.yellow(role);
-            console.log(`${roleLabel}: ${m.content}`);
-          }
-          lastMessageCount = allMessages.length;
-        }
-      }
-      if (finished) {
-        console.log("--- Verdict ---");
-        console.log(`Status: ${finished.status}`);
-        if (finished.results) {
-          console.log(`Verdict: ${finished.results.verdict}`);
-          if (finished.results.reasoning)
-            console.log(`Reasoning: ${finished.results.reasoning}`);
-          if ((_h = finished.results.metCriteria) == null ? void 0 : _h.length)
-            console.log(
-              `Met criteria: ${finished.results.metCriteria.join(", ")}`
-            );
-          if ((_i = finished.results.unmetCriteria) == null ? void 0 : _i.length)
-            console.log(
-              `Unmet criteria: ${finished.results.unmetCriteria.join(", ")}`
-            );
-          if (finished.results.error)
-            console.log(`Error: ${finished.results.error}`);
-        }
-      }
-      console.log("-----------------------------\n");
-    }
-    fs.unlinkSync(filePath);
-  }
-  async onTestRunEnd() {
-    if (this.results.length === 0) return;
-    const total = this.results.length;
-    const passed = this.results.filter((r) => r.status === "SUCCESS").length;
-    const failed = this.results.filter((r) => r.status !== "SUCCESS").length;
-    const successRate = (passed / total * 100).toFixed(1);
-    console.log();
-    console.log(chalk.bold.cyan("=== Scenario Test Report ==="));
-    console.log(`Total Scenarios: ${total}`);
-    console.log(chalk.green(`Passed: ${passed}`));
-    console.log(chalk.red(`Failed: ${failed}`));
-    console.log(`Success Rate: ${chalk.bold(`${successRate}%`)}`);
-    this.results.forEach((r, i) => {
-      const statusColor = r.status === "SUCCESS" ? chalk.green : chalk.red;
-      console.log();
-      console.log(
-        `${i + 1}. ${r.name} - ${statusColor(r.status)} in ${(r.duration / 1e3).toFixed(2)}s`
-      );
-      if (r.reasoning) {
-        console.log(chalk.greenBright("  Reasoning: ") + r.reasoning);
-      }
-      if (r.criteria) {
-        console.log(chalk.bold("  " + r.criteria));
-      }
-    });
-    console.log();
-  }
-};
-
-export {
-  VitestReporter
-};