@langwatch/scenario 0.2.6 → 0.2.11

package/dist/integrations/vitest/config.d.mts

@@ -0,0 +1,42 @@
+import { ViteUserConfig } from 'vitest/config';
+
+/**
+ * Enhances a Vitest configuration with Scenario testing framework setup.
+ *
+ * This function wraps the provided Vitest configuration and automatically adds
+ * the necessary setup files for Scenario testing. It ensures that both local
+ * and global setup files are properly configured while preserving any existing
+ * setup configuration.
+ *
+ * The function normalizes setup file configurations to handle different input formats:
+ * - `undefined` → empty array
+ * - string → array with single item
+ * - array → array as-is
+ *
+ * @param config - The base Vitest configuration to enhance
+ * @returns Enhanced Vitest configuration with Scenario setup files prepended
+ *
+ * @example
+ * ```typescript
+ * import { defineConfig } from 'vitest/config';
+ * import { withScenario } from '@langwatch/scenario/integrations/vitest/config';
+ *
+ * export default withScenario(defineConfig({
+ *   test: {
+ *     setupFiles: ['./my-setup.ts'],
+ *     globalSetup: ['./my-global-setup.ts']
+ *   }
+ * }));
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Minimal configuration - only Scenario setup files
+ * export default withScenario(defineConfig({}));
+ * ```
+ *
+ * @since 1.0.0
+ */
+declare function withScenario(config: ViteUserConfig): ViteUserConfig;
+
+export { withScenario };

package/dist/integrations/vitest/config.d.ts

@@ -0,0 +1,42 @@
+import { ViteUserConfig } from 'vitest/config';
+
+/**
+ * Enhances a Vitest configuration with Scenario testing framework setup.
+ *
+ * This function wraps the provided Vitest configuration and automatically adds
+ * the necessary setup files for Scenario testing. It ensures that both local
+ * and global setup files are properly configured while preserving any existing
+ * setup configuration.
+ *
+ * The function normalizes setup file configurations to handle different input formats:
+ * - `undefined` → empty array
+ * - string → array with single item
+ * - array → array as-is
+ *
+ * @param config - The base Vitest configuration to enhance
+ * @returns Enhanced Vitest configuration with Scenario setup files prepended
+ *
+ * @example
+ * ```typescript
+ * import { defineConfig } from 'vitest/config';
+ * import { withScenario } from '@langwatch/scenario/integrations/vitest/config';
+ *
+ * export default withScenario(defineConfig({
+ *   test: {
+ *     setupFiles: ['./my-setup.ts'],
+ *     globalSetup: ['./my-global-setup.ts']
+ *   }
+ * }));
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Minimal configuration - only Scenario setup files
+ * export default withScenario(defineConfig({}));
+ * ```
+ *
+ * @since 1.0.0
+ */
+declare function withScenario(config: ViteUserConfig): ViteUserConfig;
+
+export { withScenario };

package/dist/integrations/vitest/config.js

@@ -0,0 +1,51 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/integrations/vitest/config.ts
+var config_exports = {};
+__export(config_exports, {
+  withScenario: () => withScenario
+});
+module.exports = __toCommonJS(config_exports);
+var import_config = require("vitest/config");
+function withScenario(config) {
+  var _a, _b, _c, _d, _e, _f, _g, _h;
+  const normalizedSetupFiles = ((_a = config.test) == null ? void 0 : _a.setupFiles) === void 0 ? [] : Array.isArray((_b = config.test) == null ? void 0 : _b.setupFiles) ? (_c = config.test) == null ? void 0 : _c.setupFiles : [(_d = config.test) == null ? void 0 : _d.setupFiles];
+  const normalizedGlobalSetup = ((_e = config.test) == null ? void 0 : _e.globalSetup) === void 0 ? [] : Array.isArray((_f = config.test) == null ? void 0 : _f.globalSetup) ? (_g = config.test) == null ? void 0 : _g.globalSetup : [(_h = config.test) == null ? void 0 : _h.globalSetup];
+  return (0, import_config.defineConfig)({
+    ...config,
+    test: {
+      ...config.test,
+      // Prepend Scenario setup files to ensure they run before user-defined setup
+      setupFiles: [
+        "@langwatch/scenario/integrations/vitest/setup",
+        ...normalizedSetupFiles
+      ],
+      // Prepend Scenario global setup files to ensure they run before user-defined global setup
+      globalSetup: [
+        "@langwatch/scenario/integrations/vitest/setup-global",
+        ...normalizedGlobalSetup
+      ]
+    }
+  });
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  withScenario
+});

package/dist/integrations/vitest/config.mjs

@@ -0,0 +1,28 @@
+import "../../chunk-7P6ASYW6.mjs";
+
+// src/integrations/vitest/config.ts
+import { defineConfig } from "vitest/config";
+function withScenario(config) {
+  var _a, _b, _c, _d, _e, _f, _g, _h;
+  const normalizedSetupFiles = ((_a = config.test) == null ? void 0 : _a.setupFiles) === void 0 ? [] : Array.isArray((_b = config.test) == null ? void 0 : _b.setupFiles) ? (_c = config.test) == null ? void 0 : _c.setupFiles : [(_d = config.test) == null ? void 0 : _d.setupFiles];
+  const normalizedGlobalSetup = ((_e = config.test) == null ? void 0 : _e.globalSetup) === void 0 ? [] : Array.isArray((_f = config.test) == null ? void 0 : _f.globalSetup) ? (_g = config.test) == null ? void 0 : _g.globalSetup : [(_h = config.test) == null ? void 0 : _h.globalSetup];
+  return defineConfig({
+    ...config,
+    test: {
+      ...config.test,
+      // Prepend Scenario setup files to ensure they run before user-defined setup
+      setupFiles: [
+        "@langwatch/scenario/integrations/vitest/setup",
+        ...normalizedSetupFiles
+      ],
+      // Prepend Scenario global setup files to ensure they run before user-defined global setup
+      globalSetup: [
+        "@langwatch/scenario/integrations/vitest/setup-global",
+        ...normalizedGlobalSetup
+      ]
+    }
+  });
+}
+export {
+  withScenario
+};

package/dist/integrations/vitest/reporter.js

@@ -30,12 +30,144 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/integrations/vitest/reporter.ts
 var reporter_exports = {};
 __export(reporter_exports, {
-  default: () => VitestReporter
+  default: () => reporter_default
 });
 module.exports = __toCommonJS(reporter_exports);
 var import_fs = __toESM(require("fs"));
 var import_path = __toESM(require("path"));
 var import_chalk = __toESM(require("chalk"));
+
+// src/config/env.ts
+var import_zod = require("zod");
+
+// src/config/log-levels.ts
+var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
+  LogLevel2["ERROR"] = "ERROR";
+  LogLevel2["WARN"] = "WARN";
+  LogLevel2["INFO"] = "INFO";
+  LogLevel2["DEBUG"] = "DEBUG";
+  return LogLevel2;
+})(LogLevel || {});
+var LOG_LEVELS = Object.values(LogLevel);
+
+// src/config/env.ts
+var envSchema = import_zod.z.object({
+  /**
+   * LangWatch API key for event reporting.
+   * If not provided, events will not be sent to LangWatch.
+   */
+  LANGWATCH_API_KEY: import_zod.z.string().optional(),
+  /**
+   * LangWatch endpoint URL for event reporting.
+   * Defaults to the production LangWatch endpoint.
+   */
+  LANGWATCH_ENDPOINT: import_zod.z.string().url().optional().default("https://app.langwatch.ai"),
+  /**
+   * Disables simulation report info messages when set to any truthy value.
+   * Useful for CI/CD environments or when you want cleaner output.
+   */
+  SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod.z.string().optional().transform((val) => Boolean(val)),
+  /**
+   * Node environment - affects logging and behavior.
+   * Defaults to 'development' if not specified.
+   */
+  NODE_ENV: import_zod.z.enum(["development", "production", "test"]).default("development"),
+  /**
+   * Case-insensitive log level for the scenario package.
+   * Defaults to 'info' if not specified.
+   */
+  LOG_LEVEL: import_zod.z.string().toUpperCase().pipe(import_zod.z.nativeEnum(LogLevel)).optional().default("INFO" /* INFO */),
+  /**
+   * Scenario batch run ID.
+   * If not provided, a random ID will be generated.
+   */
+  SCENARIO_BATCH_RUN_ID: import_zod.z.string().optional()
+});
+function getEnv() {
+  return envSchema.parse(process.env);
+}
+
+// src/utils/logger.ts
+var Logger = class _Logger {
+  constructor(context) {
+    this.context = context;
+  }
+  /**
+   * Creates a logger with context (e.g., class name)
+   */
+  static create(context) {
+    return new _Logger(context);
+  }
+  /**
+   * Returns the current log level from environment.
+   * Uses a getter for clarity and idiomatic usage.
+   */
+  get LOG_LEVEL() {
+    return getEnv().LOG_LEVEL;
+  }
+  /**
+   * Returns the index of the given log level in the LOG_LEVELS array.
+   * @param level - The log level to get the index for.
+   * @returns The index of the log level in the LOG_LEVELS array.
+   */
+  getLogLevelIndexFor(level) {
+    return LOG_LEVELS.indexOf(level);
+  }
+  /**
+   * Checks if logging should occur based on LOG_LEVEL env var
+   */
+  shouldLog(level) {
+    const currentLevelIndex = this.getLogLevelIndexFor(this.LOG_LEVEL);
+    const requestedLevelIndex = this.getLogLevelIndexFor(level);
+    return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
+  }
+  formatMessage(message) {
+    return this.context ? `[${this.context}] ${message}` : message;
+  }
+  error(message, data) {
+    if (this.shouldLog("ERROR" /* ERROR */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.error(formattedMessage, data);
+      } else {
+        console.error(formattedMessage);
+      }
+    }
+  }
+  warn(message, data) {
+    if (this.shouldLog("WARN" /* WARN */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.warn(formattedMessage, data);
+      } else {
+        console.warn(formattedMessage);
+      }
+    }
+  }
+  info(message, data) {
+    if (this.shouldLog("INFO" /* INFO */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.info(formattedMessage, data);
+      } else {
+        console.info(formattedMessage);
+      }
+    }
+  }
+  debug(message, data) {
+    if (this.shouldLog("DEBUG" /* DEBUG */)) {
+      const formattedMessage = this.formatMessage(message);
+      if (data) {
+        console.log(formattedMessage, data);
+      } else {
+        console.log(formattedMessage);
+      }
+    }
+  }
+};
+
+// src/integrations/vitest/reporter.ts
+var logger = Logger.create("integrations:vitest:reporter");
 function getProjectRoot() {
   return process.cwd();
 }
@@ -54,14 +186,20 @@ function getFullTestName(task) {
   }
   return name;
 }
+function indent(str, n = 2) {
+  return str.replace(/^/gm, " ".repeat(n));
+}
 var VitestReporter = class {
   results = [];
   async onTestCaseResult(test) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _i;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
     const fullName = getFullTestName(test);
     const filePath = getLogFilePath(test.id);
     if (!import_fs.default.existsSync(filePath)) {
-      console.log(`No log file found ${filePath} for test ${fullName}`);
+      logger.warn(
+        `No log file found ${filePath} for test ${fullName}`,
+        test.id
+      );
       return;
     }
     const lines = import_fs.default.readFileSync(filePath, "utf-8").split("\n").filter(Boolean);
@@ -97,19 +235,55 @@ var VitestReporter = class {
         console.log(`Description: ${((_g = started.metadata) == null ? void 0 : _g.description) ?? ""}`);
       }
       if (messages.length) {
-        console.log("Chat log:");
+        console.log("Chat log:\n");
         let lastMessageCount = 0;
         for (const msg of messages) {
           const allMessages = msg.messages ?? [];
           for (const m of allMessages.slice(lastMessageCount)) {
             const role = m.role;
+            if (role.toLowerCase() === "assistant" && "toolCalls" in m && Array.isArray(m.toolCalls) && m.toolCalls.length > 0) {
+              for (const toolCall of m.toolCalls) {
+                const functionName = toolCall.function.name;
+                let parsedJson = "";
+                try {
+                  parsedJson = JSON.stringify(
+                    JSON.parse(toolCall.function.arguments),
+                    null,
+                    2
+                  );
+                } catch {
+                  parsedJson = toolCall.function.arguments;
+                }
+                const role2 = import_chalk.default.magenta(`ToolCall(${functionName}):`);
+                console.log(`${role2}:
+
+${indent(parsedJson)}
+`);
+              }
+              continue;
+            }
             let roleLabel = role;
             if (role.toLowerCase() === "user") roleLabel = import_chalk.default.green("User");
             else if (role.toLowerCase() === "agent")
               roleLabel = import_chalk.default.cyan("Agent");
             else if (role.toLowerCase() === "assistant")
-              roleLabel = import_chalk.default.cyan("Assistant");
-            else roleLabel = import_chalk.default.yellow(role);
+              if (Array.isArray(m.content) && typeof m.content.at(0) === "object" && ((_h = m.content.at(0)) == null ? void 0 : _h.type) === "tool-call")
+                roleLabel = import_chalk.default.cyan("ToolCall");
+              else roleLabel = import_chalk.default.cyan("Assistant");
+            else if (role.toLowerCase() === "tool") {
+              roleLabel = import_chalk.default.magenta("ToolResult");
+              let parsedJson = "";
+              try {
+                parsedJson = JSON.stringify(JSON.parse(m.content), null, 2);
+              } catch {
+                parsedJson = m.content;
+              }
+              console.log(`${roleLabel}:
+
+${indent(parsedJson)}
+`);
+              continue;
+            } else roleLabel = import_chalk.default.yellow(role);
             console.log(`${roleLabel}: ${m.content}`);
           }
           lastMessageCount = allMessages.length;
@@ -122,11 +296,11 @@ var VitestReporter = class {
           console.log(`Verdict: ${finished.results.verdict}`);
           if (finished.results.reasoning)
             console.log(`Reasoning: ${finished.results.reasoning}`);
-          if ((_h = finished.results.metCriteria) == null ? void 0 : _h.length)
+          if ((_i = finished.results.metCriteria) == null ? void 0 : _i.length)
             console.log(
               `Met criteria: ${finished.results.metCriteria.join(", ")}`
             );
-          if ((_i = finished.results.unmetCriteria) == null ? void 0 : _i.length)
+          if ((_j = finished.results.unmetCriteria) == null ? void 0 : _j.length)
             console.log(
               `Unmet criteria: ${finished.results.unmetCriteria.join(", ")}`
             );
@@ -166,3 +340,4 @@ var VitestReporter = class {
     console.log();
   }
 };
+var reporter_default = VitestReporter;

package/dist/integrations/vitest/reporter.mjs

@@ -1,9 +1,13 @@
+import {
+  Logger
+} from "../../chunk-OL4RFXV4.mjs";
 import "../../chunk-7P6ASYW6.mjs";
 
 // src/integrations/vitest/reporter.ts
 import fs from "fs";
 import path from "path";
 import chalk from "chalk";
+var logger = Logger.create("integrations:vitest:reporter");
 function getProjectRoot() {
   return process.cwd();
 }
@@ -22,14 +26,20 @@ function getFullTestName(task) {
   }
   return name;
 }
+function indent(str, n = 2) {
+  return str.replace(/^/gm, " ".repeat(n));
+}
 var VitestReporter = class {
   results = [];
   async onTestCaseResult(test) {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _i;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
     const fullName = getFullTestName(test);
     const filePath = getLogFilePath(test.id);
     if (!fs.existsSync(filePath)) {
-      console.log(`No log file found ${filePath} for test ${fullName}`);
+      logger.warn(
+        `No log file found ${filePath} for test ${fullName}`,
+        test.id
+      );
       return;
     }
     const lines = fs.readFileSync(filePath, "utf-8").split("\n").filter(Boolean);
@@ -65,19 +75,55 @@ var VitestReporter = class {
         console.log(`Description: ${((_g = started.metadata) == null ? void 0 : _g.description) ?? ""}`);
       }
       if (messages.length) {
-        console.log("Chat log:");
+        console.log("Chat log:\n");
         let lastMessageCount = 0;
         for (const msg of messages) {
           const allMessages = msg.messages ?? [];
           for (const m of allMessages.slice(lastMessageCount)) {
             const role = m.role;
+            if (role.toLowerCase() === "assistant" && "toolCalls" in m && Array.isArray(m.toolCalls) && m.toolCalls.length > 0) {
+              for (const toolCall of m.toolCalls) {
+                const functionName = toolCall.function.name;
+                let parsedJson = "";
+                try {
+                  parsedJson = JSON.stringify(
+                    JSON.parse(toolCall.function.arguments),
+                    null,
+                    2
+                  );
+                } catch {
+                  parsedJson = toolCall.function.arguments;
+                }
+                const role2 = chalk.magenta(`ToolCall(${functionName}):`);
+                console.log(`${role2}:
+
+${indent(parsedJson)}
+`);
+              }
+              continue;
+            }
             let roleLabel = role;
             if (role.toLowerCase() === "user") roleLabel = chalk.green("User");
             else if (role.toLowerCase() === "agent")
               roleLabel = chalk.cyan("Agent");
             else if (role.toLowerCase() === "assistant")
-              roleLabel = chalk.cyan("Assistant");
-            else roleLabel = chalk.yellow(role);
+              if (Array.isArray(m.content) && typeof m.content.at(0) === "object" && ((_h = m.content.at(0)) == null ? void 0 : _h.type) === "tool-call")
+                roleLabel = chalk.cyan("ToolCall");
+              else roleLabel = chalk.cyan("Assistant");
+            else if (role.toLowerCase() === "tool") {
+              roleLabel = chalk.magenta("ToolResult");
+              let parsedJson = "";
+              try {
+                parsedJson = JSON.stringify(JSON.parse(m.content), null, 2);
+              } catch {
+                parsedJson = m.content;
+              }
+              console.log(`${roleLabel}:
+
+${indent(parsedJson)}
+`);
+              continue;
+            } else roleLabel = chalk.yellow(role);
             console.log(`${roleLabel}: ${m.content}`);
           }
           lastMessageCount = allMessages.length;
@@ -90,11 +136,11 @@ var VitestReporter = class {
           console.log(`Verdict: ${finished.results.verdict}`);
           if (finished.results.reasoning)
             console.log(`Reasoning: ${finished.results.reasoning}`);
-          if ((_h = finished.results.metCriteria) == null ? void 0 : _h.length)
+          if ((_i = finished.results.metCriteria) == null ? void 0 : _i.length)
             console.log(
               `Met criteria: ${finished.results.metCriteria.join(", ")}`
             );
-          if ((_i = finished.results.unmetCriteria) == null ? void 0 : _i.length)
+          if ((_j = finished.results.unmetCriteria) == null ? void 0 : _j.length)
             console.log(
               `Unmet criteria: ${finished.results.unmetCriteria.join(", ")}`
             );
@@ -134,6 +180,7 @@ var VitestReporter = class {
     console.log();
   }
 };
+var reporter_default = VitestReporter;
 export {
-  VitestReporter as default
+  reporter_default as default
 };

package/dist/integrations/vitest/setup-global.d.mts

@@ -0,0 +1,3 @@
+declare function setup(): void;
+
+export { setup as default };

package/dist/integrations/vitest/setup-global.d.ts

@@ -0,0 +1,3 @@
+declare function setup(): void;
+
+export { setup as default };

package/dist/integrations/vitest/setup-global.js

@@ -0,0 +1,30 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/integrations/vitest/setup-global.ts
+var setup_global_exports = {};
+__export(setup_global_exports, {
+  default: () => setup
+});
+module.exports = __toCommonJS(setup_global_exports);
+var import_xksuid = require("xksuid");
+function setup() {
+  const scenarioBatchRunId = `scenariobatchrun_${(0, import_xksuid.generate)()}`;
+  process.env.SCENARIO_BATCH_RUN_ID = scenarioBatchRunId;
+}

package/dist/integrations/vitest/setup-global.mjs

@@ -0,0 +1,11 @@
+import "../../chunk-7P6ASYW6.mjs";
+
+// src/integrations/vitest/setup-global.ts
+import { generate } from "xksuid";
+function setup() {
+  const scenarioBatchRunId = `scenariobatchrun_${generate()}`;
+  process.env.SCENARIO_BATCH_RUN_ID = scenarioBatchRunId;
+}
+export {
+  setup as default
+};