@langwatch/scenario 0.2.6 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -183,6 +183,53 @@ var criterionToParamName = (criterion) => {
183
183
  return criterion.replace(/"/g, "").replace(/[^a-zA-Z0-9]/g, "_").replace(/ /g, "_").toLowerCase().substring(0, 70);
184
184
  };
185
185
 
186
+ // src/config/env.ts
187
+ var import_zod2 = require("zod");
188
+
189
+ // src/config/log-levels.ts
190
+ var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
191
+ LogLevel2["ERROR"] = "ERROR";
192
+ LogLevel2["WARN"] = "WARN";
193
+ LogLevel2["INFO"] = "INFO";
194
+ LogLevel2["DEBUG"] = "DEBUG";
195
+ return LogLevel2;
196
+ })(LogLevel || {});
197
+
198
+ // src/config/env.ts
199
+ var envSchema = import_zod2.z.object({
200
+ /**
201
+ * LangWatch API key for event reporting.
202
+ * If not provided, events will not be sent to LangWatch.
203
+ */
204
+ LANGWATCH_API_KEY: import_zod2.z.string().optional(),
205
+ /**
206
+ * LangWatch endpoint URL for event reporting.
207
+ * Defaults to the production LangWatch endpoint.
208
+ */
209
+ LANGWATCH_ENDPOINT: import_zod2.z.string().url().default("https://app.langwatch.ai"),
210
+ /**
211
+ * Disables simulation report info messages when set to any truthy value.
212
+ * Useful for CI/CD environments or when you want cleaner output.
213
+ */
214
+ SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod2.z.string().optional().transform((val) => Boolean(val)),
215
+ /**
216
+ * Node environment - affects logging and behavior.
217
+ * Defaults to 'development' if not specified.
218
+ */
219
+ NODE_ENV: import_zod2.z.enum(["development", "production", "test"]).default("development"),
220
+ /**
221
+ * Log level for the scenario package.
222
+ * Defaults to 'info' if not specified.
223
+ */
224
+ LOG_LEVEL: import_zod2.z.nativeEnum(LogLevel).optional(),
225
+ /**
226
+ * Scenario batch run ID.
227
+ * If not provided, a random ID will be generated.
228
+ */
229
+ SCENARIO_BATCH_RUN_ID: import_zod2.z.string().optional()
230
+ });
231
+ var env = envSchema.parse(process.env);
232
+
186
233
  // src/config/load.ts
187
234
  var import_promises = __toESM(require("fs/promises"));
188
235
  var import_node_path = __toESM(require("path"));
@@ -228,7 +275,7 @@ var Logger = class _Logger {
228
275
  return new _Logger(context);
229
276
  }
230
277
  getLogLevel() {
231
- return env.SCENARIO_LOG_LEVEL ?? "INFO" /* INFO */;
278
+ return env.LOG_LEVEL ?? "INFO" /* INFO */;
232
279
  }
233
280
  getLogLevelIndex(level) {
234
281
  return Object.values(LogLevel).indexOf(level);
@@ -286,54 +333,7 @@ var Logger = class _Logger {
286
333
  }
287
334
  };
288
335
 
289
- // src/config/env.ts
290
- var import_zod2 = require("zod");
291
-
292
- // src/config/log-levels.ts
293
- var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
294
- LogLevel2["ERROR"] = "ERROR";
295
- LogLevel2["WARN"] = "WARN";
296
- LogLevel2["INFO"] = "INFO";
297
- LogLevel2["DEBUG"] = "DEBUG";
298
- return LogLevel2;
299
- })(LogLevel || {});
300
-
301
- // src/config/env.ts
302
- var envSchema = import_zod2.z.object({
303
- /**
304
- * LangWatch API key for event reporting.
305
- * If not provided, events will not be sent to LangWatch.
306
- */
307
- LANGWATCH_API_KEY: import_zod2.z.string().optional(),
308
- /**
309
- * LangWatch endpoint URL for event reporting.
310
- * Defaults to the production LangWatch endpoint.
311
- */
312
- LANGWATCH_ENDPOINT: import_zod2.z.string().url().default("https://app.langwatch.ai"),
313
- /**
314
- * Disables simulation report info messages when set to any truthy value.
315
- * Useful for CI/CD environments or when you want cleaner output.
316
- */
317
- SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod2.z.string().optional().transform((val) => Boolean(val)),
318
- /**
319
- * Node environment - affects logging and behavior.
320
- * Defaults to 'development' if not specified.
321
- */
322
- NODE_ENV: import_zod2.z.enum(["development", "production", "test"]).default("development"),
323
- /**
324
- * Log level for the scenario package.
325
- * Defaults to 'info' if not specified.
326
- */
327
- SCENARIO_LOG_LEVEL: import_zod2.z.nativeEnum(LogLevel).optional(),
328
- /**
329
- * Scenario batch run ID.
330
- * If not provided, a random ID will be generated.
331
- */
332
- SCENARIO_BATCH_RUN_ID: import_zod2.z.string().optional()
333
- });
334
- var env = envSchema.parse(process.env);
335
-
336
- // src/config/index.ts
336
+ // src/config/get-project-config.ts
337
337
  var logger = new Logger("scenario.config");
338
338
  var configLoaded = false;
339
339
  var config = null;
@@ -581,7 +581,10 @@ __export(execution_exports, {
581
581
  var import_rxjs = require("rxjs");
582
582
 
583
583
  // src/utils/ids.ts
584
+ var import_node_crypto = __toESM(require("crypto"));
585
+ var import_node_process = __toESM(require("process"));
584
586
  var import_xksuid = require("xksuid");
587
+ var batchRunId;
585
588
  function generateThreadId() {
586
589
  return `thread_${(0, import_xksuid.generate)()}`;
587
590
  }
@@ -592,10 +595,31 @@ function generateScenarioId() {
592
595
  return `scenario_${(0, import_xksuid.generate)()}`;
593
596
  }
594
597
  function getBatchRunId() {
595
- if (!env.SCENARIO_BATCH_RUN_ID) {
596
- env.SCENARIO_BATCH_RUN_ID = `scenariobatchrun_${(0, import_xksuid.generate)()}`;
597
- }
598
- return env.SCENARIO_BATCH_RUN_ID;
598
+ if (batchRunId) {
599
+ return batchRunId;
600
+ }
601
+ if (import_node_process.default.env.SCENARIO_BATCH_RUN_ID) {
602
+ console.log("process.env.SCENARIO_BATCH_RUN_ID", import_node_process.default.env.SCENARIO_BATCH_RUN_ID);
603
+ return batchRunId = import_node_process.default.env.SCENARIO_BATCH_RUN_ID;
604
+ }
605
+ if (import_node_process.default.env.VITEST_WORKER_ID || import_node_process.default.env.JEST_WORKER_ID) {
606
+ const parentProcessId = import_node_process.default.ppid;
607
+ const now = /* @__PURE__ */ new Date();
608
+ const year = now.getUTCFullYear();
609
+ const week = String(getISOWeekNumber(now)).padStart(2, "0");
610
+ const raw = `${parentProcessId}_${year}_w${week}`;
611
+ const hash = import_node_crypto.default.createHash("sha256").update(raw).digest("hex").slice(0, 12);
612
+ return batchRunId = `scenariobatchrun_${hash}`;
613
+ }
614
+ return batchRunId = `scenariobatchrun_${(0, import_xksuid.generate)()}`;
615
+ }
616
+ function getISOWeekNumber(date) {
617
+ const tmp = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
618
+ const dayNum = tmp.getUTCDay() || 7;
619
+ tmp.setUTCDate(tmp.getUTCDate() + 4 - dayNum);
620
+ const yearStart = new Date(Date.UTC(tmp.getUTCFullYear(), 0, 1));
621
+ const weekNo = Math.ceil(((tmp.getTime() - yearStart.getTime()) / 864e5 + 1) / 7);
622
+ return weekNo;
599
623
  }
600
624
  function generateMessageId() {
601
625
  return `scenariomsg_${(0, import_xksuid.generate)()}`;
@@ -651,6 +675,16 @@ var ScenarioExecutionState = class {
651
675
  }
652
676
  return lastMessage;
653
677
  }
678
+ lastAgentMessage() {
679
+ if (this._messages.length === 0) {
680
+ throw new Error("No messages in history");
681
+ }
682
+ const lastMessage = this._messages.findLast((message2) => message2.role === "assistant");
683
+ if (!lastMessage) {
684
+ throw new Error("No agent message in history");
685
+ }
686
+ return lastMessage;
687
+ }
654
688
  lastToolCall(toolName) {
655
689
  if (this._messages.length === 0) {
656
690
  throw new Error("No messages in history");
@@ -658,9 +692,6 @@ var ScenarioExecutionState = class {
658
692
  const lastMessage = this._messages.findLast((message2) => message2.role === "tool" && message2.content.find(
659
693
  (part) => part.type === "tool-result" && part.toolName === toolName
660
694
  ));
661
- if (!lastMessage) {
662
- throw new Error("No tool call message in history");
663
- }
664
695
  return lastMessage;
665
696
  }
666
697
  hasToolCall(toolName) {
@@ -816,7 +847,6 @@ function convertCoreMessagesToAguiMessages(coreMessages) {
816
847
  var message_conversion_default = convertCoreMessagesToAguiMessages;
817
848
 
818
849
  // src/execution/scenario-execution.ts
819
- var batchRunId = getBatchRunId();
820
850
  var ScenarioExecution = class {
821
851
  state;
822
852
  eventSubject = new import_rxjs.Subject();
@@ -1234,7 +1264,7 @@ var ScenarioExecution = class {
1234
1264
  type: "placeholder",
1235
1265
  // This will be replaced by the specific event type
1236
1266
  timestamp: Date.now(),
1237
- batchRunId,
1267
+ batchRunId: getBatchRunId(),
1238
1268
  scenarioId: this.config.id,
1239
1269
  scenarioRunId,
1240
1270
  scenarioSetId: this.config.setId
@@ -1332,12 +1362,11 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
1332
1362
  if (this.isGreetingDisabled()) {
1333
1363
  return;
1334
1364
  }
1335
- const batchRunId2 = getBatchRunId();
1336
- if (_EventAlertMessageLogger.shownBatchIds.has(batchRunId2)) {
1365
+ if (_EventAlertMessageLogger.shownBatchIds.has(getBatchRunId())) {
1337
1366
  return;
1338
1367
  }
1339
- _EventAlertMessageLogger.shownBatchIds.add(batchRunId2);
1340
- this.displayGreeting(batchRunId2);
1368
+ _EventAlertMessageLogger.shownBatchIds.add(getBatchRunId());
1369
+ this.displayGreeting();
1341
1370
  }
1342
1371
  /**
1343
1372
  * Shows a fancy message about how to watch the simulation.
@@ -1352,7 +1381,7 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
1352
1381
  isGreetingDisabled() {
1353
1382
  return env.SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
1354
1383
  }
1355
- displayGreeting(batchRunId2) {
1384
+ displayGreeting() {
1356
1385
  const separator = "\u2500".repeat(60);
1357
1386
  if (!env.LANGWATCH_API_KEY) {
1358
1387
  console.log(`
@@ -1366,7 +1395,10 @@ ${separator}`);
1366
1395
  console.log(" \u2022 Set LANGWATCH_API_KEY environment variable");
1367
1396
  console.log(" \u2022 Or configure apiKey in scenario.config.js");
1368
1397
  console.log("");
1369
- console.log(`\u{1F4E6} Batch Run ID: ${batchRunId2}`);
1398
+ console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
1399
+ console.log("");
1400
+ console.log("\u{1F507} To disable these messages:");
1401
+ console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
1370
1402
  console.log(`${separator}
1371
1403
  `);
1372
1404
  } else {
@@ -1380,7 +1412,10 @@ ${separator}`);
1380
1412
  ` API Key: ${env.LANGWATCH_API_KEY.length > 0 ? "Configured" : "Not configured"}`
1381
1413
  );
1382
1414
  console.log("");
1383
- console.log(`\u{1F4E6} Batch Run ID: ${batchRunId2}`);
1415
+ console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
1416
+ console.log("");
1417
+ console.log("\u{1F507} To disable these messages:");
1418
+ console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
1384
1419
  console.log(`${separator}
1385
1420
  `);
1386
1421
  }
package/dist/index.mjs CHANGED
@@ -6,12 +6,10 @@ import {
6
6
  DEFAULT_VERBOSE,
7
7
  EventBus,
8
8
  JudgeAgentAdapter,
9
- Logger,
10
9
  UserSimulatorAgentAdapter,
11
10
  allAgentRoles,
12
11
  defineConfig,
13
12
  domain_exports,
14
- env,
15
13
  generateMessageId,
16
14
  generateScenarioId,
17
15
  generateScenarioRunId,
@@ -19,7 +17,11 @@ import {
19
17
  getBatchRunId,
20
18
  getProjectConfig,
21
19
  scenarioProjectConfigSchema
22
- } from "./chunk-MOOKAYIE.mjs";
20
+ } from "./chunk-7H6OGEQ5.mjs";
21
+ import {
22
+ Logger,
23
+ env
24
+ } from "./chunk-YPJZSK4J.mjs";
23
25
  import {
24
26
  __export
25
27
  } from "./chunk-7P6ASYW6.mjs";
@@ -358,6 +360,16 @@ var ScenarioExecutionState = class {
358
360
  }
359
361
  return lastMessage;
360
362
  }
363
+ lastAgentMessage() {
364
+ if (this._messages.length === 0) {
365
+ throw new Error("No messages in history");
366
+ }
367
+ const lastMessage = this._messages.findLast((message2) => message2.role === "assistant");
368
+ if (!lastMessage) {
369
+ throw new Error("No agent message in history");
370
+ }
371
+ return lastMessage;
372
+ }
361
373
  lastToolCall(toolName) {
362
374
  if (this._messages.length === 0) {
363
375
  throw new Error("No messages in history");
@@ -365,9 +377,6 @@ var ScenarioExecutionState = class {
365
377
  const lastMessage = this._messages.findLast((message2) => message2.role === "tool" && message2.content.find(
366
378
  (part) => part.type === "tool-result" && part.toolName === toolName
367
379
  ));
368
- if (!lastMessage) {
369
- throw new Error("No tool call message in history");
370
- }
371
380
  return lastMessage;
372
381
  }
373
382
  hasToolCall(toolName) {
@@ -451,7 +460,6 @@ function convertCoreMessagesToAguiMessages(coreMessages) {
451
460
  var message_conversion_default = convertCoreMessagesToAguiMessages;
452
461
 
453
462
  // src/execution/scenario-execution.ts
454
- var batchRunId = getBatchRunId();
455
463
  var ScenarioExecution = class {
456
464
  state;
457
465
  eventSubject = new Subject();
@@ -869,7 +877,7 @@ var ScenarioExecution = class {
869
877
  type: "placeholder",
870
878
  // This will be replaced by the specific event type
871
879
  timestamp: Date.now(),
872
- batchRunId,
880
+ batchRunId: getBatchRunId(),
873
881
  scenarioId: this.config.id,
874
882
  scenarioRunId,
875
883
  scenarioSetId: this.config.setId
@@ -0,0 +1,5 @@
1
+ import { ViteUserConfig } from 'vitest/config';
2
+
3
+ declare function withScenario(config: ViteUserConfig): ViteUserConfig;
4
+
5
+ export { withScenario };
@@ -0,0 +1,5 @@
1
+ import { ViteUserConfig } from 'vitest/config';
2
+
3
+ declare function withScenario(config: ViteUserConfig): ViteUserConfig;
4
+
5
+ export { withScenario };
@@ -0,0 +1,324 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+
30
+ // src/integrations/vitest/config.ts
31
+ var config_exports = {};
32
+ __export(config_exports, {
33
+ withScenario: () => withScenario
34
+ });
35
+ module.exports = __toCommonJS(config_exports);
36
+ var import_config = require("vitest/config");
37
+
38
+ // src/integrations/vitest/reporter.ts
39
+ var import_fs = __toESM(require("fs"));
40
+ var import_path = __toESM(require("path"));
41
+ var import_chalk = __toESM(require("chalk"));
42
+
43
+ // src/config/env.ts
44
+ var import_zod = require("zod");
45
+
46
+ // src/config/log-levels.ts
47
+ var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
48
+ LogLevel2["ERROR"] = "ERROR";
49
+ LogLevel2["WARN"] = "WARN";
50
+ LogLevel2["INFO"] = "INFO";
51
+ LogLevel2["DEBUG"] = "DEBUG";
52
+ return LogLevel2;
53
+ })(LogLevel || {});
54
+
55
+ // src/config/env.ts
56
+ var envSchema = import_zod.z.object({
57
+ /**
58
+ * LangWatch API key for event reporting.
59
+ * If not provided, events will not be sent to LangWatch.
60
+ */
61
+ LANGWATCH_API_KEY: import_zod.z.string().optional(),
62
+ /**
63
+ * LangWatch endpoint URL for event reporting.
64
+ * Defaults to the production LangWatch endpoint.
65
+ */
66
+ LANGWATCH_ENDPOINT: import_zod.z.string().url().default("https://app.langwatch.ai"),
67
+ /**
68
+ * Disables simulation report info messages when set to any truthy value.
69
+ * Useful for CI/CD environments or when you want cleaner output.
70
+ */
71
+ SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod.z.string().optional().transform((val) => Boolean(val)),
72
+ /**
73
+ * Node environment - affects logging and behavior.
74
+ * Defaults to 'development' if not specified.
75
+ */
76
+ NODE_ENV: import_zod.z.enum(["development", "production", "test"]).default("development"),
77
+ /**
78
+ * Log level for the scenario package.
79
+ * Defaults to 'info' if not specified.
80
+ */
81
+ LOG_LEVEL: import_zod.z.nativeEnum(LogLevel).optional(),
82
+ /**
83
+ * Scenario batch run ID.
84
+ * If not provided, a random ID will be generated.
85
+ */
86
+ SCENARIO_BATCH_RUN_ID: import_zod.z.string().optional()
87
+ });
88
+ var env = envSchema.parse(process.env);
89
+
90
+ // src/utils/logger.ts
91
+ var Logger = class _Logger {
92
+ constructor(context) {
93
+ this.context = context;
94
+ }
95
+ /**
96
+ * Creates a logger with context (e.g., class name)
97
+ */
98
+ static create(context) {
99
+ return new _Logger(context);
100
+ }
101
+ getLogLevel() {
102
+ return env.LOG_LEVEL ?? "INFO" /* INFO */;
103
+ }
104
+ getLogLevelIndex(level) {
105
+ return Object.values(LogLevel).indexOf(level);
106
+ }
107
+ /**
108
+ * Checks if logging should occur based on LOG_LEVEL env var
109
+ */
110
+ shouldLog(level) {
111
+ const currentLevelIndex = this.getLogLevelIndex(this.getLogLevel());
112
+ const requestedLevelIndex = this.getLogLevelIndex(level);
113
+ return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
114
+ }
115
+ formatMessage(message) {
116
+ return this.context ? `[${this.context}] ${message}` : message;
117
+ }
118
+ error(message, data) {
119
+ if (this.shouldLog("ERROR" /* ERROR */)) {
120
+ const formattedMessage = this.formatMessage(message);
121
+ if (data) {
122
+ console.error(formattedMessage, data);
123
+ } else {
124
+ console.error(formattedMessage);
125
+ }
126
+ }
127
+ }
128
+ warn(message, data) {
129
+ if (this.shouldLog("WARN" /* WARN */)) {
130
+ const formattedMessage = this.formatMessage(message);
131
+ if (data) {
132
+ console.warn(formattedMessage, data);
133
+ } else {
134
+ console.warn(formattedMessage);
135
+ }
136
+ }
137
+ }
138
+ info(message, data) {
139
+ if (this.shouldLog("INFO" /* INFO */)) {
140
+ const formattedMessage = this.formatMessage(message);
141
+ if (data) {
142
+ console.info(formattedMessage, data);
143
+ } else {
144
+ console.info(formattedMessage);
145
+ }
146
+ }
147
+ }
148
+ debug(message, data) {
149
+ if (this.shouldLog("DEBUG" /* DEBUG */)) {
150
+ const formattedMessage = this.formatMessage(message);
151
+ if (data) {
152
+ console.log(formattedMessage, data);
153
+ } else {
154
+ console.log(formattedMessage);
155
+ }
156
+ }
157
+ }
158
+ };
159
+
160
+ // src/integrations/vitest/reporter.ts
161
+ var logger = Logger.create("integrations:vitest:reporter");
162
+ function getProjectRoot() {
163
+ return process.cwd();
164
+ }
165
+ var projectRoot = getProjectRoot();
166
+ var logDir = import_path.default.join(projectRoot, ".scenario");
167
+ if (!import_fs.default.existsSync(logDir)) import_fs.default.mkdirSync(logDir);
168
+ function getLogFilePath(testId) {
169
+ return import_path.default.join(logDir, `${testId}.log`);
170
+ }
171
+ function getFullTestName(task) {
172
+ let name = task.name;
173
+ let parent = task.suite;
174
+ while (parent) {
175
+ name = `${parent.name} > ${name}`;
176
+ parent = parent.suite;
177
+ }
178
+ return name;
179
+ }
180
+ var VitestReporter = class {
181
+ results = [];
182
+ async onTestCaseResult(test) {
183
+ var _a, _b, _c, _d, _e, _f, _g, _h, _i;
184
+ const fullName = getFullTestName(test);
185
+ const filePath = getLogFilePath(test.id);
186
+ if (!import_fs.default.existsSync(filePath)) {
187
+ logger.warn(
188
+ `No log file found ${filePath} for test ${fullName}`,
189
+ test.id
190
+ );
191
+ return;
192
+ }
193
+ const lines = import_fs.default.readFileSync(filePath, "utf-8").split("\n").filter(Boolean);
194
+ const events = lines.map((line) => JSON.parse(line));
195
+ const runs = /* @__PURE__ */ new Map();
196
+ for (const event of events) {
197
+ const runId = event.scenarioRunId ?? "unknown";
198
+ if (!runs.has(runId)) runs.set(runId, []);
199
+ runs.get(runId).push(event);
200
+ }
201
+ for (const [runId, runEvents] of Array.from(runs.entries())) {
202
+ const started = runEvents.find(
203
+ (e) => e.type === "SCENARIO_RUN_STARTED"
204
+ );
205
+ const finished = runEvents.find(
206
+ (e) => e.type === "SCENARIO_RUN_FINISHED"
207
+ );
208
+ const messages = runEvents.filter(
209
+ (e) => e.type === "SCENARIO_MESSAGE_SNAPSHOT"
210
+ );
211
+ this.results.push({
212
+ name: ((_a = started == null ? void 0 : started.metadata) == null ? void 0 : _a.name) ?? fullName,
213
+ status: (finished == null ? void 0 : finished.status) ?? "UNKNOWN",
214
+ duration: started && finished ? finished.timestamp - started.timestamp : 0,
215
+ reasoning: (_b = finished == null ? void 0 : finished.results) == null ? void 0 : _b.reasoning,
216
+ criteria: (finished == null ? void 0 : finished.results) ? `Success Criteria: ${((_c = finished.results.metCriteria) == null ? void 0 : _c.length) ?? 0}/${(((_d = finished.results.metCriteria) == null ? void 0 : _d.length) ?? 0) + (((_e = finished.results.unmetCriteria) == null ? void 0 : _e.length) ?? 0)}` : void 0
217
+ });
218
+ console.log(
219
+ `
220
+ --- Scenario Run: ${((_f = started == null ? void 0 : started.metadata) == null ? void 0 : _f.name) ?? runId} ---`
221
+ );
222
+ if (started) {
223
+ console.log(`Description: ${((_g = started.metadata) == null ? void 0 : _g.description) ?? ""}`);
224
+ }
225
+ if (messages.length) {
226
+ console.log("Chat log:");
227
+ let lastMessageCount = 0;
228
+ for (const msg of messages) {
229
+ const allMessages = msg.messages ?? [];
230
+ for (const m of allMessages.slice(lastMessageCount)) {
231
+ const role = m.role;
232
+ let roleLabel = role;
233
+ if (role.toLowerCase() === "user") roleLabel = import_chalk.default.green("User");
234
+ else if (role.toLowerCase() === "agent")
235
+ roleLabel = import_chalk.default.cyan("Agent");
236
+ else if (role.toLowerCase() === "assistant")
237
+ roleLabel = import_chalk.default.cyan("Assistant");
238
+ else roleLabel = import_chalk.default.yellow(role);
239
+ console.log(`${roleLabel}: ${m.content}`);
240
+ }
241
+ lastMessageCount = allMessages.length;
242
+ }
243
+ }
244
+ if (finished) {
245
+ console.log("--- Verdict ---");
246
+ console.log(`Status: ${finished.status}`);
247
+ if (finished.results) {
248
+ console.log(`Verdict: ${finished.results.verdict}`);
249
+ if (finished.results.reasoning)
250
+ console.log(`Reasoning: ${finished.results.reasoning}`);
251
+ if ((_h = finished.results.metCriteria) == null ? void 0 : _h.length)
252
+ console.log(
253
+ `Met criteria: ${finished.results.metCriteria.join(", ")}`
254
+ );
255
+ if ((_i = finished.results.unmetCriteria) == null ? void 0 : _i.length)
256
+ console.log(
257
+ `Unmet criteria: ${finished.results.unmetCriteria.join(", ")}`
258
+ );
259
+ if (finished.results.error)
260
+ console.log(`Error: ${finished.results.error}`);
261
+ }
262
+ }
263
+ console.log("-----------------------------\n");
264
+ }
265
+ import_fs.default.unlinkSync(filePath);
266
+ }
267
+ async onTestRunEnd() {
268
+ if (this.results.length === 0) return;
269
+ const total = this.results.length;
270
+ const passed = this.results.filter((r) => r.status === "SUCCESS").length;
271
+ const failed = this.results.filter((r) => r.status !== "SUCCESS").length;
272
+ const successRate = (passed / total * 100).toFixed(1);
273
+ console.log();
274
+ console.log(import_chalk.default.bold.cyan("=== Scenario Test Report ==="));
275
+ console.log(`Total Scenarios: ${total}`);
276
+ console.log(import_chalk.default.green(`Passed: ${passed}`));
277
+ console.log(import_chalk.default.red(`Failed: ${failed}`));
278
+ console.log(`Success Rate: ${import_chalk.default.bold(`${successRate}%`)}`);
279
+ this.results.forEach((r, i) => {
280
+ const statusColor = r.status === "SUCCESS" ? import_chalk.default.green : import_chalk.default.red;
281
+ console.log();
282
+ console.log(
283
+ `${i + 1}. ${r.name} - ${statusColor(r.status)} in ${(r.duration / 1e3).toFixed(2)}s`
284
+ );
285
+ if (r.reasoning) {
286
+ console.log(import_chalk.default.greenBright(" Reasoning: ") + r.reasoning);
287
+ }
288
+ if (r.criteria) {
289
+ console.log(import_chalk.default.bold(" " + r.criteria));
290
+ }
291
+ });
292
+ console.log();
293
+ }
294
+ };
295
+
296
+ // src/integrations/vitest/config.ts
297
+ function withScenario(config) {
298
+ var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l;
299
+ const normalizedSetupFiles = ((_a = config.test) == null ? void 0 : _a.setupFiles) === void 0 ? [] : Array.isArray((_b = config.test) == null ? void 0 : _b.setupFiles) ? (_c = config.test) == null ? void 0 : _c.setupFiles : [(_d = config.test) == null ? void 0 : _d.setupFiles];
300
+ const normalizedGlobalSetup = ((_e = config.test) == null ? void 0 : _e.globalSetup) === void 0 ? [] : Array.isArray((_f = config.test) == null ? void 0 : _f.globalSetup) ? (_g = config.test) == null ? void 0 : _g.globalSetup : [(_h = config.test) == null ? void 0 : _h.globalSetup];
301
+ const normalizedReporters = ((_i = config.test) == null ? void 0 : _i.reporters) === void 0 ? [] : Array.isArray((_j = config.test) == null ? void 0 : _j.reporters) ? (_k = config.test) == null ? void 0 : _k.reporters : [(_l = config.test) == null ? void 0 : _l.reporters];
302
+ return (0, import_config.defineConfig)({
303
+ ...config,
304
+ test: {
305
+ ...config.test,
306
+ setupFiles: [
307
+ "@langwatch/scenario/integrations/vitest/setup",
308
+ ...normalizedSetupFiles
309
+ ],
310
+ globalSetup: [
311
+ "@langwatch/scenario/integrations/vitest/setup-global",
312
+ ...normalizedGlobalSetup
313
+ ],
314
+ reporters: [
315
+ ...normalizedReporters,
316
+ new VitestReporter()
317
+ ]
318
+ }
319
+ });
320
+ }
321
+ // Annotate the CommonJS export names for ESM import in node:
322
+ 0 && (module.exports = {
323
+ withScenario
324
+ });