@langwatch/scenario 0.2.6 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/{chunk-MOOKAYIE.mjs → chunk-7H6OGEQ5.mjs} +76 -162
- package/dist/chunk-K7KLHTDI.mjs +146 -0
- package/dist/chunk-YPJZSK4J.mjs +121 -0
- package/dist/index.d.mts +9 -3
- package/dist/index.d.ts +9 -3
- package/dist/index.js +100 -65
- package/dist/index.mjs +16 -8
- package/dist/integrations/vitest/config.d.mts +5 -0
- package/dist/integrations/vitest/config.d.ts +5 -0
- package/dist/integrations/vitest/config.js +324 -0
- package/dist/integrations/vitest/config.mjs +35 -0
- package/dist/integrations/vitest/reporter.js +124 -1
- package/dist/integrations/vitest/reporter.mjs +4 -135
- package/dist/integrations/vitest/setup-global.d.mts +3 -0
- package/dist/integrations/vitest/setup-global.d.ts +3 -0
- package/dist/integrations/vitest/setup-global.js +30 -0
- package/dist/integrations/vitest/setup-global.mjs +11 -0
- package/dist/integrations/vitest/setup.js +97 -67
- package/dist/integrations/vitest/setup.mjs +7 -3
- package/package.json +11 -1
package/dist/index.js
CHANGED
|
@@ -183,6 +183,53 @@ var criterionToParamName = (criterion) => {
|
|
|
183
183
|
return criterion.replace(/"/g, "").replace(/[^a-zA-Z0-9]/g, "_").replace(/ /g, "_").toLowerCase().substring(0, 70);
|
|
184
184
|
};
|
|
185
185
|
|
|
186
|
+
// src/config/env.ts
|
|
187
|
+
var import_zod2 = require("zod");
|
|
188
|
+
|
|
189
|
+
// src/config/log-levels.ts
|
|
190
|
+
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
191
|
+
LogLevel2["ERROR"] = "ERROR";
|
|
192
|
+
LogLevel2["WARN"] = "WARN";
|
|
193
|
+
LogLevel2["INFO"] = "INFO";
|
|
194
|
+
LogLevel2["DEBUG"] = "DEBUG";
|
|
195
|
+
return LogLevel2;
|
|
196
|
+
})(LogLevel || {});
|
|
197
|
+
|
|
198
|
+
// src/config/env.ts
|
|
199
|
+
var envSchema = import_zod2.z.object({
|
|
200
|
+
/**
|
|
201
|
+
* LangWatch API key for event reporting.
|
|
202
|
+
* If not provided, events will not be sent to LangWatch.
|
|
203
|
+
*/
|
|
204
|
+
LANGWATCH_API_KEY: import_zod2.z.string().optional(),
|
|
205
|
+
/**
|
|
206
|
+
* LangWatch endpoint URL for event reporting.
|
|
207
|
+
* Defaults to the production LangWatch endpoint.
|
|
208
|
+
*/
|
|
209
|
+
LANGWATCH_ENDPOINT: import_zod2.z.string().url().default("https://app.langwatch.ai"),
|
|
210
|
+
/**
|
|
211
|
+
* Disables simulation report info messages when set to any truthy value.
|
|
212
|
+
* Useful for CI/CD environments or when you want cleaner output.
|
|
213
|
+
*/
|
|
214
|
+
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod2.z.string().optional().transform((val) => Boolean(val)),
|
|
215
|
+
/**
|
|
216
|
+
* Node environment - affects logging and behavior.
|
|
217
|
+
* Defaults to 'development' if not specified.
|
|
218
|
+
*/
|
|
219
|
+
NODE_ENV: import_zod2.z.enum(["development", "production", "test"]).default("development"),
|
|
220
|
+
/**
|
|
221
|
+
* Log level for the scenario package.
|
|
222
|
+
* Defaults to 'info' if not specified.
|
|
223
|
+
*/
|
|
224
|
+
LOG_LEVEL: import_zod2.z.nativeEnum(LogLevel).optional(),
|
|
225
|
+
/**
|
|
226
|
+
* Scenario batch run ID.
|
|
227
|
+
* If not provided, a random ID will be generated.
|
|
228
|
+
*/
|
|
229
|
+
SCENARIO_BATCH_RUN_ID: import_zod2.z.string().optional()
|
|
230
|
+
});
|
|
231
|
+
var env = envSchema.parse(process.env);
|
|
232
|
+
|
|
186
233
|
// src/config/load.ts
|
|
187
234
|
var import_promises = __toESM(require("fs/promises"));
|
|
188
235
|
var import_node_path = __toESM(require("path"));
|
|
@@ -228,7 +275,7 @@ var Logger = class _Logger {
|
|
|
228
275
|
return new _Logger(context);
|
|
229
276
|
}
|
|
230
277
|
getLogLevel() {
|
|
231
|
-
return env.
|
|
278
|
+
return env.LOG_LEVEL ?? "INFO" /* INFO */;
|
|
232
279
|
}
|
|
233
280
|
getLogLevelIndex(level) {
|
|
234
281
|
return Object.values(LogLevel).indexOf(level);
|
|
@@ -286,54 +333,7 @@ var Logger = class _Logger {
|
|
|
286
333
|
}
|
|
287
334
|
};
|
|
288
335
|
|
|
289
|
-
// src/config/
|
|
290
|
-
var import_zod2 = require("zod");
|
|
291
|
-
|
|
292
|
-
// src/config/log-levels.ts
|
|
293
|
-
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
294
|
-
LogLevel2["ERROR"] = "ERROR";
|
|
295
|
-
LogLevel2["WARN"] = "WARN";
|
|
296
|
-
LogLevel2["INFO"] = "INFO";
|
|
297
|
-
LogLevel2["DEBUG"] = "DEBUG";
|
|
298
|
-
return LogLevel2;
|
|
299
|
-
})(LogLevel || {});
|
|
300
|
-
|
|
301
|
-
// src/config/env.ts
|
|
302
|
-
var envSchema = import_zod2.z.object({
|
|
303
|
-
/**
|
|
304
|
-
* LangWatch API key for event reporting.
|
|
305
|
-
* If not provided, events will not be sent to LangWatch.
|
|
306
|
-
*/
|
|
307
|
-
LANGWATCH_API_KEY: import_zod2.z.string().optional(),
|
|
308
|
-
/**
|
|
309
|
-
* LangWatch endpoint URL for event reporting.
|
|
310
|
-
* Defaults to the production LangWatch endpoint.
|
|
311
|
-
*/
|
|
312
|
-
LANGWATCH_ENDPOINT: import_zod2.z.string().url().default("https://app.langwatch.ai"),
|
|
313
|
-
/**
|
|
314
|
-
* Disables simulation report info messages when set to any truthy value.
|
|
315
|
-
* Useful for CI/CD environments or when you want cleaner output.
|
|
316
|
-
*/
|
|
317
|
-
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod2.z.string().optional().transform((val) => Boolean(val)),
|
|
318
|
-
/**
|
|
319
|
-
* Node environment - affects logging and behavior.
|
|
320
|
-
* Defaults to 'development' if not specified.
|
|
321
|
-
*/
|
|
322
|
-
NODE_ENV: import_zod2.z.enum(["development", "production", "test"]).default("development"),
|
|
323
|
-
/**
|
|
324
|
-
* Log level for the scenario package.
|
|
325
|
-
* Defaults to 'info' if not specified.
|
|
326
|
-
*/
|
|
327
|
-
SCENARIO_LOG_LEVEL: import_zod2.z.nativeEnum(LogLevel).optional(),
|
|
328
|
-
/**
|
|
329
|
-
* Scenario batch run ID.
|
|
330
|
-
* If not provided, a random ID will be generated.
|
|
331
|
-
*/
|
|
332
|
-
SCENARIO_BATCH_RUN_ID: import_zod2.z.string().optional()
|
|
333
|
-
});
|
|
334
|
-
var env = envSchema.parse(process.env);
|
|
335
|
-
|
|
336
|
-
// src/config/index.ts
|
|
336
|
+
// src/config/get-project-config.ts
|
|
337
337
|
var logger = new Logger("scenario.config");
|
|
338
338
|
var configLoaded = false;
|
|
339
339
|
var config = null;
|
|
@@ -581,7 +581,10 @@ __export(execution_exports, {
|
|
|
581
581
|
var import_rxjs = require("rxjs");
|
|
582
582
|
|
|
583
583
|
// src/utils/ids.ts
|
|
584
|
+
var import_node_crypto = __toESM(require("crypto"));
|
|
585
|
+
var import_node_process = __toESM(require("process"));
|
|
584
586
|
var import_xksuid = require("xksuid");
|
|
587
|
+
var batchRunId;
|
|
585
588
|
function generateThreadId() {
|
|
586
589
|
return `thread_${(0, import_xksuid.generate)()}`;
|
|
587
590
|
}
|
|
@@ -592,10 +595,31 @@ function generateScenarioId() {
|
|
|
592
595
|
return `scenario_${(0, import_xksuid.generate)()}`;
|
|
593
596
|
}
|
|
594
597
|
function getBatchRunId() {
|
|
595
|
-
if (
|
|
596
|
-
|
|
597
|
-
}
|
|
598
|
-
|
|
598
|
+
if (batchRunId) {
|
|
599
|
+
return batchRunId;
|
|
600
|
+
}
|
|
601
|
+
if (import_node_process.default.env.SCENARIO_BATCH_RUN_ID) {
|
|
602
|
+
console.log("process.env.SCENARIO_BATCH_RUN_ID", import_node_process.default.env.SCENARIO_BATCH_RUN_ID);
|
|
603
|
+
return batchRunId = import_node_process.default.env.SCENARIO_BATCH_RUN_ID;
|
|
604
|
+
}
|
|
605
|
+
if (import_node_process.default.env.VITEST_WORKER_ID || import_node_process.default.env.JEST_WORKER_ID) {
|
|
606
|
+
const parentProcessId = import_node_process.default.ppid;
|
|
607
|
+
const now = /* @__PURE__ */ new Date();
|
|
608
|
+
const year = now.getUTCFullYear();
|
|
609
|
+
const week = String(getISOWeekNumber(now)).padStart(2, "0");
|
|
610
|
+
const raw = `${parentProcessId}_${year}_w${week}`;
|
|
611
|
+
const hash = import_node_crypto.default.createHash("sha256").update(raw).digest("hex").slice(0, 12);
|
|
612
|
+
return batchRunId = `scenariobatchrun_${hash}`;
|
|
613
|
+
}
|
|
614
|
+
return batchRunId = `scenariobatchrun_${(0, import_xksuid.generate)()}`;
|
|
615
|
+
}
|
|
616
|
+
function getISOWeekNumber(date) {
|
|
617
|
+
const tmp = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
|
|
618
|
+
const dayNum = tmp.getUTCDay() || 7;
|
|
619
|
+
tmp.setUTCDate(tmp.getUTCDate() + 4 - dayNum);
|
|
620
|
+
const yearStart = new Date(Date.UTC(tmp.getUTCFullYear(), 0, 1));
|
|
621
|
+
const weekNo = Math.ceil(((tmp.getTime() - yearStart.getTime()) / 864e5 + 1) / 7);
|
|
622
|
+
return weekNo;
|
|
599
623
|
}
|
|
600
624
|
function generateMessageId() {
|
|
601
625
|
return `scenariomsg_${(0, import_xksuid.generate)()}`;
|
|
@@ -651,6 +675,16 @@ var ScenarioExecutionState = class {
|
|
|
651
675
|
}
|
|
652
676
|
return lastMessage;
|
|
653
677
|
}
|
|
678
|
+
lastAgentMessage() {
|
|
679
|
+
if (this._messages.length === 0) {
|
|
680
|
+
throw new Error("No messages in history");
|
|
681
|
+
}
|
|
682
|
+
const lastMessage = this._messages.findLast((message2) => message2.role === "assistant");
|
|
683
|
+
if (!lastMessage) {
|
|
684
|
+
throw new Error("No agent message in history");
|
|
685
|
+
}
|
|
686
|
+
return lastMessage;
|
|
687
|
+
}
|
|
654
688
|
lastToolCall(toolName) {
|
|
655
689
|
if (this._messages.length === 0) {
|
|
656
690
|
throw new Error("No messages in history");
|
|
@@ -658,9 +692,6 @@ var ScenarioExecutionState = class {
|
|
|
658
692
|
const lastMessage = this._messages.findLast((message2) => message2.role === "tool" && message2.content.find(
|
|
659
693
|
(part) => part.type === "tool-result" && part.toolName === toolName
|
|
660
694
|
));
|
|
661
|
-
if (!lastMessage) {
|
|
662
|
-
throw new Error("No tool call message in history");
|
|
663
|
-
}
|
|
664
695
|
return lastMessage;
|
|
665
696
|
}
|
|
666
697
|
hasToolCall(toolName) {
|
|
@@ -816,7 +847,6 @@ function convertCoreMessagesToAguiMessages(coreMessages) {
|
|
|
816
847
|
var message_conversion_default = convertCoreMessagesToAguiMessages;
|
|
817
848
|
|
|
818
849
|
// src/execution/scenario-execution.ts
|
|
819
|
-
var batchRunId = getBatchRunId();
|
|
820
850
|
var ScenarioExecution = class {
|
|
821
851
|
state;
|
|
822
852
|
eventSubject = new import_rxjs.Subject();
|
|
@@ -1234,7 +1264,7 @@ var ScenarioExecution = class {
|
|
|
1234
1264
|
type: "placeholder",
|
|
1235
1265
|
// This will be replaced by the specific event type
|
|
1236
1266
|
timestamp: Date.now(),
|
|
1237
|
-
batchRunId,
|
|
1267
|
+
batchRunId: getBatchRunId(),
|
|
1238
1268
|
scenarioId: this.config.id,
|
|
1239
1269
|
scenarioRunId,
|
|
1240
1270
|
scenarioSetId: this.config.setId
|
|
@@ -1332,12 +1362,11 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
1332
1362
|
if (this.isGreetingDisabled()) {
|
|
1333
1363
|
return;
|
|
1334
1364
|
}
|
|
1335
|
-
|
|
1336
|
-
if (_EventAlertMessageLogger.shownBatchIds.has(batchRunId2)) {
|
|
1365
|
+
if (_EventAlertMessageLogger.shownBatchIds.has(getBatchRunId())) {
|
|
1337
1366
|
return;
|
|
1338
1367
|
}
|
|
1339
|
-
_EventAlertMessageLogger.shownBatchIds.add(
|
|
1340
|
-
this.displayGreeting(
|
|
1368
|
+
_EventAlertMessageLogger.shownBatchIds.add(getBatchRunId());
|
|
1369
|
+
this.displayGreeting();
|
|
1341
1370
|
}
|
|
1342
1371
|
/**
|
|
1343
1372
|
* Shows a fancy message about how to watch the simulation.
|
|
@@ -1352,7 +1381,7 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
1352
1381
|
isGreetingDisabled() {
|
|
1353
1382
|
return env.SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
|
|
1354
1383
|
}
|
|
1355
|
-
displayGreeting(
|
|
1384
|
+
displayGreeting() {
|
|
1356
1385
|
const separator = "\u2500".repeat(60);
|
|
1357
1386
|
if (!env.LANGWATCH_API_KEY) {
|
|
1358
1387
|
console.log(`
|
|
@@ -1366,7 +1395,10 @@ ${separator}`);
|
|
|
1366
1395
|
console.log(" \u2022 Set LANGWATCH_API_KEY environment variable");
|
|
1367
1396
|
console.log(" \u2022 Or configure apiKey in scenario.config.js");
|
|
1368
1397
|
console.log("");
|
|
1369
|
-
console.log(`\u{1F4E6} Batch Run ID: ${
|
|
1398
|
+
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
1399
|
+
console.log("");
|
|
1400
|
+
console.log("\u{1F507} To disable these messages:");
|
|
1401
|
+
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
1370
1402
|
console.log(`${separator}
|
|
1371
1403
|
`);
|
|
1372
1404
|
} else {
|
|
@@ -1380,7 +1412,10 @@ ${separator}`);
|
|
|
1380
1412
|
` API Key: ${env.LANGWATCH_API_KEY.length > 0 ? "Configured" : "Not configured"}`
|
|
1381
1413
|
);
|
|
1382
1414
|
console.log("");
|
|
1383
|
-
console.log(`\u{1F4E6} Batch Run ID: ${
|
|
1415
|
+
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
1416
|
+
console.log("");
|
|
1417
|
+
console.log("\u{1F507} To disable these messages:");
|
|
1418
|
+
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
1384
1419
|
console.log(`${separator}
|
|
1385
1420
|
`);
|
|
1386
1421
|
}
|
package/dist/index.mjs
CHANGED
|
@@ -6,12 +6,10 @@ import {
|
|
|
6
6
|
DEFAULT_VERBOSE,
|
|
7
7
|
EventBus,
|
|
8
8
|
JudgeAgentAdapter,
|
|
9
|
-
Logger,
|
|
10
9
|
UserSimulatorAgentAdapter,
|
|
11
10
|
allAgentRoles,
|
|
12
11
|
defineConfig,
|
|
13
12
|
domain_exports,
|
|
14
|
-
env,
|
|
15
13
|
generateMessageId,
|
|
16
14
|
generateScenarioId,
|
|
17
15
|
generateScenarioRunId,
|
|
@@ -19,7 +17,11 @@ import {
|
|
|
19
17
|
getBatchRunId,
|
|
20
18
|
getProjectConfig,
|
|
21
19
|
scenarioProjectConfigSchema
|
|
22
|
-
} from "./chunk-
|
|
20
|
+
} from "./chunk-7H6OGEQ5.mjs";
|
|
21
|
+
import {
|
|
22
|
+
Logger,
|
|
23
|
+
env
|
|
24
|
+
} from "./chunk-YPJZSK4J.mjs";
|
|
23
25
|
import {
|
|
24
26
|
__export
|
|
25
27
|
} from "./chunk-7P6ASYW6.mjs";
|
|
@@ -358,6 +360,16 @@ var ScenarioExecutionState = class {
|
|
|
358
360
|
}
|
|
359
361
|
return lastMessage;
|
|
360
362
|
}
|
|
363
|
+
lastAgentMessage() {
|
|
364
|
+
if (this._messages.length === 0) {
|
|
365
|
+
throw new Error("No messages in history");
|
|
366
|
+
}
|
|
367
|
+
const lastMessage = this._messages.findLast((message2) => message2.role === "assistant");
|
|
368
|
+
if (!lastMessage) {
|
|
369
|
+
throw new Error("No agent message in history");
|
|
370
|
+
}
|
|
371
|
+
return lastMessage;
|
|
372
|
+
}
|
|
361
373
|
lastToolCall(toolName) {
|
|
362
374
|
if (this._messages.length === 0) {
|
|
363
375
|
throw new Error("No messages in history");
|
|
@@ -365,9 +377,6 @@ var ScenarioExecutionState = class {
|
|
|
365
377
|
const lastMessage = this._messages.findLast((message2) => message2.role === "tool" && message2.content.find(
|
|
366
378
|
(part) => part.type === "tool-result" && part.toolName === toolName
|
|
367
379
|
));
|
|
368
|
-
if (!lastMessage) {
|
|
369
|
-
throw new Error("No tool call message in history");
|
|
370
|
-
}
|
|
371
380
|
return lastMessage;
|
|
372
381
|
}
|
|
373
382
|
hasToolCall(toolName) {
|
|
@@ -451,7 +460,6 @@ function convertCoreMessagesToAguiMessages(coreMessages) {
|
|
|
451
460
|
var message_conversion_default = convertCoreMessagesToAguiMessages;
|
|
452
461
|
|
|
453
462
|
// src/execution/scenario-execution.ts
|
|
454
|
-
var batchRunId = getBatchRunId();
|
|
455
463
|
var ScenarioExecution = class {
|
|
456
464
|
state;
|
|
457
465
|
eventSubject = new Subject();
|
|
@@ -869,7 +877,7 @@ var ScenarioExecution = class {
|
|
|
869
877
|
type: "placeholder",
|
|
870
878
|
// This will be replaced by the specific event type
|
|
871
879
|
timestamp: Date.now(),
|
|
872
|
-
batchRunId,
|
|
880
|
+
batchRunId: getBatchRunId(),
|
|
873
881
|
scenarioId: this.config.id,
|
|
874
882
|
scenarioRunId,
|
|
875
883
|
scenarioSetId: this.config.setId
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __export = (target, all) => {
|
|
9
|
+
for (var name in all)
|
|
10
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
11
|
+
};
|
|
12
|
+
var __copyProps = (to, from, except, desc) => {
|
|
13
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
14
|
+
for (let key of __getOwnPropNames(from))
|
|
15
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
16
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
17
|
+
}
|
|
18
|
+
return to;
|
|
19
|
+
};
|
|
20
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
+
mod
|
|
27
|
+
));
|
|
28
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
|
+
|
|
30
|
+
// src/integrations/vitest/config.ts
|
|
31
|
+
var config_exports = {};
|
|
32
|
+
__export(config_exports, {
|
|
33
|
+
withScenario: () => withScenario
|
|
34
|
+
});
|
|
35
|
+
module.exports = __toCommonJS(config_exports);
|
|
36
|
+
var import_config = require("vitest/config");
|
|
37
|
+
|
|
38
|
+
// src/integrations/vitest/reporter.ts
|
|
39
|
+
var import_fs = __toESM(require("fs"));
|
|
40
|
+
var import_path = __toESM(require("path"));
|
|
41
|
+
var import_chalk = __toESM(require("chalk"));
|
|
42
|
+
|
|
43
|
+
// src/config/env.ts
|
|
44
|
+
var import_zod = require("zod");
|
|
45
|
+
|
|
46
|
+
// src/config/log-levels.ts
|
|
47
|
+
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
48
|
+
LogLevel2["ERROR"] = "ERROR";
|
|
49
|
+
LogLevel2["WARN"] = "WARN";
|
|
50
|
+
LogLevel2["INFO"] = "INFO";
|
|
51
|
+
LogLevel2["DEBUG"] = "DEBUG";
|
|
52
|
+
return LogLevel2;
|
|
53
|
+
})(LogLevel || {});
|
|
54
|
+
|
|
55
|
+
// src/config/env.ts
|
|
56
|
+
var envSchema = import_zod.z.object({
|
|
57
|
+
/**
|
|
58
|
+
* LangWatch API key for event reporting.
|
|
59
|
+
* If not provided, events will not be sent to LangWatch.
|
|
60
|
+
*/
|
|
61
|
+
LANGWATCH_API_KEY: import_zod.z.string().optional(),
|
|
62
|
+
/**
|
|
63
|
+
* LangWatch endpoint URL for event reporting.
|
|
64
|
+
* Defaults to the production LangWatch endpoint.
|
|
65
|
+
*/
|
|
66
|
+
LANGWATCH_ENDPOINT: import_zod.z.string().url().default("https://app.langwatch.ai"),
|
|
67
|
+
/**
|
|
68
|
+
* Disables simulation report info messages when set to any truthy value.
|
|
69
|
+
* Useful for CI/CD environments or when you want cleaner output.
|
|
70
|
+
*/
|
|
71
|
+
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod.z.string().optional().transform((val) => Boolean(val)),
|
|
72
|
+
/**
|
|
73
|
+
* Node environment - affects logging and behavior.
|
|
74
|
+
* Defaults to 'development' if not specified.
|
|
75
|
+
*/
|
|
76
|
+
NODE_ENV: import_zod.z.enum(["development", "production", "test"]).default("development"),
|
|
77
|
+
/**
|
|
78
|
+
* Log level for the scenario package.
|
|
79
|
+
* Defaults to 'info' if not specified.
|
|
80
|
+
*/
|
|
81
|
+
LOG_LEVEL: import_zod.z.nativeEnum(LogLevel).optional(),
|
|
82
|
+
/**
|
|
83
|
+
* Scenario batch run ID.
|
|
84
|
+
* If not provided, a random ID will be generated.
|
|
85
|
+
*/
|
|
86
|
+
SCENARIO_BATCH_RUN_ID: import_zod.z.string().optional()
|
|
87
|
+
});
|
|
88
|
+
var env = envSchema.parse(process.env);
|
|
89
|
+
|
|
90
|
+
// src/utils/logger.ts
|
|
91
|
+
var Logger = class _Logger {
|
|
92
|
+
constructor(context) {
|
|
93
|
+
this.context = context;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Creates a logger with context (e.g., class name)
|
|
97
|
+
*/
|
|
98
|
+
static create(context) {
|
|
99
|
+
return new _Logger(context);
|
|
100
|
+
}
|
|
101
|
+
getLogLevel() {
|
|
102
|
+
return env.LOG_LEVEL ?? "INFO" /* INFO */;
|
|
103
|
+
}
|
|
104
|
+
getLogLevelIndex(level) {
|
|
105
|
+
return Object.values(LogLevel).indexOf(level);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Checks if logging should occur based on LOG_LEVEL env var
|
|
109
|
+
*/
|
|
110
|
+
shouldLog(level) {
|
|
111
|
+
const currentLevelIndex = this.getLogLevelIndex(this.getLogLevel());
|
|
112
|
+
const requestedLevelIndex = this.getLogLevelIndex(level);
|
|
113
|
+
return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
|
|
114
|
+
}
|
|
115
|
+
formatMessage(message) {
|
|
116
|
+
return this.context ? `[${this.context}] ${message}` : message;
|
|
117
|
+
}
|
|
118
|
+
error(message, data) {
|
|
119
|
+
if (this.shouldLog("ERROR" /* ERROR */)) {
|
|
120
|
+
const formattedMessage = this.formatMessage(message);
|
|
121
|
+
if (data) {
|
|
122
|
+
console.error(formattedMessage, data);
|
|
123
|
+
} else {
|
|
124
|
+
console.error(formattedMessage);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
warn(message, data) {
|
|
129
|
+
if (this.shouldLog("WARN" /* WARN */)) {
|
|
130
|
+
const formattedMessage = this.formatMessage(message);
|
|
131
|
+
if (data) {
|
|
132
|
+
console.warn(formattedMessage, data);
|
|
133
|
+
} else {
|
|
134
|
+
console.warn(formattedMessage);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
info(message, data) {
|
|
139
|
+
if (this.shouldLog("INFO" /* INFO */)) {
|
|
140
|
+
const formattedMessage = this.formatMessage(message);
|
|
141
|
+
if (data) {
|
|
142
|
+
console.info(formattedMessage, data);
|
|
143
|
+
} else {
|
|
144
|
+
console.info(formattedMessage);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
debug(message, data) {
|
|
149
|
+
if (this.shouldLog("DEBUG" /* DEBUG */)) {
|
|
150
|
+
const formattedMessage = this.formatMessage(message);
|
|
151
|
+
if (data) {
|
|
152
|
+
console.log(formattedMessage, data);
|
|
153
|
+
} else {
|
|
154
|
+
console.log(formattedMessage);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
// src/integrations/vitest/reporter.ts
|
|
161
|
+
var logger = Logger.create("integrations:vitest:reporter");
|
|
162
|
+
function getProjectRoot() {
|
|
163
|
+
return process.cwd();
|
|
164
|
+
}
|
|
165
|
+
var projectRoot = getProjectRoot();
|
|
166
|
+
var logDir = import_path.default.join(projectRoot, ".scenario");
|
|
167
|
+
if (!import_fs.default.existsSync(logDir)) import_fs.default.mkdirSync(logDir);
|
|
168
|
+
function getLogFilePath(testId) {
|
|
169
|
+
return import_path.default.join(logDir, `${testId}.log`);
|
|
170
|
+
}
|
|
171
|
+
function getFullTestName(task) {
|
|
172
|
+
let name = task.name;
|
|
173
|
+
let parent = task.suite;
|
|
174
|
+
while (parent) {
|
|
175
|
+
name = `${parent.name} > ${name}`;
|
|
176
|
+
parent = parent.suite;
|
|
177
|
+
}
|
|
178
|
+
return name;
|
|
179
|
+
}
|
|
180
|
+
var VitestReporter = class {
|
|
181
|
+
results = [];
|
|
182
|
+
async onTestCaseResult(test) {
|
|
183
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
184
|
+
const fullName = getFullTestName(test);
|
|
185
|
+
const filePath = getLogFilePath(test.id);
|
|
186
|
+
if (!import_fs.default.existsSync(filePath)) {
|
|
187
|
+
logger.warn(
|
|
188
|
+
`No log file found ${filePath} for test ${fullName}`,
|
|
189
|
+
test.id
|
|
190
|
+
);
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
const lines = import_fs.default.readFileSync(filePath, "utf-8").split("\n").filter(Boolean);
|
|
194
|
+
const events = lines.map((line) => JSON.parse(line));
|
|
195
|
+
const runs = /* @__PURE__ */ new Map();
|
|
196
|
+
for (const event of events) {
|
|
197
|
+
const runId = event.scenarioRunId ?? "unknown";
|
|
198
|
+
if (!runs.has(runId)) runs.set(runId, []);
|
|
199
|
+
runs.get(runId).push(event);
|
|
200
|
+
}
|
|
201
|
+
for (const [runId, runEvents] of Array.from(runs.entries())) {
|
|
202
|
+
const started = runEvents.find(
|
|
203
|
+
(e) => e.type === "SCENARIO_RUN_STARTED"
|
|
204
|
+
);
|
|
205
|
+
const finished = runEvents.find(
|
|
206
|
+
(e) => e.type === "SCENARIO_RUN_FINISHED"
|
|
207
|
+
);
|
|
208
|
+
const messages = runEvents.filter(
|
|
209
|
+
(e) => e.type === "SCENARIO_MESSAGE_SNAPSHOT"
|
|
210
|
+
);
|
|
211
|
+
this.results.push({
|
|
212
|
+
name: ((_a = started == null ? void 0 : started.metadata) == null ? void 0 : _a.name) ?? fullName,
|
|
213
|
+
status: (finished == null ? void 0 : finished.status) ?? "UNKNOWN",
|
|
214
|
+
duration: started && finished ? finished.timestamp - started.timestamp : 0,
|
|
215
|
+
reasoning: (_b = finished == null ? void 0 : finished.results) == null ? void 0 : _b.reasoning,
|
|
216
|
+
criteria: (finished == null ? void 0 : finished.results) ? `Success Criteria: ${((_c = finished.results.metCriteria) == null ? void 0 : _c.length) ?? 0}/${(((_d = finished.results.metCriteria) == null ? void 0 : _d.length) ?? 0) + (((_e = finished.results.unmetCriteria) == null ? void 0 : _e.length) ?? 0)}` : void 0
|
|
217
|
+
});
|
|
218
|
+
console.log(
|
|
219
|
+
`
|
|
220
|
+
--- Scenario Run: ${((_f = started == null ? void 0 : started.metadata) == null ? void 0 : _f.name) ?? runId} ---`
|
|
221
|
+
);
|
|
222
|
+
if (started) {
|
|
223
|
+
console.log(`Description: ${((_g = started.metadata) == null ? void 0 : _g.description) ?? ""}`);
|
|
224
|
+
}
|
|
225
|
+
if (messages.length) {
|
|
226
|
+
console.log("Chat log:");
|
|
227
|
+
let lastMessageCount = 0;
|
|
228
|
+
for (const msg of messages) {
|
|
229
|
+
const allMessages = msg.messages ?? [];
|
|
230
|
+
for (const m of allMessages.slice(lastMessageCount)) {
|
|
231
|
+
const role = m.role;
|
|
232
|
+
let roleLabel = role;
|
|
233
|
+
if (role.toLowerCase() === "user") roleLabel = import_chalk.default.green("User");
|
|
234
|
+
else if (role.toLowerCase() === "agent")
|
|
235
|
+
roleLabel = import_chalk.default.cyan("Agent");
|
|
236
|
+
else if (role.toLowerCase() === "assistant")
|
|
237
|
+
roleLabel = import_chalk.default.cyan("Assistant");
|
|
238
|
+
else roleLabel = import_chalk.default.yellow(role);
|
|
239
|
+
console.log(`${roleLabel}: ${m.content}`);
|
|
240
|
+
}
|
|
241
|
+
lastMessageCount = allMessages.length;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
if (finished) {
|
|
245
|
+
console.log("--- Verdict ---");
|
|
246
|
+
console.log(`Status: ${finished.status}`);
|
|
247
|
+
if (finished.results) {
|
|
248
|
+
console.log(`Verdict: ${finished.results.verdict}`);
|
|
249
|
+
if (finished.results.reasoning)
|
|
250
|
+
console.log(`Reasoning: ${finished.results.reasoning}`);
|
|
251
|
+
if ((_h = finished.results.metCriteria) == null ? void 0 : _h.length)
|
|
252
|
+
console.log(
|
|
253
|
+
`Met criteria: ${finished.results.metCriteria.join(", ")}`
|
|
254
|
+
);
|
|
255
|
+
if ((_i = finished.results.unmetCriteria) == null ? void 0 : _i.length)
|
|
256
|
+
console.log(
|
|
257
|
+
`Unmet criteria: ${finished.results.unmetCriteria.join(", ")}`
|
|
258
|
+
);
|
|
259
|
+
if (finished.results.error)
|
|
260
|
+
console.log(`Error: ${finished.results.error}`);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
console.log("-----------------------------\n");
|
|
264
|
+
}
|
|
265
|
+
import_fs.default.unlinkSync(filePath);
|
|
266
|
+
}
|
|
267
|
+
async onTestRunEnd() {
|
|
268
|
+
if (this.results.length === 0) return;
|
|
269
|
+
const total = this.results.length;
|
|
270
|
+
const passed = this.results.filter((r) => r.status === "SUCCESS").length;
|
|
271
|
+
const failed = this.results.filter((r) => r.status !== "SUCCESS").length;
|
|
272
|
+
const successRate = (passed / total * 100).toFixed(1);
|
|
273
|
+
console.log();
|
|
274
|
+
console.log(import_chalk.default.bold.cyan("=== Scenario Test Report ==="));
|
|
275
|
+
console.log(`Total Scenarios: ${total}`);
|
|
276
|
+
console.log(import_chalk.default.green(`Passed: ${passed}`));
|
|
277
|
+
console.log(import_chalk.default.red(`Failed: ${failed}`));
|
|
278
|
+
console.log(`Success Rate: ${import_chalk.default.bold(`${successRate}%`)}`);
|
|
279
|
+
this.results.forEach((r, i) => {
|
|
280
|
+
const statusColor = r.status === "SUCCESS" ? import_chalk.default.green : import_chalk.default.red;
|
|
281
|
+
console.log();
|
|
282
|
+
console.log(
|
|
283
|
+
`${i + 1}. ${r.name} - ${statusColor(r.status)} in ${(r.duration / 1e3).toFixed(2)}s`
|
|
284
|
+
);
|
|
285
|
+
if (r.reasoning) {
|
|
286
|
+
console.log(import_chalk.default.greenBright(" Reasoning: ") + r.reasoning);
|
|
287
|
+
}
|
|
288
|
+
if (r.criteria) {
|
|
289
|
+
console.log(import_chalk.default.bold(" " + r.criteria));
|
|
290
|
+
}
|
|
291
|
+
});
|
|
292
|
+
console.log();
|
|
293
|
+
}
|
|
294
|
+
};
|
|
295
|
+
|
|
296
|
+
// src/integrations/vitest/config.ts
|
|
297
|
+
function withScenario(config) {
|
|
298
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l;
|
|
299
|
+
const normalizedSetupFiles = ((_a = config.test) == null ? void 0 : _a.setupFiles) === void 0 ? [] : Array.isArray((_b = config.test) == null ? void 0 : _b.setupFiles) ? (_c = config.test) == null ? void 0 : _c.setupFiles : [(_d = config.test) == null ? void 0 : _d.setupFiles];
|
|
300
|
+
const normalizedGlobalSetup = ((_e = config.test) == null ? void 0 : _e.globalSetup) === void 0 ? [] : Array.isArray((_f = config.test) == null ? void 0 : _f.globalSetup) ? (_g = config.test) == null ? void 0 : _g.globalSetup : [(_h = config.test) == null ? void 0 : _h.globalSetup];
|
|
301
|
+
const normalizedReporters = ((_i = config.test) == null ? void 0 : _i.reporters) === void 0 ? [] : Array.isArray((_j = config.test) == null ? void 0 : _j.reporters) ? (_k = config.test) == null ? void 0 : _k.reporters : [(_l = config.test) == null ? void 0 : _l.reporters];
|
|
302
|
+
return (0, import_config.defineConfig)({
|
|
303
|
+
...config,
|
|
304
|
+
test: {
|
|
305
|
+
...config.test,
|
|
306
|
+
setupFiles: [
|
|
307
|
+
"@langwatch/scenario/integrations/vitest/setup",
|
|
308
|
+
...normalizedSetupFiles
|
|
309
|
+
],
|
|
310
|
+
globalSetup: [
|
|
311
|
+
"@langwatch/scenario/integrations/vitest/setup-global",
|
|
312
|
+
...normalizedGlobalSetup
|
|
313
|
+
],
|
|
314
|
+
reporters: [
|
|
315
|
+
...normalizedReporters,
|
|
316
|
+
new VitestReporter()
|
|
317
|
+
]
|
|
318
|
+
}
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
322
|
+
0 && (module.exports = {
|
|
323
|
+
withScenario
|
|
324
|
+
});
|