@langwatch/scenario 0.2.9 → 0.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -15
- package/dist/{chunk-7H6OGEQ5.mjs → chunk-7HLDX5EL.mjs} +9 -14
- package/dist/{chunk-YPJZSK4J.mjs → chunk-OL4RFXV4.mjs} +23 -11
- package/dist/index.d.mts +559 -72
- package/dist/index.d.ts +559 -72
- package/dist/index.js +746 -212
- package/dist/index.mjs +711 -187
- package/dist/integrations/vitest/config.d.mts +37 -0
- package/dist/integrations/vitest/config.d.ts +37 -0
- package/dist/integrations/vitest/config.js +3 -276
- package/dist/integrations/vitest/config.mjs +3 -10
- package/dist/integrations/vitest/reporter.js +69 -17
- package/dist/integrations/vitest/reporter.mjs +182 -4
- package/dist/integrations/vitest/setup.js +24 -12
- package/dist/integrations/vitest/setup.mjs +2 -2
- package/package.json +1 -2
- package/dist/chunk-K7KLHTDI.mjs +0 -146
|
@@ -1,5 +1,42 @@
|
|
|
1
1
|
import { ViteUserConfig } from 'vitest/config';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Enhances a Vitest configuration with Scenario testing framework setup.
|
|
5
|
+
*
|
|
6
|
+
* This function wraps the provided Vitest configuration and automatically adds
|
|
7
|
+
* the necessary setup files for Scenario testing. It ensures that both local
|
|
8
|
+
* and global setup files are properly configured while preserving any existing
|
|
9
|
+
* setup configuration.
|
|
10
|
+
*
|
|
11
|
+
* The function normalizes setup file configurations to handle different input formats:
|
|
12
|
+
* - `undefined` → empty array
|
|
13
|
+
* - string → array with single item
|
|
14
|
+
* - array → array as-is
|
|
15
|
+
*
|
|
16
|
+
* @param config - The base Vitest configuration to enhance
|
|
17
|
+
* @returns Enhanced Vitest configuration with Scenario setup files prepended
|
|
18
|
+
*
|
|
19
|
+
* @example
|
|
20
|
+
* ```typescript
|
|
21
|
+
* import { defineConfig } from 'vitest/config';
|
|
22
|
+
* import { withScenario } from '@langwatch/scenario/integrations/vitest/config';
|
|
23
|
+
*
|
|
24
|
+
* export default withScenario(defineConfig({
|
|
25
|
+
* test: {
|
|
26
|
+
* setupFiles: ['./my-setup.ts'],
|
|
27
|
+
* globalSetup: ['./my-global-setup.ts']
|
|
28
|
+
* }
|
|
29
|
+
* }));
|
|
30
|
+
* ```
|
|
31
|
+
*
|
|
32
|
+
* @example
|
|
33
|
+
* ```typescript
|
|
34
|
+
* // Minimal configuration - only Scenario setup files
|
|
35
|
+
* export default withScenario(defineConfig({}));
|
|
36
|
+
* ```
|
|
37
|
+
*
|
|
38
|
+
* @since 1.0.0
|
|
39
|
+
*/
|
|
3
40
|
declare function withScenario(config: ViteUserConfig): ViteUserConfig;
|
|
4
41
|
|
|
5
42
|
export { withScenario };
|
|
@@ -1,5 +1,42 @@
|
|
|
1
1
|
import { ViteUserConfig } from 'vitest/config';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Enhances a Vitest configuration with Scenario testing framework setup.
|
|
5
|
+
*
|
|
6
|
+
* This function wraps the provided Vitest configuration and automatically adds
|
|
7
|
+
* the necessary setup files for Scenario testing. It ensures that both local
|
|
8
|
+
* and global setup files are properly configured while preserving any existing
|
|
9
|
+
* setup configuration.
|
|
10
|
+
*
|
|
11
|
+
* The function normalizes setup file configurations to handle different input formats:
|
|
12
|
+
* - `undefined` → empty array
|
|
13
|
+
* - string → array with single item
|
|
14
|
+
* - array → array as-is
|
|
15
|
+
*
|
|
16
|
+
* @param config - The base Vitest configuration to enhance
|
|
17
|
+
* @returns Enhanced Vitest configuration with Scenario setup files prepended
|
|
18
|
+
*
|
|
19
|
+
* @example
|
|
20
|
+
* ```typescript
|
|
21
|
+
* import { defineConfig } from 'vitest/config';
|
|
22
|
+
* import { withScenario } from '@langwatch/scenario/integrations/vitest/config';
|
|
23
|
+
*
|
|
24
|
+
* export default withScenario(defineConfig({
|
|
25
|
+
* test: {
|
|
26
|
+
* setupFiles: ['./my-setup.ts'],
|
|
27
|
+
* globalSetup: ['./my-global-setup.ts']
|
|
28
|
+
* }
|
|
29
|
+
* }));
|
|
30
|
+
* ```
|
|
31
|
+
*
|
|
32
|
+
* @example
|
|
33
|
+
* ```typescript
|
|
34
|
+
* // Minimal configuration - only Scenario setup files
|
|
35
|
+
* export default withScenario(defineConfig({}));
|
|
36
|
+
* ```
|
|
37
|
+
*
|
|
38
|
+
* @since 1.0.0
|
|
39
|
+
*/
|
|
3
40
|
declare function withScenario(config: ViteUserConfig): ViteUserConfig;
|
|
4
41
|
|
|
5
42
|
export { withScenario };
|
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
2
|
var __defProp = Object.defineProperty;
|
|
4
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
6
|
var __export = (target, all) => {
|
|
9
7
|
for (var name in all)
|
|
@@ -17,14 +15,6 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
17
15
|
}
|
|
18
16
|
return to;
|
|
19
17
|
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
19
|
|
|
30
20
|
// src/integrations/vitest/config.ts
|
|
@@ -34,286 +24,23 @@ __export(config_exports, {
|
|
|
34
24
|
});
|
|
35
25
|
module.exports = __toCommonJS(config_exports);
|
|
36
26
|
var import_config = require("vitest/config");
|
|
37
|
-
|
|
38
|
-
// src/integrations/vitest/reporter.ts
|
|
39
|
-
var import_fs = __toESM(require("fs"));
|
|
40
|
-
var import_path = __toESM(require("path"));
|
|
41
|
-
var import_chalk = __toESM(require("chalk"));
|
|
42
|
-
|
|
43
|
-
// src/config/env.ts
|
|
44
|
-
var import_zod = require("zod");
|
|
45
|
-
|
|
46
|
-
// src/config/log-levels.ts
|
|
47
|
-
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
48
|
-
LogLevel2["ERROR"] = "ERROR";
|
|
49
|
-
LogLevel2["WARN"] = "WARN";
|
|
50
|
-
LogLevel2["INFO"] = "INFO";
|
|
51
|
-
LogLevel2["DEBUG"] = "DEBUG";
|
|
52
|
-
return LogLevel2;
|
|
53
|
-
})(LogLevel || {});
|
|
54
|
-
|
|
55
|
-
// src/config/env.ts
|
|
56
|
-
var envSchema = import_zod.z.object({
|
|
57
|
-
/**
|
|
58
|
-
* LangWatch API key for event reporting.
|
|
59
|
-
* If not provided, events will not be sent to LangWatch.
|
|
60
|
-
*/
|
|
61
|
-
LANGWATCH_API_KEY: import_zod.z.string().optional(),
|
|
62
|
-
/**
|
|
63
|
-
* LangWatch endpoint URL for event reporting.
|
|
64
|
-
* Defaults to the production LangWatch endpoint.
|
|
65
|
-
*/
|
|
66
|
-
LANGWATCH_ENDPOINT: import_zod.z.string().url().default("https://app.langwatch.ai"),
|
|
67
|
-
/**
|
|
68
|
-
* Disables simulation report info messages when set to any truthy value.
|
|
69
|
-
* Useful for CI/CD environments or when you want cleaner output.
|
|
70
|
-
*/
|
|
71
|
-
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod.z.string().optional().transform((val) => Boolean(val)),
|
|
72
|
-
/**
|
|
73
|
-
* Node environment - affects logging and behavior.
|
|
74
|
-
* Defaults to 'development' if not specified.
|
|
75
|
-
*/
|
|
76
|
-
NODE_ENV: import_zod.z.enum(["development", "production", "test"]).default("development"),
|
|
77
|
-
/**
|
|
78
|
-
* Log level for the scenario package.
|
|
79
|
-
* Defaults to 'info' if not specified.
|
|
80
|
-
*/
|
|
81
|
-
LOG_LEVEL: import_zod.z.nativeEnum(LogLevel).optional(),
|
|
82
|
-
/**
|
|
83
|
-
* Scenario batch run ID.
|
|
84
|
-
* If not provided, a random ID will be generated.
|
|
85
|
-
*/
|
|
86
|
-
SCENARIO_BATCH_RUN_ID: import_zod.z.string().optional()
|
|
87
|
-
});
|
|
88
|
-
var env = envSchema.parse(process.env);
|
|
89
|
-
|
|
90
|
-
// src/utils/logger.ts
|
|
91
|
-
var Logger = class _Logger {
|
|
92
|
-
constructor(context) {
|
|
93
|
-
this.context = context;
|
|
94
|
-
}
|
|
95
|
-
/**
|
|
96
|
-
* Creates a logger with context (e.g., class name)
|
|
97
|
-
*/
|
|
98
|
-
static create(context) {
|
|
99
|
-
return new _Logger(context);
|
|
100
|
-
}
|
|
101
|
-
getLogLevel() {
|
|
102
|
-
return env.LOG_LEVEL ?? "INFO" /* INFO */;
|
|
103
|
-
}
|
|
104
|
-
getLogLevelIndex(level) {
|
|
105
|
-
return Object.values(LogLevel).indexOf(level);
|
|
106
|
-
}
|
|
107
|
-
/**
|
|
108
|
-
* Checks if logging should occur based on LOG_LEVEL env var
|
|
109
|
-
*/
|
|
110
|
-
shouldLog(level) {
|
|
111
|
-
const currentLevelIndex = this.getLogLevelIndex(this.getLogLevel());
|
|
112
|
-
const requestedLevelIndex = this.getLogLevelIndex(level);
|
|
113
|
-
return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
|
|
114
|
-
}
|
|
115
|
-
formatMessage(message) {
|
|
116
|
-
return this.context ? `[${this.context}] ${message}` : message;
|
|
117
|
-
}
|
|
118
|
-
error(message, data) {
|
|
119
|
-
if (this.shouldLog("ERROR" /* ERROR */)) {
|
|
120
|
-
const formattedMessage = this.formatMessage(message);
|
|
121
|
-
if (data) {
|
|
122
|
-
console.error(formattedMessage, data);
|
|
123
|
-
} else {
|
|
124
|
-
console.error(formattedMessage);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
warn(message, data) {
|
|
129
|
-
if (this.shouldLog("WARN" /* WARN */)) {
|
|
130
|
-
const formattedMessage = this.formatMessage(message);
|
|
131
|
-
if (data) {
|
|
132
|
-
console.warn(formattedMessage, data);
|
|
133
|
-
} else {
|
|
134
|
-
console.warn(formattedMessage);
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
info(message, data) {
|
|
139
|
-
if (this.shouldLog("INFO" /* INFO */)) {
|
|
140
|
-
const formattedMessage = this.formatMessage(message);
|
|
141
|
-
if (data) {
|
|
142
|
-
console.info(formattedMessage, data);
|
|
143
|
-
} else {
|
|
144
|
-
console.info(formattedMessage);
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
debug(message, data) {
|
|
149
|
-
if (this.shouldLog("DEBUG" /* DEBUG */)) {
|
|
150
|
-
const formattedMessage = this.formatMessage(message);
|
|
151
|
-
if (data) {
|
|
152
|
-
console.log(formattedMessage, data);
|
|
153
|
-
} else {
|
|
154
|
-
console.log(formattedMessage);
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
};
|
|
159
|
-
|
|
160
|
-
// src/integrations/vitest/reporter.ts
|
|
161
|
-
var logger = Logger.create("integrations:vitest:reporter");
|
|
162
|
-
function getProjectRoot() {
|
|
163
|
-
return process.cwd();
|
|
164
|
-
}
|
|
165
|
-
var projectRoot = getProjectRoot();
|
|
166
|
-
var logDir = import_path.default.join(projectRoot, ".scenario");
|
|
167
|
-
if (!import_fs.default.existsSync(logDir)) import_fs.default.mkdirSync(logDir);
|
|
168
|
-
function getLogFilePath(testId) {
|
|
169
|
-
return import_path.default.join(logDir, `${testId}.log`);
|
|
170
|
-
}
|
|
171
|
-
function getFullTestName(task) {
|
|
172
|
-
let name = task.name;
|
|
173
|
-
let parent = task.suite;
|
|
174
|
-
while (parent) {
|
|
175
|
-
name = `${parent.name} > ${name}`;
|
|
176
|
-
parent = parent.suite;
|
|
177
|
-
}
|
|
178
|
-
return name;
|
|
179
|
-
}
|
|
180
|
-
var VitestReporter = class {
|
|
181
|
-
results = [];
|
|
182
|
-
async onTestCaseResult(test) {
|
|
183
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
184
|
-
const fullName = getFullTestName(test);
|
|
185
|
-
const filePath = getLogFilePath(test.id);
|
|
186
|
-
if (!import_fs.default.existsSync(filePath)) {
|
|
187
|
-
logger.warn(
|
|
188
|
-
`No log file found ${filePath} for test ${fullName}`,
|
|
189
|
-
test.id
|
|
190
|
-
);
|
|
191
|
-
return;
|
|
192
|
-
}
|
|
193
|
-
const lines = import_fs.default.readFileSync(filePath, "utf-8").split("\n").filter(Boolean);
|
|
194
|
-
const events = lines.map((line) => JSON.parse(line));
|
|
195
|
-
const runs = /* @__PURE__ */ new Map();
|
|
196
|
-
for (const event of events) {
|
|
197
|
-
const runId = event.scenarioRunId ?? "unknown";
|
|
198
|
-
if (!runs.has(runId)) runs.set(runId, []);
|
|
199
|
-
runs.get(runId).push(event);
|
|
200
|
-
}
|
|
201
|
-
for (const [runId, runEvents] of Array.from(runs.entries())) {
|
|
202
|
-
const started = runEvents.find(
|
|
203
|
-
(e) => e.type === "SCENARIO_RUN_STARTED"
|
|
204
|
-
);
|
|
205
|
-
const finished = runEvents.find(
|
|
206
|
-
(e) => e.type === "SCENARIO_RUN_FINISHED"
|
|
207
|
-
);
|
|
208
|
-
const messages = runEvents.filter(
|
|
209
|
-
(e) => e.type === "SCENARIO_MESSAGE_SNAPSHOT"
|
|
210
|
-
);
|
|
211
|
-
this.results.push({
|
|
212
|
-
name: ((_a = started == null ? void 0 : started.metadata) == null ? void 0 : _a.name) ?? fullName,
|
|
213
|
-
status: (finished == null ? void 0 : finished.status) ?? "UNKNOWN",
|
|
214
|
-
duration: started && finished ? finished.timestamp - started.timestamp : 0,
|
|
215
|
-
reasoning: (_b = finished == null ? void 0 : finished.results) == null ? void 0 : _b.reasoning,
|
|
216
|
-
criteria: (finished == null ? void 0 : finished.results) ? `Success Criteria: ${((_c = finished.results.metCriteria) == null ? void 0 : _c.length) ?? 0}/${(((_d = finished.results.metCriteria) == null ? void 0 : _d.length) ?? 0) + (((_e = finished.results.unmetCriteria) == null ? void 0 : _e.length) ?? 0)}` : void 0
|
|
217
|
-
});
|
|
218
|
-
console.log(
|
|
219
|
-
`
|
|
220
|
-
--- Scenario Run: ${((_f = started == null ? void 0 : started.metadata) == null ? void 0 : _f.name) ?? runId} ---`
|
|
221
|
-
);
|
|
222
|
-
if (started) {
|
|
223
|
-
console.log(`Description: ${((_g = started.metadata) == null ? void 0 : _g.description) ?? ""}`);
|
|
224
|
-
}
|
|
225
|
-
if (messages.length) {
|
|
226
|
-
console.log("Chat log:");
|
|
227
|
-
let lastMessageCount = 0;
|
|
228
|
-
for (const msg of messages) {
|
|
229
|
-
const allMessages = msg.messages ?? [];
|
|
230
|
-
for (const m of allMessages.slice(lastMessageCount)) {
|
|
231
|
-
const role = m.role;
|
|
232
|
-
let roleLabel = role;
|
|
233
|
-
if (role.toLowerCase() === "user") roleLabel = import_chalk.default.green("User");
|
|
234
|
-
else if (role.toLowerCase() === "agent")
|
|
235
|
-
roleLabel = import_chalk.default.cyan("Agent");
|
|
236
|
-
else if (role.toLowerCase() === "assistant")
|
|
237
|
-
roleLabel = import_chalk.default.cyan("Assistant");
|
|
238
|
-
else roleLabel = import_chalk.default.yellow(role);
|
|
239
|
-
console.log(`${roleLabel}: ${m.content}`);
|
|
240
|
-
}
|
|
241
|
-
lastMessageCount = allMessages.length;
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
|
-
if (finished) {
|
|
245
|
-
console.log("--- Verdict ---");
|
|
246
|
-
console.log(`Status: ${finished.status}`);
|
|
247
|
-
if (finished.results) {
|
|
248
|
-
console.log(`Verdict: ${finished.results.verdict}`);
|
|
249
|
-
if (finished.results.reasoning)
|
|
250
|
-
console.log(`Reasoning: ${finished.results.reasoning}`);
|
|
251
|
-
if ((_h = finished.results.metCriteria) == null ? void 0 : _h.length)
|
|
252
|
-
console.log(
|
|
253
|
-
`Met criteria: ${finished.results.metCriteria.join(", ")}`
|
|
254
|
-
);
|
|
255
|
-
if ((_i = finished.results.unmetCriteria) == null ? void 0 : _i.length)
|
|
256
|
-
console.log(
|
|
257
|
-
`Unmet criteria: ${finished.results.unmetCriteria.join(", ")}`
|
|
258
|
-
);
|
|
259
|
-
if (finished.results.error)
|
|
260
|
-
console.log(`Error: ${finished.results.error}`);
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
console.log("-----------------------------\n");
|
|
264
|
-
}
|
|
265
|
-
import_fs.default.unlinkSync(filePath);
|
|
266
|
-
}
|
|
267
|
-
async onTestRunEnd() {
|
|
268
|
-
if (this.results.length === 0) return;
|
|
269
|
-
const total = this.results.length;
|
|
270
|
-
const passed = this.results.filter((r) => r.status === "SUCCESS").length;
|
|
271
|
-
const failed = this.results.filter((r) => r.status !== "SUCCESS").length;
|
|
272
|
-
const successRate = (passed / total * 100).toFixed(1);
|
|
273
|
-
console.log();
|
|
274
|
-
console.log(import_chalk.default.bold.cyan("=== Scenario Test Report ==="));
|
|
275
|
-
console.log(`Total Scenarios: ${total}`);
|
|
276
|
-
console.log(import_chalk.default.green(`Passed: ${passed}`));
|
|
277
|
-
console.log(import_chalk.default.red(`Failed: ${failed}`));
|
|
278
|
-
console.log(`Success Rate: ${import_chalk.default.bold(`${successRate}%`)}`);
|
|
279
|
-
this.results.forEach((r, i) => {
|
|
280
|
-
const statusColor = r.status === "SUCCESS" ? import_chalk.default.green : import_chalk.default.red;
|
|
281
|
-
console.log();
|
|
282
|
-
console.log(
|
|
283
|
-
`${i + 1}. ${r.name} - ${statusColor(r.status)} in ${(r.duration / 1e3).toFixed(2)}s`
|
|
284
|
-
);
|
|
285
|
-
if (r.reasoning) {
|
|
286
|
-
console.log(import_chalk.default.greenBright(" Reasoning: ") + r.reasoning);
|
|
287
|
-
}
|
|
288
|
-
if (r.criteria) {
|
|
289
|
-
console.log(import_chalk.default.bold(" " + r.criteria));
|
|
290
|
-
}
|
|
291
|
-
});
|
|
292
|
-
console.log();
|
|
293
|
-
}
|
|
294
|
-
};
|
|
295
|
-
|
|
296
|
-
// src/integrations/vitest/config.ts
|
|
297
27
|
function withScenario(config) {
|
|
298
|
-
var _a, _b, _c, _d, _e, _f, _g, _h
|
|
28
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
299
29
|
const normalizedSetupFiles = ((_a = config.test) == null ? void 0 : _a.setupFiles) === void 0 ? [] : Array.isArray((_b = config.test) == null ? void 0 : _b.setupFiles) ? (_c = config.test) == null ? void 0 : _c.setupFiles : [(_d = config.test) == null ? void 0 : _d.setupFiles];
|
|
300
30
|
const normalizedGlobalSetup = ((_e = config.test) == null ? void 0 : _e.globalSetup) === void 0 ? [] : Array.isArray((_f = config.test) == null ? void 0 : _f.globalSetup) ? (_g = config.test) == null ? void 0 : _g.globalSetup : [(_h = config.test) == null ? void 0 : _h.globalSetup];
|
|
301
|
-
const normalizedReporters = ((_i = config.test) == null ? void 0 : _i.reporters) === void 0 ? [] : Array.isArray((_j = config.test) == null ? void 0 : _j.reporters) ? (_k = config.test) == null ? void 0 : _k.reporters : [(_l = config.test) == null ? void 0 : _l.reporters];
|
|
302
31
|
return (0, import_config.defineConfig)({
|
|
303
32
|
...config,
|
|
304
33
|
test: {
|
|
305
34
|
...config.test,
|
|
35
|
+
// Prepend Scenario setup files to ensure they run before user-defined setup
|
|
306
36
|
setupFiles: [
|
|
307
37
|
"@langwatch/scenario/integrations/vitest/setup",
|
|
308
38
|
...normalizedSetupFiles
|
|
309
39
|
],
|
|
40
|
+
// Prepend Scenario global setup files to ensure they run before user-defined global setup
|
|
310
41
|
globalSetup: [
|
|
311
42
|
"@langwatch/scenario/integrations/vitest/setup-global",
|
|
312
43
|
...normalizedGlobalSetup
|
|
313
|
-
],
|
|
314
|
-
reporters: [
|
|
315
|
-
...normalizedReporters,
|
|
316
|
-
new VitestReporter()
|
|
317
44
|
]
|
|
318
45
|
}
|
|
319
46
|
});
|
|
@@ -1,31 +1,24 @@
|
|
|
1
|
-
import {
|
|
2
|
-
VitestReporter
|
|
3
|
-
} from "../../chunk-K7KLHTDI.mjs";
|
|
4
|
-
import "../../chunk-YPJZSK4J.mjs";
|
|
5
1
|
import "../../chunk-7P6ASYW6.mjs";
|
|
6
2
|
|
|
7
3
|
// src/integrations/vitest/config.ts
|
|
8
4
|
import { defineConfig } from "vitest/config";
|
|
9
5
|
function withScenario(config) {
|
|
10
|
-
var _a, _b, _c, _d, _e, _f, _g, _h
|
|
6
|
+
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
11
7
|
const normalizedSetupFiles = ((_a = config.test) == null ? void 0 : _a.setupFiles) === void 0 ? [] : Array.isArray((_b = config.test) == null ? void 0 : _b.setupFiles) ? (_c = config.test) == null ? void 0 : _c.setupFiles : [(_d = config.test) == null ? void 0 : _d.setupFiles];
|
|
12
8
|
const normalizedGlobalSetup = ((_e = config.test) == null ? void 0 : _e.globalSetup) === void 0 ? [] : Array.isArray((_f = config.test) == null ? void 0 : _f.globalSetup) ? (_g = config.test) == null ? void 0 : _g.globalSetup : [(_h = config.test) == null ? void 0 : _h.globalSetup];
|
|
13
|
-
const normalizedReporters = ((_i = config.test) == null ? void 0 : _i.reporters) === void 0 ? [] : Array.isArray((_j = config.test) == null ? void 0 : _j.reporters) ? (_k = config.test) == null ? void 0 : _k.reporters : [(_l = config.test) == null ? void 0 : _l.reporters];
|
|
14
9
|
return defineConfig({
|
|
15
10
|
...config,
|
|
16
11
|
test: {
|
|
17
12
|
...config.test,
|
|
13
|
+
// Prepend Scenario setup files to ensure they run before user-defined setup
|
|
18
14
|
setupFiles: [
|
|
19
15
|
"@langwatch/scenario/integrations/vitest/setup",
|
|
20
16
|
...normalizedSetupFiles
|
|
21
17
|
],
|
|
18
|
+
// Prepend Scenario global setup files to ensure they run before user-defined global setup
|
|
22
19
|
globalSetup: [
|
|
23
20
|
"@langwatch/scenario/integrations/vitest/setup-global",
|
|
24
21
|
...normalizedGlobalSetup
|
|
25
|
-
],
|
|
26
|
-
reporters: [
|
|
27
|
-
...normalizedReporters,
|
|
28
|
-
new VitestReporter()
|
|
29
22
|
]
|
|
30
23
|
}
|
|
31
24
|
});
|
|
@@ -30,7 +30,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
30
30
|
// src/integrations/vitest/reporter.ts
|
|
31
31
|
var reporter_exports = {};
|
|
32
32
|
__export(reporter_exports, {
|
|
33
|
-
default: () =>
|
|
33
|
+
default: () => reporter_default
|
|
34
34
|
});
|
|
35
35
|
module.exports = __toCommonJS(reporter_exports);
|
|
36
36
|
var import_fs = __toESM(require("fs"));
|
|
@@ -48,6 +48,7 @@ var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
|
48
48
|
LogLevel2["DEBUG"] = "DEBUG";
|
|
49
49
|
return LogLevel2;
|
|
50
50
|
})(LogLevel || {});
|
|
51
|
+
var LOG_LEVELS = Object.values(LogLevel);
|
|
51
52
|
|
|
52
53
|
// src/config/env.ts
|
|
53
54
|
var envSchema = import_zod.z.object({
|
|
@@ -60,7 +61,7 @@ var envSchema = import_zod.z.object({
|
|
|
60
61
|
* LangWatch endpoint URL for event reporting.
|
|
61
62
|
* Defaults to the production LangWatch endpoint.
|
|
62
63
|
*/
|
|
63
|
-
LANGWATCH_ENDPOINT: import_zod.z.string().url().default("https://app.langwatch.ai"),
|
|
64
|
+
LANGWATCH_ENDPOINT: import_zod.z.string().url().optional().default("https://app.langwatch.ai"),
|
|
64
65
|
/**
|
|
65
66
|
* Disables simulation report info messages when set to any truthy value.
|
|
66
67
|
* Useful for CI/CD environments or when you want cleaner output.
|
|
@@ -72,17 +73,19 @@ var envSchema = import_zod.z.object({
|
|
|
72
73
|
*/
|
|
73
74
|
NODE_ENV: import_zod.z.enum(["development", "production", "test"]).default("development"),
|
|
74
75
|
/**
|
|
75
|
-
*
|
|
76
|
+
* Case-insensitive log level for the scenario package.
|
|
76
77
|
* Defaults to 'info' if not specified.
|
|
77
78
|
*/
|
|
78
|
-
LOG_LEVEL: import_zod.z.nativeEnum(LogLevel).optional(),
|
|
79
|
+
LOG_LEVEL: import_zod.z.string().toUpperCase().pipe(import_zod.z.nativeEnum(LogLevel)).optional().default("INFO" /* INFO */),
|
|
79
80
|
/**
|
|
80
81
|
* Scenario batch run ID.
|
|
81
82
|
* If not provided, a random ID will be generated.
|
|
82
83
|
*/
|
|
83
84
|
SCENARIO_BATCH_RUN_ID: import_zod.z.string().optional()
|
|
84
85
|
});
|
|
85
|
-
|
|
86
|
+
function getEnv() {
|
|
87
|
+
return envSchema.parse(process.env);
|
|
88
|
+
}
|
|
86
89
|
|
|
87
90
|
// src/utils/logger.ts
|
|
88
91
|
var Logger = class _Logger {
|
|
@@ -95,18 +98,27 @@ var Logger = class _Logger {
|
|
|
95
98
|
static create(context) {
|
|
96
99
|
return new _Logger(context);
|
|
97
100
|
}
|
|
98
|
-
|
|
99
|
-
|
|
101
|
+
/**
|
|
102
|
+
* Returns the current log level from environment.
|
|
103
|
+
* Uses a getter for clarity and idiomatic usage.
|
|
104
|
+
*/
|
|
105
|
+
get LOG_LEVEL() {
|
|
106
|
+
return getEnv().LOG_LEVEL;
|
|
100
107
|
}
|
|
101
|
-
|
|
102
|
-
|
|
108
|
+
/**
|
|
109
|
+
* Returns the index of the given log level in the LOG_LEVELS array.
|
|
110
|
+
* @param level - The log level to get the index for.
|
|
111
|
+
* @returns The index of the log level in the LOG_LEVELS array.
|
|
112
|
+
*/
|
|
113
|
+
getLogLevelIndexFor(level) {
|
|
114
|
+
return LOG_LEVELS.indexOf(level);
|
|
103
115
|
}
|
|
104
116
|
/**
|
|
105
117
|
* Checks if logging should occur based on LOG_LEVEL env var
|
|
106
118
|
*/
|
|
107
119
|
shouldLog(level) {
|
|
108
|
-
const currentLevelIndex = this.
|
|
109
|
-
const requestedLevelIndex = this.
|
|
120
|
+
const currentLevelIndex = this.getLogLevelIndexFor(this.LOG_LEVEL);
|
|
121
|
+
const requestedLevelIndex = this.getLogLevelIndexFor(level);
|
|
110
122
|
return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
|
|
111
123
|
}
|
|
112
124
|
formatMessage(message) {
|
|
@@ -174,10 +186,13 @@ function getFullTestName(task) {
|
|
|
174
186
|
}
|
|
175
187
|
return name;
|
|
176
188
|
}
|
|
189
|
+
function indent(str, n = 2) {
|
|
190
|
+
return str.replace(/^/gm, " ".repeat(n));
|
|
191
|
+
}
|
|
177
192
|
var VitestReporter = class {
|
|
178
193
|
results = [];
|
|
179
194
|
async onTestCaseResult(test) {
|
|
180
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
195
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j;
|
|
181
196
|
const fullName = getFullTestName(test);
|
|
182
197
|
const filePath = getLogFilePath(test.id);
|
|
183
198
|
if (!import_fs.default.existsSync(filePath)) {
|
|
@@ -220,19 +235,55 @@ var VitestReporter = class {
|
|
|
220
235
|
console.log(`Description: ${((_g = started.metadata) == null ? void 0 : _g.description) ?? ""}`);
|
|
221
236
|
}
|
|
222
237
|
if (messages.length) {
|
|
223
|
-
console.log("Chat log
|
|
238
|
+
console.log("Chat log:\n");
|
|
224
239
|
let lastMessageCount = 0;
|
|
225
240
|
for (const msg of messages) {
|
|
226
241
|
const allMessages = msg.messages ?? [];
|
|
227
242
|
for (const m of allMessages.slice(lastMessageCount)) {
|
|
228
243
|
const role = m.role;
|
|
244
|
+
if (role.toLowerCase() === "assistant" && "toolCalls" in m && Array.isArray(m.toolCalls) && m.toolCalls.length > 0) {
|
|
245
|
+
for (const toolCall of m.toolCalls) {
|
|
246
|
+
const functionName = toolCall.function.name;
|
|
247
|
+
let parsedJson = "";
|
|
248
|
+
try {
|
|
249
|
+
parsedJson = JSON.stringify(
|
|
250
|
+
JSON.parse(toolCall.function.arguments),
|
|
251
|
+
null,
|
|
252
|
+
2
|
|
253
|
+
);
|
|
254
|
+
} catch {
|
|
255
|
+
parsedJson = toolCall.function.arguments;
|
|
256
|
+
}
|
|
257
|
+
const role2 = import_chalk.default.magenta(`ToolCall(${functionName}):`);
|
|
258
|
+
console.log(`${role2}:
|
|
259
|
+
|
|
260
|
+
${indent(parsedJson)}
|
|
261
|
+
`);
|
|
262
|
+
}
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
229
265
|
let roleLabel = role;
|
|
230
266
|
if (role.toLowerCase() === "user") roleLabel = import_chalk.default.green("User");
|
|
231
267
|
else if (role.toLowerCase() === "agent")
|
|
232
268
|
roleLabel = import_chalk.default.cyan("Agent");
|
|
233
269
|
else if (role.toLowerCase() === "assistant")
|
|
234
|
-
|
|
235
|
-
|
|
270
|
+
if (Array.isArray(m.content) && typeof m.content.at(0) === "object" && ((_h = m.content.at(0)) == null ? void 0 : _h.type) === "tool-call")
|
|
271
|
+
roleLabel = import_chalk.default.cyan("ToolCall");
|
|
272
|
+
else roleLabel = import_chalk.default.cyan("Assistant");
|
|
273
|
+
else if (role.toLowerCase() === "tool") {
|
|
274
|
+
roleLabel = import_chalk.default.magenta("ToolResult");
|
|
275
|
+
let parsedJson = "";
|
|
276
|
+
try {
|
|
277
|
+
parsedJson = JSON.stringify(JSON.parse(m.content), null, 2);
|
|
278
|
+
} catch {
|
|
279
|
+
parsedJson = m.content;
|
|
280
|
+
}
|
|
281
|
+
console.log(`${roleLabel}:
|
|
282
|
+
|
|
283
|
+
${indent(parsedJson)}
|
|
284
|
+
`);
|
|
285
|
+
continue;
|
|
286
|
+
} else roleLabel = import_chalk.default.yellow(role);
|
|
236
287
|
console.log(`${roleLabel}: ${m.content}`);
|
|
237
288
|
}
|
|
238
289
|
lastMessageCount = allMessages.length;
|
|
@@ -245,11 +296,11 @@ var VitestReporter = class {
|
|
|
245
296
|
console.log(`Verdict: ${finished.results.verdict}`);
|
|
246
297
|
if (finished.results.reasoning)
|
|
247
298
|
console.log(`Reasoning: ${finished.results.reasoning}`);
|
|
248
|
-
if ((
|
|
299
|
+
if ((_i = finished.results.metCriteria) == null ? void 0 : _i.length)
|
|
249
300
|
console.log(
|
|
250
301
|
`Met criteria: ${finished.results.metCriteria.join(", ")}`
|
|
251
302
|
);
|
|
252
|
-
if ((
|
|
303
|
+
if ((_j = finished.results.unmetCriteria) == null ? void 0 : _j.length)
|
|
253
304
|
console.log(
|
|
254
305
|
`Unmet criteria: ${finished.results.unmetCriteria.join(", ")}`
|
|
255
306
|
);
|
|
@@ -289,3 +340,4 @@ var VitestReporter = class {
|
|
|
289
340
|
console.log();
|
|
290
341
|
}
|
|
291
342
|
};
|
|
343
|
+
var reporter_default = VitestReporter;
|