@langwatch/scenario 0.2.6 → 0.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -18
- package/dist/{chunk-MOOKAYIE.mjs → chunk-7HLDX5EL.mjs} +82 -173
- package/dist/chunk-OL4RFXV4.mjs +133 -0
- package/dist/index.d.mts +568 -75
- package/dist/index.d.ts +568 -75
- package/dist/index.js +838 -269
- package/dist/index.mjs +722 -190
- package/dist/integrations/vitest/config.d.mts +42 -0
- package/dist/integrations/vitest/config.d.ts +42 -0
- package/dist/integrations/vitest/config.js +51 -0
- package/dist/integrations/vitest/config.mjs +28 -0
- package/dist/integrations/vitest/reporter.js +183 -8
- package/dist/integrations/vitest/reporter.mjs +55 -8
- package/dist/integrations/vitest/setup-global.d.mts +3 -0
- package/dist/integrations/vitest/setup-global.d.ts +3 -0
- package/dist/integrations/vitest/setup-global.js +30 -0
- package/dist/integrations/vitest/setup-global.mjs +11 -0
- package/dist/integrations/vitest/setup.js +115 -73
- package/dist/integrations/vitest/setup.mjs +7 -3
- package/package.json +11 -2
package/README.md
CHANGED
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|

|
|
4
4
|
|
|
5
|
-
|
|
6
5
|
[](https://badge.fury.io/js/%40langwatch%2Fscenario)
|
|
7
6
|
|
|
8
7
|
A powerful TypeScript library for testing AI agents in realistic, scripted scenarios.
|
|
@@ -89,7 +88,8 @@ describe("Weather Agent", () => {
|
|
|
89
88
|
parameters: z.object({
|
|
90
89
|
city: z.string().describe("The city to get the weather for."),
|
|
91
90
|
}),
|
|
92
|
-
execute: async ({ city }) =>
|
|
91
|
+
execute: async ({ city }) =>
|
|
92
|
+
`The weather in ${city} is cloudy with a temperature of 24°C.`,
|
|
93
93
|
});
|
|
94
94
|
|
|
95
95
|
// 2. Create an adapter for your agent
|
|
@@ -97,7 +97,7 @@ describe("Weather Agent", () => {
|
|
|
97
97
|
role: AgentRole.AGENT,
|
|
98
98
|
call: async (input) => {
|
|
99
99
|
const response = await generateText({
|
|
100
|
-
model: openai("gpt-4.1
|
|
100
|
+
model: openai("gpt-4.1"),
|
|
101
101
|
system: `You are a helpful assistant that may help the user with weather information.`,
|
|
102
102
|
messages: input.messages,
|
|
103
103
|
tools: { get_current_weather: getCurrentWeather },
|
|
@@ -119,10 +119,11 @@ describe("Weather Agent", () => {
|
|
|
119
119
|
// 3. Define and run your scenario
|
|
120
120
|
const result = await scenario.run({
|
|
121
121
|
name: "Checking the weather",
|
|
122
|
-
description:
|
|
122
|
+
description:
|
|
123
|
+
"The user asks for the weather in a specific city, and the agent should use the weather tool to find it.",
|
|
123
124
|
agents: [
|
|
124
125
|
weatherAgent,
|
|
125
|
-
scenario.userSimulatorAgent({ model: openai("gpt-4.1
|
|
126
|
+
scenario.userSimulatorAgent({ model: openai("gpt-4.1") }),
|
|
126
127
|
],
|
|
127
128
|
script: [
|
|
128
129
|
scenario.user("What's the weather like in Barcelona?"),
|
|
@@ -240,7 +241,7 @@ The following configuration options are all optional. You can specify any combin
|
|
|
240
241
|
|
|
241
242
|
You can control the library's behavior with the following environment variables:
|
|
242
243
|
|
|
243
|
-
- `
|
|
244
|
+
- `LOG_LEVEL`: Sets the verbosity of the internal logger. Can be `error`, `warn`, `info`, or `debug`. By default, logging is silent.
|
|
244
245
|
- `SCENARIO_DISABLE_SIMULATION_REPORT_INFO`: Set to `true` to disable the "Scenario Simulation Reporting" banner that is printed to the console when a test run starts.
|
|
245
246
|
- `LANGWATCH_API_KEY`: Your LangWatch API key. This is used as a fallback if `langwatchApiKey` is not set in your config file.
|
|
246
247
|
- `LANGWATCH_ENDPOINT`: The LangWatch reporting endpoint. This is used as a fallback if `langwatchEndpoint` is not set in your config file.
|
|
@@ -254,10 +255,7 @@ const result = await scenario.run({
|
|
|
254
255
|
name: "my first scenario",
|
|
255
256
|
description: "A simple test to see if the agent responds.",
|
|
256
257
|
setId: "my-test-suite", // Group this scenario into a set
|
|
257
|
-
agents: [
|
|
258
|
-
myAgent,
|
|
259
|
-
scenario.userSimulatorAgent(),
|
|
260
|
-
],
|
|
258
|
+
agents: [myAgent, scenario.userSimulatorAgent()],
|
|
261
259
|
});
|
|
262
260
|
```
|
|
263
261
|
|
|
@@ -266,28 +264,64 @@ This will group all scenarios with the same `setId` together in the LangWatch UI
|
|
|
266
264
|
- The `setupFiles` entry enables Scenario's event logging for each test.
|
|
267
265
|
- The custom `VitestReporter` provides detailed scenario test reports in your test output.
|
|
268
266
|
|
|
269
|
-
|
|
270
267
|
## Vitest Integration
|
|
271
268
|
|
|
272
|
-
|
|
269
|
+
Scenario provides a convenient helper function to enhance your Vitest configuration with all the necessary setup files.
|
|
270
|
+
|
|
271
|
+
### Using the withScenario Helper (Recommended)
|
|
272
|
+
|
|
273
|
+
```typescript
|
|
274
|
+
// vitest.config.ts
|
|
275
|
+
import { defineConfig } from "vitest/config";
|
|
276
|
+
import { withScenario } from "@langwatch/scenario/integrations/vitest/config";
|
|
277
|
+
import VitestReporter from "@langwatch/scenario/integrations/vitest/reporter";
|
|
278
|
+
|
|
279
|
+
export default withScenario(
|
|
280
|
+
defineConfig({
|
|
281
|
+
test: {
|
|
282
|
+
testTimeout: 180000, // 3 minutes, or however long you want to wait for the scenario to run
|
|
283
|
+
// Your existing setup files will be preserved and run after Scenario's setup
|
|
284
|
+
setupFiles: ["./my-custom-setup.ts"],
|
|
285
|
+
// Your existing global setup files will be preserved and run after Scenario's global setup
|
|
286
|
+
globalSetup: ["./my-global-setup.ts"],
|
|
287
|
+
// Optional: Add the Scenario reporter for detailed test reports
|
|
288
|
+
reporters: ["default", new VitestReporter()],
|
|
289
|
+
},
|
|
290
|
+
})
|
|
291
|
+
);
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
The `withScenario` helper automatically:
|
|
295
|
+
|
|
296
|
+
- Adds Scenario's setup files for event logging
|
|
297
|
+
- Adds Scenario's global setup files
|
|
298
|
+
- Preserves any existing setup configuration you have
|
|
299
|
+
- Handles both string and array configurations for setup files
|
|
300
|
+
|
|
301
|
+
### Manual Configuration
|
|
302
|
+
|
|
303
|
+
If you prefer to configure Vitest manually, you can add the Scenario setup files directly:
|
|
273
304
|
|
|
274
305
|
```typescript
|
|
275
306
|
// vitest.config.ts
|
|
276
307
|
import { defineConfig } from "vitest/config";
|
|
277
|
-
import VitestReporter from
|
|
308
|
+
import VitestReporter from "@langwatch/scenario/integrations/vitest/reporter";
|
|
278
309
|
|
|
279
310
|
export default defineConfig({
|
|
280
311
|
test: {
|
|
281
312
|
testTimeout: 180000, // 3 minutes, or however long you want to wait for the scenario to run
|
|
282
|
-
setupFiles: [
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
new VitestReporter(),
|
|
286
|
-
],
|
|
313
|
+
setupFiles: ["@langwatch/scenario/integrations/vitest/setup"],
|
|
314
|
+
// Optional: Add the Scenario reporter for detailed test reports
|
|
315
|
+
reporters: ["default", new VitestReporter()],
|
|
287
316
|
},
|
|
288
317
|
});
|
|
289
318
|
```
|
|
290
319
|
|
|
320
|
+
This configuration:
|
|
321
|
+
|
|
322
|
+
- The `setupFiles` entry enables Scenario's event logging for each test
|
|
323
|
+
- The custom `VitestReporter` provides detailed scenario test reports in your test output (optional)
|
|
324
|
+
|
|
291
325
|
## Development
|
|
292
326
|
|
|
293
327
|
This project uses `pnpm` for package management.
|
|
@@ -314,6 +348,7 @@ MIT
|
|
|
314
348
|
When running scenario tests, you can set the `SCENARIO_BATCH_RUN_ID` environment variable to uniquely identify a batch of test runs. This is especially useful for grouping results in reporting tools and CI pipelines.
|
|
315
349
|
|
|
316
350
|
Example:
|
|
351
|
+
|
|
317
352
|
```bash
|
|
318
353
|
SCENARIO_BATCH_RUN_ID=my-ci-run-123 pnpm test
|
|
319
354
|
```
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import {
|
|
2
|
+
Logger,
|
|
3
|
+
getEnv
|
|
4
|
+
} from "./chunk-OL4RFXV4.mjs";
|
|
1
5
|
import {
|
|
2
6
|
__export
|
|
3
7
|
} from "./chunk-7P6ASYW6.mjs";
|
|
@@ -23,24 +27,19 @@ var AgentRole = /* @__PURE__ */ ((AgentRole2) => {
|
|
|
23
27
|
AgentRole2["JUDGE"] = "Judge";
|
|
24
28
|
return AgentRole2;
|
|
25
29
|
})(AgentRole || {});
|
|
26
|
-
var allAgentRoles = [
|
|
30
|
+
var allAgentRoles = [
|
|
31
|
+
"User" /* USER */,
|
|
32
|
+
"Agent" /* AGENT */,
|
|
33
|
+
"Judge" /* JUDGE */
|
|
34
|
+
];
|
|
27
35
|
var AgentAdapter = class {
|
|
28
36
|
role = "Agent" /* AGENT */;
|
|
29
|
-
constructor(input) {
|
|
30
|
-
void input;
|
|
31
|
-
}
|
|
32
37
|
};
|
|
33
38
|
var UserSimulatorAgentAdapter = class {
|
|
34
39
|
role = "User" /* USER */;
|
|
35
|
-
constructor(input) {
|
|
36
|
-
void input;
|
|
37
|
-
}
|
|
38
40
|
};
|
|
39
41
|
var JudgeAgentAdapter = class {
|
|
40
42
|
role = "Judge" /* JUDGE */;
|
|
41
|
-
constructor(input) {
|
|
42
|
-
void input;
|
|
43
|
-
}
|
|
44
43
|
};
|
|
45
44
|
|
|
46
45
|
// src/domain/scenarios/index.ts
|
|
@@ -105,124 +104,7 @@ async function loadScenarioProjectConfig() {
|
|
|
105
104
|
return await scenarioProjectConfigSchema.parseAsync({});
|
|
106
105
|
}
|
|
107
106
|
|
|
108
|
-
// src/
|
|
109
|
-
var Logger = class _Logger {
|
|
110
|
-
constructor(context) {
|
|
111
|
-
this.context = context;
|
|
112
|
-
}
|
|
113
|
-
/**
|
|
114
|
-
* Creates a logger with context (e.g., class name)
|
|
115
|
-
*/
|
|
116
|
-
static create(context) {
|
|
117
|
-
return new _Logger(context);
|
|
118
|
-
}
|
|
119
|
-
getLogLevel() {
|
|
120
|
-
return env.SCENARIO_LOG_LEVEL ?? "INFO" /* INFO */;
|
|
121
|
-
}
|
|
122
|
-
getLogLevelIndex(level) {
|
|
123
|
-
return Object.values(LogLevel).indexOf(level);
|
|
124
|
-
}
|
|
125
|
-
/**
|
|
126
|
-
* Checks if logging should occur based on LOG_LEVEL env var
|
|
127
|
-
*/
|
|
128
|
-
shouldLog(level) {
|
|
129
|
-
const currentLevelIndex = this.getLogLevelIndex(this.getLogLevel());
|
|
130
|
-
const requestedLevelIndex = this.getLogLevelIndex(level);
|
|
131
|
-
return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
|
|
132
|
-
}
|
|
133
|
-
formatMessage(message) {
|
|
134
|
-
return this.context ? `[${this.context}] ${message}` : message;
|
|
135
|
-
}
|
|
136
|
-
error(message, data) {
|
|
137
|
-
if (this.shouldLog("ERROR" /* ERROR */)) {
|
|
138
|
-
const formattedMessage = this.formatMessage(message);
|
|
139
|
-
if (data) {
|
|
140
|
-
console.error(formattedMessage, data);
|
|
141
|
-
} else {
|
|
142
|
-
console.error(formattedMessage);
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
warn(message, data) {
|
|
147
|
-
if (this.shouldLog("WARN" /* WARN */)) {
|
|
148
|
-
const formattedMessage = this.formatMessage(message);
|
|
149
|
-
if (data) {
|
|
150
|
-
console.warn(formattedMessage, data);
|
|
151
|
-
} else {
|
|
152
|
-
console.warn(formattedMessage);
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
info(message, data) {
|
|
157
|
-
if (this.shouldLog("INFO" /* INFO */)) {
|
|
158
|
-
const formattedMessage = this.formatMessage(message);
|
|
159
|
-
if (data) {
|
|
160
|
-
console.info(formattedMessage, data);
|
|
161
|
-
} else {
|
|
162
|
-
console.info(formattedMessage);
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
}
|
|
166
|
-
debug(message, data) {
|
|
167
|
-
if (this.shouldLog("DEBUG" /* DEBUG */)) {
|
|
168
|
-
const formattedMessage = this.formatMessage(message);
|
|
169
|
-
if (data) {
|
|
170
|
-
console.log(formattedMessage, data);
|
|
171
|
-
} else {
|
|
172
|
-
console.log(formattedMessage);
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
};
|
|
177
|
-
|
|
178
|
-
// src/config/env.ts
|
|
179
|
-
import { z as z2 } from "zod";
|
|
180
|
-
|
|
181
|
-
// src/config/log-levels.ts
|
|
182
|
-
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
183
|
-
LogLevel2["ERROR"] = "ERROR";
|
|
184
|
-
LogLevel2["WARN"] = "WARN";
|
|
185
|
-
LogLevel2["INFO"] = "INFO";
|
|
186
|
-
LogLevel2["DEBUG"] = "DEBUG";
|
|
187
|
-
return LogLevel2;
|
|
188
|
-
})(LogLevel || {});
|
|
189
|
-
|
|
190
|
-
// src/config/env.ts
|
|
191
|
-
var envSchema = z2.object({
|
|
192
|
-
/**
|
|
193
|
-
* LangWatch API key for event reporting.
|
|
194
|
-
* If not provided, events will not be sent to LangWatch.
|
|
195
|
-
*/
|
|
196
|
-
LANGWATCH_API_KEY: z2.string().optional(),
|
|
197
|
-
/**
|
|
198
|
-
* LangWatch endpoint URL for event reporting.
|
|
199
|
-
* Defaults to the production LangWatch endpoint.
|
|
200
|
-
*/
|
|
201
|
-
LANGWATCH_ENDPOINT: z2.string().url().default("https://app.langwatch.ai"),
|
|
202
|
-
/**
|
|
203
|
-
* Disables simulation report info messages when set to any truthy value.
|
|
204
|
-
* Useful for CI/CD environments or when you want cleaner output.
|
|
205
|
-
*/
|
|
206
|
-
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: z2.string().optional().transform((val) => Boolean(val)),
|
|
207
|
-
/**
|
|
208
|
-
* Node environment - affects logging and behavior.
|
|
209
|
-
* Defaults to 'development' if not specified.
|
|
210
|
-
*/
|
|
211
|
-
NODE_ENV: z2.enum(["development", "production", "test"]).default("development"),
|
|
212
|
-
/**
|
|
213
|
-
* Log level for the scenario package.
|
|
214
|
-
* Defaults to 'info' if not specified.
|
|
215
|
-
*/
|
|
216
|
-
SCENARIO_LOG_LEVEL: z2.nativeEnum(LogLevel).optional(),
|
|
217
|
-
/**
|
|
218
|
-
* Scenario batch run ID.
|
|
219
|
-
* If not provided, a random ID will be generated.
|
|
220
|
-
*/
|
|
221
|
-
SCENARIO_BATCH_RUN_ID: z2.string().optional()
|
|
222
|
-
});
|
|
223
|
-
var env = envSchema.parse(process.env);
|
|
224
|
-
|
|
225
|
-
// src/config/index.ts
|
|
107
|
+
// src/config/get-project-config.ts
|
|
226
108
|
var logger = new Logger("scenario.config");
|
|
227
109
|
var configLoaded = false;
|
|
228
110
|
var config = null;
|
|
@@ -252,7 +134,10 @@ async function getProjectConfig() {
|
|
|
252
134
|
}
|
|
253
135
|
|
|
254
136
|
// src/utils/ids.ts
|
|
137
|
+
import crypto from "node:crypto";
|
|
138
|
+
import process2 from "node:process";
|
|
255
139
|
import { generate, parse } from "xksuid";
|
|
140
|
+
var batchRunId;
|
|
256
141
|
function generateThreadId() {
|
|
257
142
|
return `thread_${generate()}`;
|
|
258
143
|
}
|
|
@@ -263,10 +148,30 @@ function generateScenarioId() {
|
|
|
263
148
|
return `scenario_${generate()}`;
|
|
264
149
|
}
|
|
265
150
|
function getBatchRunId() {
|
|
266
|
-
if (
|
|
267
|
-
|
|
268
|
-
}
|
|
269
|
-
|
|
151
|
+
if (batchRunId) {
|
|
152
|
+
return batchRunId;
|
|
153
|
+
}
|
|
154
|
+
if (process2.env.SCENARIO_BATCH_RUN_ID) {
|
|
155
|
+
return batchRunId = process2.env.SCENARIO_BATCH_RUN_ID;
|
|
156
|
+
}
|
|
157
|
+
if (process2.env.VITEST_WORKER_ID || process2.env.JEST_WORKER_ID) {
|
|
158
|
+
const parentProcessId = process2.ppid;
|
|
159
|
+
const now = /* @__PURE__ */ new Date();
|
|
160
|
+
const year = now.getUTCFullYear();
|
|
161
|
+
const week = String(getISOWeekNumber(now)).padStart(2, "0");
|
|
162
|
+
const raw = `${parentProcessId}_${year}_w${week}`;
|
|
163
|
+
const hash = crypto.createHash("sha256").update(raw).digest("hex").slice(0, 12);
|
|
164
|
+
return batchRunId = `scenariobatchrun_${hash}`;
|
|
165
|
+
}
|
|
166
|
+
return batchRunId = `scenariobatchrun_${generate()}`;
|
|
167
|
+
}
|
|
168
|
+
function getISOWeekNumber(date) {
|
|
169
|
+
const tmp = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
|
|
170
|
+
const dayNum = tmp.getUTCDay() || 7;
|
|
171
|
+
tmp.setUTCDate(tmp.getUTCDate() + 4 - dayNum);
|
|
172
|
+
const yearStart = new Date(Date.UTC(tmp.getUTCFullYear(), 0, 1));
|
|
173
|
+
const weekNo = Math.ceil(((tmp.getTime() - yearStart.getTime()) / 864e5 + 1) / 7);
|
|
174
|
+
return weekNo;
|
|
270
175
|
}
|
|
271
176
|
function generateMessageId() {
|
|
272
177
|
return `scenariomsg_${generate()}`;
|
|
@@ -283,12 +188,11 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
283
188
|
if (this.isGreetingDisabled()) {
|
|
284
189
|
return;
|
|
285
190
|
}
|
|
286
|
-
|
|
287
|
-
if (_EventAlertMessageLogger.shownBatchIds.has(batchRunId)) {
|
|
191
|
+
if (_EventAlertMessageLogger.shownBatchIds.has(getBatchRunId())) {
|
|
288
192
|
return;
|
|
289
193
|
}
|
|
290
|
-
_EventAlertMessageLogger.shownBatchIds.add(
|
|
291
|
-
this.displayGreeting(
|
|
194
|
+
_EventAlertMessageLogger.shownBatchIds.add(getBatchRunId());
|
|
195
|
+
this.displayGreeting();
|
|
292
196
|
}
|
|
293
197
|
/**
|
|
294
198
|
* Shows a fancy message about how to watch the simulation.
|
|
@@ -301,10 +205,11 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
301
205
|
this.displayWatchMessage(params);
|
|
302
206
|
}
|
|
303
207
|
isGreetingDisabled() {
|
|
304
|
-
return
|
|
208
|
+
return getEnv().SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
|
|
305
209
|
}
|
|
306
|
-
displayGreeting(
|
|
210
|
+
displayGreeting() {
|
|
307
211
|
const separator = "\u2500".repeat(60);
|
|
212
|
+
const env = getEnv();
|
|
308
213
|
if (!env.LANGWATCH_API_KEY) {
|
|
309
214
|
console.log(`
|
|
310
215
|
${separator}`);
|
|
@@ -317,7 +222,10 @@ ${separator}`);
|
|
|
317
222
|
console.log(" \u2022 Set LANGWATCH_API_KEY environment variable");
|
|
318
223
|
console.log(" \u2022 Or configure apiKey in scenario.config.js");
|
|
319
224
|
console.log("");
|
|
320
|
-
console.log(`\u{1F4E6} Batch Run ID: ${
|
|
225
|
+
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
226
|
+
console.log("");
|
|
227
|
+
console.log("\u{1F507} To disable these messages:");
|
|
228
|
+
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
321
229
|
console.log(`${separator}
|
|
322
230
|
`);
|
|
323
231
|
} else {
|
|
@@ -331,7 +239,10 @@ ${separator}`);
|
|
|
331
239
|
` API Key: ${env.LANGWATCH_API_KEY.length > 0 ? "Configured" : "Not configured"}`
|
|
332
240
|
);
|
|
333
241
|
console.log("");
|
|
334
|
-
console.log(`\u{1F4E6} Batch Run ID: ${
|
|
242
|
+
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
243
|
+
console.log("");
|
|
244
|
+
console.log("\u{1F507} To disable these messages:");
|
|
245
|
+
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
335
246
|
console.log(`${separator}
|
|
336
247
|
`);
|
|
337
248
|
}
|
|
@@ -355,7 +266,7 @@ ${separator}`);
|
|
|
355
266
|
|
|
356
267
|
// src/events/schema.ts
|
|
357
268
|
import { EventType, MessagesSnapshotEventSchema } from "@ag-ui/core";
|
|
358
|
-
import { z as
|
|
269
|
+
import { z as z2 } from "zod";
|
|
359
270
|
var Verdict = /* @__PURE__ */ ((Verdict2) => {
|
|
360
271
|
Verdict2["SUCCESS"] = "success";
|
|
361
272
|
Verdict2["FAILURE"] = "failure";
|
|
@@ -371,59 +282,59 @@ var ScenarioRunStatus = /* @__PURE__ */ ((ScenarioRunStatus2) => {
|
|
|
371
282
|
ScenarioRunStatus2["FAILED"] = "FAILED";
|
|
372
283
|
return ScenarioRunStatus2;
|
|
373
284
|
})(ScenarioRunStatus || {});
|
|
374
|
-
var baseEventSchema =
|
|
375
|
-
type:
|
|
376
|
-
timestamp:
|
|
377
|
-
rawEvent:
|
|
285
|
+
var baseEventSchema = z2.object({
|
|
286
|
+
type: z2.nativeEnum(EventType),
|
|
287
|
+
timestamp: z2.number(),
|
|
288
|
+
rawEvent: z2.any().optional()
|
|
378
289
|
});
|
|
379
|
-
var batchRunIdSchema =
|
|
380
|
-
var scenarioRunIdSchema =
|
|
381
|
-
var scenarioIdSchema =
|
|
290
|
+
var batchRunIdSchema = z2.string();
|
|
291
|
+
var scenarioRunIdSchema = z2.string();
|
|
292
|
+
var scenarioIdSchema = z2.string();
|
|
382
293
|
var baseScenarioEventSchema = baseEventSchema.extend({
|
|
383
294
|
batchRunId: batchRunIdSchema,
|
|
384
295
|
scenarioId: scenarioIdSchema,
|
|
385
296
|
scenarioRunId: scenarioRunIdSchema,
|
|
386
|
-
scenarioSetId:
|
|
297
|
+
scenarioSetId: z2.string().optional().default("default")
|
|
387
298
|
});
|
|
388
299
|
var scenarioRunStartedSchema = baseScenarioEventSchema.extend({
|
|
389
|
-
type:
|
|
390
|
-
metadata:
|
|
391
|
-
name:
|
|
392
|
-
description:
|
|
300
|
+
type: z2.literal("SCENARIO_RUN_STARTED" /* RUN_STARTED */),
|
|
301
|
+
metadata: z2.object({
|
|
302
|
+
name: z2.string().optional(),
|
|
303
|
+
description: z2.string().optional()
|
|
393
304
|
})
|
|
394
305
|
});
|
|
395
|
-
var scenarioResultsSchema =
|
|
396
|
-
verdict:
|
|
397
|
-
reasoning:
|
|
398
|
-
metCriteria:
|
|
399
|
-
unmetCriteria:
|
|
400
|
-
error:
|
|
306
|
+
var scenarioResultsSchema = z2.object({
|
|
307
|
+
verdict: z2.nativeEnum(Verdict),
|
|
308
|
+
reasoning: z2.string().optional(),
|
|
309
|
+
metCriteria: z2.array(z2.string()),
|
|
310
|
+
unmetCriteria: z2.array(z2.string()),
|
|
311
|
+
error: z2.string().optional()
|
|
401
312
|
});
|
|
402
313
|
var scenarioRunFinishedSchema = baseScenarioEventSchema.extend({
|
|
403
|
-
type:
|
|
404
|
-
status:
|
|
314
|
+
type: z2.literal("SCENARIO_RUN_FINISHED" /* RUN_FINISHED */),
|
|
315
|
+
status: z2.nativeEnum(ScenarioRunStatus),
|
|
405
316
|
results: scenarioResultsSchema.optional().nullable()
|
|
406
317
|
});
|
|
407
318
|
var scenarioMessageSnapshotSchema = MessagesSnapshotEventSchema.merge(
|
|
408
319
|
baseScenarioEventSchema.extend({
|
|
409
|
-
type:
|
|
320
|
+
type: z2.literal("SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */)
|
|
410
321
|
})
|
|
411
322
|
);
|
|
412
|
-
var scenarioEventSchema =
|
|
323
|
+
var scenarioEventSchema = z2.discriminatedUnion("type", [
|
|
413
324
|
scenarioRunStartedSchema,
|
|
414
325
|
scenarioRunFinishedSchema,
|
|
415
326
|
scenarioMessageSnapshotSchema
|
|
416
327
|
]);
|
|
417
|
-
var successSchema =
|
|
418
|
-
var errorSchema =
|
|
419
|
-
var stateSchema =
|
|
420
|
-
state:
|
|
421
|
-
messages:
|
|
422
|
-
status:
|
|
328
|
+
var successSchema = z2.object({ success: z2.boolean() });
|
|
329
|
+
var errorSchema = z2.object({ error: z2.string() });
|
|
330
|
+
var stateSchema = z2.object({
|
|
331
|
+
state: z2.object({
|
|
332
|
+
messages: z2.array(z2.any()),
|
|
333
|
+
status: z2.string()
|
|
423
334
|
})
|
|
424
335
|
});
|
|
425
|
-
var runsSchema =
|
|
426
|
-
var eventsSchema =
|
|
336
|
+
var runsSchema = z2.object({ runs: z2.array(z2.string()) });
|
|
337
|
+
var eventsSchema = z2.object({ events: z2.array(scenarioEventSchema) });
|
|
427
338
|
|
|
428
339
|
// src/events/event-reporter.ts
|
|
429
340
|
var EventReporter = class {
|
|
@@ -618,8 +529,6 @@ export {
|
|
|
618
529
|
DEFAULT_MAX_TURNS,
|
|
619
530
|
DEFAULT_VERBOSE,
|
|
620
531
|
domain_exports,
|
|
621
|
-
Logger,
|
|
622
|
-
env,
|
|
623
532
|
getProjectConfig,
|
|
624
533
|
generateThreadId,
|
|
625
534
|
generateScenarioRunId,
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
// src/config/env.ts
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
|
|
4
|
+
// src/config/log-levels.ts
|
|
5
|
+
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
6
|
+
LogLevel2["ERROR"] = "ERROR";
|
|
7
|
+
LogLevel2["WARN"] = "WARN";
|
|
8
|
+
LogLevel2["INFO"] = "INFO";
|
|
9
|
+
LogLevel2["DEBUG"] = "DEBUG";
|
|
10
|
+
return LogLevel2;
|
|
11
|
+
})(LogLevel || {});
|
|
12
|
+
var LOG_LEVELS = Object.values(LogLevel);
|
|
13
|
+
|
|
14
|
+
// src/config/env.ts
|
|
15
|
+
var envSchema = z.object({
|
|
16
|
+
/**
|
|
17
|
+
* LangWatch API key for event reporting.
|
|
18
|
+
* If not provided, events will not be sent to LangWatch.
|
|
19
|
+
*/
|
|
20
|
+
LANGWATCH_API_KEY: z.string().optional(),
|
|
21
|
+
/**
|
|
22
|
+
* LangWatch endpoint URL for event reporting.
|
|
23
|
+
* Defaults to the production LangWatch endpoint.
|
|
24
|
+
*/
|
|
25
|
+
LANGWATCH_ENDPOINT: z.string().url().optional().default("https://app.langwatch.ai"),
|
|
26
|
+
/**
|
|
27
|
+
* Disables simulation report info messages when set to any truthy value.
|
|
28
|
+
* Useful for CI/CD environments or when you want cleaner output.
|
|
29
|
+
*/
|
|
30
|
+
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: z.string().optional().transform((val) => Boolean(val)),
|
|
31
|
+
/**
|
|
32
|
+
* Node environment - affects logging and behavior.
|
|
33
|
+
* Defaults to 'development' if not specified.
|
|
34
|
+
*/
|
|
35
|
+
NODE_ENV: z.enum(["development", "production", "test"]).default("development"),
|
|
36
|
+
/**
|
|
37
|
+
* Case-insensitive log level for the scenario package.
|
|
38
|
+
* Defaults to 'info' if not specified.
|
|
39
|
+
*/
|
|
40
|
+
LOG_LEVEL: z.string().toUpperCase().pipe(z.nativeEnum(LogLevel)).optional().default("INFO" /* INFO */),
|
|
41
|
+
/**
|
|
42
|
+
* Scenario batch run ID.
|
|
43
|
+
* If not provided, a random ID will be generated.
|
|
44
|
+
*/
|
|
45
|
+
SCENARIO_BATCH_RUN_ID: z.string().optional()
|
|
46
|
+
});
|
|
47
|
+
function getEnv() {
|
|
48
|
+
return envSchema.parse(process.env);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// src/utils/logger.ts
|
|
52
|
+
var Logger = class _Logger {
|
|
53
|
+
constructor(context) {
|
|
54
|
+
this.context = context;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Creates a logger with context (e.g., class name)
|
|
58
|
+
*/
|
|
59
|
+
static create(context) {
|
|
60
|
+
return new _Logger(context);
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Returns the current log level from environment.
|
|
64
|
+
* Uses a getter for clarity and idiomatic usage.
|
|
65
|
+
*/
|
|
66
|
+
get LOG_LEVEL() {
|
|
67
|
+
return getEnv().LOG_LEVEL;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Returns the index of the given log level in the LOG_LEVELS array.
|
|
71
|
+
* @param level - The log level to get the index for.
|
|
72
|
+
* @returns The index of the log level in the LOG_LEVELS array.
|
|
73
|
+
*/
|
|
74
|
+
getLogLevelIndexFor(level) {
|
|
75
|
+
return LOG_LEVELS.indexOf(level);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Checks if logging should occur based on LOG_LEVEL env var
|
|
79
|
+
*/
|
|
80
|
+
shouldLog(level) {
|
|
81
|
+
const currentLevelIndex = this.getLogLevelIndexFor(this.LOG_LEVEL);
|
|
82
|
+
const requestedLevelIndex = this.getLogLevelIndexFor(level);
|
|
83
|
+
return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
|
|
84
|
+
}
|
|
85
|
+
formatMessage(message) {
|
|
86
|
+
return this.context ? `[${this.context}] ${message}` : message;
|
|
87
|
+
}
|
|
88
|
+
error(message, data) {
|
|
89
|
+
if (this.shouldLog("ERROR" /* ERROR */)) {
|
|
90
|
+
const formattedMessage = this.formatMessage(message);
|
|
91
|
+
if (data) {
|
|
92
|
+
console.error(formattedMessage, data);
|
|
93
|
+
} else {
|
|
94
|
+
console.error(formattedMessage);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
warn(message, data) {
|
|
99
|
+
if (this.shouldLog("WARN" /* WARN */)) {
|
|
100
|
+
const formattedMessage = this.formatMessage(message);
|
|
101
|
+
if (data) {
|
|
102
|
+
console.warn(formattedMessage, data);
|
|
103
|
+
} else {
|
|
104
|
+
console.warn(formattedMessage);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
info(message, data) {
|
|
109
|
+
if (this.shouldLog("INFO" /* INFO */)) {
|
|
110
|
+
const formattedMessage = this.formatMessage(message);
|
|
111
|
+
if (data) {
|
|
112
|
+
console.info(formattedMessage, data);
|
|
113
|
+
} else {
|
|
114
|
+
console.info(formattedMessage);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
debug(message, data) {
|
|
119
|
+
if (this.shouldLog("DEBUG" /* DEBUG */)) {
|
|
120
|
+
const formattedMessage = this.formatMessage(message);
|
|
121
|
+
if (data) {
|
|
122
|
+
console.log(formattedMessage, data);
|
|
123
|
+
} else {
|
|
124
|
+
console.log(formattedMessage);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
export {
|
|
131
|
+
getEnv,
|
|
132
|
+
Logger
|
|
133
|
+
};
|