@langwatch/scenario 0.2.6 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/{chunk-MOOKAYIE.mjs → chunk-7H6OGEQ5.mjs} +76 -162
- package/dist/chunk-K7KLHTDI.mjs +146 -0
- package/dist/chunk-YPJZSK4J.mjs +121 -0
- package/dist/index.d.mts +9 -3
- package/dist/index.d.ts +9 -3
- package/dist/index.js +100 -65
- package/dist/index.mjs +16 -8
- package/dist/integrations/vitest/config.d.mts +5 -0
- package/dist/integrations/vitest/config.d.ts +5 -0
- package/dist/integrations/vitest/config.js +324 -0
- package/dist/integrations/vitest/config.mjs +35 -0
- package/dist/integrations/vitest/reporter.js +124 -1
- package/dist/integrations/vitest/reporter.mjs +4 -135
- package/dist/integrations/vitest/setup-global.d.mts +3 -0
- package/dist/integrations/vitest/setup-global.d.ts +3 -0
- package/dist/integrations/vitest/setup-global.js +30 -0
- package/dist/integrations/vitest/setup-global.mjs +11 -0
- package/dist/integrations/vitest/setup.js +97 -67
- package/dist/integrations/vitest/setup.mjs +7 -3
- package/package.json +11 -1
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import {
|
|
2
|
+
VitestReporter
|
|
3
|
+
} from "../../chunk-K7KLHTDI.mjs";
|
|
4
|
+
import "../../chunk-YPJZSK4J.mjs";
|
|
5
|
+
import "../../chunk-7P6ASYW6.mjs";
|
|
6
|
+
|
|
7
|
+
// src/integrations/vitest/config.ts
|
|
8
|
+
import { defineConfig } from "vitest/config";
|
|
9
|
+
function withScenario(config) {
|
|
10
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l;
|
|
11
|
+
const normalizedSetupFiles = ((_a = config.test) == null ? void 0 : _a.setupFiles) === void 0 ? [] : Array.isArray((_b = config.test) == null ? void 0 : _b.setupFiles) ? (_c = config.test) == null ? void 0 : _c.setupFiles : [(_d = config.test) == null ? void 0 : _d.setupFiles];
|
|
12
|
+
const normalizedGlobalSetup = ((_e = config.test) == null ? void 0 : _e.globalSetup) === void 0 ? [] : Array.isArray((_f = config.test) == null ? void 0 : _f.globalSetup) ? (_g = config.test) == null ? void 0 : _g.globalSetup : [(_h = config.test) == null ? void 0 : _h.globalSetup];
|
|
13
|
+
const normalizedReporters = ((_i = config.test) == null ? void 0 : _i.reporters) === void 0 ? [] : Array.isArray((_j = config.test) == null ? void 0 : _j.reporters) ? (_k = config.test) == null ? void 0 : _k.reporters : [(_l = config.test) == null ? void 0 : _l.reporters];
|
|
14
|
+
return defineConfig({
|
|
15
|
+
...config,
|
|
16
|
+
test: {
|
|
17
|
+
...config.test,
|
|
18
|
+
setupFiles: [
|
|
19
|
+
"@langwatch/scenario/integrations/vitest/setup",
|
|
20
|
+
...normalizedSetupFiles
|
|
21
|
+
],
|
|
22
|
+
globalSetup: [
|
|
23
|
+
"@langwatch/scenario/integrations/vitest/setup-global",
|
|
24
|
+
...normalizedGlobalSetup
|
|
25
|
+
],
|
|
26
|
+
reporters: [
|
|
27
|
+
...normalizedReporters,
|
|
28
|
+
new VitestReporter()
|
|
29
|
+
]
|
|
30
|
+
}
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
export {
|
|
34
|
+
withScenario
|
|
35
|
+
};
|
|
@@ -36,6 +36,126 @@ module.exports = __toCommonJS(reporter_exports);
|
|
|
36
36
|
var import_fs = __toESM(require("fs"));
|
|
37
37
|
var import_path = __toESM(require("path"));
|
|
38
38
|
var import_chalk = __toESM(require("chalk"));
|
|
39
|
+
|
|
40
|
+
// src/config/env.ts
|
|
41
|
+
var import_zod = require("zod");
|
|
42
|
+
|
|
43
|
+
// src/config/log-levels.ts
|
|
44
|
+
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
45
|
+
LogLevel2["ERROR"] = "ERROR";
|
|
46
|
+
LogLevel2["WARN"] = "WARN";
|
|
47
|
+
LogLevel2["INFO"] = "INFO";
|
|
48
|
+
LogLevel2["DEBUG"] = "DEBUG";
|
|
49
|
+
return LogLevel2;
|
|
50
|
+
})(LogLevel || {});
|
|
51
|
+
|
|
52
|
+
// src/config/env.ts
|
|
53
|
+
var envSchema = import_zod.z.object({
|
|
54
|
+
/**
|
|
55
|
+
* LangWatch API key for event reporting.
|
|
56
|
+
* If not provided, events will not be sent to LangWatch.
|
|
57
|
+
*/
|
|
58
|
+
LANGWATCH_API_KEY: import_zod.z.string().optional(),
|
|
59
|
+
/**
|
|
60
|
+
* LangWatch endpoint URL for event reporting.
|
|
61
|
+
* Defaults to the production LangWatch endpoint.
|
|
62
|
+
*/
|
|
63
|
+
LANGWATCH_ENDPOINT: import_zod.z.string().url().default("https://app.langwatch.ai"),
|
|
64
|
+
/**
|
|
65
|
+
* Disables simulation report info messages when set to any truthy value.
|
|
66
|
+
* Useful for CI/CD environments or when you want cleaner output.
|
|
67
|
+
*/
|
|
68
|
+
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod.z.string().optional().transform((val) => Boolean(val)),
|
|
69
|
+
/**
|
|
70
|
+
* Node environment - affects logging and behavior.
|
|
71
|
+
* Defaults to 'development' if not specified.
|
|
72
|
+
*/
|
|
73
|
+
NODE_ENV: import_zod.z.enum(["development", "production", "test"]).default("development"),
|
|
74
|
+
/**
|
|
75
|
+
* Log level for the scenario package.
|
|
76
|
+
* Defaults to 'info' if not specified.
|
|
77
|
+
*/
|
|
78
|
+
LOG_LEVEL: import_zod.z.nativeEnum(LogLevel).optional(),
|
|
79
|
+
/**
|
|
80
|
+
* Scenario batch run ID.
|
|
81
|
+
* If not provided, a random ID will be generated.
|
|
82
|
+
*/
|
|
83
|
+
SCENARIO_BATCH_RUN_ID: import_zod.z.string().optional()
|
|
84
|
+
});
|
|
85
|
+
var env = envSchema.parse(process.env);
|
|
86
|
+
|
|
87
|
+
// src/utils/logger.ts
|
|
88
|
+
var Logger = class _Logger {
|
|
89
|
+
constructor(context) {
|
|
90
|
+
this.context = context;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Creates a logger with context (e.g., class name)
|
|
94
|
+
*/
|
|
95
|
+
static create(context) {
|
|
96
|
+
return new _Logger(context);
|
|
97
|
+
}
|
|
98
|
+
getLogLevel() {
|
|
99
|
+
return env.LOG_LEVEL ?? "INFO" /* INFO */;
|
|
100
|
+
}
|
|
101
|
+
getLogLevelIndex(level) {
|
|
102
|
+
return Object.values(LogLevel).indexOf(level);
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Checks if logging should occur based on LOG_LEVEL env var
|
|
106
|
+
*/
|
|
107
|
+
shouldLog(level) {
|
|
108
|
+
const currentLevelIndex = this.getLogLevelIndex(this.getLogLevel());
|
|
109
|
+
const requestedLevelIndex = this.getLogLevelIndex(level);
|
|
110
|
+
return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
|
|
111
|
+
}
|
|
112
|
+
formatMessage(message) {
|
|
113
|
+
return this.context ? `[${this.context}] ${message}` : message;
|
|
114
|
+
}
|
|
115
|
+
error(message, data) {
|
|
116
|
+
if (this.shouldLog("ERROR" /* ERROR */)) {
|
|
117
|
+
const formattedMessage = this.formatMessage(message);
|
|
118
|
+
if (data) {
|
|
119
|
+
console.error(formattedMessage, data);
|
|
120
|
+
} else {
|
|
121
|
+
console.error(formattedMessage);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
warn(message, data) {
|
|
126
|
+
if (this.shouldLog("WARN" /* WARN */)) {
|
|
127
|
+
const formattedMessage = this.formatMessage(message);
|
|
128
|
+
if (data) {
|
|
129
|
+
console.warn(formattedMessage, data);
|
|
130
|
+
} else {
|
|
131
|
+
console.warn(formattedMessage);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
info(message, data) {
|
|
136
|
+
if (this.shouldLog("INFO" /* INFO */)) {
|
|
137
|
+
const formattedMessage = this.formatMessage(message);
|
|
138
|
+
if (data) {
|
|
139
|
+
console.info(formattedMessage, data);
|
|
140
|
+
} else {
|
|
141
|
+
console.info(formattedMessage);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
debug(message, data) {
|
|
146
|
+
if (this.shouldLog("DEBUG" /* DEBUG */)) {
|
|
147
|
+
const formattedMessage = this.formatMessage(message);
|
|
148
|
+
if (data) {
|
|
149
|
+
console.log(formattedMessage, data);
|
|
150
|
+
} else {
|
|
151
|
+
console.log(formattedMessage);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
// src/integrations/vitest/reporter.ts
|
|
158
|
+
var logger = Logger.create("integrations:vitest:reporter");
|
|
39
159
|
function getProjectRoot() {
|
|
40
160
|
return process.cwd();
|
|
41
161
|
}
|
|
@@ -61,7 +181,10 @@ var VitestReporter = class {
|
|
|
61
181
|
const fullName = getFullTestName(test);
|
|
62
182
|
const filePath = getLogFilePath(test.id);
|
|
63
183
|
if (!import_fs.default.existsSync(filePath)) {
|
|
64
|
-
|
|
184
|
+
logger.warn(
|
|
185
|
+
`No log file found ${filePath} for test ${fullName}`,
|
|
186
|
+
test.id
|
|
187
|
+
);
|
|
65
188
|
return;
|
|
66
189
|
}
|
|
67
190
|
const lines = import_fs.default.readFileSync(filePath, "utf-8").split("\n").filter(Boolean);
|
|
@@ -1,139 +1,8 @@
|
|
|
1
|
+
import {
|
|
2
|
+
VitestReporter
|
|
3
|
+
} from "../../chunk-K7KLHTDI.mjs";
|
|
4
|
+
import "../../chunk-YPJZSK4J.mjs";
|
|
1
5
|
import "../../chunk-7P6ASYW6.mjs";
|
|
2
|
-
|
|
3
|
-
// src/integrations/vitest/reporter.ts
|
|
4
|
-
import fs from "fs";
|
|
5
|
-
import path from "path";
|
|
6
|
-
import chalk from "chalk";
|
|
7
|
-
function getProjectRoot() {
|
|
8
|
-
return process.cwd();
|
|
9
|
-
}
|
|
10
|
-
var projectRoot = getProjectRoot();
|
|
11
|
-
var logDir = path.join(projectRoot, ".scenario");
|
|
12
|
-
if (!fs.existsSync(logDir)) fs.mkdirSync(logDir);
|
|
13
|
-
function getLogFilePath(testId) {
|
|
14
|
-
return path.join(logDir, `${testId}.log`);
|
|
15
|
-
}
|
|
16
|
-
function getFullTestName(task) {
|
|
17
|
-
let name = task.name;
|
|
18
|
-
let parent = task.suite;
|
|
19
|
-
while (parent) {
|
|
20
|
-
name = `${parent.name} > ${name}`;
|
|
21
|
-
parent = parent.suite;
|
|
22
|
-
}
|
|
23
|
-
return name;
|
|
24
|
-
}
|
|
25
|
-
var VitestReporter = class {
|
|
26
|
-
results = [];
|
|
27
|
-
async onTestCaseResult(test) {
|
|
28
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i;
|
|
29
|
-
const fullName = getFullTestName(test);
|
|
30
|
-
const filePath = getLogFilePath(test.id);
|
|
31
|
-
if (!fs.existsSync(filePath)) {
|
|
32
|
-
console.log(`No log file found ${filePath} for test ${fullName}`);
|
|
33
|
-
return;
|
|
34
|
-
}
|
|
35
|
-
const lines = fs.readFileSync(filePath, "utf-8").split("\n").filter(Boolean);
|
|
36
|
-
const events = lines.map((line) => JSON.parse(line));
|
|
37
|
-
const runs = /* @__PURE__ */ new Map();
|
|
38
|
-
for (const event of events) {
|
|
39
|
-
const runId = event.scenarioRunId ?? "unknown";
|
|
40
|
-
if (!runs.has(runId)) runs.set(runId, []);
|
|
41
|
-
runs.get(runId).push(event);
|
|
42
|
-
}
|
|
43
|
-
for (const [runId, runEvents] of Array.from(runs.entries())) {
|
|
44
|
-
const started = runEvents.find(
|
|
45
|
-
(e) => e.type === "SCENARIO_RUN_STARTED"
|
|
46
|
-
);
|
|
47
|
-
const finished = runEvents.find(
|
|
48
|
-
(e) => e.type === "SCENARIO_RUN_FINISHED"
|
|
49
|
-
);
|
|
50
|
-
const messages = runEvents.filter(
|
|
51
|
-
(e) => e.type === "SCENARIO_MESSAGE_SNAPSHOT"
|
|
52
|
-
);
|
|
53
|
-
this.results.push({
|
|
54
|
-
name: ((_a = started == null ? void 0 : started.metadata) == null ? void 0 : _a.name) ?? fullName,
|
|
55
|
-
status: (finished == null ? void 0 : finished.status) ?? "UNKNOWN",
|
|
56
|
-
duration: started && finished ? finished.timestamp - started.timestamp : 0,
|
|
57
|
-
reasoning: (_b = finished == null ? void 0 : finished.results) == null ? void 0 : _b.reasoning,
|
|
58
|
-
criteria: (finished == null ? void 0 : finished.results) ? `Success Criteria: ${((_c = finished.results.metCriteria) == null ? void 0 : _c.length) ?? 0}/${(((_d = finished.results.metCriteria) == null ? void 0 : _d.length) ?? 0) + (((_e = finished.results.unmetCriteria) == null ? void 0 : _e.length) ?? 0)}` : void 0
|
|
59
|
-
});
|
|
60
|
-
console.log(
|
|
61
|
-
`
|
|
62
|
-
--- Scenario Run: ${((_f = started == null ? void 0 : started.metadata) == null ? void 0 : _f.name) ?? runId} ---`
|
|
63
|
-
);
|
|
64
|
-
if (started) {
|
|
65
|
-
console.log(`Description: ${((_g = started.metadata) == null ? void 0 : _g.description) ?? ""}`);
|
|
66
|
-
}
|
|
67
|
-
if (messages.length) {
|
|
68
|
-
console.log("Chat log:");
|
|
69
|
-
let lastMessageCount = 0;
|
|
70
|
-
for (const msg of messages) {
|
|
71
|
-
const allMessages = msg.messages ?? [];
|
|
72
|
-
for (const m of allMessages.slice(lastMessageCount)) {
|
|
73
|
-
const role = m.role;
|
|
74
|
-
let roleLabel = role;
|
|
75
|
-
if (role.toLowerCase() === "user") roleLabel = chalk.green("User");
|
|
76
|
-
else if (role.toLowerCase() === "agent")
|
|
77
|
-
roleLabel = chalk.cyan("Agent");
|
|
78
|
-
else if (role.toLowerCase() === "assistant")
|
|
79
|
-
roleLabel = chalk.cyan("Assistant");
|
|
80
|
-
else roleLabel = chalk.yellow(role);
|
|
81
|
-
console.log(`${roleLabel}: ${m.content}`);
|
|
82
|
-
}
|
|
83
|
-
lastMessageCount = allMessages.length;
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
if (finished) {
|
|
87
|
-
console.log("--- Verdict ---");
|
|
88
|
-
console.log(`Status: ${finished.status}`);
|
|
89
|
-
if (finished.results) {
|
|
90
|
-
console.log(`Verdict: ${finished.results.verdict}`);
|
|
91
|
-
if (finished.results.reasoning)
|
|
92
|
-
console.log(`Reasoning: ${finished.results.reasoning}`);
|
|
93
|
-
if ((_h = finished.results.metCriteria) == null ? void 0 : _h.length)
|
|
94
|
-
console.log(
|
|
95
|
-
`Met criteria: ${finished.results.metCriteria.join(", ")}`
|
|
96
|
-
);
|
|
97
|
-
if ((_i = finished.results.unmetCriteria) == null ? void 0 : _i.length)
|
|
98
|
-
console.log(
|
|
99
|
-
`Unmet criteria: ${finished.results.unmetCriteria.join(", ")}`
|
|
100
|
-
);
|
|
101
|
-
if (finished.results.error)
|
|
102
|
-
console.log(`Error: ${finished.results.error}`);
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
console.log("-----------------------------\n");
|
|
106
|
-
}
|
|
107
|
-
fs.unlinkSync(filePath);
|
|
108
|
-
}
|
|
109
|
-
async onTestRunEnd() {
|
|
110
|
-
if (this.results.length === 0) return;
|
|
111
|
-
const total = this.results.length;
|
|
112
|
-
const passed = this.results.filter((r) => r.status === "SUCCESS").length;
|
|
113
|
-
const failed = this.results.filter((r) => r.status !== "SUCCESS").length;
|
|
114
|
-
const successRate = (passed / total * 100).toFixed(1);
|
|
115
|
-
console.log();
|
|
116
|
-
console.log(chalk.bold.cyan("=== Scenario Test Report ==="));
|
|
117
|
-
console.log(`Total Scenarios: ${total}`);
|
|
118
|
-
console.log(chalk.green(`Passed: ${passed}`));
|
|
119
|
-
console.log(chalk.red(`Failed: ${failed}`));
|
|
120
|
-
console.log(`Success Rate: ${chalk.bold(`${successRate}%`)}`);
|
|
121
|
-
this.results.forEach((r, i) => {
|
|
122
|
-
const statusColor = r.status === "SUCCESS" ? chalk.green : chalk.red;
|
|
123
|
-
console.log();
|
|
124
|
-
console.log(
|
|
125
|
-
`${i + 1}. ${r.name} - ${statusColor(r.status)} in ${(r.duration / 1e3).toFixed(2)}s`
|
|
126
|
-
);
|
|
127
|
-
if (r.reasoning) {
|
|
128
|
-
console.log(chalk.greenBright(" Reasoning: ") + r.reasoning);
|
|
129
|
-
}
|
|
130
|
-
if (r.criteria) {
|
|
131
|
-
console.log(chalk.bold(" " + r.criteria));
|
|
132
|
-
}
|
|
133
|
-
});
|
|
134
|
-
console.log();
|
|
135
|
-
}
|
|
136
|
-
};
|
|
137
6
|
export {
|
|
138
7
|
VitestReporter as default
|
|
139
8
|
};
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
|
|
20
|
+
// src/integrations/vitest/setup-global.ts
|
|
21
|
+
var setup_global_exports = {};
|
|
22
|
+
__export(setup_global_exports, {
|
|
23
|
+
default: () => setup
|
|
24
|
+
});
|
|
25
|
+
module.exports = __toCommonJS(setup_global_exports);
|
|
26
|
+
var import_xksuid = require("xksuid");
|
|
27
|
+
function setup() {
|
|
28
|
+
const scenarioBatchRunId = `scenariobatchrun_${(0, import_xksuid.generate)()}`;
|
|
29
|
+
process.env.SCENARIO_BATCH_RUN_ID = scenarioBatchRunId;
|
|
30
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import "../../chunk-7P6ASYW6.mjs";
|
|
2
|
+
|
|
3
|
+
// src/integrations/vitest/setup-global.ts
|
|
4
|
+
import { generate } from "xksuid";
|
|
5
|
+
function setup() {
|
|
6
|
+
const scenarioBatchRunId = `scenariobatchrun_${generate()}`;
|
|
7
|
+
process.env.SCENARIO_BATCH_RUN_ID = scenarioBatchRunId;
|
|
8
|
+
}
|
|
9
|
+
export {
|
|
10
|
+
setup as default
|
|
11
|
+
};
|
|
@@ -30,19 +30,66 @@ var import_vitest = require("vitest");
|
|
|
30
30
|
// src/events/event-bus.ts
|
|
31
31
|
var import_rxjs = require("rxjs");
|
|
32
32
|
|
|
33
|
+
// src/config/env.ts
|
|
34
|
+
var import_zod = require("zod");
|
|
35
|
+
|
|
36
|
+
// src/config/log-levels.ts
|
|
37
|
+
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
38
|
+
LogLevel2["ERROR"] = "ERROR";
|
|
39
|
+
LogLevel2["WARN"] = "WARN";
|
|
40
|
+
LogLevel2["INFO"] = "INFO";
|
|
41
|
+
LogLevel2["DEBUG"] = "DEBUG";
|
|
42
|
+
return LogLevel2;
|
|
43
|
+
})(LogLevel || {});
|
|
44
|
+
|
|
45
|
+
// src/config/env.ts
|
|
46
|
+
var envSchema = import_zod.z.object({
|
|
47
|
+
/**
|
|
48
|
+
* LangWatch API key for event reporting.
|
|
49
|
+
* If not provided, events will not be sent to LangWatch.
|
|
50
|
+
*/
|
|
51
|
+
LANGWATCH_API_KEY: import_zod.z.string().optional(),
|
|
52
|
+
/**
|
|
53
|
+
* LangWatch endpoint URL for event reporting.
|
|
54
|
+
* Defaults to the production LangWatch endpoint.
|
|
55
|
+
*/
|
|
56
|
+
LANGWATCH_ENDPOINT: import_zod.z.string().url().default("https://app.langwatch.ai"),
|
|
57
|
+
/**
|
|
58
|
+
* Disables simulation report info messages when set to any truthy value.
|
|
59
|
+
* Useful for CI/CD environments or when you want cleaner output.
|
|
60
|
+
*/
|
|
61
|
+
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod.z.string().optional().transform((val) => Boolean(val)),
|
|
62
|
+
/**
|
|
63
|
+
* Node environment - affects logging and behavior.
|
|
64
|
+
* Defaults to 'development' if not specified.
|
|
65
|
+
*/
|
|
66
|
+
NODE_ENV: import_zod.z.enum(["development", "production", "test"]).default("development"),
|
|
67
|
+
/**
|
|
68
|
+
* Log level for the scenario package.
|
|
69
|
+
* Defaults to 'info' if not specified.
|
|
70
|
+
*/
|
|
71
|
+
LOG_LEVEL: import_zod.z.nativeEnum(LogLevel).optional(),
|
|
72
|
+
/**
|
|
73
|
+
* Scenario batch run ID.
|
|
74
|
+
* If not provided, a random ID will be generated.
|
|
75
|
+
*/
|
|
76
|
+
SCENARIO_BATCH_RUN_ID: import_zod.z.string().optional()
|
|
77
|
+
});
|
|
78
|
+
var env = envSchema.parse(process.env);
|
|
79
|
+
|
|
33
80
|
// src/config/load.ts
|
|
34
81
|
var import_promises = __toESM(require("fs/promises"));
|
|
35
82
|
var import_node_path = __toESM(require("path"));
|
|
36
83
|
var import_node_url = require("url");
|
|
37
84
|
|
|
38
85
|
// src/domain/core/config.ts
|
|
39
|
-
var
|
|
86
|
+
var import_zod2 = require("zod");
|
|
40
87
|
var DEFAULT_TEMPERATURE = 0;
|
|
41
|
-
var scenarioProjectConfigSchema =
|
|
42
|
-
defaultModel:
|
|
43
|
-
model:
|
|
44
|
-
temperature:
|
|
45
|
-
maxTokens:
|
|
88
|
+
var scenarioProjectConfigSchema = import_zod2.z.object({
|
|
89
|
+
defaultModel: import_zod2.z.object({
|
|
90
|
+
model: import_zod2.z.custom(),
|
|
91
|
+
temperature: import_zod2.z.number().min(0).max(1).optional().default(DEFAULT_TEMPERATURE),
|
|
92
|
+
maxTokens: import_zod2.z.number().optional()
|
|
46
93
|
}).optional()
|
|
47
94
|
}).strict();
|
|
48
95
|
|
|
@@ -58,7 +105,7 @@ var Logger = class _Logger {
|
|
|
58
105
|
return new _Logger(context);
|
|
59
106
|
}
|
|
60
107
|
getLogLevel() {
|
|
61
|
-
return env.
|
|
108
|
+
return env.LOG_LEVEL ?? "INFO" /* INFO */;
|
|
62
109
|
}
|
|
63
110
|
getLogLevelIndex(level) {
|
|
64
111
|
return Object.values(LogLevel).indexOf(level);
|
|
@@ -116,63 +163,40 @@ var Logger = class _Logger {
|
|
|
116
163
|
}
|
|
117
164
|
};
|
|
118
165
|
|
|
119
|
-
// src/config/
|
|
120
|
-
var import_zod2 = require("zod");
|
|
121
|
-
|
|
122
|
-
// src/config/log-levels.ts
|
|
123
|
-
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
124
|
-
LogLevel2["ERROR"] = "ERROR";
|
|
125
|
-
LogLevel2["WARN"] = "WARN";
|
|
126
|
-
LogLevel2["INFO"] = "INFO";
|
|
127
|
-
LogLevel2["DEBUG"] = "DEBUG";
|
|
128
|
-
return LogLevel2;
|
|
129
|
-
})(LogLevel || {});
|
|
130
|
-
|
|
131
|
-
// src/config/env.ts
|
|
132
|
-
var envSchema = import_zod2.z.object({
|
|
133
|
-
/**
|
|
134
|
-
* LangWatch API key for event reporting.
|
|
135
|
-
* If not provided, events will not be sent to LangWatch.
|
|
136
|
-
*/
|
|
137
|
-
LANGWATCH_API_KEY: import_zod2.z.string().optional(),
|
|
138
|
-
/**
|
|
139
|
-
* LangWatch endpoint URL for event reporting.
|
|
140
|
-
* Defaults to the production LangWatch endpoint.
|
|
141
|
-
*/
|
|
142
|
-
LANGWATCH_ENDPOINT: import_zod2.z.string().url().default("https://app.langwatch.ai"),
|
|
143
|
-
/**
|
|
144
|
-
* Disables simulation report info messages when set to any truthy value.
|
|
145
|
-
* Useful for CI/CD environments or when you want cleaner output.
|
|
146
|
-
*/
|
|
147
|
-
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod2.z.string().optional().transform((val) => Boolean(val)),
|
|
148
|
-
/**
|
|
149
|
-
* Node environment - affects logging and behavior.
|
|
150
|
-
* Defaults to 'development' if not specified.
|
|
151
|
-
*/
|
|
152
|
-
NODE_ENV: import_zod2.z.enum(["development", "production", "test"]).default("development"),
|
|
153
|
-
/**
|
|
154
|
-
* Log level for the scenario package.
|
|
155
|
-
* Defaults to 'info' if not specified.
|
|
156
|
-
*/
|
|
157
|
-
SCENARIO_LOG_LEVEL: import_zod2.z.nativeEnum(LogLevel).optional(),
|
|
158
|
-
/**
|
|
159
|
-
* Scenario batch run ID.
|
|
160
|
-
* If not provided, a random ID will be generated.
|
|
161
|
-
*/
|
|
162
|
-
SCENARIO_BATCH_RUN_ID: import_zod2.z.string().optional()
|
|
163
|
-
});
|
|
164
|
-
var env = envSchema.parse(process.env);
|
|
165
|
-
|
|
166
|
-
// src/config/index.ts
|
|
166
|
+
// src/config/get-project-config.ts
|
|
167
167
|
var logger = new Logger("scenario.config");
|
|
168
168
|
|
|
169
169
|
// src/utils/ids.ts
|
|
170
|
+
var import_node_crypto = __toESM(require("crypto"));
|
|
171
|
+
var import_node_process = __toESM(require("process"));
|
|
170
172
|
var import_xksuid = require("xksuid");
|
|
173
|
+
var batchRunId;
|
|
171
174
|
function getBatchRunId() {
|
|
172
|
-
if (
|
|
173
|
-
|
|
175
|
+
if (batchRunId) {
|
|
176
|
+
return batchRunId;
|
|
174
177
|
}
|
|
175
|
-
|
|
178
|
+
if (import_node_process.default.env.SCENARIO_BATCH_RUN_ID) {
|
|
179
|
+
console.log("process.env.SCENARIO_BATCH_RUN_ID", import_node_process.default.env.SCENARIO_BATCH_RUN_ID);
|
|
180
|
+
return batchRunId = import_node_process.default.env.SCENARIO_BATCH_RUN_ID;
|
|
181
|
+
}
|
|
182
|
+
if (import_node_process.default.env.VITEST_WORKER_ID || import_node_process.default.env.JEST_WORKER_ID) {
|
|
183
|
+
const parentProcessId = import_node_process.default.ppid;
|
|
184
|
+
const now = /* @__PURE__ */ new Date();
|
|
185
|
+
const year = now.getUTCFullYear();
|
|
186
|
+
const week = String(getISOWeekNumber(now)).padStart(2, "0");
|
|
187
|
+
const raw = `${parentProcessId}_${year}_w${week}`;
|
|
188
|
+
const hash = import_node_crypto.default.createHash("sha256").update(raw).digest("hex").slice(0, 12);
|
|
189
|
+
return batchRunId = `scenariobatchrun_${hash}`;
|
|
190
|
+
}
|
|
191
|
+
return batchRunId = `scenariobatchrun_${(0, import_xksuid.generate)()}`;
|
|
192
|
+
}
|
|
193
|
+
function getISOWeekNumber(date) {
|
|
194
|
+
const tmp = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
|
|
195
|
+
const dayNum = tmp.getUTCDay() || 7;
|
|
196
|
+
tmp.setUTCDate(tmp.getUTCDate() + 4 - dayNum);
|
|
197
|
+
const yearStart = new Date(Date.UTC(tmp.getUTCFullYear(), 0, 1));
|
|
198
|
+
const weekNo = Math.ceil(((tmp.getTime() - yearStart.getTime()) / 864e5 + 1) / 7);
|
|
199
|
+
return weekNo;
|
|
176
200
|
}
|
|
177
201
|
|
|
178
202
|
// src/events/event-alert-message-logger.ts
|
|
@@ -186,12 +210,11 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
186
210
|
if (this.isGreetingDisabled()) {
|
|
187
211
|
return;
|
|
188
212
|
}
|
|
189
|
-
|
|
190
|
-
if (_EventAlertMessageLogger.shownBatchIds.has(batchRunId)) {
|
|
213
|
+
if (_EventAlertMessageLogger.shownBatchIds.has(getBatchRunId())) {
|
|
191
214
|
return;
|
|
192
215
|
}
|
|
193
|
-
_EventAlertMessageLogger.shownBatchIds.add(
|
|
194
|
-
this.displayGreeting(
|
|
216
|
+
_EventAlertMessageLogger.shownBatchIds.add(getBatchRunId());
|
|
217
|
+
this.displayGreeting();
|
|
195
218
|
}
|
|
196
219
|
/**
|
|
197
220
|
* Shows a fancy message about how to watch the simulation.
|
|
@@ -206,7 +229,7 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
206
229
|
isGreetingDisabled() {
|
|
207
230
|
return env.SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
|
|
208
231
|
}
|
|
209
|
-
displayGreeting(
|
|
232
|
+
displayGreeting() {
|
|
210
233
|
const separator = "\u2500".repeat(60);
|
|
211
234
|
if (!env.LANGWATCH_API_KEY) {
|
|
212
235
|
console.log(`
|
|
@@ -220,7 +243,10 @@ ${separator}`);
|
|
|
220
243
|
console.log(" \u2022 Set LANGWATCH_API_KEY environment variable");
|
|
221
244
|
console.log(" \u2022 Or configure apiKey in scenario.config.js");
|
|
222
245
|
console.log("");
|
|
223
|
-
console.log(`\u{1F4E6} Batch Run ID: ${
|
|
246
|
+
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
247
|
+
console.log("");
|
|
248
|
+
console.log("\u{1F507} To disable these messages:");
|
|
249
|
+
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
224
250
|
console.log(`${separator}
|
|
225
251
|
`);
|
|
226
252
|
} else {
|
|
@@ -234,7 +260,10 @@ ${separator}`);
|
|
|
234
260
|
` API Key: ${env.LANGWATCH_API_KEY.length > 0 ? "Configured" : "Not configured"}`
|
|
235
261
|
);
|
|
236
262
|
console.log("");
|
|
237
|
-
console.log(`\u{1F4E6} Batch Run ID: ${
|
|
263
|
+
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
264
|
+
console.log("");
|
|
265
|
+
console.log("\u{1F507} To disable these messages:");
|
|
266
|
+
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
238
267
|
console.log(`${separator}
|
|
239
268
|
`);
|
|
240
269
|
}
|
|
@@ -510,6 +539,7 @@ var EventBus = class _EventBus {
|
|
|
510
539
|
};
|
|
511
540
|
|
|
512
541
|
// src/integrations/vitest/setup.ts
|
|
542
|
+
var logger2 = Logger.create("integrations:vitest:setup");
|
|
513
543
|
function getProjectRoot() {
|
|
514
544
|
return process.cwd();
|
|
515
545
|
}
|
|
@@ -530,7 +560,7 @@ var subs = [];
|
|
|
530
560
|
try {
|
|
531
561
|
import_fs.default.appendFileSync(filePath, JSON.stringify(event) + "\n");
|
|
532
562
|
} catch (error) {
|
|
533
|
-
|
|
563
|
+
logger2.error("Error writing to log file:", error);
|
|
534
564
|
}
|
|
535
565
|
})
|
|
536
566
|
);
|
|
@@ -542,7 +572,7 @@ EventBus.addGlobalListener((bus) => {
|
|
|
542
572
|
try {
|
|
543
573
|
import_fs.default.appendFileSync(filePath, JSON.stringify(event) + "\n");
|
|
544
574
|
} catch (error) {
|
|
545
|
-
|
|
575
|
+
logger2.error("Error writing to log file:", error);
|
|
546
576
|
}
|
|
547
577
|
})
|
|
548
578
|
);
|
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import {
|
|
2
2
|
EventBus
|
|
3
|
-
} from "../../chunk-
|
|
3
|
+
} from "../../chunk-7H6OGEQ5.mjs";
|
|
4
|
+
import {
|
|
5
|
+
Logger
|
|
6
|
+
} from "../../chunk-YPJZSK4J.mjs";
|
|
4
7
|
import "../../chunk-7P6ASYW6.mjs";
|
|
5
8
|
|
|
6
9
|
// src/integrations/vitest/setup.ts
|
|
7
10
|
import fs from "fs";
|
|
8
11
|
import path from "path";
|
|
9
12
|
import { beforeEach, afterEach } from "vitest";
|
|
13
|
+
var logger = Logger.create("integrations:vitest:setup");
|
|
10
14
|
function getProjectRoot() {
|
|
11
15
|
return process.cwd();
|
|
12
16
|
}
|
|
@@ -27,7 +31,7 @@ beforeEach((ctx) => {
|
|
|
27
31
|
try {
|
|
28
32
|
fs.appendFileSync(filePath, JSON.stringify(event) + "\n");
|
|
29
33
|
} catch (error) {
|
|
30
|
-
|
|
34
|
+
logger.error("Error writing to log file:", error);
|
|
31
35
|
}
|
|
32
36
|
})
|
|
33
37
|
);
|
|
@@ -39,7 +43,7 @@ EventBus.addGlobalListener((bus) => {
|
|
|
39
43
|
try {
|
|
40
44
|
fs.appendFileSync(filePath, JSON.stringify(event) + "\n");
|
|
41
45
|
} catch (error) {
|
|
42
|
-
|
|
46
|
+
logger.error("Error writing to log file:", error);
|
|
43
47
|
}
|
|
44
48
|
})
|
|
45
49
|
);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@langwatch/scenario",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.9",
|
|
4
4
|
"description": "A TypeScript library for testing AI agents using scenarios",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -72,6 +72,16 @@
|
|
|
72
72
|
"types": "./dist/integrations/vitest/setup.d.ts",
|
|
73
73
|
"require": "./dist/integrations/vitest/setup.js",
|
|
74
74
|
"import": "./dist/integrations/vitest/setup.mjs"
|
|
75
|
+
},
|
|
76
|
+
"./integrations/vitest/setup-global": {
|
|
77
|
+
"types": "./dist/integrations/vitest/setup-global.d.ts",
|
|
78
|
+
"require": "./dist/integrations/vitest/setup-global.js",
|
|
79
|
+
"import": "./dist/integrations/vitest/setup-global.mjs"
|
|
80
|
+
},
|
|
81
|
+
"./integrations/vitest/config": {
|
|
82
|
+
"types": "./dist/integrations/vitest/config.d.ts",
|
|
83
|
+
"require": "./dist/integrations/vitest/config.js",
|
|
84
|
+
"import": "./dist/integrations/vitest/config.mjs"
|
|
75
85
|
}
|
|
76
86
|
},
|
|
77
87
|
"peerDependencies": {
|