@langwatch/scenario 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.d.mts +348 -150
- package/dist/index.d.ts +348 -150
- package/dist/index.js +2148 -462
- package/dist/index.mjs +2580 -271
- package/dist/integrations/vitest/config.mjs +0 -2
- package/dist/integrations/vitest/reporter.js +22 -1
- package/dist/integrations/vitest/reporter.mjs +153 -6
- package/dist/integrations/vitest/setup-global.mjs +0 -2
- package/dist/integrations/vitest/setup.js +21 -9
- package/dist/integrations/vitest/setup.mjs +619 -18
- package/package.json +46 -31
- package/dist/chunk-3Z7E24UI.mjs +0 -548
- package/dist/chunk-7P6ASYW6.mjs +0 -9
- package/dist/chunk-RHTLQKEJ.mjs +0 -133
|
@@ -186,6 +186,27 @@ function getFullTestName(task) {
|
|
|
186
186
|
}
|
|
187
187
|
return name;
|
|
188
188
|
}
|
|
189
|
+
function formatContentForLogging(content) {
|
|
190
|
+
if (typeof content !== "string") {
|
|
191
|
+
return "[Non-text content]";
|
|
192
|
+
}
|
|
193
|
+
if (/^[A-Za-z0-9+/]+=*$/.test(content) && content.length > 50) {
|
|
194
|
+
return "[Binary data]";
|
|
195
|
+
}
|
|
196
|
+
try {
|
|
197
|
+
const parsed = JSON.parse(content);
|
|
198
|
+
if (Array.isArray(parsed)) {
|
|
199
|
+
const hasBinaryData = parsed.some(
|
|
200
|
+
(part) => (part == null ? void 0 : part.type) === "file" && (part == null ? void 0 : part.data) && typeof part.data === "string" && /^[A-Za-z0-9+/]+=*$/.test(part.data) && part.data.length > 50
|
|
201
|
+
);
|
|
202
|
+
if (hasBinaryData) {
|
|
203
|
+
return "[Content with binary data]";
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
} catch {
|
|
207
|
+
}
|
|
208
|
+
return content.length > 100 ? `${content.slice(0, 100)}...` : content;
|
|
209
|
+
}
|
|
189
210
|
function indent(str, n = 2) {
|
|
190
211
|
return str.replace(/^/gm, " ".repeat(n));
|
|
191
212
|
}
|
|
@@ -284,7 +305,7 @@ ${indent(parsedJson)}
|
|
|
284
305
|
`);
|
|
285
306
|
continue;
|
|
286
307
|
} else roleLabel = import_chalk.default.yellow(role);
|
|
287
|
-
console.log(`${roleLabel}: ${m.content}`);
|
|
308
|
+
console.log(`${roleLabel}: ${formatContentForLogging(m.content)}`);
|
|
288
309
|
}
|
|
289
310
|
lastMessageCount = allMessages.length;
|
|
290
311
|
}
|
|
@@ -1,12 +1,138 @@
|
|
|
1
|
-
import {
|
|
2
|
-
Logger
|
|
3
|
-
} from "../../chunk-RHTLQKEJ.mjs";
|
|
4
|
-
import "../../chunk-7P6ASYW6.mjs";
|
|
5
|
-
|
|
6
1
|
// src/integrations/vitest/reporter.ts
|
|
7
2
|
import fs from "fs";
|
|
8
3
|
import path from "path";
|
|
9
4
|
import chalk from "chalk";
|
|
5
|
+
|
|
6
|
+
// src/config/env.ts
|
|
7
|
+
import { z } from "zod/v4";
|
|
8
|
+
|
|
9
|
+
// src/config/log-levels.ts
|
|
10
|
+
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
11
|
+
LogLevel2["ERROR"] = "ERROR";
|
|
12
|
+
LogLevel2["WARN"] = "WARN";
|
|
13
|
+
LogLevel2["INFO"] = "INFO";
|
|
14
|
+
LogLevel2["DEBUG"] = "DEBUG";
|
|
15
|
+
return LogLevel2;
|
|
16
|
+
})(LogLevel || {});
|
|
17
|
+
var LOG_LEVELS = Object.values(LogLevel);
|
|
18
|
+
|
|
19
|
+
// src/config/env.ts
|
|
20
|
+
var envSchema = z.object({
|
|
21
|
+
/**
|
|
22
|
+
* LangWatch API key for event reporting.
|
|
23
|
+
* If not provided, events will not be sent to LangWatch.
|
|
24
|
+
*/
|
|
25
|
+
LANGWATCH_API_KEY: z.string().optional(),
|
|
26
|
+
/**
|
|
27
|
+
* LangWatch endpoint URL for event reporting.
|
|
28
|
+
* Defaults to the production LangWatch endpoint.
|
|
29
|
+
*/
|
|
30
|
+
LANGWATCH_ENDPOINT: z.string().url().optional().default("https://app.langwatch.ai"),
|
|
31
|
+
/**
|
|
32
|
+
* Disables simulation report info messages when set to any truthy value.
|
|
33
|
+
* Useful for CI/CD environments or when you want cleaner output.
|
|
34
|
+
*/
|
|
35
|
+
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: z.string().optional().transform((val) => Boolean(val)),
|
|
36
|
+
/**
|
|
37
|
+
* Node environment - affects logging and behavior.
|
|
38
|
+
* Defaults to 'development' if not specified.
|
|
39
|
+
*/
|
|
40
|
+
NODE_ENV: z.enum(["development", "production", "test"]).default("development"),
|
|
41
|
+
/**
|
|
42
|
+
* Case-insensitive log level for the scenario package.
|
|
43
|
+
* Defaults to 'info' if not specified.
|
|
44
|
+
*/
|
|
45
|
+
LOG_LEVEL: z.string().toUpperCase().pipe(z.nativeEnum(LogLevel)).optional().default("INFO" /* INFO */),
|
|
46
|
+
/**
|
|
47
|
+
* Scenario batch run ID.
|
|
48
|
+
* If not provided, a random ID will be generated.
|
|
49
|
+
*/
|
|
50
|
+
SCENARIO_BATCH_RUN_ID: z.string().optional()
|
|
51
|
+
});
|
|
52
|
+
function getEnv() {
|
|
53
|
+
return envSchema.parse(process.env);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// src/utils/logger.ts
|
|
57
|
+
var Logger = class _Logger {
|
|
58
|
+
constructor(context) {
|
|
59
|
+
this.context = context;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Creates a logger with context (e.g., class name)
|
|
63
|
+
*/
|
|
64
|
+
static create(context) {
|
|
65
|
+
return new _Logger(context);
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Returns the current log level from environment.
|
|
69
|
+
* Uses a getter for clarity and idiomatic usage.
|
|
70
|
+
*/
|
|
71
|
+
get LOG_LEVEL() {
|
|
72
|
+
return getEnv().LOG_LEVEL;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Returns the index of the given log level in the LOG_LEVELS array.
|
|
76
|
+
* @param level - The log level to get the index for.
|
|
77
|
+
* @returns The index of the log level in the LOG_LEVELS array.
|
|
78
|
+
*/
|
|
79
|
+
getLogLevelIndexFor(level) {
|
|
80
|
+
return LOG_LEVELS.indexOf(level);
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Checks if logging should occur based on LOG_LEVEL env var
|
|
84
|
+
*/
|
|
85
|
+
shouldLog(level) {
|
|
86
|
+
const currentLevelIndex = this.getLogLevelIndexFor(this.LOG_LEVEL);
|
|
87
|
+
const requestedLevelIndex = this.getLogLevelIndexFor(level);
|
|
88
|
+
return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
|
|
89
|
+
}
|
|
90
|
+
formatMessage(message) {
|
|
91
|
+
return this.context ? `[${this.context}] ${message}` : message;
|
|
92
|
+
}
|
|
93
|
+
error(message, data) {
|
|
94
|
+
if (this.shouldLog("ERROR" /* ERROR */)) {
|
|
95
|
+
const formattedMessage = this.formatMessage(message);
|
|
96
|
+
if (data) {
|
|
97
|
+
console.error(formattedMessage, data);
|
|
98
|
+
} else {
|
|
99
|
+
console.error(formattedMessage);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
warn(message, data) {
|
|
104
|
+
if (this.shouldLog("WARN" /* WARN */)) {
|
|
105
|
+
const formattedMessage = this.formatMessage(message);
|
|
106
|
+
if (data) {
|
|
107
|
+
console.warn(formattedMessage, data);
|
|
108
|
+
} else {
|
|
109
|
+
console.warn(formattedMessage);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
info(message, data) {
|
|
114
|
+
if (this.shouldLog("INFO" /* INFO */)) {
|
|
115
|
+
const formattedMessage = this.formatMessage(message);
|
|
116
|
+
if (data) {
|
|
117
|
+
console.info(formattedMessage, data);
|
|
118
|
+
} else {
|
|
119
|
+
console.info(formattedMessage);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
debug(message, data) {
|
|
124
|
+
if (this.shouldLog("DEBUG" /* DEBUG */)) {
|
|
125
|
+
const formattedMessage = this.formatMessage(message);
|
|
126
|
+
if (data) {
|
|
127
|
+
console.log(formattedMessage, data);
|
|
128
|
+
} else {
|
|
129
|
+
console.log(formattedMessage);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
// src/integrations/vitest/reporter.ts
|
|
10
136
|
var logger = Logger.create("integrations:vitest:reporter");
|
|
11
137
|
function getProjectRoot() {
|
|
12
138
|
return process.cwd();
|
|
@@ -26,6 +152,27 @@ function getFullTestName(task) {
|
|
|
26
152
|
}
|
|
27
153
|
return name;
|
|
28
154
|
}
|
|
155
|
+
function formatContentForLogging(content) {
|
|
156
|
+
if (typeof content !== "string") {
|
|
157
|
+
return "[Non-text content]";
|
|
158
|
+
}
|
|
159
|
+
if (/^[A-Za-z0-9+/]+=*$/.test(content) && content.length > 50) {
|
|
160
|
+
return "[Binary data]";
|
|
161
|
+
}
|
|
162
|
+
try {
|
|
163
|
+
const parsed = JSON.parse(content);
|
|
164
|
+
if (Array.isArray(parsed)) {
|
|
165
|
+
const hasBinaryData = parsed.some(
|
|
166
|
+
(part) => (part == null ? void 0 : part.type) === "file" && (part == null ? void 0 : part.data) && typeof part.data === "string" && /^[A-Za-z0-9+/]+=*$/.test(part.data) && part.data.length > 50
|
|
167
|
+
);
|
|
168
|
+
if (hasBinaryData) {
|
|
169
|
+
return "[Content with binary data]";
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
} catch {
|
|
173
|
+
}
|
|
174
|
+
return content.length > 100 ? `${content.slice(0, 100)}...` : content;
|
|
175
|
+
}
|
|
29
176
|
function indent(str, n = 2) {
|
|
30
177
|
return str.replace(/^/gm, " ".repeat(n));
|
|
31
178
|
}
|
|
@@ -124,7 +271,7 @@ ${indent(parsedJson)}
|
|
|
124
271
|
`);
|
|
125
272
|
continue;
|
|
126
273
|
} else roleLabel = chalk.yellow(role);
|
|
127
|
-
console.log(`${roleLabel}: ${m.content}`);
|
|
274
|
+
console.log(`${roleLabel}: ${formatContentForLogging(m.content)}`);
|
|
128
275
|
}
|
|
129
276
|
lastMessageCount = allMessages.length;
|
|
130
277
|
}
|
|
@@ -92,17 +92,28 @@ var import_node_path = __toESM(require("path"));
|
|
|
92
92
|
var import_node_url = require("url");
|
|
93
93
|
|
|
94
94
|
// src/domain/core/config.ts
|
|
95
|
+
var import_v43 = require("zod/v4");
|
|
96
|
+
|
|
97
|
+
// src/domain/core/schemas/model.schema.ts
|
|
95
98
|
var import_v42 = require("zod/v4");
|
|
99
|
+
|
|
100
|
+
// src/domain/core/constants.ts
|
|
96
101
|
var DEFAULT_TEMPERATURE = 0;
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
}).
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
102
|
+
|
|
103
|
+
// src/domain/core/schemas/model.schema.ts
|
|
104
|
+
var modelSchema = import_v42.z.object({
|
|
105
|
+
model: import_v42.z.custom((val) => Boolean(val), {
|
|
106
|
+
message: "A model is required. Configure it in scenario.config.js defaultModel or pass directly to the agent."
|
|
107
|
+
}).describe("Language model that is used by the AI SDK Core functions."),
|
|
108
|
+
temperature: import_v42.z.number().min(0).max(1).optional().describe("The temperature for the language model.").default(DEFAULT_TEMPERATURE),
|
|
109
|
+
maxTokens: import_v42.z.number().optional().describe("The maximum number of tokens to generate.")
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
// src/domain/core/config.ts
|
|
113
|
+
var headless = typeof process !== "undefined" ? process.env.SCENARIO_HEADLESS === "true" : false;
|
|
114
|
+
var scenarioProjectConfigSchema = import_v43.z.object({
|
|
115
|
+
defaultModel: modelSchema.optional(),
|
|
116
|
+
headless: import_v43.z.boolean().optional().default(headless)
|
|
106
117
|
}).strict();
|
|
107
118
|
|
|
108
119
|
// src/config/load.ts
|
|
@@ -476,6 +487,7 @@ var EventReporter = class {
|
|
|
476
487
|
} else {
|
|
477
488
|
const errorText = await response.text();
|
|
478
489
|
this.logger.error(`[${event.type}] Event POST failed:`, {
|
|
490
|
+
endpoint: this.eventsEndpoint.href,
|
|
479
491
|
status: response.status,
|
|
480
492
|
statusText: response.statusText,
|
|
481
493
|
error: errorText,
|