@langwatch/scenario 0.2.2 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -9
- package/dist/{chunk-NUZZAQV2.mjs → chunk-7H6OGEQ5.mjs} +85 -163
- package/dist/chunk-K7KLHTDI.mjs +146 -0
- package/dist/chunk-YPJZSK4J.mjs +121 -0
- package/dist/index.d.mts +86 -72
- package/dist/index.d.ts +86 -72
- package/dist/index.js +131 -82
- package/dist/index.mjs +40 -24
- package/dist/integrations/vitest/config.d.mts +5 -0
- package/dist/integrations/vitest/config.d.ts +5 -0
- package/dist/integrations/vitest/config.js +324 -0
- package/dist/integrations/vitest/config.mjs +35 -0
- package/dist/integrations/vitest/reporter.js +124 -1
- package/dist/integrations/vitest/reporter.mjs +4 -135
- package/dist/integrations/vitest/setup-global.d.mts +3 -0
- package/dist/integrations/vitest/setup-global.d.ts +3 -0
- package/dist/integrations/vitest/setup-global.js +30 -0
- package/dist/integrations/vitest/setup-global.mjs +11 -0
- package/dist/integrations/vitest/setup.js +97 -67
- package/dist/integrations/vitest/setup.mjs +7 -3
- package/package.json +13 -4
|
@@ -30,19 +30,66 @@ var import_vitest = require("vitest");
|
|
|
30
30
|
// src/events/event-bus.ts
|
|
31
31
|
var import_rxjs = require("rxjs");
|
|
32
32
|
|
|
33
|
+
// src/config/env.ts
|
|
34
|
+
var import_zod = require("zod");
|
|
35
|
+
|
|
36
|
+
// src/config/log-levels.ts
|
|
37
|
+
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
38
|
+
LogLevel2["ERROR"] = "ERROR";
|
|
39
|
+
LogLevel2["WARN"] = "WARN";
|
|
40
|
+
LogLevel2["INFO"] = "INFO";
|
|
41
|
+
LogLevel2["DEBUG"] = "DEBUG";
|
|
42
|
+
return LogLevel2;
|
|
43
|
+
})(LogLevel || {});
|
|
44
|
+
|
|
45
|
+
// src/config/env.ts
|
|
46
|
+
var envSchema = import_zod.z.object({
|
|
47
|
+
/**
|
|
48
|
+
* LangWatch API key for event reporting.
|
|
49
|
+
* If not provided, events will not be sent to LangWatch.
|
|
50
|
+
*/
|
|
51
|
+
LANGWATCH_API_KEY: import_zod.z.string().optional(),
|
|
52
|
+
/**
|
|
53
|
+
* LangWatch endpoint URL for event reporting.
|
|
54
|
+
* Defaults to the production LangWatch endpoint.
|
|
55
|
+
*/
|
|
56
|
+
LANGWATCH_ENDPOINT: import_zod.z.string().url().default("https://app.langwatch.ai"),
|
|
57
|
+
/**
|
|
58
|
+
* Disables simulation report info messages when set to any truthy value.
|
|
59
|
+
* Useful for CI/CD environments or when you want cleaner output.
|
|
60
|
+
*/
|
|
61
|
+
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod.z.string().optional().transform((val) => Boolean(val)),
|
|
62
|
+
/**
|
|
63
|
+
* Node environment - affects logging and behavior.
|
|
64
|
+
* Defaults to 'development' if not specified.
|
|
65
|
+
*/
|
|
66
|
+
NODE_ENV: import_zod.z.enum(["development", "production", "test"]).default("development"),
|
|
67
|
+
/**
|
|
68
|
+
* Log level for the scenario package.
|
|
69
|
+
* Defaults to 'info' if not specified.
|
|
70
|
+
*/
|
|
71
|
+
LOG_LEVEL: import_zod.z.nativeEnum(LogLevel).optional(),
|
|
72
|
+
/**
|
|
73
|
+
* Scenario batch run ID.
|
|
74
|
+
* If not provided, a random ID will be generated.
|
|
75
|
+
*/
|
|
76
|
+
SCENARIO_BATCH_RUN_ID: import_zod.z.string().optional()
|
|
77
|
+
});
|
|
78
|
+
var env = envSchema.parse(process.env);
|
|
79
|
+
|
|
33
80
|
// src/config/load.ts
|
|
34
81
|
var import_promises = __toESM(require("fs/promises"));
|
|
35
82
|
var import_node_path = __toESM(require("path"));
|
|
36
83
|
var import_node_url = require("url");
|
|
37
84
|
|
|
38
85
|
// src/domain/core/config.ts
|
|
39
|
-
var
|
|
86
|
+
var import_zod2 = require("zod");
|
|
40
87
|
var DEFAULT_TEMPERATURE = 0;
|
|
41
|
-
var scenarioProjectConfigSchema =
|
|
42
|
-
defaultModel:
|
|
43
|
-
model:
|
|
44
|
-
temperature:
|
|
45
|
-
maxTokens:
|
|
88
|
+
var scenarioProjectConfigSchema = import_zod2.z.object({
|
|
89
|
+
defaultModel: import_zod2.z.object({
|
|
90
|
+
model: import_zod2.z.custom(),
|
|
91
|
+
temperature: import_zod2.z.number().min(0).max(1).optional().default(DEFAULT_TEMPERATURE),
|
|
92
|
+
maxTokens: import_zod2.z.number().optional()
|
|
46
93
|
}).optional()
|
|
47
94
|
}).strict();
|
|
48
95
|
|
|
@@ -58,7 +105,7 @@ var Logger = class _Logger {
|
|
|
58
105
|
return new _Logger(context);
|
|
59
106
|
}
|
|
60
107
|
getLogLevel() {
|
|
61
|
-
return env.
|
|
108
|
+
return env.LOG_LEVEL ?? "INFO" /* INFO */;
|
|
62
109
|
}
|
|
63
110
|
getLogLevelIndex(level) {
|
|
64
111
|
return Object.values(LogLevel).indexOf(level);
|
|
@@ -116,63 +163,40 @@ var Logger = class _Logger {
|
|
|
116
163
|
}
|
|
117
164
|
};
|
|
118
165
|
|
|
119
|
-
// src/config/
|
|
120
|
-
var import_zod2 = require("zod");
|
|
121
|
-
|
|
122
|
-
// src/config/log-levels.ts
|
|
123
|
-
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
124
|
-
LogLevel2["ERROR"] = "ERROR";
|
|
125
|
-
LogLevel2["WARN"] = "WARN";
|
|
126
|
-
LogLevel2["INFO"] = "INFO";
|
|
127
|
-
LogLevel2["DEBUG"] = "DEBUG";
|
|
128
|
-
return LogLevel2;
|
|
129
|
-
})(LogLevel || {});
|
|
130
|
-
|
|
131
|
-
// src/config/env.ts
|
|
132
|
-
var envSchema = import_zod2.z.object({
|
|
133
|
-
/**
|
|
134
|
-
* LangWatch API key for event reporting.
|
|
135
|
-
* If not provided, events will not be sent to LangWatch.
|
|
136
|
-
*/
|
|
137
|
-
LANGWATCH_API_KEY: import_zod2.z.string().optional(),
|
|
138
|
-
/**
|
|
139
|
-
* LangWatch endpoint URL for event reporting.
|
|
140
|
-
* Defaults to the production LangWatch endpoint.
|
|
141
|
-
*/
|
|
142
|
-
LANGWATCH_ENDPOINT: import_zod2.z.string().url().default("https://app.langwatch.ai"),
|
|
143
|
-
/**
|
|
144
|
-
* Disables simulation report info messages when set to any truthy value.
|
|
145
|
-
* Useful for CI/CD environments or when you want cleaner output.
|
|
146
|
-
*/
|
|
147
|
-
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_zod2.z.string().optional().transform((val) => Boolean(val)),
|
|
148
|
-
/**
|
|
149
|
-
* Node environment - affects logging and behavior.
|
|
150
|
-
* Defaults to 'development' if not specified.
|
|
151
|
-
*/
|
|
152
|
-
NODE_ENV: import_zod2.z.enum(["development", "production", "test"]).default("development"),
|
|
153
|
-
/**
|
|
154
|
-
* Log level for the scenario package.
|
|
155
|
-
* Defaults to 'info' if not specified.
|
|
156
|
-
*/
|
|
157
|
-
SCENARIO_LOG_LEVEL: import_zod2.z.nativeEnum(LogLevel).optional(),
|
|
158
|
-
/**
|
|
159
|
-
* Scenario batch run ID.
|
|
160
|
-
* If not provided, a random ID will be generated.
|
|
161
|
-
*/
|
|
162
|
-
SCENARIO_BATCH_RUN_ID: import_zod2.z.string().optional()
|
|
163
|
-
});
|
|
164
|
-
var env = envSchema.parse(process.env);
|
|
165
|
-
|
|
166
|
-
// src/config/index.ts
|
|
166
|
+
// src/config/get-project-config.ts
|
|
167
167
|
var logger = new Logger("scenario.config");
|
|
168
168
|
|
|
169
169
|
// src/utils/ids.ts
|
|
170
|
+
var import_node_crypto = __toESM(require("crypto"));
|
|
171
|
+
var import_node_process = __toESM(require("process"));
|
|
170
172
|
var import_xksuid = require("xksuid");
|
|
173
|
+
var batchRunId;
|
|
171
174
|
function getBatchRunId() {
|
|
172
|
-
if (
|
|
173
|
-
|
|
175
|
+
if (batchRunId) {
|
|
176
|
+
return batchRunId;
|
|
174
177
|
}
|
|
175
|
-
|
|
178
|
+
if (import_node_process.default.env.SCENARIO_BATCH_RUN_ID) {
|
|
179
|
+
console.log("process.env.SCENARIO_BATCH_RUN_ID", import_node_process.default.env.SCENARIO_BATCH_RUN_ID);
|
|
180
|
+
return batchRunId = import_node_process.default.env.SCENARIO_BATCH_RUN_ID;
|
|
181
|
+
}
|
|
182
|
+
if (import_node_process.default.env.VITEST_WORKER_ID || import_node_process.default.env.JEST_WORKER_ID) {
|
|
183
|
+
const parentProcessId = import_node_process.default.ppid;
|
|
184
|
+
const now = /* @__PURE__ */ new Date();
|
|
185
|
+
const year = now.getUTCFullYear();
|
|
186
|
+
const week = String(getISOWeekNumber(now)).padStart(2, "0");
|
|
187
|
+
const raw = `${parentProcessId}_${year}_w${week}`;
|
|
188
|
+
const hash = import_node_crypto.default.createHash("sha256").update(raw).digest("hex").slice(0, 12);
|
|
189
|
+
return batchRunId = `scenariobatchrun_${hash}`;
|
|
190
|
+
}
|
|
191
|
+
return batchRunId = `scenariobatchrun_${(0, import_xksuid.generate)()}`;
|
|
192
|
+
}
|
|
193
|
+
function getISOWeekNumber(date) {
|
|
194
|
+
const tmp = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
|
|
195
|
+
const dayNum = tmp.getUTCDay() || 7;
|
|
196
|
+
tmp.setUTCDate(tmp.getUTCDate() + 4 - dayNum);
|
|
197
|
+
const yearStart = new Date(Date.UTC(tmp.getUTCFullYear(), 0, 1));
|
|
198
|
+
const weekNo = Math.ceil(((tmp.getTime() - yearStart.getTime()) / 864e5 + 1) / 7);
|
|
199
|
+
return weekNo;
|
|
176
200
|
}
|
|
177
201
|
|
|
178
202
|
// src/events/event-alert-message-logger.ts
|
|
@@ -186,12 +210,11 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
186
210
|
if (this.isGreetingDisabled()) {
|
|
187
211
|
return;
|
|
188
212
|
}
|
|
189
|
-
|
|
190
|
-
if (_EventAlertMessageLogger.shownBatchIds.has(batchRunId)) {
|
|
213
|
+
if (_EventAlertMessageLogger.shownBatchIds.has(getBatchRunId())) {
|
|
191
214
|
return;
|
|
192
215
|
}
|
|
193
|
-
_EventAlertMessageLogger.shownBatchIds.add(
|
|
194
|
-
this.displayGreeting(
|
|
216
|
+
_EventAlertMessageLogger.shownBatchIds.add(getBatchRunId());
|
|
217
|
+
this.displayGreeting();
|
|
195
218
|
}
|
|
196
219
|
/**
|
|
197
220
|
* Shows a fancy message about how to watch the simulation.
|
|
@@ -206,7 +229,7 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
206
229
|
isGreetingDisabled() {
|
|
207
230
|
return env.SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
|
|
208
231
|
}
|
|
209
|
-
displayGreeting(
|
|
232
|
+
displayGreeting() {
|
|
210
233
|
const separator = "\u2500".repeat(60);
|
|
211
234
|
if (!env.LANGWATCH_API_KEY) {
|
|
212
235
|
console.log(`
|
|
@@ -220,7 +243,10 @@ ${separator}`);
|
|
|
220
243
|
console.log(" \u2022 Set LANGWATCH_API_KEY environment variable");
|
|
221
244
|
console.log(" \u2022 Or configure apiKey in scenario.config.js");
|
|
222
245
|
console.log("");
|
|
223
|
-
console.log(`\u{1F4E6} Batch Run ID: ${
|
|
246
|
+
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
247
|
+
console.log("");
|
|
248
|
+
console.log("\u{1F507} To disable these messages:");
|
|
249
|
+
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
224
250
|
console.log(`${separator}
|
|
225
251
|
`);
|
|
226
252
|
} else {
|
|
@@ -234,7 +260,10 @@ ${separator}`);
|
|
|
234
260
|
` API Key: ${env.LANGWATCH_API_KEY.length > 0 ? "Configured" : "Not configured"}`
|
|
235
261
|
);
|
|
236
262
|
console.log("");
|
|
237
|
-
console.log(`\u{1F4E6} Batch Run ID: ${
|
|
263
|
+
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
264
|
+
console.log("");
|
|
265
|
+
console.log("\u{1F507} To disable these messages:");
|
|
266
|
+
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
238
267
|
console.log(`${separator}
|
|
239
268
|
`);
|
|
240
269
|
}
|
|
@@ -510,6 +539,7 @@ var EventBus = class _EventBus {
|
|
|
510
539
|
};
|
|
511
540
|
|
|
512
541
|
// src/integrations/vitest/setup.ts
|
|
542
|
+
var logger2 = Logger.create("integrations:vitest:setup");
|
|
513
543
|
function getProjectRoot() {
|
|
514
544
|
return process.cwd();
|
|
515
545
|
}
|
|
@@ -530,7 +560,7 @@ var subs = [];
|
|
|
530
560
|
try {
|
|
531
561
|
import_fs.default.appendFileSync(filePath, JSON.stringify(event) + "\n");
|
|
532
562
|
} catch (error) {
|
|
533
|
-
|
|
563
|
+
logger2.error("Error writing to log file:", error);
|
|
534
564
|
}
|
|
535
565
|
})
|
|
536
566
|
);
|
|
@@ -542,7 +572,7 @@ EventBus.addGlobalListener((bus) => {
|
|
|
542
572
|
try {
|
|
543
573
|
import_fs.default.appendFileSync(filePath, JSON.stringify(event) + "\n");
|
|
544
574
|
} catch (error) {
|
|
545
|
-
|
|
575
|
+
logger2.error("Error writing to log file:", error);
|
|
546
576
|
}
|
|
547
577
|
})
|
|
548
578
|
);
|
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import {
|
|
2
2
|
EventBus
|
|
3
|
-
} from "../../chunk-
|
|
3
|
+
} from "../../chunk-7H6OGEQ5.mjs";
|
|
4
|
+
import {
|
|
5
|
+
Logger
|
|
6
|
+
} from "../../chunk-YPJZSK4J.mjs";
|
|
4
7
|
import "../../chunk-7P6ASYW6.mjs";
|
|
5
8
|
|
|
6
9
|
// src/integrations/vitest/setup.ts
|
|
7
10
|
import fs from "fs";
|
|
8
11
|
import path from "path";
|
|
9
12
|
import { beforeEach, afterEach } from "vitest";
|
|
13
|
+
var logger = Logger.create("integrations:vitest:setup");
|
|
10
14
|
function getProjectRoot() {
|
|
11
15
|
return process.cwd();
|
|
12
16
|
}
|
|
@@ -27,7 +31,7 @@ beforeEach((ctx) => {
|
|
|
27
31
|
try {
|
|
28
32
|
fs.appendFileSync(filePath, JSON.stringify(event) + "\n");
|
|
29
33
|
} catch (error) {
|
|
30
|
-
|
|
34
|
+
logger.error("Error writing to log file:", error);
|
|
31
35
|
}
|
|
32
36
|
})
|
|
33
37
|
);
|
|
@@ -39,7 +43,7 @@ EventBus.addGlobalListener((bus) => {
|
|
|
39
43
|
try {
|
|
40
44
|
fs.appendFileSync(filePath, JSON.stringify(event) + "\n");
|
|
41
45
|
} catch (error) {
|
|
42
|
-
|
|
46
|
+
logger.error("Error writing to log file:", error);
|
|
43
47
|
}
|
|
44
48
|
})
|
|
45
49
|
);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@langwatch/scenario",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.9",
|
|
4
4
|
"description": "A TypeScript library for testing AI agents using scenarios",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -72,6 +72,16 @@
|
|
|
72
72
|
"types": "./dist/integrations/vitest/setup.d.ts",
|
|
73
73
|
"require": "./dist/integrations/vitest/setup.js",
|
|
74
74
|
"import": "./dist/integrations/vitest/setup.mjs"
|
|
75
|
+
},
|
|
76
|
+
"./integrations/vitest/setup-global": {
|
|
77
|
+
"types": "./dist/integrations/vitest/setup-global.d.ts",
|
|
78
|
+
"require": "./dist/integrations/vitest/setup-global.js",
|
|
79
|
+
"import": "./dist/integrations/vitest/setup-global.mjs"
|
|
80
|
+
},
|
|
81
|
+
"./integrations/vitest/config": {
|
|
82
|
+
"types": "./dist/integrations/vitest/config.d.ts",
|
|
83
|
+
"require": "./dist/integrations/vitest/config.js",
|
|
84
|
+
"import": "./dist/integrations/vitest/config.mjs"
|
|
75
85
|
}
|
|
76
86
|
},
|
|
77
87
|
"peerDependencies": {
|
|
@@ -84,8 +94,7 @@
|
|
|
84
94
|
"test": "vitest",
|
|
85
95
|
"test:ci": "vitest run",
|
|
86
96
|
"lint": "eslint .",
|
|
87
|
-
"examples:vitest:run": "export SCENARIO_BATCH_ID=scenariobatch_$(uuidgen) &&
|
|
88
|
-
"
|
|
89
|
-
"generate:api-reference": "npx typedoc src --out api-reference-docs && rm -rf ../docs/docs/public/reference/javascript/scenario && mv api-reference-docs ../docs/docs/public/reference/javascript/scenario && pnpm run hash-source > ../docs/docs/public/reference/javascript/.docs-source-hash"
|
|
97
|
+
"examples:vitest:run": "export SCENARIO_BATCH_ID=scenariobatch_$(uuidgen) && (cd examples/vitest && pnpm install) && pnpm -F vitest-example run test",
|
|
98
|
+
"generate:api-reference": "npx typedoc src --out api-reference-docs && rm -rf ../docs/docs/public/reference/javascript/scenario && mv api-reference-docs ../docs/docs/public/reference/javascript/scenario"
|
|
90
99
|
}
|
|
91
100
|
}
|