@langwatch/scenario 0.2.13 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -8
- package/dist/{chunk-6SKQWXT7.mjs → chunk-3Z7E24UI.mjs} +26 -6
- package/dist/{chunk-OL4RFXV4.mjs → chunk-RHTLQKEJ.mjs} +1 -1
- package/dist/index.d.mts +109 -130
- package/dist/index.d.ts +109 -130
- package/dist/index.js +95 -75
- package/dist/index.mjs +23 -23
- package/dist/integrations/vitest/reporter.js +14 -10
- package/dist/integrations/vitest/reporter.mjs +7 -3
- package/dist/integrations/vitest/setup.js +70 -50
- package/dist/integrations/vitest/setup.mjs +2 -2
- package/package.json +6 -5
|
@@ -31,10 +31,13 @@ var import_vitest = require("vitest");
|
|
|
31
31
|
var import_rxjs = require("rxjs");
|
|
32
32
|
|
|
33
33
|
// src/events/event-alert-message-logger.ts
|
|
34
|
+
var fs2 = __toESM(require("fs"));
|
|
35
|
+
var os = __toESM(require("os"));
|
|
36
|
+
var path2 = __toESM(require("path"));
|
|
34
37
|
var import_open = __toESM(require("open"));
|
|
35
38
|
|
|
36
39
|
// src/config/env.ts
|
|
37
|
-
var
|
|
40
|
+
var import_v4 = require("zod/v4");
|
|
38
41
|
|
|
39
42
|
// src/config/log-levels.ts
|
|
40
43
|
var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
@@ -47,37 +50,37 @@ var LogLevel = /* @__PURE__ */ ((LogLevel2) => {
|
|
|
47
50
|
var LOG_LEVELS = Object.values(LogLevel);
|
|
48
51
|
|
|
49
52
|
// src/config/env.ts
|
|
50
|
-
var envSchema =
|
|
53
|
+
var envSchema = import_v4.z.object({
|
|
51
54
|
/**
|
|
52
55
|
* LangWatch API key for event reporting.
|
|
53
56
|
* If not provided, events will not be sent to LangWatch.
|
|
54
57
|
*/
|
|
55
|
-
LANGWATCH_API_KEY:
|
|
58
|
+
LANGWATCH_API_KEY: import_v4.z.string().optional(),
|
|
56
59
|
/**
|
|
57
60
|
* LangWatch endpoint URL for event reporting.
|
|
58
61
|
* Defaults to the production LangWatch endpoint.
|
|
59
62
|
*/
|
|
60
|
-
LANGWATCH_ENDPOINT:
|
|
63
|
+
LANGWATCH_ENDPOINT: import_v4.z.string().url().optional().default("https://app.langwatch.ai"),
|
|
61
64
|
/**
|
|
62
65
|
* Disables simulation report info messages when set to any truthy value.
|
|
63
66
|
* Useful for CI/CD environments or when you want cleaner output.
|
|
64
67
|
*/
|
|
65
|
-
SCENARIO_DISABLE_SIMULATION_REPORT_INFO:
|
|
68
|
+
SCENARIO_DISABLE_SIMULATION_REPORT_INFO: import_v4.z.string().optional().transform((val) => Boolean(val)),
|
|
66
69
|
/**
|
|
67
70
|
* Node environment - affects logging and behavior.
|
|
68
71
|
* Defaults to 'development' if not specified.
|
|
69
72
|
*/
|
|
70
|
-
NODE_ENV:
|
|
73
|
+
NODE_ENV: import_v4.z.enum(["development", "production", "test"]).default("development"),
|
|
71
74
|
/**
|
|
72
75
|
* Case-insensitive log level for the scenario package.
|
|
73
76
|
* Defaults to 'info' if not specified.
|
|
74
77
|
*/
|
|
75
|
-
LOG_LEVEL:
|
|
78
|
+
LOG_LEVEL: import_v4.z.string().toUpperCase().pipe(import_v4.z.nativeEnum(LogLevel)).optional().default("INFO" /* INFO */),
|
|
76
79
|
/**
|
|
77
80
|
* Scenario batch run ID.
|
|
78
81
|
* If not provided, a random ID will be generated.
|
|
79
82
|
*/
|
|
80
|
-
SCENARIO_BATCH_RUN_ID:
|
|
83
|
+
SCENARIO_BATCH_RUN_ID: import_v4.z.string().optional()
|
|
81
84
|
});
|
|
82
85
|
function getEnv() {
|
|
83
86
|
return envSchema.parse(process.env);
|
|
@@ -89,15 +92,15 @@ var import_node_path = __toESM(require("path"));
|
|
|
89
92
|
var import_node_url = require("url");
|
|
90
93
|
|
|
91
94
|
// src/domain/core/config.ts
|
|
92
|
-
var
|
|
95
|
+
var import_v42 = require("zod/v4");
|
|
93
96
|
var DEFAULT_TEMPERATURE = 0;
|
|
94
|
-
var scenarioProjectConfigSchema =
|
|
95
|
-
defaultModel:
|
|
96
|
-
model:
|
|
97
|
-
temperature:
|
|
98
|
-
maxTokens:
|
|
97
|
+
var scenarioProjectConfigSchema = import_v42.z.object({
|
|
98
|
+
defaultModel: import_v42.z.object({
|
|
99
|
+
model: import_v42.z.custom(),
|
|
100
|
+
temperature: import_v42.z.number().min(0).max(1).optional().default(DEFAULT_TEMPERATURE),
|
|
101
|
+
maxTokens: import_v42.z.number().optional()
|
|
99
102
|
}).optional(),
|
|
100
|
-
headless:
|
|
103
|
+
headless: import_v42.z.boolean().optional().default(
|
|
101
104
|
typeof process !== "undefined" ? !["false", "0"].includes(process.env.SCENARIO_HEADLESS || "false") : false
|
|
102
105
|
)
|
|
103
106
|
}).strict();
|
|
@@ -273,8 +276,23 @@ function getISOWeekNumber(date) {
|
|
|
273
276
|
}
|
|
274
277
|
|
|
275
278
|
// src/events/event-alert-message-logger.ts
|
|
276
|
-
var EventAlertMessageLogger = class
|
|
277
|
-
|
|
279
|
+
var EventAlertMessageLogger = class {
|
|
280
|
+
/**
|
|
281
|
+
* Creates a coordination file to prevent duplicate messages across processes.
|
|
282
|
+
* Returns true if this process should show the message (first one to create the file).
|
|
283
|
+
*/
|
|
284
|
+
createCoordinationFile(type) {
|
|
285
|
+
try {
|
|
286
|
+
const batchId = getBatchRunId();
|
|
287
|
+
const tmpDir = os.tmpdir();
|
|
288
|
+
const fileName = `scenario-${type}-${batchId}`;
|
|
289
|
+
const filePath = path2.join(tmpDir, fileName);
|
|
290
|
+
fs2.writeFileSync(filePath, process.pid.toString(), { flag: "wx" });
|
|
291
|
+
return true;
|
|
292
|
+
} catch {
|
|
293
|
+
return false;
|
|
294
|
+
}
|
|
295
|
+
}
|
|
278
296
|
/**
|
|
279
297
|
* Shows a fancy greeting message about simulation reporting status.
|
|
280
298
|
* Only shows once per batch run to avoid spam.
|
|
@@ -283,10 +301,9 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
283
301
|
if (this.isGreetingDisabled()) {
|
|
284
302
|
return;
|
|
285
303
|
}
|
|
286
|
-
if (
|
|
304
|
+
if (!this.createCoordinationFile("greeting")) {
|
|
287
305
|
return;
|
|
288
306
|
}
|
|
289
|
-
_EventAlertMessageLogger.shownBatchIds.add(getBatchRunId());
|
|
290
307
|
this.displayGreeting();
|
|
291
308
|
}
|
|
292
309
|
/**
|
|
@@ -297,6 +314,9 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
297
314
|
if (this.isGreetingDisabled()) {
|
|
298
315
|
return;
|
|
299
316
|
}
|
|
317
|
+
if (!this.createCoordinationFile(`watch-${params.scenarioSetId}`)) {
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
300
320
|
await this.displayWatchMessage(params);
|
|
301
321
|
}
|
|
302
322
|
isGreetingDisabled() {
|
|
@@ -344,7 +364,7 @@ ${separator}`);
|
|
|
344
364
|
|
|
345
365
|
// src/events/schema.ts
|
|
346
366
|
var import_core = require("@ag-ui/core");
|
|
347
|
-
var
|
|
367
|
+
var import_zod = require("zod");
|
|
348
368
|
var Verdict = /* @__PURE__ */ ((Verdict2) => {
|
|
349
369
|
Verdict2["SUCCESS"] = "success";
|
|
350
370
|
Verdict2["FAILURE"] = "failure";
|
|
@@ -360,59 +380,59 @@ var ScenarioRunStatus = /* @__PURE__ */ ((ScenarioRunStatus2) => {
|
|
|
360
380
|
ScenarioRunStatus2["FAILED"] = "FAILED";
|
|
361
381
|
return ScenarioRunStatus2;
|
|
362
382
|
})(ScenarioRunStatus || {});
|
|
363
|
-
var baseEventSchema =
|
|
364
|
-
type:
|
|
365
|
-
timestamp:
|
|
366
|
-
rawEvent:
|
|
383
|
+
var baseEventSchema = import_zod.z.object({
|
|
384
|
+
type: import_zod.z.nativeEnum(import_core.EventType),
|
|
385
|
+
timestamp: import_zod.z.number(),
|
|
386
|
+
rawEvent: import_zod.z.any().optional()
|
|
367
387
|
});
|
|
368
|
-
var batchRunIdSchema =
|
|
369
|
-
var scenarioRunIdSchema =
|
|
370
|
-
var scenarioIdSchema =
|
|
388
|
+
var batchRunIdSchema = import_zod.z.string();
|
|
389
|
+
var scenarioRunIdSchema = import_zod.z.string();
|
|
390
|
+
var scenarioIdSchema = import_zod.z.string();
|
|
371
391
|
var baseScenarioEventSchema = baseEventSchema.extend({
|
|
372
392
|
batchRunId: batchRunIdSchema,
|
|
373
393
|
scenarioId: scenarioIdSchema,
|
|
374
394
|
scenarioRunId: scenarioRunIdSchema,
|
|
375
|
-
scenarioSetId:
|
|
395
|
+
scenarioSetId: import_zod.z.string().optional().default("default")
|
|
376
396
|
});
|
|
377
397
|
var scenarioRunStartedSchema = baseScenarioEventSchema.extend({
|
|
378
|
-
type:
|
|
379
|
-
metadata:
|
|
380
|
-
name:
|
|
381
|
-
description:
|
|
398
|
+
type: import_zod.z.literal("SCENARIO_RUN_STARTED" /* RUN_STARTED */),
|
|
399
|
+
metadata: import_zod.z.object({
|
|
400
|
+
name: import_zod.z.string().optional(),
|
|
401
|
+
description: import_zod.z.string().optional()
|
|
382
402
|
})
|
|
383
403
|
});
|
|
384
|
-
var scenarioResultsSchema =
|
|
385
|
-
verdict:
|
|
386
|
-
reasoning:
|
|
387
|
-
metCriteria:
|
|
388
|
-
unmetCriteria:
|
|
389
|
-
error:
|
|
404
|
+
var scenarioResultsSchema = import_zod.z.object({
|
|
405
|
+
verdict: import_zod.z.nativeEnum(Verdict),
|
|
406
|
+
reasoning: import_zod.z.string().optional(),
|
|
407
|
+
metCriteria: import_zod.z.array(import_zod.z.string()),
|
|
408
|
+
unmetCriteria: import_zod.z.array(import_zod.z.string()),
|
|
409
|
+
error: import_zod.z.string().optional()
|
|
390
410
|
});
|
|
391
411
|
var scenarioRunFinishedSchema = baseScenarioEventSchema.extend({
|
|
392
|
-
type:
|
|
393
|
-
status:
|
|
412
|
+
type: import_zod.z.literal("SCENARIO_RUN_FINISHED" /* RUN_FINISHED */),
|
|
413
|
+
status: import_zod.z.nativeEnum(ScenarioRunStatus),
|
|
394
414
|
results: scenarioResultsSchema.optional().nullable()
|
|
395
415
|
});
|
|
396
416
|
var scenarioMessageSnapshotSchema = import_core.MessagesSnapshotEventSchema.merge(
|
|
397
417
|
baseScenarioEventSchema.extend({
|
|
398
|
-
type:
|
|
418
|
+
type: import_zod.z.literal("SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */)
|
|
399
419
|
})
|
|
400
420
|
);
|
|
401
|
-
var scenarioEventSchema =
|
|
421
|
+
var scenarioEventSchema = import_zod.z.discriminatedUnion("type", [
|
|
402
422
|
scenarioRunStartedSchema,
|
|
403
423
|
scenarioRunFinishedSchema,
|
|
404
424
|
scenarioMessageSnapshotSchema
|
|
405
425
|
]);
|
|
406
|
-
var successSchema =
|
|
407
|
-
var errorSchema =
|
|
408
|
-
var stateSchema =
|
|
409
|
-
state:
|
|
410
|
-
messages:
|
|
411
|
-
status:
|
|
426
|
+
var successSchema = import_zod.z.object({ success: import_zod.z.boolean() });
|
|
427
|
+
var errorSchema = import_zod.z.object({ error: import_zod.z.string() });
|
|
428
|
+
var stateSchema = import_zod.z.object({
|
|
429
|
+
state: import_zod.z.object({
|
|
430
|
+
messages: import_zod.z.array(import_zod.z.any()),
|
|
431
|
+
status: import_zod.z.string()
|
|
412
432
|
})
|
|
413
433
|
});
|
|
414
|
-
var runsSchema =
|
|
415
|
-
var eventsSchema =
|
|
434
|
+
var runsSchema = import_zod.z.object({ runs: import_zod.z.array(import_zod.z.string()) });
|
|
435
|
+
var eventsSchema = import_zod.z.object({ events: import_zod.z.array(scenarioEventSchema) });
|
|
416
436
|
|
|
417
437
|
// src/events/event-reporter.ts
|
|
418
438
|
var EventReporter = class {
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
EventBus
|
|
3
|
-
} from "../../chunk-
|
|
3
|
+
} from "../../chunk-3Z7E24UI.mjs";
|
|
4
4
|
import {
|
|
5
5
|
Logger
|
|
6
|
-
} from "../../chunk-
|
|
6
|
+
} from "../../chunk-RHTLQKEJ.mjs";
|
|
7
7
|
import "../../chunk-7P6ASYW6.mjs";
|
|
8
8
|
|
|
9
9
|
// src/integrations/vitest/setup.ts
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@langwatch/scenario",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "A TypeScript library for testing AI agents using scenarios",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -29,13 +29,13 @@
|
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
31
|
"@ag-ui/core": "^0.0.28",
|
|
32
|
-
"@ai-sdk/openai": "^
|
|
33
|
-
"ai": ">=
|
|
32
|
+
"@ai-sdk/openai": "^2.0.23",
|
|
33
|
+
"ai": ">=5.0.0",
|
|
34
34
|
"chalk": "^5.4.1",
|
|
35
35
|
"open": "10.2.0",
|
|
36
36
|
"rxjs": "^7.8.2",
|
|
37
37
|
"xksuid": "^0.0.4",
|
|
38
|
-
"zod": "^3.
|
|
38
|
+
"zod": "^3.25.76"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"@eslint/js": "^9.26.0",
|
|
@@ -85,7 +85,8 @@
|
|
|
85
85
|
}
|
|
86
86
|
},
|
|
87
87
|
"peerDependencies": {
|
|
88
|
-
"vitest": ">=3.2.4"
|
|
88
|
+
"vitest": ">=3.2.4",
|
|
89
|
+
"ai": ">=5.0.0"
|
|
89
90
|
},
|
|
90
91
|
"scripts": {
|
|
91
92
|
"build": "tsup src/index.ts src/integrations/vitest/*.ts --format cjs,esm --dts --clean --external vitest",
|