@langwatch/scenario 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,309 +0,0 @@
1
- // src/events/event-bus.ts
2
- import { concatMap, EMPTY, catchError, Subject } from "rxjs";
3
-
4
- // src/utils/logger.ts
5
- var Logger = class _Logger {
6
- constructor(context) {
7
- this.context = context;
8
- }
9
- /**
10
- * Creates a logger with context (e.g., class name)
11
- */
12
- static create(context) {
13
- return new _Logger(context);
14
- }
15
- /**
16
- * Checks if logging should occur based on LOG_LEVEL env var
17
- */
18
- shouldLog(level) {
19
- const logLevel = (process.env.SCENARIO_LOG_LEVEL || "").toLowerCase();
20
- const levels = ["error", "warn", "info", "debug"];
21
- const currentLevelIndex = levels.indexOf(logLevel);
22
- const requestedLevelIndex = levels.indexOf(level);
23
- return currentLevelIndex >= 0 && requestedLevelIndex <= currentLevelIndex;
24
- }
25
- formatMessage(message) {
26
- return this.context ? `[${this.context}] ${message}` : message;
27
- }
28
- error(message, data) {
29
- if (this.shouldLog("error")) {
30
- const formattedMessage = this.formatMessage(message);
31
- if (data) {
32
- console.error(formattedMessage, data);
33
- } else {
34
- console.error(formattedMessage);
35
- }
36
- }
37
- }
38
- warn(message, data) {
39
- if (this.shouldLog("warn")) {
40
- const formattedMessage = this.formatMessage(message);
41
- if (data) {
42
- console.warn(formattedMessage, data);
43
- } else {
44
- console.warn(formattedMessage);
45
- }
46
- }
47
- }
48
- info(message, data) {
49
- if (this.shouldLog("info")) {
50
- const formattedMessage = this.formatMessage(message);
51
- if (data) {
52
- console.info(formattedMessage, data);
53
- } else {
54
- console.info(formattedMessage);
55
- }
56
- }
57
- }
58
- debug(message, data) {
59
- if (this.shouldLog("debug")) {
60
- const formattedMessage = this.formatMessage(message);
61
- if (data) {
62
- console.log(formattedMessage, data);
63
- } else {
64
- console.log(formattedMessage);
65
- }
66
- }
67
- }
68
- };
69
-
70
- // src/events/event-reporter.ts
71
- var EventReporter = class {
72
- eventsEndpoint;
73
- apiKey;
74
- logger = new Logger("scenario.events.EventReporter");
75
- constructor(config) {
76
- this.eventsEndpoint = new URL("/api/scenario-events", config.endpoint);
77
- this.apiKey = config.apiKey ?? "";
78
- if (!process.env.SCENARIO_DISABLE_SIMULATION_REPORT_INFO) {
79
- if (!this.apiKey) {
80
- console.log(
81
- "\u27A1\uFE0F LangWatch API key not configured, simulations will only output the final result"
82
- );
83
- console.log(
84
- "To visualize the conversations in real time, configure your LangWatch API key (via LANGWATCH_API_KEY, or scenario.config.js)"
85
- );
86
- } else {
87
- console.log(`simulation reporting is enabled, endpoint:(${this.eventsEndpoint}) api_key_configured:(${this.apiKey.length > 0 ? "true" : "false"})`);
88
- }
89
- }
90
- }
91
- /**
92
- * Posts an event to the configured endpoint.
93
- * Logs success/failure but doesn't throw - event posting shouldn't break scenario execution.
94
- */
95
- async postEvent(event) {
96
- this.logger.debug(`[${event.type}] Posting event`, {
97
- event
98
- });
99
- if (!this.eventsEndpoint) {
100
- this.logger.warn(
101
- "No LANGWATCH_ENDPOINT configured, skipping event posting"
102
- );
103
- return;
104
- }
105
- try {
106
- const response = await fetch(this.eventsEndpoint.href, {
107
- method: "POST",
108
- body: JSON.stringify(event),
109
- headers: {
110
- "Content-Type": "application/json",
111
- "X-Auth-Token": this.apiKey
112
- }
113
- });
114
- this.logger.debug(
115
- `[${event.type}] Event POST response status: ${response.status}`
116
- );
117
- if (response.ok) {
118
- const data = await response.json();
119
- this.logger.debug(`[${event.type}] Event POST response:`, data);
120
- } else {
121
- const errorText = await response.text();
122
- this.logger.error(`[${event.type}] Event POST failed:`, {
123
- status: response.status,
124
- statusText: response.statusText,
125
- error: errorText,
126
- event
127
- });
128
- }
129
- } catch (error) {
130
- this.logger.error(`[${event.type}] Event POST error:`, {
131
- error,
132
- event,
133
- endpoint: this.eventsEndpoint
134
- });
135
- }
136
- }
137
- };
138
-
139
- // src/events/schema.ts
140
- import { EventType, MessagesSnapshotEventSchema } from "@ag-ui/core";
141
- import { z } from "zod";
142
- var Verdict = /* @__PURE__ */ ((Verdict2) => {
143
- Verdict2["SUCCESS"] = "success";
144
- Verdict2["FAILURE"] = "failure";
145
- Verdict2["INCONCLUSIVE"] = "inconclusive";
146
- return Verdict2;
147
- })(Verdict || {});
148
- var ScenarioRunStatus = /* @__PURE__ */ ((ScenarioRunStatus2) => {
149
- ScenarioRunStatus2["SUCCESS"] = "SUCCESS";
150
- ScenarioRunStatus2["ERROR"] = "ERROR";
151
- ScenarioRunStatus2["CANCELLED"] = "CANCELLED";
152
- ScenarioRunStatus2["IN_PROGRESS"] = "IN_PROGRESS";
153
- ScenarioRunStatus2["PENDING"] = "PENDING";
154
- ScenarioRunStatus2["FAILED"] = "FAILED";
155
- return ScenarioRunStatus2;
156
- })(ScenarioRunStatus || {});
157
- var baseEventSchema = z.object({
158
- type: z.nativeEnum(EventType),
159
- timestamp: z.number(),
160
- rawEvent: z.any().optional()
161
- });
162
- var batchRunIdSchema = z.string();
163
- var scenarioRunIdSchema = z.string();
164
- var scenarioIdSchema = z.string();
165
- var baseScenarioEventSchema = baseEventSchema.extend({
166
- batchRunId: batchRunIdSchema,
167
- scenarioId: scenarioIdSchema,
168
- scenarioRunId: scenarioRunIdSchema,
169
- scenarioSetId: z.string().optional().default("default")
170
- });
171
- var scenarioRunStartedSchema = baseScenarioEventSchema.extend({
172
- type: z.literal("SCENARIO_RUN_STARTED" /* RUN_STARTED */),
173
- metadata: z.object({
174
- name: z.string().optional(),
175
- description: z.string().optional()
176
- })
177
- });
178
- var scenarioResultsSchema = z.object({
179
- verdict: z.nativeEnum(Verdict),
180
- reasoning: z.string().optional(),
181
- metCriteria: z.array(z.string()),
182
- unmetCriteria: z.array(z.string()),
183
- error: z.string().optional()
184
- });
185
- var scenarioRunFinishedSchema = baseScenarioEventSchema.extend({
186
- type: z.literal("SCENARIO_RUN_FINISHED" /* RUN_FINISHED */),
187
- status: z.nativeEnum(ScenarioRunStatus),
188
- results: scenarioResultsSchema.optional().nullable()
189
- });
190
- var scenarioMessageSnapshotSchema = MessagesSnapshotEventSchema.merge(
191
- baseScenarioEventSchema.extend({
192
- type: z.literal("SCENARIO_MESSAGE_SNAPSHOT" /* MESSAGE_SNAPSHOT */)
193
- })
194
- );
195
- var scenarioEventSchema = z.discriminatedUnion("type", [
196
- scenarioRunStartedSchema,
197
- scenarioRunFinishedSchema,
198
- scenarioMessageSnapshotSchema
199
- ]);
200
- var successSchema = z.object({ success: z.boolean() });
201
- var errorSchema = z.object({ error: z.string() });
202
- var stateSchema = z.object({
203
- state: z.object({
204
- messages: z.array(z.any()),
205
- status: z.string()
206
- })
207
- });
208
- var runsSchema = z.object({ runs: z.array(z.string()) });
209
- var eventsSchema = z.object({ events: z.array(scenarioEventSchema) });
210
-
211
- // src/events/event-bus.ts
212
- var EventBus = class _EventBus {
213
- static registry = /* @__PURE__ */ new Set();
214
- events$ = new Subject();
215
- eventReporter;
216
- processingPromise = null;
217
- logger = new Logger("scenario.events.EventBus");
218
- static globalListeners = [];
219
- constructor(config) {
220
- this.eventReporter = new EventReporter(config);
221
- _EventBus.registry.add(this);
222
- for (const listener of _EventBus.globalListeners) {
223
- listener(this);
224
- }
225
- }
226
- static getAllBuses() {
227
- return _EventBus.registry;
228
- }
229
- static addGlobalListener(listener) {
230
- _EventBus.globalListeners.push(listener);
231
- }
232
- /**
233
- * Publishes an event into the processing pipeline.
234
- */
235
- publish(event) {
236
- this.logger.debug(`[${event.type}] Publishing event`, {
237
- event
238
- });
239
- this.events$.next(event);
240
- }
241
- /**
242
- * Begins listening for and processing events.
243
- * Returns a promise that resolves when a RUN_FINISHED event is fully processed.
244
- */
245
- listen() {
246
- this.logger.debug("Listening for events");
247
- if (this.processingPromise) {
248
- return this.processingPromise;
249
- }
250
- this.processingPromise = new Promise((resolve, reject) => {
251
- this.events$.pipe(
252
- concatMap(async (event) => {
253
- this.logger.debug(`[${event.type}] Processing event`, {
254
- event
255
- });
256
- await this.eventReporter.postEvent(event);
257
- return event;
258
- }),
259
- catchError((error) => {
260
- this.logger.error("Error in event stream:", error);
261
- return EMPTY;
262
- })
263
- ).subscribe({
264
- next: (event) => {
265
- this.logger.debug(`[${event.type}] Event processed`, {
266
- event
267
- });
268
- if (event.type === "SCENARIO_RUN_FINISHED" /* RUN_FINISHED */) {
269
- resolve();
270
- }
271
- },
272
- error: (error) => {
273
- this.logger.error("Error in event stream:", error);
274
- reject(error);
275
- }
276
- });
277
- });
278
- return this.processingPromise;
279
- }
280
- /**
281
- * Stops accepting new events and drains the processing queue.
282
- */
283
- async drain() {
284
- this.logger.debug("Draining event stream");
285
- this.events$.complete();
286
- if (this.processingPromise) {
287
- await this.processingPromise;
288
- }
289
- }
290
- /**
291
- * Subscribes to an event stream.
292
- * @param source$ - The event stream to subscribe to.
293
- */
294
- subscribeTo(source$) {
295
- this.logger.debug("Subscribing to event stream");
296
- return source$.subscribe(this.events$);
297
- }
298
- /**
299
- * Expose the events$ observable for external subscription (read-only).
300
- */
301
- get eventsObservable() {
302
- return this.events$.asObservable();
303
- }
304
- };
305
-
306
- export {
307
- Logger,
308
- EventBus
309
- };