@langwatch/scenario 0.2.12 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-7HLDX5EL.mjs → chunk-6SKQWXT7.mjs} +26 -37
- package/dist/index.d.mts +3 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +24 -37
- package/dist/index.mjs +1 -1
- package/dist/integrations/vitest/setup-global.js +1 -1
- package/dist/integrations/vitest/setup-global.mjs +1 -1
- package/dist/integrations/vitest/setup.js +86 -41
- package/dist/integrations/vitest/setup.mjs +1 -1
- package/package.json +3 -2
|
@@ -14,7 +14,10 @@ var scenarioProjectConfigSchema = z.object({
|
|
|
14
14
|
model: z.custom(),
|
|
15
15
|
temperature: z.number().min(0).max(1).optional().default(DEFAULT_TEMPERATURE),
|
|
16
16
|
maxTokens: z.number().optional()
|
|
17
|
-
}).optional()
|
|
17
|
+
}).optional(),
|
|
18
|
+
headless: z.boolean().optional().default(
|
|
19
|
+
typeof process !== "undefined" ? !["false", "0"].includes(process.env.SCENARIO_HEADLESS || "false") : false
|
|
20
|
+
)
|
|
18
21
|
}).strict();
|
|
19
22
|
function defineConfig(config2) {
|
|
20
23
|
return config2;
|
|
@@ -71,6 +74,9 @@ import {
|
|
|
71
74
|
map
|
|
72
75
|
} from "rxjs";
|
|
73
76
|
|
|
77
|
+
// src/events/event-alert-message-logger.ts
|
|
78
|
+
import open from "open";
|
|
79
|
+
|
|
74
80
|
// src/config/load.ts
|
|
75
81
|
import fs from "node:fs/promises";
|
|
76
82
|
import path from "node:path";
|
|
@@ -139,7 +145,7 @@ import process2 from "node:process";
|
|
|
139
145
|
import { generate, parse } from "xksuid";
|
|
140
146
|
var batchRunId;
|
|
141
147
|
function generateThreadId() {
|
|
142
|
-
return `
|
|
148
|
+
return `scenariothread_${generate()}`;
|
|
143
149
|
}
|
|
144
150
|
function generateScenarioRunId() {
|
|
145
151
|
return `scenariorun_${generate()}`;
|
|
@@ -161,9 +167,9 @@ function getBatchRunId() {
|
|
|
161
167
|
const week = String(getISOWeekNumber(now)).padStart(2, "0");
|
|
162
168
|
const raw = `${parentProcessId}_${year}_w${week}`;
|
|
163
169
|
const hash = crypto.createHash("sha256").update(raw).digest("hex").slice(0, 12);
|
|
164
|
-
return batchRunId = `
|
|
170
|
+
return batchRunId = `scenariobatch_${hash}`;
|
|
165
171
|
}
|
|
166
|
-
return batchRunId = `
|
|
172
|
+
return batchRunId = `scenariobatch_${generate()}`;
|
|
167
173
|
}
|
|
168
174
|
function getISOWeekNumber(date) {
|
|
169
175
|
const tmp = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
|
|
@@ -198,11 +204,11 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
198
204
|
* Shows a fancy message about how to watch the simulation.
|
|
199
205
|
* Called when a run started event is received with a session ID.
|
|
200
206
|
*/
|
|
201
|
-
handleWatchMessage(params) {
|
|
207
|
+
async handleWatchMessage(params) {
|
|
202
208
|
if (this.isGreetingDisabled()) {
|
|
203
209
|
return;
|
|
204
210
|
}
|
|
205
|
-
this.displayWatchMessage(params);
|
|
211
|
+
await this.displayWatchMessage(params);
|
|
206
212
|
}
|
|
207
213
|
isGreetingDisabled() {
|
|
208
214
|
return getEnv().SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
|
|
@@ -213,54 +219,37 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
213
219
|
if (!env.LANGWATCH_API_KEY) {
|
|
214
220
|
console.log(`
|
|
215
221
|
${separator}`);
|
|
216
|
-
console.log("\u{
|
|
222
|
+
console.log("\u{1F3AD} Running Scenario Tests");
|
|
217
223
|
console.log(`${separator}`);
|
|
218
|
-
console.log("\u27A1\uFE0F API key not configured");
|
|
224
|
+
console.log("\u27A1\uFE0F LangWatch API key not configured");
|
|
219
225
|
console.log(" Simulations will only output final results");
|
|
220
226
|
console.log("");
|
|
221
227
|
console.log("\u{1F4A1} To visualize conversations in real time:");
|
|
222
228
|
console.log(" \u2022 Set LANGWATCH_API_KEY environment variable");
|
|
223
229
|
console.log(" \u2022 Or configure apiKey in scenario.config.js");
|
|
224
230
|
console.log("");
|
|
225
|
-
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
226
|
-
console.log("");
|
|
227
|
-
console.log("\u{1F507} To disable these messages:");
|
|
228
|
-
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
229
|
-
console.log(`${separator}
|
|
230
|
-
`);
|
|
231
|
-
} else {
|
|
232
|
-
console.log(`
|
|
233
|
-
${separator}`);
|
|
234
|
-
console.log("\u{1F680} LangWatch Simulation Reporting");
|
|
235
|
-
console.log(`${separator}`);
|
|
236
|
-
console.log("\u2705 Simulation reporting enabled");
|
|
237
|
-
console.log(` Endpoint: ${env.LANGWATCH_ENDPOINT}`);
|
|
238
|
-
console.log(
|
|
239
|
-
` API Key: ${env.LANGWATCH_API_KEY.length > 0 ? "Configured" : "Not configured"}`
|
|
240
|
-
);
|
|
241
|
-
console.log("");
|
|
242
|
-
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
243
|
-
console.log("");
|
|
244
|
-
console.log("\u{1F507} To disable these messages:");
|
|
245
|
-
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
246
231
|
console.log(`${separator}
|
|
247
232
|
`);
|
|
248
233
|
}
|
|
249
234
|
}
|
|
250
|
-
displayWatchMessage(params) {
|
|
235
|
+
async displayWatchMessage(params) {
|
|
251
236
|
const separator = "\u2500".repeat(60);
|
|
252
237
|
const setUrl = params.setUrl;
|
|
253
238
|
const batchUrl = `${setUrl}/${getBatchRunId()}`;
|
|
254
239
|
console.log(`
|
|
255
240
|
${separator}`);
|
|
256
|
-
console.log("\u{
|
|
241
|
+
console.log("\u{1F3AD} Running Scenario Tests");
|
|
257
242
|
console.log(`${separator}`);
|
|
258
|
-
console.log(
|
|
259
|
-
console.log(` Scenario Set: ${setUrl}`);
|
|
260
|
-
console.log(` Batch Run: ${batchUrl}`);
|
|
261
|
-
console.log("");
|
|
243
|
+
console.log(`Follow it live: ${batchUrl}`);
|
|
262
244
|
console.log(`${separator}
|
|
263
245
|
`);
|
|
246
|
+
const projectConfig = await getProjectConfig();
|
|
247
|
+
if (!(projectConfig == null ? void 0 : projectConfig.headless)) {
|
|
248
|
+
try {
|
|
249
|
+
open(batchUrl);
|
|
250
|
+
} catch (_) {
|
|
251
|
+
}
|
|
252
|
+
}
|
|
264
253
|
}
|
|
265
254
|
};
|
|
266
255
|
|
|
@@ -461,9 +450,9 @@ var EventBus = class _EventBus {
|
|
|
461
450
|
return { event, result };
|
|
462
451
|
}),
|
|
463
452
|
// Handle watch messages reactively
|
|
464
|
-
tap(({ event, result }) => {
|
|
453
|
+
tap(async ({ event, result }) => {
|
|
465
454
|
if (event.type === "SCENARIO_RUN_STARTED" /* RUN_STARTED */ && result.setUrl) {
|
|
466
|
-
this.eventAlertMessageLogger.handleWatchMessage({
|
|
455
|
+
await this.eventAlertMessageLogger.handleWatchMessage({
|
|
467
456
|
scenarioSetId: event.scenarioSetId,
|
|
468
457
|
scenarioRunId: event.scenarioRunId,
|
|
469
458
|
setUrl: result.setUrl
|
package/dist/index.d.mts
CHANGED
|
@@ -364,7 +364,9 @@ declare const scenarioProjectConfigSchema: z.ZodObject<{
|
|
|
364
364
|
temperature?: number | undefined;
|
|
365
365
|
maxTokens?: number | undefined;
|
|
366
366
|
}>>;
|
|
367
|
+
headless: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
367
368
|
}, "strict", z.ZodTypeAny, {
|
|
369
|
+
headless: boolean;
|
|
368
370
|
defaultModel?: {
|
|
369
371
|
model: ai.LanguageModelV1;
|
|
370
372
|
temperature: number;
|
|
@@ -376,6 +378,7 @@ declare const scenarioProjectConfigSchema: z.ZodObject<{
|
|
|
376
378
|
temperature?: number | undefined;
|
|
377
379
|
maxTokens?: number | undefined;
|
|
378
380
|
} | undefined;
|
|
381
|
+
headless?: boolean | undefined;
|
|
379
382
|
}>;
|
|
380
383
|
type ScenarioProjectConfig = z.infer<typeof scenarioProjectConfigSchema>;
|
|
381
384
|
declare function defineConfig(config: ScenarioProjectConfig): ScenarioProjectConfig;
|
package/dist/index.d.ts
CHANGED
|
@@ -364,7 +364,9 @@ declare const scenarioProjectConfigSchema: z.ZodObject<{
|
|
|
364
364
|
temperature?: number | undefined;
|
|
365
365
|
maxTokens?: number | undefined;
|
|
366
366
|
}>>;
|
|
367
|
+
headless: z.ZodDefault<z.ZodOptional<z.ZodBoolean>>;
|
|
367
368
|
}, "strict", z.ZodTypeAny, {
|
|
369
|
+
headless: boolean;
|
|
368
370
|
defaultModel?: {
|
|
369
371
|
model: ai.LanguageModelV1;
|
|
370
372
|
temperature: number;
|
|
@@ -376,6 +378,7 @@ declare const scenarioProjectConfigSchema: z.ZodObject<{
|
|
|
376
378
|
temperature?: number | undefined;
|
|
377
379
|
maxTokens?: number | undefined;
|
|
378
380
|
} | undefined;
|
|
381
|
+
headless?: boolean | undefined;
|
|
379
382
|
}>;
|
|
380
383
|
type ScenarioProjectConfig = z.infer<typeof scenarioProjectConfigSchema>;
|
|
381
384
|
declare function defineConfig(config: ScenarioProjectConfig): ScenarioProjectConfig;
|
package/dist/index.js
CHANGED
|
@@ -92,7 +92,10 @@ var scenarioProjectConfigSchema = import_zod.z.object({
|
|
|
92
92
|
model: import_zod.z.custom(),
|
|
93
93
|
temperature: import_zod.z.number().min(0).max(1).optional().default(DEFAULT_TEMPERATURE),
|
|
94
94
|
maxTokens: import_zod.z.number().optional()
|
|
95
|
-
}).optional()
|
|
95
|
+
}).optional(),
|
|
96
|
+
headless: import_zod.z.boolean().optional().default(
|
|
97
|
+
typeof process !== "undefined" ? !["false", "0"].includes(process.env.SCENARIO_HEADLESS || "false") : false
|
|
98
|
+
)
|
|
96
99
|
}).strict();
|
|
97
100
|
function defineConfig(config2) {
|
|
98
101
|
return config2;
|
|
@@ -633,7 +636,7 @@ var import_node_process = __toESM(require("process"));
|
|
|
633
636
|
var import_xksuid = require("xksuid");
|
|
634
637
|
var batchRunId;
|
|
635
638
|
function generateThreadId() {
|
|
636
|
-
return `
|
|
639
|
+
return `scenariothread_${(0, import_xksuid.generate)()}`;
|
|
637
640
|
}
|
|
638
641
|
function generateScenarioRunId() {
|
|
639
642
|
return `scenariorun_${(0, import_xksuid.generate)()}`;
|
|
@@ -655,9 +658,9 @@ function getBatchRunId() {
|
|
|
655
658
|
const week = String(getISOWeekNumber(now)).padStart(2, "0");
|
|
656
659
|
const raw = `${parentProcessId}_${year}_w${week}`;
|
|
657
660
|
const hash = import_node_crypto.default.createHash("sha256").update(raw).digest("hex").slice(0, 12);
|
|
658
|
-
return batchRunId = `
|
|
661
|
+
return batchRunId = `scenariobatch_${hash}`;
|
|
659
662
|
}
|
|
660
|
-
return batchRunId = `
|
|
663
|
+
return batchRunId = `scenariobatch_${(0, import_xksuid.generate)()}`;
|
|
661
664
|
}
|
|
662
665
|
function getISOWeekNumber(date) {
|
|
663
666
|
const tmp = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
|
|
@@ -1883,6 +1886,7 @@ __export(runner_exports, {
|
|
|
1883
1886
|
var import_rxjs3 = require("rxjs");
|
|
1884
1887
|
|
|
1885
1888
|
// src/events/event-alert-message-logger.ts
|
|
1889
|
+
var import_open = __toESM(require("open"));
|
|
1886
1890
|
var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
1887
1891
|
static shownBatchIds = /* @__PURE__ */ new Set();
|
|
1888
1892
|
/**
|
|
@@ -1903,11 +1907,11 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
1903
1907
|
* Shows a fancy message about how to watch the simulation.
|
|
1904
1908
|
* Called when a run started event is received with a session ID.
|
|
1905
1909
|
*/
|
|
1906
|
-
handleWatchMessage(params) {
|
|
1910
|
+
async handleWatchMessage(params) {
|
|
1907
1911
|
if (this.isGreetingDisabled()) {
|
|
1908
1912
|
return;
|
|
1909
1913
|
}
|
|
1910
|
-
this.displayWatchMessage(params);
|
|
1914
|
+
await this.displayWatchMessage(params);
|
|
1911
1915
|
}
|
|
1912
1916
|
isGreetingDisabled() {
|
|
1913
1917
|
return getEnv().SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
|
|
@@ -1918,54 +1922,37 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
1918
1922
|
if (!env.LANGWATCH_API_KEY) {
|
|
1919
1923
|
console.log(`
|
|
1920
1924
|
${separator}`);
|
|
1921
|
-
console.log("\u{
|
|
1925
|
+
console.log("\u{1F3AD} Running Scenario Tests");
|
|
1922
1926
|
console.log(`${separator}`);
|
|
1923
|
-
console.log("\u27A1\uFE0F API key not configured");
|
|
1927
|
+
console.log("\u27A1\uFE0F LangWatch API key not configured");
|
|
1924
1928
|
console.log(" Simulations will only output final results");
|
|
1925
1929
|
console.log("");
|
|
1926
1930
|
console.log("\u{1F4A1} To visualize conversations in real time:");
|
|
1927
1931
|
console.log(" \u2022 Set LANGWATCH_API_KEY environment variable");
|
|
1928
1932
|
console.log(" \u2022 Or configure apiKey in scenario.config.js");
|
|
1929
1933
|
console.log("");
|
|
1930
|
-
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
1931
|
-
console.log("");
|
|
1932
|
-
console.log("\u{1F507} To disable these messages:");
|
|
1933
|
-
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
1934
|
-
console.log(`${separator}
|
|
1935
|
-
`);
|
|
1936
|
-
} else {
|
|
1937
|
-
console.log(`
|
|
1938
|
-
${separator}`);
|
|
1939
|
-
console.log("\u{1F680} LangWatch Simulation Reporting");
|
|
1940
|
-
console.log(`${separator}`);
|
|
1941
|
-
console.log("\u2705 Simulation reporting enabled");
|
|
1942
|
-
console.log(` Endpoint: ${env.LANGWATCH_ENDPOINT}`);
|
|
1943
|
-
console.log(
|
|
1944
|
-
` API Key: ${env.LANGWATCH_API_KEY.length > 0 ? "Configured" : "Not configured"}`
|
|
1945
|
-
);
|
|
1946
|
-
console.log("");
|
|
1947
|
-
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
1948
|
-
console.log("");
|
|
1949
|
-
console.log("\u{1F507} To disable these messages:");
|
|
1950
|
-
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
1951
1934
|
console.log(`${separator}
|
|
1952
1935
|
`);
|
|
1953
1936
|
}
|
|
1954
1937
|
}
|
|
1955
|
-
displayWatchMessage(params) {
|
|
1938
|
+
async displayWatchMessage(params) {
|
|
1956
1939
|
const separator = "\u2500".repeat(60);
|
|
1957
1940
|
const setUrl = params.setUrl;
|
|
1958
1941
|
const batchUrl = `${setUrl}/${getBatchRunId()}`;
|
|
1959
1942
|
console.log(`
|
|
1960
1943
|
${separator}`);
|
|
1961
|
-
console.log("\u{
|
|
1944
|
+
console.log("\u{1F3AD} Running Scenario Tests");
|
|
1962
1945
|
console.log(`${separator}`);
|
|
1963
|
-
console.log(
|
|
1964
|
-
console.log(` Scenario Set: ${setUrl}`);
|
|
1965
|
-
console.log(` Batch Run: ${batchUrl}`);
|
|
1966
|
-
console.log("");
|
|
1946
|
+
console.log(`Follow it live: ${batchUrl}`);
|
|
1967
1947
|
console.log(`${separator}
|
|
1968
1948
|
`);
|
|
1949
|
+
const projectConfig = await getProjectConfig();
|
|
1950
|
+
if (!(projectConfig == null ? void 0 : projectConfig.headless)) {
|
|
1951
|
+
try {
|
|
1952
|
+
(0, import_open.default)(batchUrl);
|
|
1953
|
+
} catch (_) {
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1969
1956
|
}
|
|
1970
1957
|
};
|
|
1971
1958
|
|
|
@@ -2094,9 +2081,9 @@ var EventBus = class _EventBus {
|
|
|
2094
2081
|
return { event, result };
|
|
2095
2082
|
}),
|
|
2096
2083
|
// Handle watch messages reactively
|
|
2097
|
-
(0, import_rxjs3.tap)(({ event, result }) => {
|
|
2084
|
+
(0, import_rxjs3.tap)(async ({ event, result }) => {
|
|
2098
2085
|
if (event.type === "SCENARIO_RUN_STARTED" /* RUN_STARTED */ && result.setUrl) {
|
|
2099
|
-
this.eventAlertMessageLogger.handleWatchMessage({
|
|
2086
|
+
await this.eventAlertMessageLogger.handleWatchMessage({
|
|
2100
2087
|
scenarioSetId: event.scenarioSetId,
|
|
2101
2088
|
scenarioRunId: event.scenarioRunId,
|
|
2102
2089
|
setUrl: result.setUrl
|
package/dist/index.mjs
CHANGED
|
@@ -25,6 +25,6 @@ __export(setup_global_exports, {
|
|
|
25
25
|
module.exports = __toCommonJS(setup_global_exports);
|
|
26
26
|
var import_xksuid = require("xksuid");
|
|
27
27
|
function setup() {
|
|
28
|
-
const scenarioBatchRunId = `
|
|
28
|
+
const scenarioBatchRunId = `scenariobatch_${(0, import_xksuid.generate)()}`;
|
|
29
29
|
process.env.SCENARIO_BATCH_RUN_ID = scenarioBatchRunId;
|
|
30
30
|
}
|
|
@@ -3,7 +3,7 @@ import "../../chunk-7P6ASYW6.mjs";
|
|
|
3
3
|
// src/integrations/vitest/setup-global.ts
|
|
4
4
|
import { generate } from "xksuid";
|
|
5
5
|
function setup() {
|
|
6
|
-
const scenarioBatchRunId = `
|
|
6
|
+
const scenarioBatchRunId = `scenariobatch_${generate()}`;
|
|
7
7
|
process.env.SCENARIO_BATCH_RUN_ID = scenarioBatchRunId;
|
|
8
8
|
}
|
|
9
9
|
export {
|
|
@@ -30,6 +30,9 @@ var import_vitest = require("vitest");
|
|
|
30
30
|
// src/events/event-bus.ts
|
|
31
31
|
var import_rxjs = require("rxjs");
|
|
32
32
|
|
|
33
|
+
// src/events/event-alert-message-logger.ts
|
|
34
|
+
var import_open = __toESM(require("open"));
|
|
35
|
+
|
|
33
36
|
// src/config/env.ts
|
|
34
37
|
var import_zod = require("zod");
|
|
35
38
|
|
|
@@ -93,9 +96,42 @@ var scenarioProjectConfigSchema = import_zod2.z.object({
|
|
|
93
96
|
model: import_zod2.z.custom(),
|
|
94
97
|
temperature: import_zod2.z.number().min(0).max(1).optional().default(DEFAULT_TEMPERATURE),
|
|
95
98
|
maxTokens: import_zod2.z.number().optional()
|
|
96
|
-
}).optional()
|
|
99
|
+
}).optional(),
|
|
100
|
+
headless: import_zod2.z.boolean().optional().default(
|
|
101
|
+
typeof process !== "undefined" ? !["false", "0"].includes(process.env.SCENARIO_HEADLESS || "false") : false
|
|
102
|
+
)
|
|
97
103
|
}).strict();
|
|
98
104
|
|
|
105
|
+
// src/config/load.ts
|
|
106
|
+
async function loadScenarioProjectConfig() {
|
|
107
|
+
const cwd = process.cwd();
|
|
108
|
+
const configNames = [
|
|
109
|
+
"scenario.config.js",
|
|
110
|
+
"scenario.config.mjs"
|
|
111
|
+
];
|
|
112
|
+
for (const name of configNames) {
|
|
113
|
+
const fullPath = import_node_path.default.join(cwd, name);
|
|
114
|
+
try {
|
|
115
|
+
await import_promises.default.access(fullPath);
|
|
116
|
+
const configModule = await import((0, import_node_url.pathToFileURL)(fullPath).href);
|
|
117
|
+
const config2 = configModule.default || configModule;
|
|
118
|
+
const parsed = scenarioProjectConfigSchema.safeParse(config2);
|
|
119
|
+
if (!parsed.success) {
|
|
120
|
+
throw new Error(
|
|
121
|
+
`Invalid config file ${name}: ${JSON.stringify(parsed.error.format(), null, 2)}`
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
return parsed.data;
|
|
125
|
+
} catch (error) {
|
|
126
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT") {
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
throw error;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
return await scenarioProjectConfigSchema.parseAsync({});
|
|
133
|
+
}
|
|
134
|
+
|
|
99
135
|
// src/utils/logger.ts
|
|
100
136
|
var Logger = class _Logger {
|
|
101
137
|
constructor(context) {
|
|
@@ -177,6 +213,32 @@ var Logger = class _Logger {
|
|
|
177
213
|
|
|
178
214
|
// src/config/get-project-config.ts
|
|
179
215
|
var logger = new Logger("scenario.config");
|
|
216
|
+
var configLoaded = false;
|
|
217
|
+
var config = null;
|
|
218
|
+
var configLoadPromise = null;
|
|
219
|
+
async function loadProjectConfig() {
|
|
220
|
+
if (configLoaded) {
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
if (configLoadPromise) {
|
|
224
|
+
return configLoadPromise;
|
|
225
|
+
}
|
|
226
|
+
configLoadPromise = (async () => {
|
|
227
|
+
try {
|
|
228
|
+
config = await loadScenarioProjectConfig();
|
|
229
|
+
logger.debug("loaded scenario project config", { config });
|
|
230
|
+
} catch (error) {
|
|
231
|
+
logger.error("error loading scenario project config", { error });
|
|
232
|
+
} finally {
|
|
233
|
+
configLoaded = true;
|
|
234
|
+
}
|
|
235
|
+
})();
|
|
236
|
+
return configLoadPromise;
|
|
237
|
+
}
|
|
238
|
+
async function getProjectConfig() {
|
|
239
|
+
await loadProjectConfig();
|
|
240
|
+
return config;
|
|
241
|
+
}
|
|
180
242
|
|
|
181
243
|
// src/utils/ids.ts
|
|
182
244
|
var import_node_crypto = __toESM(require("crypto"));
|
|
@@ -197,9 +259,9 @@ function getBatchRunId() {
|
|
|
197
259
|
const week = String(getISOWeekNumber(now)).padStart(2, "0");
|
|
198
260
|
const raw = `${parentProcessId}_${year}_w${week}`;
|
|
199
261
|
const hash = import_node_crypto.default.createHash("sha256").update(raw).digest("hex").slice(0, 12);
|
|
200
|
-
return batchRunId = `
|
|
262
|
+
return batchRunId = `scenariobatch_${hash}`;
|
|
201
263
|
}
|
|
202
|
-
return batchRunId = `
|
|
264
|
+
return batchRunId = `scenariobatch_${(0, import_xksuid.generate)()}`;
|
|
203
265
|
}
|
|
204
266
|
function getISOWeekNumber(date) {
|
|
205
267
|
const tmp = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
|
|
@@ -231,11 +293,11 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
231
293
|
* Shows a fancy message about how to watch the simulation.
|
|
232
294
|
* Called when a run started event is received with a session ID.
|
|
233
295
|
*/
|
|
234
|
-
handleWatchMessage(params) {
|
|
296
|
+
async handleWatchMessage(params) {
|
|
235
297
|
if (this.isGreetingDisabled()) {
|
|
236
298
|
return;
|
|
237
299
|
}
|
|
238
|
-
this.displayWatchMessage(params);
|
|
300
|
+
await this.displayWatchMessage(params);
|
|
239
301
|
}
|
|
240
302
|
isGreetingDisabled() {
|
|
241
303
|
return getEnv().SCENARIO_DISABLE_SIMULATION_REPORT_INFO === true;
|
|
@@ -246,54 +308,37 @@ var EventAlertMessageLogger = class _EventAlertMessageLogger {
|
|
|
246
308
|
if (!env.LANGWATCH_API_KEY) {
|
|
247
309
|
console.log(`
|
|
248
310
|
${separator}`);
|
|
249
|
-
console.log("\u{
|
|
311
|
+
console.log("\u{1F3AD} Running Scenario Tests");
|
|
250
312
|
console.log(`${separator}`);
|
|
251
|
-
console.log("\u27A1\uFE0F API key not configured");
|
|
313
|
+
console.log("\u27A1\uFE0F LangWatch API key not configured");
|
|
252
314
|
console.log(" Simulations will only output final results");
|
|
253
315
|
console.log("");
|
|
254
316
|
console.log("\u{1F4A1} To visualize conversations in real time:");
|
|
255
317
|
console.log(" \u2022 Set LANGWATCH_API_KEY environment variable");
|
|
256
318
|
console.log(" \u2022 Or configure apiKey in scenario.config.js");
|
|
257
319
|
console.log("");
|
|
258
|
-
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
259
|
-
console.log("");
|
|
260
|
-
console.log("\u{1F507} To disable these messages:");
|
|
261
|
-
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
262
|
-
console.log(`${separator}
|
|
263
|
-
`);
|
|
264
|
-
} else {
|
|
265
|
-
console.log(`
|
|
266
|
-
${separator}`);
|
|
267
|
-
console.log("\u{1F680} LangWatch Simulation Reporting");
|
|
268
|
-
console.log(`${separator}`);
|
|
269
|
-
console.log("\u2705 Simulation reporting enabled");
|
|
270
|
-
console.log(` Endpoint: ${env.LANGWATCH_ENDPOINT}`);
|
|
271
|
-
console.log(
|
|
272
|
-
` API Key: ${env.LANGWATCH_API_KEY.length > 0 ? "Configured" : "Not configured"}`
|
|
273
|
-
);
|
|
274
|
-
console.log("");
|
|
275
|
-
console.log(`\u{1F4E6} Batch Run ID: ${getBatchRunId()}`);
|
|
276
|
-
console.log("");
|
|
277
|
-
console.log("\u{1F507} To disable these messages:");
|
|
278
|
-
console.log(" \u2022 Set SCENARIO_DISABLE_SIMULATION_REPORT_INFO=true");
|
|
279
320
|
console.log(`${separator}
|
|
280
321
|
`);
|
|
281
322
|
}
|
|
282
323
|
}
|
|
283
|
-
displayWatchMessage(params) {
|
|
324
|
+
async displayWatchMessage(params) {
|
|
284
325
|
const separator = "\u2500".repeat(60);
|
|
285
326
|
const setUrl = params.setUrl;
|
|
286
327
|
const batchUrl = `${setUrl}/${getBatchRunId()}`;
|
|
287
328
|
console.log(`
|
|
288
329
|
${separator}`);
|
|
289
|
-
console.log("\u{
|
|
330
|
+
console.log("\u{1F3AD} Running Scenario Tests");
|
|
290
331
|
console.log(`${separator}`);
|
|
291
|
-
console.log(
|
|
292
|
-
console.log(` Scenario Set: ${setUrl}`);
|
|
293
|
-
console.log(` Batch Run: ${batchUrl}`);
|
|
294
|
-
console.log("");
|
|
332
|
+
console.log(`Follow it live: ${batchUrl}`);
|
|
295
333
|
console.log(`${separator}
|
|
296
334
|
`);
|
|
335
|
+
const projectConfig = await getProjectConfig();
|
|
336
|
+
if (!(projectConfig == null ? void 0 : projectConfig.headless)) {
|
|
337
|
+
try {
|
|
338
|
+
(0, import_open.default)(batchUrl);
|
|
339
|
+
} catch (_) {
|
|
340
|
+
}
|
|
341
|
+
}
|
|
297
342
|
}
|
|
298
343
|
};
|
|
299
344
|
|
|
@@ -376,9 +421,9 @@ var EventReporter = class {
|
|
|
376
421
|
eventAlertMessageLogger;
|
|
377
422
|
logger = new Logger("scenario.events.EventReporter");
|
|
378
423
|
isEnabled;
|
|
379
|
-
constructor(
|
|
380
|
-
this.apiKey =
|
|
381
|
-
this.eventsEndpoint = new URL("/api/scenario-events",
|
|
424
|
+
constructor(config2) {
|
|
425
|
+
this.apiKey = config2.apiKey ?? "";
|
|
426
|
+
this.eventsEndpoint = new URL("/api/scenario-events", config2.endpoint);
|
|
382
427
|
this.eventAlertMessageLogger = new EventAlertMessageLogger();
|
|
383
428
|
this.eventAlertMessageLogger.handleGreeting();
|
|
384
429
|
this.isEnabled = this.apiKey.length > 0 && this.eventsEndpoint.href.length > 0;
|
|
@@ -453,8 +498,8 @@ var EventBus = class _EventBus {
|
|
|
453
498
|
processingPromise = null;
|
|
454
499
|
logger = new Logger("scenario.events.EventBus");
|
|
455
500
|
static globalListeners = [];
|
|
456
|
-
constructor(
|
|
457
|
-
this.eventReporter = new EventReporter(
|
|
501
|
+
constructor(config2) {
|
|
502
|
+
this.eventReporter = new EventReporter(config2);
|
|
458
503
|
this.eventAlertMessageLogger = new EventAlertMessageLogger();
|
|
459
504
|
_EventBus.registry.add(this);
|
|
460
505
|
for (const listener of _EventBus.globalListeners) {
|
|
@@ -494,9 +539,9 @@ var EventBus = class _EventBus {
|
|
|
494
539
|
return { event, result };
|
|
495
540
|
}),
|
|
496
541
|
// Handle watch messages reactively
|
|
497
|
-
(0, import_rxjs.tap)(({ event, result }) => {
|
|
542
|
+
(0, import_rxjs.tap)(async ({ event, result }) => {
|
|
498
543
|
if (event.type === "SCENARIO_RUN_STARTED" /* RUN_STARTED */ && result.setUrl) {
|
|
499
|
-
this.eventAlertMessageLogger.handleWatchMessage({
|
|
544
|
+
await this.eventAlertMessageLogger.handleWatchMessage({
|
|
500
545
|
scenarioSetId: event.scenarioSetId,
|
|
501
546
|
scenarioRunId: event.scenarioRunId,
|
|
502
547
|
setUrl: result.setUrl
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@langwatch/scenario",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.13",
|
|
4
4
|
"description": "A TypeScript library for testing AI agents using scenarios",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"module": "dist/index.mjs",
|
|
@@ -32,6 +32,7 @@
|
|
|
32
32
|
"@ai-sdk/openai": "^1.3.22",
|
|
33
33
|
"ai": ">=4.0.0",
|
|
34
34
|
"chalk": "^5.4.1",
|
|
35
|
+
"open": "10.2.0",
|
|
35
36
|
"rxjs": "^7.8.2",
|
|
36
37
|
"xksuid": "^0.0.4",
|
|
37
38
|
"zod": "^3.24.4"
|
|
@@ -93,7 +94,7 @@
|
|
|
93
94
|
"test": "vitest",
|
|
94
95
|
"test:ci": "vitest run",
|
|
95
96
|
"lint": "eslint .",
|
|
96
|
-
"examples:vitest:run": "
|
|
97
|
+
"examples:vitest:run": "(cd examples/vitest && pnpm install) && pnpm -F vitest-example run test",
|
|
97
98
|
"generate:api-reference": "npx typedoc src --out api-reference-docs && rm -rf ../docs/docs/public/reference/javascript/scenario && mv api-reference-docs ../docs/docs/public/reference/javascript/scenario"
|
|
98
99
|
}
|
|
99
100
|
}
|