elasticdash-test 0.1.11 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -0
- package/dist/capture/event.d.ts +4 -0
- package/dist/capture/event.d.ts.map +1 -1
- package/dist/capture/recorder.d.ts +5 -0
- package/dist/capture/recorder.d.ts.map +1 -1
- package/dist/capture/recorder.js +10 -0
- package/dist/capture/recorder.js.map +1 -1
- package/dist/dashboard-server.d.ts +12 -0
- package/dist/dashboard-server.d.ts.map +1 -1
- package/dist/dashboard-server.js +269 -46
- package/dist/dashboard-server.js.map +1 -1
- package/dist/index.cjs +2526 -0
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/interceptors/ai-interceptor.d.ts.map +1 -1
- package/dist/interceptors/ai-interceptor.js +101 -7
- package/dist/interceptors/ai-interceptor.js.map +1 -1
- package/dist/interceptors/http.d.ts +20 -0
- package/dist/interceptors/http.d.ts.map +1 -1
- package/dist/interceptors/http.js +184 -17
- package/dist/interceptors/http.js.map +1 -1
- package/dist/interceptors/tool.d.ts.map +1 -1
- package/dist/interceptors/tool.js +91 -0
- package/dist/interceptors/tool.js.map +1 -1
- package/dist/internals/mock-resolver.d.ts +25 -0
- package/dist/internals/mock-resolver.d.ts.map +1 -0
- package/dist/internals/mock-resolver.js +82 -0
- package/dist/internals/mock-resolver.js.map +1 -0
- package/dist/workflow-runner-worker.js +50 -3
- package/dist/workflow-runner-worker.js.map +1 -1
- package/dist/workflow-runner.d.ts.map +1 -1
- package/dist/workflow-runner.js +1 -0
- package/dist/workflow-runner.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,2526 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __create = Object.create;
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
7
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
|
+
var __esm = (fn, res) => function __init() {
|
|
9
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
10
|
+
};
|
|
11
|
+
var __export = (target, all) => {
|
|
12
|
+
for (var name in all)
|
|
13
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
14
|
+
};
|
|
15
|
+
var __copyProps = (to, from, except, desc) => {
|
|
16
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
17
|
+
for (let key of __getOwnPropNames(from))
|
|
18
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
19
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
20
|
+
}
|
|
21
|
+
return to;
|
|
22
|
+
};
|
|
23
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
24
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
25
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
26
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
27
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
28
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
29
|
+
mod
|
|
30
|
+
));
|
|
31
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
32
|
+
|
|
33
|
+
// src/capture/recorder.ts
|
|
34
|
+
function setCaptureContext(ctx) {
|
|
35
|
+
captureAls.enterWith(ctx);
|
|
36
|
+
}
|
|
37
|
+
function getCaptureContext() {
|
|
38
|
+
return captureAls.getStore();
|
|
39
|
+
}
|
|
40
|
+
var import_node_async_hooks, import_node_crypto, TraceRecorder, g, CAPTURE_ALS_KEY, captureAls;
|
|
41
|
+
var init_recorder = __esm({
|
|
42
|
+
"src/capture/recorder.ts"() {
|
|
43
|
+
"use strict";
|
|
44
|
+
import_node_async_hooks = require("node:async_hooks");
|
|
45
|
+
import_node_crypto = require("node:crypto");
|
|
46
|
+
TraceRecorder = class {
|
|
47
|
+
events = [];
|
|
48
|
+
_counter = 0;
|
|
49
|
+
_sideEffectCounter = 0;
|
|
50
|
+
record(event) {
|
|
51
|
+
this.events.push(event);
|
|
52
|
+
}
|
|
53
|
+
nextId() {
|
|
54
|
+
return ++this._counter;
|
|
55
|
+
}
|
|
56
|
+
/** Separate counter for Date.now / Math.random — never shares IDs with main events. */
|
|
57
|
+
nextSideEffectId() {
|
|
58
|
+
return ++this._sideEffectCounter;
|
|
59
|
+
}
|
|
60
|
+
toTrace(traceId) {
|
|
61
|
+
return {
|
|
62
|
+
traceId: traceId ?? (0, import_node_crypto.randomUUID)(),
|
|
63
|
+
events: [...this.events]
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
g = globalThis;
|
|
68
|
+
CAPTURE_ALS_KEY = "__elasticdash_capture_als__";
|
|
69
|
+
captureAls = g[CAPTURE_ALS_KEY] ?? new import_node_async_hooks.AsyncLocalStorage();
|
|
70
|
+
if (!g[CAPTURE_ALS_KEY]) g[CAPTURE_ALS_KEY] = captureAls;
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
// src/interceptors/side-effects.ts
|
|
75
|
+
function rawDateNow() {
|
|
76
|
+
return originalDateNow ? originalDateNow() : Date.now();
|
|
77
|
+
}
|
|
78
|
+
function interceptRandom() {
|
|
79
|
+
if (originalRandom) return;
|
|
80
|
+
originalRandom = Math.random;
|
|
81
|
+
Math.random = () => {
|
|
82
|
+
const ctx = getCaptureContext();
|
|
83
|
+
if (!ctx) return originalRandom();
|
|
84
|
+
const { recorder, replay } = ctx;
|
|
85
|
+
const n = recorder.nextSideEffectId();
|
|
86
|
+
if (replay.shouldReplaySideEffectOfType(n, "Math.random")) {
|
|
87
|
+
return replay.getSideEffectResultOfType(n, "Math.random");
|
|
88
|
+
}
|
|
89
|
+
const value = originalRandom();
|
|
90
|
+
recorder.record({
|
|
91
|
+
id: n,
|
|
92
|
+
type: "side_effect",
|
|
93
|
+
name: "Math.random",
|
|
94
|
+
input: null,
|
|
95
|
+
output: value,
|
|
96
|
+
timestamp: rawDateNow(),
|
|
97
|
+
durationMs: 0
|
|
98
|
+
});
|
|
99
|
+
return value;
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
function restoreRandom() {
|
|
103
|
+
if (originalRandom) {
|
|
104
|
+
Math.random = originalRandom;
|
|
105
|
+
originalRandom = void 0;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
function interceptDateNow() {
|
|
109
|
+
if (originalDateNow) return;
|
|
110
|
+
originalDateNow = Date.now;
|
|
111
|
+
Date.now = () => {
|
|
112
|
+
const ctx = getCaptureContext();
|
|
113
|
+
if (!ctx) return originalDateNow();
|
|
114
|
+
const { recorder, replay } = ctx;
|
|
115
|
+
const n = recorder.nextSideEffectId();
|
|
116
|
+
if (replay.shouldReplaySideEffectOfType(n, "Date.now")) {
|
|
117
|
+
return replay.getSideEffectResultOfType(n, "Date.now");
|
|
118
|
+
}
|
|
119
|
+
const value = originalDateNow();
|
|
120
|
+
recorder.record({
|
|
121
|
+
id: n,
|
|
122
|
+
type: "side_effect",
|
|
123
|
+
name: "Date.now",
|
|
124
|
+
input: null,
|
|
125
|
+
output: value,
|
|
126
|
+
timestamp: value,
|
|
127
|
+
durationMs: 0
|
|
128
|
+
});
|
|
129
|
+
return value;
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
function restoreDateNow() {
|
|
133
|
+
if (originalDateNow) {
|
|
134
|
+
Date.now = originalDateNow;
|
|
135
|
+
originalDateNow = void 0;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
var originalRandom, originalDateNow;
|
|
139
|
+
var init_side_effects = __esm({
|
|
140
|
+
"src/interceptors/side-effects.ts"() {
|
|
141
|
+
"use strict";
|
|
142
|
+
init_recorder();
|
|
143
|
+
}
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
// src/trace-adapter/context.ts
|
|
147
|
+
function setCurrentTrace(trace) {
|
|
148
|
+
traceAls.enterWith(trace);
|
|
149
|
+
}
|
|
150
|
+
function getCurrentTrace() {
|
|
151
|
+
return traceAls.getStore();
|
|
152
|
+
}
|
|
153
|
+
function createTraceHandle() {
|
|
154
|
+
const steps = [];
|
|
155
|
+
const llmSteps = [];
|
|
156
|
+
const toolCalls = [];
|
|
157
|
+
const customSteps = [];
|
|
158
|
+
return {
|
|
159
|
+
getSteps() {
|
|
160
|
+
return steps;
|
|
161
|
+
},
|
|
162
|
+
getLLMSteps() {
|
|
163
|
+
return llmSteps;
|
|
164
|
+
},
|
|
165
|
+
getToolCalls() {
|
|
166
|
+
return toolCalls;
|
|
167
|
+
},
|
|
168
|
+
getCustomSteps() {
|
|
169
|
+
return customSteps;
|
|
170
|
+
},
|
|
171
|
+
recordLLMStep(step) {
|
|
172
|
+
llmSteps.push(step);
|
|
173
|
+
steps.push({
|
|
174
|
+
type: "llm",
|
|
175
|
+
timestamp: rawDateNow(),
|
|
176
|
+
durationMs: 0,
|
|
177
|
+
data: step
|
|
178
|
+
});
|
|
179
|
+
},
|
|
180
|
+
recordToolCall(call) {
|
|
181
|
+
toolCalls.push(call);
|
|
182
|
+
steps.push({
|
|
183
|
+
type: "tool",
|
|
184
|
+
timestamp: rawDateNow(),
|
|
185
|
+
durationMs: 0,
|
|
186
|
+
data: call
|
|
187
|
+
});
|
|
188
|
+
},
|
|
189
|
+
recordCustomStep(step) {
|
|
190
|
+
customSteps.push(step);
|
|
191
|
+
steps.push({
|
|
192
|
+
type: "custom",
|
|
193
|
+
timestamp: rawDateNow(),
|
|
194
|
+
durationMs: 0,
|
|
195
|
+
data: step
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
function startTraceSession() {
|
|
201
|
+
const trace = createTraceHandle();
|
|
202
|
+
const context = { trace };
|
|
203
|
+
return {
|
|
204
|
+
context,
|
|
205
|
+
finalise() {
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
var import_node_async_hooks2, g2, TRACE_ALS_KEY, traceAls;
|
|
210
|
+
var init_context = __esm({
|
|
211
|
+
"src/trace-adapter/context.ts"() {
|
|
212
|
+
"use strict";
|
|
213
|
+
import_node_async_hooks2 = require("node:async_hooks");
|
|
214
|
+
init_side_effects();
|
|
215
|
+
g2 = globalThis;
|
|
216
|
+
TRACE_ALS_KEY = "__elasticdash_trace_als__";
|
|
217
|
+
traceAls = g2[TRACE_ALS_KEY] ?? new import_node_async_hooks2.AsyncLocalStorage();
|
|
218
|
+
if (!g2[TRACE_ALS_KEY]) g2[TRACE_ALS_KEY] = traceAls;
|
|
219
|
+
}
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
// src/tracing.ts
|
|
223
|
+
var tracing_exports = {};
|
|
224
|
+
__export(tracing_exports, {
|
|
225
|
+
recordToolCall: () => recordToolCall
|
|
226
|
+
});
|
|
227
|
+
function wrapperRecordingActive() {
|
|
228
|
+
return globalThis[TOOL_WRAPPER_ACTIVE_KEY] === true;
|
|
229
|
+
}
|
|
230
|
+
function recordToolCall(name, args, result) {
|
|
231
|
+
if (!globalThis.__ELASTICDASH_WORKER__) return;
|
|
232
|
+
try {
|
|
233
|
+
if (wrapperRecordingActive()) return;
|
|
234
|
+
const trace = getCurrentTrace();
|
|
235
|
+
if (!trace || typeof trace.recordToolCall !== "function") return;
|
|
236
|
+
const ctx = getCaptureContext();
|
|
237
|
+
if (!ctx) {
|
|
238
|
+
trace.recordToolCall({ name, args, result });
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
const { recorder, replay } = ctx;
|
|
242
|
+
const id = recorder.nextId();
|
|
243
|
+
if (replay.shouldReplay(id)) {
|
|
244
|
+
const historical = replay.getRecordedEvent(id);
|
|
245
|
+
if (historical) recorder.record(historical);
|
|
246
|
+
const replayed = replay.getRecordedResult(id);
|
|
247
|
+
trace.recordToolCall({ name, args, result: replayed, workflowEventId: id });
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
const output = result instanceof Error ? { error: String(result) } : result;
|
|
251
|
+
recorder.record({
|
|
252
|
+
id,
|
|
253
|
+
type: "tool",
|
|
254
|
+
name,
|
|
255
|
+
input: args,
|
|
256
|
+
output,
|
|
257
|
+
timestamp: rawDateNow(),
|
|
258
|
+
durationMs: 0
|
|
259
|
+
});
|
|
260
|
+
trace.recordToolCall({ name, args, result: output, workflowEventId: id });
|
|
261
|
+
} catch {
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
var TOOL_WRAPPER_ACTIVE_KEY;
|
|
265
|
+
var init_tracing = __esm({
|
|
266
|
+
"src/tracing.ts"() {
|
|
267
|
+
"use strict";
|
|
268
|
+
init_context();
|
|
269
|
+
init_recorder();
|
|
270
|
+
init_side_effects();
|
|
271
|
+
TOOL_WRAPPER_ACTIVE_KEY = "__elasticdash_tool_wrapper_active__";
|
|
272
|
+
}
|
|
273
|
+
});
|
|
274
|
+
|
|
275
|
+
// src/index.ts
|
|
276
|
+
var index_exports = {};
|
|
277
|
+
__export(index_exports, {
|
|
278
|
+
ReplayController: () => ReplayController,
|
|
279
|
+
TraceRecorder: () => TraceRecorder,
|
|
280
|
+
afterAll: () => afterAll,
|
|
281
|
+
afterEach: () => afterEach,
|
|
282
|
+
aiTest: () => aiTest,
|
|
283
|
+
beforeAll: () => beforeAll,
|
|
284
|
+
beforeEach: () => beforeEach,
|
|
285
|
+
clearRegistry: () => clearRegistry,
|
|
286
|
+
createTraceHandle: () => createTraceHandle,
|
|
287
|
+
deserializeAgentState: () => deserializeAgentState,
|
|
288
|
+
expect: () => import_expect.expect,
|
|
289
|
+
extractTaskOutputs: () => extractTaskOutputs,
|
|
290
|
+
fetchCapturedTrace: () => fetchCapturedTrace,
|
|
291
|
+
getCaptureContext: () => getCaptureContext,
|
|
292
|
+
getCurrentTrace: () => getCurrentTrace,
|
|
293
|
+
getRegistry: () => getRegistry,
|
|
294
|
+
installAIInterceptor: () => installAIInterceptor,
|
|
295
|
+
installDBAutoInterceptor: () => installDBAutoInterceptor,
|
|
296
|
+
interceptDateNow: () => interceptDateNow,
|
|
297
|
+
interceptFetch: () => interceptFetch,
|
|
298
|
+
interceptRandom: () => interceptRandom,
|
|
299
|
+
isWorker: () => isWorker,
|
|
300
|
+
recordToolCall: () => recordToolCall,
|
|
301
|
+
registerMatchers: () => registerMatchers,
|
|
302
|
+
reportResults: () => reportResults,
|
|
303
|
+
resolveTaskInput: () => resolveTaskInput,
|
|
304
|
+
restoreDateNow: () => restoreDateNow,
|
|
305
|
+
restoreFetch: () => restoreFetch,
|
|
306
|
+
restoreRandom: () => restoreRandom,
|
|
307
|
+
runFiles: () => runFiles,
|
|
308
|
+
runWorkflow: () => runWorkflow,
|
|
309
|
+
safeRecordToolCall: () => safeRecordToolCall,
|
|
310
|
+
serializeAgentState: () => serializeAgentState,
|
|
311
|
+
setCaptureContext: () => setCaptureContext,
|
|
312
|
+
setCurrentTrace: () => setCurrentTrace,
|
|
313
|
+
startLLMProxy: () => startLLMProxy,
|
|
314
|
+
startTraceSession: () => startTraceSession,
|
|
315
|
+
uninstallAIInterceptor: () => uninstallAIInterceptor,
|
|
316
|
+
uninstallDBAutoInterceptor: () => uninstallDBAutoInterceptor,
|
|
317
|
+
wrapAI: () => wrapAI,
|
|
318
|
+
wrapDB: () => wrapDB,
|
|
319
|
+
wrapKnex: () => wrapKnex,
|
|
320
|
+
wrapMongoCollection: () => wrapMongoCollection,
|
|
321
|
+
wrapPgClient: () => wrapPgClient,
|
|
322
|
+
wrapRedisClient: () => wrapRedisClient,
|
|
323
|
+
wrapTool: () => wrapTool
|
|
324
|
+
});
|
|
325
|
+
module.exports = __toCommonJS(index_exports);
|
|
326
|
+
|
|
327
|
+
// src/core/registry.ts
|
|
328
|
+
var REGISTRY_KEY = "__elasticdash_registry__";
|
|
329
|
+
function getGlobalRegistry() {
|
|
330
|
+
if (!globalThis[REGISTRY_KEY]) {
|
|
331
|
+
globalThis[REGISTRY_KEY] = createEmptyRegistry();
|
|
332
|
+
}
|
|
333
|
+
return globalThis[REGISTRY_KEY];
|
|
334
|
+
}
|
|
335
|
+
function createEmptyRegistry() {
|
|
336
|
+
return {
|
|
337
|
+
tests: [],
|
|
338
|
+
beforeAllHooks: [],
|
|
339
|
+
afterAllHooks: [],
|
|
340
|
+
beforeEachHooks: [],
|
|
341
|
+
afterEachHooks: []
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
function clearRegistry() {
|
|
345
|
+
globalThis[REGISTRY_KEY] = createEmptyRegistry();
|
|
346
|
+
console.log("[elasticdash] clearRegistry called. Registry reset.");
|
|
347
|
+
}
|
|
348
|
+
function getRegistry() {
|
|
349
|
+
const registry = getGlobalRegistry();
|
|
350
|
+
console.log("[elasticdash] getRegistry called. Current tests:", registry.tests.map((t) => t.name));
|
|
351
|
+
return registry;
|
|
352
|
+
}
|
|
353
|
+
function aiTest(name, fn) {
|
|
354
|
+
const registry = getGlobalRegistry();
|
|
355
|
+
registry.tests.push({ name, fn });
|
|
356
|
+
console.log(`[elasticdash] Registered test: ${name}`);
|
|
357
|
+
}
|
|
358
|
+
function beforeAll(fn) {
|
|
359
|
+
const registry = getGlobalRegistry();
|
|
360
|
+
registry.beforeAllHooks.push(fn);
|
|
361
|
+
}
|
|
362
|
+
function afterAll(fn) {
|
|
363
|
+
const registry = getGlobalRegistry();
|
|
364
|
+
registry.afterAllHooks.push(fn);
|
|
365
|
+
}
|
|
366
|
+
function beforeEach(fn) {
|
|
367
|
+
const registry = getGlobalRegistry();
|
|
368
|
+
registry.beforeEachHooks.push(fn);
|
|
369
|
+
}
|
|
370
|
+
function afterEach(fn) {
|
|
371
|
+
const registry = getGlobalRegistry();
|
|
372
|
+
registry.afterEachHooks.push(fn);
|
|
373
|
+
}
|
|
374
|
+
globalThis.aiTest = aiTest;
|
|
375
|
+
globalThis.beforeAll = beforeAll;
|
|
376
|
+
globalThis.afterAll = afterAll;
|
|
377
|
+
globalThis.beforeEach = beforeEach;
|
|
378
|
+
globalThis.afterEach = afterEach;
|
|
379
|
+
|
|
380
|
+
// src/index.ts
|
|
381
|
+
init_context();
|
|
382
|
+
|
|
383
|
+
// src/matchers/index.ts
|
|
384
|
+
var import_expect = require("expect");
|
|
385
|
+
function isTraceHandle(value) {
|
|
386
|
+
return value !== null && typeof value === "object" && typeof value.getLLMSteps === "function" && typeof value.getToolCalls === "function";
|
|
387
|
+
}
|
|
388
|
+
var defaultModels = {
|
|
389
|
+
openai: "gpt-4.1",
|
|
390
|
+
claude: "claude-3-opus-20240229",
|
|
391
|
+
gemini: "gemini-1.5-pro",
|
|
392
|
+
grok: "grok-beta",
|
|
393
|
+
kimi: "kimi-k2-turbo-preview"
|
|
394
|
+
};
|
|
395
|
+
async function callProviderLLM(prompt, options = {}, systemPrompt = "You are an expert test judge.", maxTokens = 32, temperature = 0) {
|
|
396
|
+
const provider = options.provider ?? "openai";
|
|
397
|
+
const sdk = options.sdk;
|
|
398
|
+
const resolvedModel = options.model ?? defaultModels[provider];
|
|
399
|
+
switch (provider) {
|
|
400
|
+
case "openai": {
|
|
401
|
+
if (sdk && sdk.chat?.completions?.create) {
|
|
402
|
+
const resp = await sdk.chat.completions.create({
|
|
403
|
+
model: resolvedModel,
|
|
404
|
+
messages: [
|
|
405
|
+
{ role: "system", content: systemPrompt },
|
|
406
|
+
{ role: "user", content: prompt }
|
|
407
|
+
],
|
|
408
|
+
max_tokens: maxTokens,
|
|
409
|
+
temperature
|
|
410
|
+
});
|
|
411
|
+
return resp?.choices?.[0]?.message?.content?.trim() ?? "";
|
|
412
|
+
}
|
|
413
|
+
const apiKey = options.apiKey ?? process.env.OPENAI_API_KEY;
|
|
414
|
+
if (!apiKey) throw new Error("Provide apiKey or set OPENAI_API_KEY for OpenAI-compatible endpoint.");
|
|
415
|
+
const baseURL = (options.baseURL ?? "https://api.openai.com/v1").replace(/\/$/, "");
|
|
416
|
+
const response = await fetch(`${baseURL}/chat/completions`, {
|
|
417
|
+
method: "POST",
|
|
418
|
+
headers: {
|
|
419
|
+
Authorization: `Bearer ${apiKey}`,
|
|
420
|
+
"Content-Type": "application/json"
|
|
421
|
+
},
|
|
422
|
+
body: JSON.stringify({
|
|
423
|
+
model: resolvedModel,
|
|
424
|
+
messages: [
|
|
425
|
+
{ role: "system", content: systemPrompt },
|
|
426
|
+
{ role: "user", content: prompt }
|
|
427
|
+
],
|
|
428
|
+
max_tokens: maxTokens,
|
|
429
|
+
temperature
|
|
430
|
+
})
|
|
431
|
+
});
|
|
432
|
+
if (!response.ok) {
|
|
433
|
+
throw new Error(`OpenAI API error: ${response.status} ${response.statusText}`);
|
|
434
|
+
}
|
|
435
|
+
const data = await response.json();
|
|
436
|
+
return data.choices?.[0]?.message?.content?.trim() ?? "";
|
|
437
|
+
}
|
|
438
|
+
case "claude": {
|
|
439
|
+
if (sdk && sdk.messages?.create) {
|
|
440
|
+
const resp = await sdk.messages.create({
|
|
441
|
+
model: resolvedModel,
|
|
442
|
+
max_tokens: maxTokens,
|
|
443
|
+
temperature,
|
|
444
|
+
messages: [{ role: "user", content: `${systemPrompt}
|
|
445
|
+
|
|
446
|
+
${prompt}` }]
|
|
447
|
+
});
|
|
448
|
+
return resp?.content?.[0]?.text?.trim() ?? "";
|
|
449
|
+
}
|
|
450
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
451
|
+
if (!apiKey) throw new Error("ANTHROPIC_API_KEY is not set in environment.");
|
|
452
|
+
const response = await fetch("https://api.anthropic.com/v1/messages", {
|
|
453
|
+
method: "POST",
|
|
454
|
+
headers: {
|
|
455
|
+
"x-api-key": apiKey,
|
|
456
|
+
"anthropic-version": "2023-06-01",
|
|
457
|
+
"content-type": "application/json"
|
|
458
|
+
},
|
|
459
|
+
body: JSON.stringify({
|
|
460
|
+
model: resolvedModel,
|
|
461
|
+
max_tokens: maxTokens,
|
|
462
|
+
temperature,
|
|
463
|
+
messages: [{ role: "user", content: `${systemPrompt}
|
|
464
|
+
|
|
465
|
+
${prompt}` }]
|
|
466
|
+
})
|
|
467
|
+
});
|
|
468
|
+
if (!response.ok) {
|
|
469
|
+
throw new Error(`Claude API error: ${response.status} ${response.statusText}`);
|
|
470
|
+
}
|
|
471
|
+
const data = await response.json();
|
|
472
|
+
return data?.content?.[0]?.text?.trim() ?? "";
|
|
473
|
+
}
|
|
474
|
+
case "gemini": {
|
|
475
|
+
if (sdk && sdk.models?.generateContent) {
|
|
476
|
+
const resp = await sdk.models.generateContent({
|
|
477
|
+
model: resolvedModel,
|
|
478
|
+
contents: [{ role: "user", parts: [{ text: `${systemPrompt}
|
|
479
|
+
|
|
480
|
+
${prompt}` }] }],
|
|
481
|
+
generationConfig: { temperature, maxOutputTokens: maxTokens }
|
|
482
|
+
});
|
|
483
|
+
return resp?.response?.candidates?.[0]?.content?.parts?.[0]?.text?.trim() ?? "";
|
|
484
|
+
}
|
|
485
|
+
const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
|
|
486
|
+
if (!apiKey) throw new Error("GEMINI_API_KEY (or GOOGLE_API_KEY) is not set in environment.");
|
|
487
|
+
const response = await fetch(
|
|
488
|
+
`https://generativelanguage.googleapis.com/v1beta/models/${resolvedModel}:generateContent?key=${apiKey}`,
|
|
489
|
+
{
|
|
490
|
+
method: "POST",
|
|
491
|
+
headers: { "Content-Type": "application/json" },
|
|
492
|
+
body: JSON.stringify({
|
|
493
|
+
contents: [{ role: "user", parts: [{ text: `${systemPrompt}
|
|
494
|
+
|
|
495
|
+
${prompt}` }] }],
|
|
496
|
+
generationConfig: { temperature, maxOutputTokens: maxTokens }
|
|
497
|
+
})
|
|
498
|
+
}
|
|
499
|
+
);
|
|
500
|
+
if (!response.ok) {
|
|
501
|
+
throw new Error(`Gemini API error: ${response.status} ${response.statusText}`);
|
|
502
|
+
}
|
|
503
|
+
const data = await response.json();
|
|
504
|
+
return data?.candidates?.[0]?.content?.parts?.[0]?.text?.trim() ?? "";
|
|
505
|
+
}
|
|
506
|
+
case "grok": {
|
|
507
|
+
if (sdk && sdk.chat?.completions?.create) {
|
|
508
|
+
const resp = await sdk.chat.completions.create({
|
|
509
|
+
model: resolvedModel,
|
|
510
|
+
messages: [
|
|
511
|
+
{ role: "system", content: systemPrompt },
|
|
512
|
+
{ role: "user", content: prompt }
|
|
513
|
+
],
|
|
514
|
+
max_tokens: maxTokens,
|
|
515
|
+
temperature
|
|
516
|
+
});
|
|
517
|
+
return resp?.choices?.[0]?.message?.content?.trim() ?? "";
|
|
518
|
+
}
|
|
519
|
+
const apiKey = process.env.GROK_API_KEY;
|
|
520
|
+
if (!apiKey) throw new Error("GROK_API_KEY is not set in environment.");
|
|
521
|
+
const response = await fetch("https://api.x.ai/v1/chat/completions", {
|
|
522
|
+
method: "POST",
|
|
523
|
+
headers: {
|
|
524
|
+
Authorization: `Bearer ${apiKey}`,
|
|
525
|
+
"Content-Type": "application/json"
|
|
526
|
+
},
|
|
527
|
+
body: JSON.stringify({
|
|
528
|
+
model: resolvedModel,
|
|
529
|
+
messages: [
|
|
530
|
+
{ role: "system", content: systemPrompt },
|
|
531
|
+
{ role: "user", content: prompt }
|
|
532
|
+
],
|
|
533
|
+
max_tokens: maxTokens,
|
|
534
|
+
temperature
|
|
535
|
+
})
|
|
536
|
+
});
|
|
537
|
+
if (!response.ok) {
|
|
538
|
+
throw new Error(`Grok API error: ${response.status} ${response.statusText}`);
|
|
539
|
+
}
|
|
540
|
+
const data = await response.json();
|
|
541
|
+
return data.choices?.[0]?.message?.content?.trim() ?? "";
|
|
542
|
+
}
|
|
543
|
+
case "kimi": {
|
|
544
|
+
const apiKey = process.env.KIMI_API_KEY;
|
|
545
|
+
if (!apiKey) throw new Error("KIMI_API_KEY is not set in environment.");
|
|
546
|
+
const response = await fetch("https://api.moonshot.ai/v1/chat/completions", {
|
|
547
|
+
method: "POST",
|
|
548
|
+
headers: {
|
|
549
|
+
Authorization: `Bearer ${apiKey}`,
|
|
550
|
+
"Content-Type": "application/json"
|
|
551
|
+
},
|
|
552
|
+
body: JSON.stringify({
|
|
553
|
+
model: resolvedModel,
|
|
554
|
+
messages: [
|
|
555
|
+
{ role: "system", content: systemPrompt },
|
|
556
|
+
{ role: "user", content: prompt }
|
|
557
|
+
],
|
|
558
|
+
max_tokens: maxTokens,
|
|
559
|
+
temperature
|
|
560
|
+
})
|
|
561
|
+
});
|
|
562
|
+
if (!response.ok) {
|
|
563
|
+
throw new Error(`Kimi API error: ${response.status} ${response.statusText}`);
|
|
564
|
+
}
|
|
565
|
+
const data = await response.json();
|
|
566
|
+
return data.choices?.[0]?.message?.content?.trim() ?? "";
|
|
567
|
+
}
|
|
568
|
+
default:
|
|
569
|
+
throw new Error(`Unsupported provider: ${provider}`);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
async function llmJudgeSemanticMatch(traceOutput, expected, options = {}) {
|
|
573
|
+
const prompt = `
|
|
574
|
+
You are an expert test judge. Given the following AI trace output and an expected semantic result, answer "YES" if the trace output semantically matches the expectation, otherwise answer "NO".
|
|
575
|
+
|
|
576
|
+
Trace Output:
|
|
577
|
+
${traceOutput}
|
|
578
|
+
|
|
579
|
+
Expected:
|
|
580
|
+
${expected}
|
|
581
|
+
|
|
582
|
+
Answer only "YES" or "NO".
|
|
583
|
+
`.trim();
|
|
584
|
+
const content = (await callProviderLLM(prompt, options, "You are an expert test judge.", 8, 0)).trim().toUpperCase();
|
|
585
|
+
return content.startsWith("YES");
|
|
586
|
+
}
|
|
587
|
+
function parseFirstNumber(text) {
|
|
588
|
+
const match = text.match(/-?\d+(?:\.\d+)?/);
|
|
589
|
+
if (!match) return null;
|
|
590
|
+
const num = Number.parseFloat(match[0]);
|
|
591
|
+
return Number.isFinite(num) ? num : null;
|
|
592
|
+
}
|
|
593
|
+
function resolveCondition(config) {
|
|
594
|
+
const entries = Object.entries(config || {}).filter(([, v]) => typeof v === "number" && Number.isFinite(v));
|
|
595
|
+
if (entries.length === 0) return { kind: "atLeast", value: 0.7 };
|
|
596
|
+
if (entries.length > 1) {
|
|
597
|
+
throw new Error("Provide only one metric condition (greaterThan, lessThan, atLeast, atMost, equals).");
|
|
598
|
+
}
|
|
599
|
+
return { kind: entries[0][0], value: entries[0][1] };
|
|
600
|
+
}
|
|
601
|
+
function checkCondition(score, condition) {
|
|
602
|
+
switch (condition.kind) {
|
|
603
|
+
case "greaterThan":
|
|
604
|
+
return score > condition.value;
|
|
605
|
+
case "lessThan":
|
|
606
|
+
return score < condition.value;
|
|
607
|
+
case "atLeast":
|
|
608
|
+
return score >= condition.value;
|
|
609
|
+
case "atMost":
|
|
610
|
+
return score <= condition.value;
|
|
611
|
+
case "equals":
|
|
612
|
+
return score === condition.value;
|
|
613
|
+
default:
|
|
614
|
+
return false;
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
function registerMatchers() {
|
|
618
|
+
import_expect.expect.extend({
|
|
619
|
+
toHaveLLMStep(trace, config = {}) {
|
|
620
|
+
if (!isTraceHandle(trace)) {
|
|
621
|
+
return {
|
|
622
|
+
pass: false,
|
|
623
|
+
message: () => `Expected a TraceHandle (ctx.trace) but received ${typeof trace}.
|
|
624
|
+
Use: expect(ctx.trace).toHaveLLMStep(...)`
|
|
625
|
+
};
|
|
626
|
+
}
|
|
627
|
+
const steps = trace.getLLMSteps();
|
|
628
|
+
const matching = steps.filter((step) => {
|
|
629
|
+
if (config.model && step.model !== config.model) return false;
|
|
630
|
+
if (config.provider && step.provider !== config.provider) return false;
|
|
631
|
+
if (config.contains) {
|
|
632
|
+
const haystack = [step.completion, step.prompt, step.contains].filter(Boolean).join(" ").toLowerCase();
|
|
633
|
+
if (!haystack.includes(config.contains.toLowerCase())) return false;
|
|
634
|
+
}
|
|
635
|
+
if (config.promptContains) {
|
|
636
|
+
const promptHaystack = (step.prompt ?? "").toLowerCase();
|
|
637
|
+
if (!promptHaystack.includes(config.promptContains.toLowerCase())) return false;
|
|
638
|
+
}
|
|
639
|
+
if (config.outputContains) {
|
|
640
|
+
const outputHaystack = (step.completion ?? "").toLowerCase();
|
|
641
|
+
if (!outputHaystack.includes(config.outputContains.toLowerCase())) return false;
|
|
642
|
+
}
|
|
643
|
+
return true;
|
|
644
|
+
});
|
|
645
|
+
const count = matching.length;
|
|
646
|
+
let pass;
|
|
647
|
+
if (config.times !== void 0) {
|
|
648
|
+
pass = count === config.times;
|
|
649
|
+
} else if (config.minTimes !== void 0 || config.maxTimes !== void 0) {
|
|
650
|
+
const min = config.minTimes ?? 0;
|
|
651
|
+
const max = config.maxTimes ?? Infinity;
|
|
652
|
+
pass = count >= min && count <= max;
|
|
653
|
+
} else {
|
|
654
|
+
pass = count > 0;
|
|
655
|
+
}
|
|
656
|
+
return {
|
|
657
|
+
pass,
|
|
658
|
+
message: () => {
|
|
659
|
+
if (pass) {
|
|
660
|
+
return `Expected trace NOT to have LLM step matching ${JSON.stringify(config)}`;
|
|
661
|
+
}
|
|
662
|
+
const stepSummary = steps.length === 0 ? "no LLM steps were recorded" : `${count} matching step(s) found; recorded steps: ${JSON.stringify(steps)}`;
|
|
663
|
+
return `Expected trace to have LLM step matching ${JSON.stringify(config)}, but ${stepSummary}`;
|
|
664
|
+
}
|
|
665
|
+
};
|
|
666
|
+
},
|
|
667
|
+
toCallTool(trace, toolName) {
|
|
668
|
+
if (!isTraceHandle(trace)) {
|
|
669
|
+
return {
|
|
670
|
+
pass: false,
|
|
671
|
+
message: () => `Expected a TraceHandle (ctx.trace) but received ${typeof trace}.
|
|
672
|
+
Use: expect(ctx.trace).toCallTool(...)`
|
|
673
|
+
};
|
|
674
|
+
}
|
|
675
|
+
const calls = trace.getToolCalls();
|
|
676
|
+
const pass = calls.some((c) => c.name === toolName);
|
|
677
|
+
return {
|
|
678
|
+
pass,
|
|
679
|
+
message: () => {
|
|
680
|
+
if (pass) {
|
|
681
|
+
return `Expected trace NOT to call tool "${toolName}"`;
|
|
682
|
+
}
|
|
683
|
+
const names = calls.map((c) => c.name);
|
|
684
|
+
const recorded = names.length === 0 ? "no tool calls were recorded" : `recorded: [${names.join(", ")}]`;
|
|
685
|
+
return `Expected tool "${toolName}" to be called, but ${recorded}`;
|
|
686
|
+
}
|
|
687
|
+
};
|
|
688
|
+
},
|
|
689
|
+
async toMatchSemanticOutput(trace, expected, options) {
|
|
690
|
+
if (!isTraceHandle(trace)) {
|
|
691
|
+
return {
|
|
692
|
+
pass: false,
|
|
693
|
+
message: () => `Expected a TraceHandle (ctx.trace) but received ${typeof trace}.
|
|
694
|
+
Use: expect(ctx.trace).toMatchSemanticOutput(...)`
|
|
695
|
+
};
|
|
696
|
+
}
|
|
697
|
+
const steps = trace.getLLMSteps();
|
|
698
|
+
const fullOutput = steps.map((s) => [s.completion, s.contains].filter(Boolean).join(" ")).join(" ").trim();
|
|
699
|
+
try {
|
|
700
|
+
const pass = await llmJudgeSemanticMatch(fullOutput, expected, options);
|
|
701
|
+
return {
|
|
702
|
+
pass,
|
|
703
|
+
message: () => {
|
|
704
|
+
if (pass) {
|
|
705
|
+
return `Expected trace output NOT to semantically match "${expected}" (LLM judged YES)`;
|
|
706
|
+
}
|
|
707
|
+
return `Expected trace output to semantically match "${expected}", but LLM judged NO. Trace output: "${fullOutput || "(empty)"}"`;
|
|
708
|
+
}
|
|
709
|
+
};
|
|
710
|
+
} catch (err) {
|
|
711
|
+
return {
|
|
712
|
+
pass: false,
|
|
713
|
+
message: () => `LLM semantic match failed: ${err.message}`
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
},
|
|
717
|
+
async toEvaluateOutputMetric(trace, config) {
|
|
718
|
+
if (!isTraceHandle(trace)) {
|
|
719
|
+
return {
|
|
720
|
+
pass: false,
|
|
721
|
+
message: () => `Expected a TraceHandle (ctx.trace) but received ${typeof trace}.
|
|
722
|
+
Use: expect(ctx.trace).toEvaluateOutputMetric(...)`
|
|
723
|
+
};
|
|
724
|
+
}
|
|
725
|
+
if (!config || !config.evaluationPrompt) {
|
|
726
|
+
return {
|
|
727
|
+
pass: false,
|
|
728
|
+
message: () => "toEvaluateOutputMetric requires evaluationPrompt"
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
const steps = trace.getLLMSteps();
|
|
732
|
+
if (steps.length === 0) {
|
|
733
|
+
return {
|
|
734
|
+
pass: false,
|
|
735
|
+
message: () => "No LLM steps recorded; cannot evaluate output metric."
|
|
736
|
+
};
|
|
737
|
+
}
|
|
738
|
+
const targetIdx = config.index ?? (config.nth !== void 0 ? config.nth - 1 : steps.length - 1);
|
|
739
|
+
if (targetIdx < 0 || targetIdx >= steps.length) {
|
|
740
|
+
return {
|
|
741
|
+
pass: false,
|
|
742
|
+
message: () => `LLM steps length ${steps.length}, but index/nth points to ${targetIdx}.`
|
|
743
|
+
};
|
|
744
|
+
}
|
|
745
|
+
const targetStep = steps[targetIdx];
|
|
746
|
+
const targetField = config.target ?? "result";
|
|
747
|
+
const targetText = targetField === "prompt" ? targetStep.prompt ?? "" : targetStep.completion ?? "";
|
|
748
|
+
if (!targetText) {
|
|
749
|
+
return {
|
|
750
|
+
pass: false,
|
|
751
|
+
message: () => `Selected LLM step has empty ${targetField}; cannot evaluate.`
|
|
752
|
+
};
|
|
753
|
+
}
|
|
754
|
+
const condition = (() => {
|
|
755
|
+
try {
|
|
756
|
+
return resolveCondition(config.condition);
|
|
757
|
+
} catch (err) {
|
|
758
|
+
return err;
|
|
759
|
+
}
|
|
760
|
+
})();
|
|
761
|
+
if (condition instanceof Error) {
|
|
762
|
+
return {
|
|
763
|
+
pass: false,
|
|
764
|
+
message: () => condition.message
|
|
765
|
+
};
|
|
766
|
+
}
|
|
767
|
+
const evalPrompt = `
|
|
768
|
+
Evaluation prompt (from user):
|
|
769
|
+
${config.evaluationPrompt}
|
|
770
|
+
|
|
771
|
+
Score the following text strictly between 0 and 1 (inclusive). Respond with only the number.
|
|
772
|
+
|
|
773
|
+
Text:
|
|
774
|
+
${targetText}
|
|
775
|
+
`.trim();
|
|
776
|
+
try {
|
|
777
|
+
const raw = await callProviderLLM(
|
|
778
|
+
evalPrompt,
|
|
779
|
+
{ provider: config.provider, model: config.model, sdk: config.sdk, apiKey: config.apiKey, baseURL: config.baseURL },
|
|
780
|
+
"You are an evaluation assistant. Return only a number between 0 and 1.",
|
|
781
|
+
16,
|
|
782
|
+
0
|
|
783
|
+
);
|
|
784
|
+
const score = parseFirstNumber(raw);
|
|
785
|
+
if (score === null) {
|
|
786
|
+
return {
|
|
787
|
+
pass: false,
|
|
788
|
+
message: () => `Could not parse numeric metric from model response: "${raw}"`
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
if (score < 0 || score > 1) {
|
|
792
|
+
return {
|
|
793
|
+
pass: false,
|
|
794
|
+
message: () => `Metric ${score} is out of allowed range 0.0\u20131.0 (raw: "${raw}")`
|
|
795
|
+
};
|
|
796
|
+
}
|
|
797
|
+
const pass = checkCondition(score, condition);
|
|
798
|
+
return {
|
|
799
|
+
pass,
|
|
800
|
+
message: () => {
|
|
801
|
+
if (pass) {
|
|
802
|
+
return `Expected metric NOT to satisfy ${condition.kind} ${condition.value} (score ${score})`;
|
|
803
|
+
}
|
|
804
|
+
return `Metric check failed: score ${score} did not satisfy ${condition.kind} ${condition.value}. Raw response: "${raw}"`;
|
|
805
|
+
}
|
|
806
|
+
};
|
|
807
|
+
} catch (err) {
|
|
808
|
+
return {
|
|
809
|
+
pass: false,
|
|
810
|
+
message: () => `LLM evaluation failed: ${err.message}`
|
|
811
|
+
};
|
|
812
|
+
}
|
|
813
|
+
},
|
|
814
|
+
toHaveCustomStep(trace, config = {}) {
|
|
815
|
+
if (!isTraceHandle(trace) || typeof trace.getCustomSteps !== "function") {
|
|
816
|
+
return {
|
|
817
|
+
pass: false,
|
|
818
|
+
message: () => `Expected a TraceHandle (ctx.trace with getCustomSteps) but received ${typeof trace}.
|
|
819
|
+
Use: expect(ctx.trace).toHaveCustomStep(...)`
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
const steps = trace.getCustomSteps();
|
|
823
|
+
const matchString = (val) => {
|
|
824
|
+
if (val === void 0 || val === null) return "";
|
|
825
|
+
if (typeof val === "string") return val;
|
|
826
|
+
try {
|
|
827
|
+
return JSON.stringify(val);
|
|
828
|
+
} catch {
|
|
829
|
+
return String(val);
|
|
830
|
+
}
|
|
831
|
+
};
|
|
832
|
+
const matching = steps.filter((step) => {
|
|
833
|
+
if (config.kind && step.kind !== config.kind) return false;
|
|
834
|
+
if (config.name && step.name !== config.name) return false;
|
|
835
|
+
if (config.tag && !(step.tags || []).includes(config.tag)) return false;
|
|
836
|
+
const payloadStr = matchString(step.payload).toLowerCase();
|
|
837
|
+
const resultStr = matchString(step.result).toLowerCase();
|
|
838
|
+
const metaStr = matchString(step.metadata).toLowerCase();
|
|
839
|
+
const combined = [payloadStr, resultStr, metaStr].filter(Boolean).join(" ");
|
|
840
|
+
if (config.contains && !combined.includes(config.contains.toLowerCase())) return false;
|
|
841
|
+
if (config.payloadContains && !payloadStr.includes(config.payloadContains.toLowerCase())) return false;
|
|
842
|
+
if (config.resultContains && !resultStr.includes(config.resultContains.toLowerCase())) return false;
|
|
843
|
+
if (config.metadataContains && !metaStr.includes(config.metadataContains.toLowerCase())) return false;
|
|
844
|
+
return true;
|
|
845
|
+
});
|
|
846
|
+
const count = matching.length;
|
|
847
|
+
let pass;
|
|
848
|
+
if (config.times !== void 0) {
|
|
849
|
+
pass = count === config.times;
|
|
850
|
+
} else if (config.minTimes !== void 0 || config.maxTimes !== void 0) {
|
|
851
|
+
const min = config.minTimes ?? 0;
|
|
852
|
+
const max = config.maxTimes ?? Infinity;
|
|
853
|
+
pass = count >= min && count <= max;
|
|
854
|
+
} else {
|
|
855
|
+
pass = count > 0;
|
|
856
|
+
}
|
|
857
|
+
return {
|
|
858
|
+
pass,
|
|
859
|
+
message: () => {
|
|
860
|
+
if (pass) {
|
|
861
|
+
return `Expected trace NOT to have custom step matching ${JSON.stringify(config)}`;
|
|
862
|
+
}
|
|
863
|
+
const stepSummary = steps.length === 0 ? "no custom steps were recorded" : `${count} matching step(s) found; recorded custom steps: ${JSON.stringify(steps)}`;
|
|
864
|
+
return `Expected trace to have custom step matching ${JSON.stringify(config)}, but ${stepSummary}`;
|
|
865
|
+
}
|
|
866
|
+
};
|
|
867
|
+
},
|
|
868
|
+
toHavePromptWhere(trace, config) {
|
|
869
|
+
if (!isTraceHandle(trace)) {
|
|
870
|
+
return {
|
|
871
|
+
pass: false,
|
|
872
|
+
message: () => `Expected a TraceHandle (ctx.trace) but received ${typeof trace}.
|
|
873
|
+
Use: expect(ctx.trace).toHavePromptWhere(...)`
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
if (!config || !config.filterContains) {
|
|
877
|
+
return {
|
|
878
|
+
pass: false,
|
|
879
|
+
message: () => "toHavePromptWhere requires filterContains"
|
|
880
|
+
};
|
|
881
|
+
}
|
|
882
|
+
const filterNeedle = config.filterContains.toLowerCase();
|
|
883
|
+
const requireNeedle = config.requireContains?.toLowerCase();
|
|
884
|
+
const forbidNeedle = config.requireNotContains?.toLowerCase();
|
|
885
|
+
const prompts = trace.getLLMSteps().map((s) => s.prompt ?? "");
|
|
886
|
+
const filtered = prompts.filter((p) => p.toLowerCase().includes(filterNeedle));
|
|
887
|
+
const targetIdx = config.index ?? (config.nth !== void 0 ? config.nth - 1 : void 0);
|
|
888
|
+
let checked = [];
|
|
889
|
+
let count = 0;
|
|
890
|
+
let pass = true;
|
|
891
|
+
if (targetIdx !== void 0) {
|
|
892
|
+
if (targetIdx < 0 || targetIdx >= filtered.length) {
|
|
893
|
+
return {
|
|
894
|
+
pass: false,
|
|
895
|
+
message: () => `Filtered prompts length ${filtered.length}, but index/nth points to ${targetIdx}. Config: ${JSON.stringify(config)}`
|
|
896
|
+
};
|
|
897
|
+
}
|
|
898
|
+
const p = filtered[targetIdx];
|
|
899
|
+
const lower = p.toLowerCase();
|
|
900
|
+
const okRequire = requireNeedle ? lower.includes(requireNeedle) : true;
|
|
901
|
+
const okForbid = forbidNeedle ? !lower.includes(forbidNeedle) : true;
|
|
902
|
+
pass = okRequire && okForbid;
|
|
903
|
+
checked = okRequire && okForbid ? [p] : [];
|
|
904
|
+
count = checked.length;
|
|
905
|
+
} else {
|
|
906
|
+
checked = filtered.filter((p) => {
|
|
907
|
+
const lower = p.toLowerCase();
|
|
908
|
+
if (requireNeedle && !lower.includes(requireNeedle)) return false;
|
|
909
|
+
if (forbidNeedle && lower.includes(forbidNeedle)) return false;
|
|
910
|
+
return true;
|
|
911
|
+
});
|
|
912
|
+
count = checked.length;
|
|
913
|
+
if (config.times !== void 0) {
|
|
914
|
+
pass = count === config.times;
|
|
915
|
+
} else {
|
|
916
|
+
const min = config.minTimes ?? 0;
|
|
917
|
+
const max = config.maxTimes ?? Infinity;
|
|
918
|
+
pass = count >= min && count <= max;
|
|
919
|
+
}
|
|
920
|
+
if (requireNeedle) {
|
|
921
|
+
const violating = filtered.filter((p) => !p.toLowerCase().includes(requireNeedle));
|
|
922
|
+
if (violating.length > 0) pass = false;
|
|
923
|
+
}
|
|
924
|
+
if (forbidNeedle) {
|
|
925
|
+
const violating = filtered.filter((p) => p.toLowerCase().includes(forbidNeedle));
|
|
926
|
+
if (violating.length > 0) pass = false;
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
return {
|
|
930
|
+
pass,
|
|
931
|
+
message: () => {
|
|
932
|
+
if (pass) {
|
|
933
|
+
return `Expected prompts NOT to satisfy filter/require combo: ${JSON.stringify(config)}`;
|
|
934
|
+
}
|
|
935
|
+
const base = [`Expected prompts filtered by "${config.filterContains}" to satisfy requirements`];
|
|
936
|
+
if (config.requireContains) base.push(`requireContains: "${config.requireContains}"`);
|
|
937
|
+
if (config.requireNotContains) base.push(`requireNotContains: "${config.requireNotContains}"`);
|
|
938
|
+
if (targetIdx !== void 0) {
|
|
939
|
+
base.push(`checked index: ${targetIdx}`, `filtered count: ${filtered.length}`);
|
|
940
|
+
} else {
|
|
941
|
+
base.push(`filtered count: ${filtered.length}, passing count: ${checked.length}`);
|
|
942
|
+
base.push(
|
|
943
|
+
config.times !== void 0 ? `expected exactly ${config.times}` : `expected between ${config.minTimes ?? 0} and ${config.maxTimes ?? Infinity}`
|
|
944
|
+
);
|
|
945
|
+
}
|
|
946
|
+
return base.filter(Boolean).join("; ");
|
|
947
|
+
}
|
|
948
|
+
};
|
|
949
|
+
}
|
|
950
|
+
});
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
// src/runner.ts
|
|
954
|
+
init_context();
|
|
955
|
+
|
|
956
|
+
// src/proxy/llm-capture.ts
|
|
957
|
+
var import_node_http = __toESM(require("node:http"), 1);
|
|
958
|
+
var import_node_url = require("node:url");
|
|
959
|
+
var import_node_stream = require("node:stream");
|
|
960
|
+
var DEFAULT_PORT = 8787;
|
|
961
|
+
var HEADER_TRACE_ID = "x-trace-id";
|
|
962
|
+
var DEFAULT_UPSTREAM = {
|
|
963
|
+
openai: "https://api.openai.com",
|
|
964
|
+
gemini: "https://generativelanguage.googleapis.com",
|
|
965
|
+
grok: "https://api.x.ai",
|
|
966
|
+
anthropic: "https://api.anthropic.com"
|
|
967
|
+
};
|
|
968
|
+
var AI_PATTERNS = {
|
|
969
|
+
openai: /\/v1\/(chat\/)?completions/,
|
|
970
|
+
// also covers legacy /v1/completions
|
|
971
|
+
gemini: /\/v1beta\/models\/[^/:]+:(generateContent|streamGenerateContent)/,
|
|
972
|
+
grok: /\/v1\/(chat\/)?completions/,
|
|
973
|
+
anthropic: /\/v1\/messages/
|
|
974
|
+
};
|
|
975
|
+
function detectProvider(pathname) {
|
|
976
|
+
for (const [provider, pattern] of Object.entries(AI_PATTERNS)) {
|
|
977
|
+
if (pattern.test(pathname)) return provider;
|
|
978
|
+
}
|
|
979
|
+
return null;
|
|
980
|
+
}
|
|
981
|
+
function extractModel(provider, body, url) {
|
|
982
|
+
if (provider === "gemini") {
|
|
983
|
+
const match = /\/models\/([^/:]+):/.exec(url);
|
|
984
|
+
return match ? match[1] : "unknown";
|
|
985
|
+
}
|
|
986
|
+
if (provider === "anthropic") {
|
|
987
|
+
return typeof body.model === "string" ? body.model : "unknown";
|
|
988
|
+
}
|
|
989
|
+
return typeof body.model === "string" ? body.model : "unknown";
|
|
990
|
+
}
|
|
991
|
+
function extractPrompt(provider, body) {
|
|
992
|
+
if (provider === "openai" || provider === "grok") {
|
|
993
|
+
const messages = body.messages;
|
|
994
|
+
if (Array.isArray(messages)) {
|
|
995
|
+
return messages.map((m) => {
|
|
996
|
+
if (m && typeof m === "object") {
|
|
997
|
+
const msg = m;
|
|
998
|
+
return `${msg.role}: ${msg.content}`;
|
|
999
|
+
}
|
|
1000
|
+
return String(m);
|
|
1001
|
+
}).join("\n");
|
|
1002
|
+
}
|
|
1003
|
+
return typeof body.prompt === "string" ? body.prompt : "";
|
|
1004
|
+
}
|
|
1005
|
+
if (provider === "gemini") {
|
|
1006
|
+
const contents = body.contents;
|
|
1007
|
+
if (Array.isArray(contents)) {
|
|
1008
|
+
return contents.flatMap((c) => {
|
|
1009
|
+
if (c && typeof c === "object") {
|
|
1010
|
+
const parts = c.parts;
|
|
1011
|
+
if (Array.isArray(parts)) {
|
|
1012
|
+
return parts.map((p) => {
|
|
1013
|
+
if (p && typeof p === "object") {
|
|
1014
|
+
return String(p.text ?? "");
|
|
1015
|
+
}
|
|
1016
|
+
return "";
|
|
1017
|
+
});
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
return [];
|
|
1021
|
+
}).join("\n");
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
if (provider === "anthropic") {
|
|
1025
|
+
const messages = body.messages;
|
|
1026
|
+
if (Array.isArray(messages)) {
|
|
1027
|
+
return messages.map((m) => {
|
|
1028
|
+
if (m && typeof m === "object") {
|
|
1029
|
+
const msg = m;
|
|
1030
|
+
return `${msg.role ?? "user"}: ${msg.content ?? ""}`;
|
|
1031
|
+
}
|
|
1032
|
+
return String(m);
|
|
1033
|
+
}).join("\n");
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
return "";
|
|
1037
|
+
}
|
|
1038
|
+
function extractCompletion(provider, responseBody) {
|
|
1039
|
+
if (provider === "openai" || provider === "grok") {
|
|
1040
|
+
const choices = responseBody.choices;
|
|
1041
|
+
if (Array.isArray(choices) && choices.length > 0) {
|
|
1042
|
+
const first = choices[0];
|
|
1043
|
+
if (first.message && typeof first.message === "object") {
|
|
1044
|
+
return String(first.message.content ?? "");
|
|
1045
|
+
}
|
|
1046
|
+
if (typeof first.text === "string") return first.text;
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
if (provider === "gemini") {
|
|
1050
|
+
const candidates = responseBody.candidates;
|
|
1051
|
+
if (Array.isArray(candidates) && candidates.length > 0) {
|
|
1052
|
+
const first = candidates[0];
|
|
1053
|
+
if (first.content && typeof first.content === "object") {
|
|
1054
|
+
const parts = first.content.parts;
|
|
1055
|
+
if (Array.isArray(parts) && parts.length > 0) {
|
|
1056
|
+
return String(parts[0].text ?? "");
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
}
|
|
1061
|
+
if (provider === "anthropic") {
|
|
1062
|
+
const content = responseBody.content;
|
|
1063
|
+
if (Array.isArray(content) && content.length > 0) {
|
|
1064
|
+
const first = content[0];
|
|
1065
|
+
if (typeof first.text === "string") return first.text;
|
|
1066
|
+
if (first.type === "text" && typeof first.text === "string") return first.text;
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
return "";
|
|
1070
|
+
}
|
|
1071
|
+
function cloneHeaders(headers) {
|
|
1072
|
+
const result = {};
|
|
1073
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
1074
|
+
if (Array.isArray(value)) {
|
|
1075
|
+
result[key] = value.join(", ");
|
|
1076
|
+
} else if (typeof value === "string") {
|
|
1077
|
+
result[key] = value;
|
|
1078
|
+
}
|
|
1079
|
+
}
|
|
1080
|
+
return result;
|
|
1081
|
+
}
|
|
1082
|
+
function normalizeUpstream(provider, userBase) {
|
|
1083
|
+
const base = userBase || DEFAULT_UPSTREAM[provider];
|
|
1084
|
+
return base.endsWith("/") ? base.slice(0, -1) : base;
|
|
1085
|
+
}
|
|
1086
|
+
function recordStep(store, traceId, step) {
|
|
1087
|
+
if (!store.has(traceId)) {
|
|
1088
|
+
store.set(traceId, []);
|
|
1089
|
+
}
|
|
1090
|
+
store.get(traceId).push(step);
|
|
1091
|
+
}
|
|
1092
|
+
async function readBody(req) {
|
|
1093
|
+
const chunks = [];
|
|
1094
|
+
for await (const chunk of req) {
|
|
1095
|
+
chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
|
|
1096
|
+
}
|
|
1097
|
+
return Buffer.concat(chunks);
|
|
1098
|
+
}
|
|
1099
|
+
function sendUpstreamResponse(upstreamRes, res) {
|
|
1100
|
+
res.statusCode = upstreamRes.status;
|
|
1101
|
+
for (const [key, value] of upstreamRes.headers.entries()) {
|
|
1102
|
+
res.setHeader(key, value);
|
|
1103
|
+
}
|
|
1104
|
+
const body = upstreamRes.body;
|
|
1105
|
+
if (!body) {
|
|
1106
|
+
res.end();
|
|
1107
|
+
return;
|
|
1108
|
+
}
|
|
1109
|
+
const nodeStream = import_node_stream.Readable.fromWeb(body);
|
|
1110
|
+
nodeStream.pipe(res);
|
|
1111
|
+
}
|
|
1112
|
+
async function startLLMProxy(options = {}) {
|
|
1113
|
+
const port = options.port ?? DEFAULT_PORT;
|
|
1114
|
+
const store = /* @__PURE__ */ new Map();
|
|
1115
|
+
const upstreamOverride = options.upstream || {};
|
|
1116
|
+
const server = import_node_http.default.createServer(async (req, res) => {
|
|
1117
|
+
try {
|
|
1118
|
+
if (!req.url || !req.method) {
|
|
1119
|
+
res.statusCode = 400;
|
|
1120
|
+
res.end("Bad request");
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
1123
|
+
const parsed = new import_node_url.URL(req.url, `http://localhost:${port}`);
|
|
1124
|
+
if (req.method === "GET" && parsed.pathname.startsWith("/traces/")) {
|
|
1125
|
+
const traceId2 = decodeURIComponent(parsed.pathname.replace("/traces/", ""));
|
|
1126
|
+
const steps = store.get(traceId2) ?? [];
|
|
1127
|
+
store.delete(traceId2);
|
|
1128
|
+
res.setHeader("content-type", "application/json");
|
|
1129
|
+
res.end(JSON.stringify({ steps }));
|
|
1130
|
+
return;
|
|
1131
|
+
}
|
|
1132
|
+
if (req.method === "GET" && parsed.pathname === "/health") {
|
|
1133
|
+
res.statusCode = 200;
|
|
1134
|
+
res.end("ok");
|
|
1135
|
+
return;
|
|
1136
|
+
}
|
|
1137
|
+
const bodyBuf = await readBody(req);
|
|
1138
|
+
const bodyText = bodyBuf.toString() || "{}";
|
|
1139
|
+
const requestBody = (() => {
|
|
1140
|
+
try {
|
|
1141
|
+
return JSON.parse(bodyText);
|
|
1142
|
+
} catch {
|
|
1143
|
+
return {};
|
|
1144
|
+
}
|
|
1145
|
+
})();
|
|
1146
|
+
const provider = detectProvider(parsed.pathname);
|
|
1147
|
+
const traceId = req.headers[HEADER_TRACE_ID]?.toString();
|
|
1148
|
+
const isStreaming = requestBody && typeof requestBody === "object" ? requestBody.stream === true : false;
|
|
1149
|
+
const headers = cloneHeaders(req.headers);
|
|
1150
|
+
const upstreamBase = provider ? normalizeUpstream(provider, upstreamOverride[provider]) : void 0;
|
|
1151
|
+
if (!provider || !upstreamBase) {
|
|
1152
|
+
const passthrough = await fetch(parsed.toString(), {
|
|
1153
|
+
method: req.method,
|
|
1154
|
+
headers,
|
|
1155
|
+
body: bodyBuf.length > 0 ? bodyBuf : void 0
|
|
1156
|
+
});
|
|
1157
|
+
sendUpstreamResponse(passthrough, res);
|
|
1158
|
+
return;
|
|
1159
|
+
}
|
|
1160
|
+
const targetUrl = `${upstreamBase}${parsed.pathname}${parsed.search}`;
|
|
1161
|
+
const upstreamRes = await fetch(targetUrl, {
|
|
1162
|
+
method: req.method,
|
|
1163
|
+
headers,
|
|
1164
|
+
body: bodyBuf.length > 0 ? bodyBuf : void 0
|
|
1165
|
+
});
|
|
1166
|
+
if (traceId) {
|
|
1167
|
+
const model = extractModel(provider, requestBody, targetUrl);
|
|
1168
|
+
const prompt = extractPrompt(provider, requestBody);
|
|
1169
|
+
if (isStreaming) {
|
|
1170
|
+
recordStep(store, traceId, { model, provider, prompt, completion: "(streamed)" });
|
|
1171
|
+
} else {
|
|
1172
|
+
try {
|
|
1173
|
+
const clone = upstreamRes.clone();
|
|
1174
|
+
const responseBody = await clone.json();
|
|
1175
|
+
const completion = extractCompletion(provider, responseBody);
|
|
1176
|
+
recordStep(store, traceId, { model, provider, prompt, completion });
|
|
1177
|
+
} catch {
|
|
1178
|
+
recordStep(store, traceId, { model, provider, prompt, completion: "" });
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
}
|
|
1182
|
+
sendUpstreamResponse(upstreamRes, res);
|
|
1183
|
+
} catch (err) {
|
|
1184
|
+
res.statusCode = 500;
|
|
1185
|
+
res.end(`proxy error: ${err.message}`);
|
|
1186
|
+
}
|
|
1187
|
+
});
|
|
1188
|
+
await new Promise((resolve) => server.listen(port, resolve));
|
|
1189
|
+
return {
|
|
1190
|
+
url: `http://localhost:${port}`,
|
|
1191
|
+
async stop() {
|
|
1192
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
1193
|
+
}
|
|
1194
|
+
};
|
|
1195
|
+
}
|
|
1196
|
+
async function fetchCapturedTrace(proxyUrl, traceId) {
|
|
1197
|
+
const url = `${proxyUrl.replace(/\/$/, "")}/traces/${encodeURIComponent(traceId)}`;
|
|
1198
|
+
const res = await fetch(url);
|
|
1199
|
+
if (!res.ok) {
|
|
1200
|
+
throw new Error(`failed to fetch trace ${traceId} from proxy: ${res.status}`);
|
|
1201
|
+
}
|
|
1202
|
+
const data = await res.json();
|
|
1203
|
+
return data.steps || [];
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
// src/runner.ts
|
|
1207
|
+
var import_node_url2 = require("node:url");
|
|
1208
|
+
var import_node_crypto2 = require("node:crypto");
|
|
1209
|
+
var import_node_path = __toESM(require("node:path"), 1);
|
|
1210
|
+
async function runFiles(files, options = {}) {
|
|
1211
|
+
const proxyOptIn = process.env.ELASTICDASH_LLM_PROXY === "1" || Boolean(process.env.ELASTICDASH_LLM_PROXY_URL);
|
|
1212
|
+
const proxyPort = Number.parseInt(process.env.ELASTICDASH_LLM_PROXY_PORT || "8787", 10);
|
|
1213
|
+
let proxyUrl = process.env.ELASTICDASH_LLM_PROXY_URL;
|
|
1214
|
+
const proxyHandle = proxyOptIn && !proxyUrl ? await startLLMProxy({ port: proxyPort }) : null;
|
|
1215
|
+
if (proxyHandle) {
|
|
1216
|
+
proxyUrl = proxyHandle.url;
|
|
1217
|
+
}
|
|
1218
|
+
const fileResults = [];
|
|
1219
|
+
for (const file of files) {
|
|
1220
|
+
const result = await runFile(file, options, { proxyOptIn, proxyUrl });
|
|
1221
|
+
fileResults.push(result);
|
|
1222
|
+
}
|
|
1223
|
+
if (proxyHandle) {
|
|
1224
|
+
await proxyHandle.stop();
|
|
1225
|
+
}
|
|
1226
|
+
return fileResults;
|
|
1227
|
+
}
|
|
1228
|
+
async function runFile(file, options, proxyCtx) {
|
|
1229
|
+
const { hooks = {} } = options;
|
|
1230
|
+
clearRegistry();
|
|
1231
|
+
const resolvedPath = file.startsWith("file://") ? file : (0, import_node_url2.pathToFileURL)(import_node_path.default.resolve(file)).href;
|
|
1232
|
+
if (resolvedPath.endsWith(".ts") && typeof globalThis.Deno === "undefined") {
|
|
1233
|
+
await import("tsx/esm");
|
|
1234
|
+
await import("tsx/cjs");
|
|
1235
|
+
}
|
|
1236
|
+
await import(resolvedPath);
|
|
1237
|
+
const registry = getRegistry();
|
|
1238
|
+
const results = [];
|
|
1239
|
+
let currentTestName = null;
|
|
1240
|
+
let pendingUnhandled;
|
|
1241
|
+
const onUnhandled = (reason) => {
|
|
1242
|
+
if (!pendingUnhandled) pendingUnhandled = reason instanceof Error ? reason : new Error(String(reason));
|
|
1243
|
+
};
|
|
1244
|
+
process.on("unhandledRejection", onUnhandled);
|
|
1245
|
+
process.on("uncaughtException", onUnhandled);
|
|
1246
|
+
for (const hook of registry.beforeAllHooks) {
|
|
1247
|
+
await hook();
|
|
1248
|
+
}
|
|
1249
|
+
for (const entry of registry.tests) {
|
|
1250
|
+
const { context, finalise } = startTraceSession();
|
|
1251
|
+
const traceId = proxyCtx.proxyOptIn ? (0, import_node_crypto2.randomUUID)() : null;
|
|
1252
|
+
setCurrentTrace(context.trace);
|
|
1253
|
+
if (traceId) {
|
|
1254
|
+
process.env.ELASTICDASH_TRACE_ID = traceId;
|
|
1255
|
+
}
|
|
1256
|
+
if (hooks.onTestStart) {
|
|
1257
|
+
await hooks.onTestStart(entry.name);
|
|
1258
|
+
}
|
|
1259
|
+
const startTime = Date.now();
|
|
1260
|
+
let passed = false;
|
|
1261
|
+
let error;
|
|
1262
|
+
pendingUnhandled = void 0;
|
|
1263
|
+
currentTestName = entry.name;
|
|
1264
|
+
setCurrentTrace(context.trace);
|
|
1265
|
+
try {
|
|
1266
|
+
for (const hook of registry.beforeEachHooks) {
|
|
1267
|
+
await hook();
|
|
1268
|
+
}
|
|
1269
|
+
await entry.fn(context);
|
|
1270
|
+
passed = true;
|
|
1271
|
+
} catch (err) {
|
|
1272
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
1273
|
+
} finally {
|
|
1274
|
+
try {
|
|
1275
|
+
for (const hook of registry.afterEachHooks) {
|
|
1276
|
+
await hook();
|
|
1277
|
+
}
|
|
1278
|
+
} catch (afterErr) {
|
|
1279
|
+
if (!error) {
|
|
1280
|
+
error = afterErr instanceof Error ? afterErr : new Error(String(afterErr));
|
|
1281
|
+
passed = false;
|
|
1282
|
+
}
|
|
1283
|
+
}
|
|
1284
|
+
setCurrentTrace(void 0);
|
|
1285
|
+
if (!error && pendingUnhandled) {
|
|
1286
|
+
error = pendingUnhandled;
|
|
1287
|
+
passed = false;
|
|
1288
|
+
}
|
|
1289
|
+
currentTestName = null;
|
|
1290
|
+
}
|
|
1291
|
+
const durationMs = Date.now() - startTime;
|
|
1292
|
+
if (hooks.onTestFinish) {
|
|
1293
|
+
await hooks.onTestFinish(entry.name, passed, durationMs, error);
|
|
1294
|
+
}
|
|
1295
|
+
if (hooks.onTraceComplete) {
|
|
1296
|
+
await hooks.onTraceComplete(entry.name, context.trace);
|
|
1297
|
+
}
|
|
1298
|
+
if (traceId && proxyCtx.proxyUrl) {
|
|
1299
|
+
try {
|
|
1300
|
+
const captured = await fetchCapturedTrace(proxyCtx.proxyUrl, traceId);
|
|
1301
|
+
for (const step of captured) {
|
|
1302
|
+
context.trace.recordLLMStep(step);
|
|
1303
|
+
}
|
|
1304
|
+
} catch (proxyErr) {
|
|
1305
|
+
console.warn("[elasticdash] Failed to fetch proxy-captured steps:", proxyErr);
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
finalise();
|
|
1309
|
+
setCurrentTrace(void 0);
|
|
1310
|
+
if (traceId) {
|
|
1311
|
+
delete process.env.ELASTICDASH_TRACE_ID;
|
|
1312
|
+
}
|
|
1313
|
+
results.push({ name: entry.name, passed, durationMs, error });
|
|
1314
|
+
}
|
|
1315
|
+
for (const hook of registry.afterAllHooks) {
|
|
1316
|
+
await hook();
|
|
1317
|
+
}
|
|
1318
|
+
process.off("unhandledRejection", onUnhandled);
|
|
1319
|
+
process.off("uncaughtException", onUnhandled);
|
|
1320
|
+
return { file, results };
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
// src/reporter.ts
|
|
1324
|
+
var import_chalk = __toESM(require("chalk"), 1);
|
|
1325
|
+
function reportResults(fileResults) {
|
|
1326
|
+
let totalPassed = 0;
|
|
1327
|
+
let totalFailed = 0;
|
|
1328
|
+
let totalDurationMs = 0;
|
|
1329
|
+
for (const fileResult of fileResults) {
|
|
1330
|
+
if (fileResults.length > 1) {
|
|
1331
|
+
console.log(import_chalk.default.dim(`
|
|
1332
|
+
${fileResult.file}`));
|
|
1333
|
+
}
|
|
1334
|
+
for (const result of fileResult.results) {
|
|
1335
|
+
printTestResult(result);
|
|
1336
|
+
totalDurationMs += result.durationMs;
|
|
1337
|
+
if (result.passed) {
|
|
1338
|
+
totalPassed++;
|
|
1339
|
+
} else {
|
|
1340
|
+
totalFailed++;
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1343
|
+
}
|
|
1344
|
+
printSummary(totalPassed, totalFailed, totalDurationMs);
|
|
1345
|
+
}
|
|
1346
|
+
function printTestResult(result) {
|
|
1347
|
+
const duration = import_chalk.default.dim(`(${formatDuration(result.durationMs)})`);
|
|
1348
|
+
if (result.passed) {
|
|
1349
|
+
console.log(` ${import_chalk.default.green("\u2713")} ${result.name} ${duration}`);
|
|
1350
|
+
} else {
|
|
1351
|
+
console.log(` ${import_chalk.default.red("\u2717")} ${result.name} ${duration}`);
|
|
1352
|
+
if (result.error) {
|
|
1353
|
+
const errorLines = formatError(result.error);
|
|
1354
|
+
for (const line of errorLines) {
|
|
1355
|
+
console.log(` ${import_chalk.default.red("\u2192")} ${line}`);
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
}
|
|
1360
|
+
function printSummary(passed, failed, totalMs) {
|
|
1361
|
+
const total = passed + failed;
|
|
1362
|
+
console.log("");
|
|
1363
|
+
if (passed > 0) {
|
|
1364
|
+
console.log(import_chalk.default.green(`${passed} passed`));
|
|
1365
|
+
}
|
|
1366
|
+
if (failed > 0) {
|
|
1367
|
+
console.log(import_chalk.default.red(`${failed} failed`));
|
|
1368
|
+
}
|
|
1369
|
+
console.log(import_chalk.default.dim(`Total: ${total}`));
|
|
1370
|
+
console.log(import_chalk.default.dim(`Duration: ${formatDuration(totalMs)}`));
|
|
1371
|
+
}
|
|
1372
|
+
function formatDuration(ms) {
|
|
1373
|
+
if (ms >= 1e3) {
|
|
1374
|
+
return `${(ms / 1e3).toFixed(1)}s`;
|
|
1375
|
+
}
|
|
1376
|
+
return `${ms}ms`;
|
|
1377
|
+
}
|
|
1378
|
+
function formatError(error) {
|
|
1379
|
+
const lines = [];
|
|
1380
|
+
if (error.message) {
|
|
1381
|
+
lines.push(error.message);
|
|
1382
|
+
}
|
|
1383
|
+
if (error.stack) {
|
|
1384
|
+
const stackLines = error.stack.split("\n").slice(1).map((l) => l.trim()).filter((l) => l.startsWith("at ")).slice(0, 3);
|
|
1385
|
+
lines.push(...stackLines);
|
|
1386
|
+
}
|
|
1387
|
+
return lines;
|
|
1388
|
+
}
|
|
1389
|
+
|
|
1390
|
+
// src/interceptors/ai-interceptor.ts
|
|
1391
|
+
init_context();
|
|
1392
|
+
init_recorder();
|
|
1393
|
+
init_side_effects();
|
|
1394
|
+
var AI_PATTERNS2 = {
|
|
1395
|
+
openai: /https?:\/\/api\.openai\.com\/v1\/((chat\/)?completions|embeddings)/,
|
|
1396
|
+
gemini: /https?:\/\/generativelanguage\.googleapis\.com\/.*\/models\/[^/:]+:(generateContent|streamGenerateContent)/,
|
|
1397
|
+
grok: /https?:\/\/api\.x\.ai\/v1\/(chat\/)?completions/,
|
|
1398
|
+
kimi: /https?:\/\/api\.moonshot\.ai\/v1\/(chat\/)?completions/
|
|
1399
|
+
};
|
|
1400
|
+
function detectProvider2(url) {
|
|
1401
|
+
for (const [provider, pattern] of Object.entries(AI_PATTERNS2)) {
|
|
1402
|
+
if (pattern.test(url)) return provider;
|
|
1403
|
+
}
|
|
1404
|
+
return null;
|
|
1405
|
+
}
|
|
1406
|
+
function extractModel2(provider, body, url) {
|
|
1407
|
+
if (provider === "gemini") {
|
|
1408
|
+
const match = /\/models\/([^/:]+):/.exec(url);
|
|
1409
|
+
return match ? match[1] : "unknown";
|
|
1410
|
+
}
|
|
1411
|
+
return typeof body.model === "string" ? body.model : "unknown";
|
|
1412
|
+
}
|
|
1413
|
+
function extractPrompt2(provider, body) {
|
|
1414
|
+
if (provider === "openai" || provider === "grok" || provider === "kimi") {
|
|
1415
|
+
const messages = body.messages;
|
|
1416
|
+
if (Array.isArray(messages)) {
|
|
1417
|
+
return messages.map((m) => {
|
|
1418
|
+
if (m && typeof m === "object") {
|
|
1419
|
+
const msg = m;
|
|
1420
|
+
return `${msg.role}: ${msg.content}`;
|
|
1421
|
+
}
|
|
1422
|
+
return String(m);
|
|
1423
|
+
}).join("\n");
|
|
1424
|
+
}
|
|
1425
|
+
if (typeof body.prompt === "string") return body.prompt;
|
|
1426
|
+
if (typeof body.input === "string") return body.input;
|
|
1427
|
+
if (Array.isArray(body.input)) return body.input.map((v) => String(v)).join("\n");
|
|
1428
|
+
return "";
|
|
1429
|
+
}
|
|
1430
|
+
if (provider === "gemini") {
|
|
1431
|
+
const contents = body.contents;
|
|
1432
|
+
if (Array.isArray(contents)) {
|
|
1433
|
+
return contents.flatMap((c) => {
|
|
1434
|
+
if (c && typeof c === "object") {
|
|
1435
|
+
const parts = c.parts;
|
|
1436
|
+
if (Array.isArray(parts)) {
|
|
1437
|
+
return parts.map((p) => {
|
|
1438
|
+
if (p && typeof p === "object") {
|
|
1439
|
+
return String(p.text ?? "");
|
|
1440
|
+
}
|
|
1441
|
+
return "";
|
|
1442
|
+
});
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
return [];
|
|
1446
|
+
}).join("\n");
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1449
|
+
return "";
|
|
1450
|
+
}
|
|
1451
|
+
function extractCompletion2(provider, responseBody) {
|
|
1452
|
+
if (responseBody.streamed === true && typeof responseBody.completion === "string") {
|
|
1453
|
+
return responseBody.completion;
|
|
1454
|
+
}
|
|
1455
|
+
if (provider === "openai" || provider === "grok" || provider === "kimi") {
|
|
1456
|
+
const choices = responseBody.choices;
|
|
1457
|
+
if (Array.isArray(choices) && choices.length > 0) {
|
|
1458
|
+
const first = choices[0];
|
|
1459
|
+
if (first.message && typeof first.message === "object") {
|
|
1460
|
+
return String(first.message.content ?? "");
|
|
1461
|
+
}
|
|
1462
|
+
if (typeof first.text === "string") return first.text;
|
|
1463
|
+
}
|
|
1464
|
+
const data = responseBody.data;
|
|
1465
|
+
if (Array.isArray(data) && data.length > 0) {
|
|
1466
|
+
const first = data[0];
|
|
1467
|
+
if (Array.isArray(first?.embedding)) {
|
|
1468
|
+
return `[${data.length} embedding(s), ${first.embedding.length} dimensions]`;
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
}
|
|
1472
|
+
if (provider === "gemini") {
|
|
1473
|
+
const candidates = responseBody.candidates;
|
|
1474
|
+
if (Array.isArray(candidates) && candidates.length > 0) {
|
|
1475
|
+
const first = candidates[0];
|
|
1476
|
+
if (first.content && typeof first.content === "object") {
|
|
1477
|
+
const parts = first.content.parts;
|
|
1478
|
+
if (Array.isArray(parts) && parts.length > 0) {
|
|
1479
|
+
return String(parts[0].text ?? "");
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
1483
|
+
const embeddings = responseBody.embeddings;
|
|
1484
|
+
if (Array.isArray(embeddings) && embeddings.length > 0) {
|
|
1485
|
+
const first = embeddings[0];
|
|
1486
|
+
if (Array.isArray(first?.values)) {
|
|
1487
|
+
return `[${embeddings.length} embedding(s), ${first.values.length} dimensions]`;
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
return "";
|
|
1492
|
+
}
|
|
1493
|
+
async function bufferSSEStream(provider, stream) {
|
|
1494
|
+
const decoder = new TextDecoder();
|
|
1495
|
+
const reader = stream.getReader();
|
|
1496
|
+
let raw = "";
|
|
1497
|
+
try {
|
|
1498
|
+
for (; ; ) {
|
|
1499
|
+
const { done, value } = await reader.read();
|
|
1500
|
+
if (done) break;
|
|
1501
|
+
raw += decoder.decode(value, { stream: true });
|
|
1502
|
+
}
|
|
1503
|
+
} finally {
|
|
1504
|
+
reader.releaseLock();
|
|
1505
|
+
}
|
|
1506
|
+
const lines = raw.split("\n");
|
|
1507
|
+
let completion = "";
|
|
1508
|
+
if (provider === "gemini") {
|
|
1509
|
+
for (const line of lines) {
|
|
1510
|
+
const trimmed = line.trim().replace(/^[,\[]/, "").replace(/[,\]]$/, "");
|
|
1511
|
+
if (!trimmed) continue;
|
|
1512
|
+
try {
|
|
1513
|
+
const obj = JSON.parse(trimmed);
|
|
1514
|
+
const candidates = obj.candidates;
|
|
1515
|
+
if (Array.isArray(candidates) && candidates.length > 0) {
|
|
1516
|
+
const first = candidates[0];
|
|
1517
|
+
if (first.content && typeof first.content === "object") {
|
|
1518
|
+
const parts = first.content.parts;
|
|
1519
|
+
if (Array.isArray(parts) && parts.length > 0) {
|
|
1520
|
+
completion += String(parts[0].text ?? "");
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
} catch {
|
|
1525
|
+
}
|
|
1526
|
+
}
|
|
1527
|
+
} else {
|
|
1528
|
+
for (const line of lines) {
|
|
1529
|
+
if (!line.startsWith("data: ")) continue;
|
|
1530
|
+
const data = line.slice(6).trim();
|
|
1531
|
+
if (data === "[DONE]") continue;
|
|
1532
|
+
try {
|
|
1533
|
+
const obj = JSON.parse(data);
|
|
1534
|
+
const choices = obj.choices;
|
|
1535
|
+
if (Array.isArray(choices) && choices.length > 0) {
|
|
1536
|
+
const first = choices[0];
|
|
1537
|
+
if (first.delta && typeof first.delta === "object") {
|
|
1538
|
+
completion += String(first.delta.content ?? "");
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1541
|
+
} catch {
|
|
1542
|
+
}
|
|
1543
|
+
}
|
|
1544
|
+
}
|
|
1545
|
+
return completion;
|
|
1546
|
+
}
|
|
1547
|
+
function synthesizeCompletionJSON(provider, completion) {
|
|
1548
|
+
if (provider === "gemini") {
|
|
1549
|
+
return {
|
|
1550
|
+
candidates: [{ content: { parts: [{ text: completion }], role: "model" }, finishReason: "STOP" }]
|
|
1551
|
+
};
|
|
1552
|
+
}
|
|
1553
|
+
return {
|
|
1554
|
+
id: "replay",
|
|
1555
|
+
object: "chat.completion",
|
|
1556
|
+
choices: [{ index: 0, message: { role: "assistant", content: completion }, finish_reason: "stop" }]
|
|
1557
|
+
};
|
|
1558
|
+
}
|
|
1559
|
+
function synthesizeSSEStream(provider, completion) {
|
|
1560
|
+
const encoder = new TextEncoder();
|
|
1561
|
+
return new ReadableStream({
|
|
1562
|
+
start(ctrl) {
|
|
1563
|
+
if (provider === "gemini") {
|
|
1564
|
+
const chunk = `[{"candidates":[{"content":{"parts":[{"text":${JSON.stringify(completion)}}],"role":"model"},"finishReason":"STOP"}]}]
|
|
1565
|
+
`;
|
|
1566
|
+
ctrl.enqueue(encoder.encode(chunk));
|
|
1567
|
+
} else {
|
|
1568
|
+
const frame1 = `data: ${JSON.stringify({ id: "replay", choices: [{ delta: { content: completion }, index: 0, finish_reason: null }] })}
|
|
1569
|
+
|
|
1570
|
+
`;
|
|
1571
|
+
const frame2 = "data: [DONE]\n\n";
|
|
1572
|
+
ctrl.enqueue(encoder.encode(frame1));
|
|
1573
|
+
ctrl.enqueue(encoder.encode(frame2));
|
|
1574
|
+
}
|
|
1575
|
+
ctrl.close();
|
|
1576
|
+
}
|
|
1577
|
+
});
|
|
1578
|
+
}
|
|
1579
|
+
var originalFetch = null;
|
|
1580
|
+
function installAIInterceptor() {
|
|
1581
|
+
if (originalFetch) return;
|
|
1582
|
+
originalFetch = globalThis.fetch;
|
|
1583
|
+
globalThis.fetch = async function patchedFetch(input, init) {
|
|
1584
|
+
const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
|
|
1585
|
+
const provider = detectProvider2(url);
|
|
1586
|
+
const traceAtCall = getCurrentTrace();
|
|
1587
|
+
if (!provider || !traceAtCall) {
|
|
1588
|
+
return originalFetch(input, init);
|
|
1589
|
+
}
|
|
1590
|
+
let model = "unknown";
|
|
1591
|
+
let prompt = "";
|
|
1592
|
+
let isStreaming = false;
|
|
1593
|
+
try {
|
|
1594
|
+
const rawBody = init?.body;
|
|
1595
|
+
if (rawBody && typeof rawBody === "string") {
|
|
1596
|
+
const body = JSON.parse(rawBody);
|
|
1597
|
+
model = extractModel2(provider, body, url);
|
|
1598
|
+
prompt = extractPrompt2(provider, body);
|
|
1599
|
+
isStreaming = body.stream === true;
|
|
1600
|
+
}
|
|
1601
|
+
} catch {
|
|
1602
|
+
}
|
|
1603
|
+
const ctx = getCaptureContext();
|
|
1604
|
+
if (ctx) {
|
|
1605
|
+
const { recorder, replay } = ctx;
|
|
1606
|
+
const id = recorder.nextId();
|
|
1607
|
+
const start = rawDateNow();
|
|
1608
|
+
if (replay.shouldReplay(id)) {
|
|
1609
|
+
const historicalEvent = replay.getRecordedEvent(id);
|
|
1610
|
+
const historicalInput = historicalEvent?.input;
|
|
1611
|
+
const historicalUrl = typeof historicalInput?.url === "string" ? historicalInput.url : void 0;
|
|
1612
|
+
const historicalProvider = typeof historicalInput?.provider === "string" ? historicalInput.provider : void 0;
|
|
1613
|
+
const isReplayMatch = !!historicalEvent && historicalEvent.type === "ai" && historicalProvider === provider && historicalUrl === url;
|
|
1614
|
+
if (isReplayMatch && historicalEvent) {
|
|
1615
|
+
recorder.record(historicalEvent);
|
|
1616
|
+
const historicalOutput = historicalEvent.output;
|
|
1617
|
+
const completion = historicalOutput ? extractCompletion2(provider, historicalOutput) : "(replayed)";
|
|
1618
|
+
traceAtCall.recordLLMStep({ model, provider, prompt, completion, workflowEventId: id });
|
|
1619
|
+
if (isStreaming) {
|
|
1620
|
+
return new Response(synthesizeSSEStream(provider, completion), {
|
|
1621
|
+
status: 200,
|
|
1622
|
+
headers: { "Content-Type": provider === "gemini" ? "application/json" : "text/event-stream" }
|
|
1623
|
+
});
|
|
1624
|
+
}
|
|
1625
|
+
if (historicalOutput?.streamed === true) {
|
|
1626
|
+
return new Response(JSON.stringify(synthesizeCompletionJSON(provider, completion)), {
|
|
1627
|
+
status: 200,
|
|
1628
|
+
headers: { "Content-Type": "application/json" }
|
|
1629
|
+
});
|
|
1630
|
+
}
|
|
1631
|
+
return new Response(
|
|
1632
|
+
historicalOutput != null ? JSON.stringify(historicalOutput) : null,
|
|
1633
|
+
{ status: 200, headers: { "Content-Type": "application/json" } }
|
|
1634
|
+
);
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
const response2 = await originalFetch(input, init);
|
|
1638
|
+
const durationMs = rawDateNow() - start;
|
|
1639
|
+
if (isStreaming) {
|
|
1640
|
+
if (response2.body) {
|
|
1641
|
+
const [streamForCaller, streamForRecorder] = response2.body.tee();
|
|
1642
|
+
bufferSSEStream(provider, streamForRecorder).then((completion) => {
|
|
1643
|
+
const durationMs2 = rawDateNow() - start;
|
|
1644
|
+
traceAtCall.recordLLMStep({ model, provider, prompt, completion, workflowEventId: id });
|
|
1645
|
+
recorder.record({ id, type: "ai", name: model, input: { url, provider, model, prompt }, output: { streamed: true, completion }, timestamp: start, durationMs: durationMs2 });
|
|
1646
|
+
}).catch(() => {
|
|
1647
|
+
traceAtCall.recordLLMStep({ model, provider, prompt, completion: "(streamed-error)", workflowEventId: id });
|
|
1648
|
+
recorder.record({ id, type: "ai", name: model, input: { url, provider, model, prompt }, output: null, timestamp: start, durationMs: rawDateNow() - start });
|
|
1649
|
+
});
|
|
1650
|
+
return new Response(streamForCaller, {
|
|
1651
|
+
status: response2.status,
|
|
1652
|
+
statusText: response2.statusText,
|
|
1653
|
+
headers: response2.headers
|
|
1654
|
+
});
|
|
1655
|
+
} else {
|
|
1656
|
+
traceAtCall.recordLLMStep({ model, provider, prompt, completion: "(streamed)", workflowEventId: id });
|
|
1657
|
+
recorder.record({ id, type: "ai", name: model, input: { url, provider, model, prompt }, output: null, timestamp: start, durationMs });
|
|
1658
|
+
}
|
|
1659
|
+
} else {
|
|
1660
|
+
try {
|
|
1661
|
+
const cloned = response2.clone();
|
|
1662
|
+
const responseBody = await cloned.json();
|
|
1663
|
+
const completion = extractCompletion2(provider, responseBody);
|
|
1664
|
+
traceAtCall.recordLLMStep({ model, provider, prompt, completion, workflowEventId: id });
|
|
1665
|
+
recorder.record({ id, type: "ai", name: model, input: { url, provider, model, prompt }, output: responseBody, timestamp: start, durationMs });
|
|
1666
|
+
} catch {
|
|
1667
|
+
traceAtCall.recordLLMStep({ model, provider, prompt, completion: "", workflowEventId: id });
|
|
1668
|
+
recorder.record({ id, type: "ai", name: model, input: { url, provider, model, prompt }, output: null, timestamp: start, durationMs });
|
|
1669
|
+
}
|
|
1670
|
+
}
|
|
1671
|
+
return response2;
|
|
1672
|
+
}
|
|
1673
|
+
const response = await originalFetch(input, init);
|
|
1674
|
+
if (isStreaming && response.body) {
|
|
1675
|
+
const [streamForCaller, streamForRecorder] = response.body.tee();
|
|
1676
|
+
bufferSSEStream(provider, streamForRecorder).then((completion) => {
|
|
1677
|
+
traceAtCall.recordLLMStep({ model, provider, prompt, completion });
|
|
1678
|
+
}).catch(() => {
|
|
1679
|
+
traceAtCall.recordLLMStep({ model, provider, prompt, completion: "(streamed-error)" });
|
|
1680
|
+
});
|
|
1681
|
+
return new Response(streamForCaller, { status: response.status, statusText: response.statusText, headers: response.headers });
|
|
1682
|
+
} else if (!isStreaming) {
|
|
1683
|
+
try {
|
|
1684
|
+
const cloned = response.clone();
|
|
1685
|
+
const responseBody = await cloned.json();
|
|
1686
|
+
const completion = extractCompletion2(provider, responseBody);
|
|
1687
|
+
traceAtCall.recordLLMStep({ model, provider, prompt, completion });
|
|
1688
|
+
} catch {
|
|
1689
|
+
traceAtCall.recordLLMStep({ model, provider, prompt, completion: "" });
|
|
1690
|
+
}
|
|
1691
|
+
}
|
|
1692
|
+
return response;
|
|
1693
|
+
};
|
|
1694
|
+
}
|
|
1695
|
+
function uninstallAIInterceptor() {
|
|
1696
|
+
if (originalFetch) {
|
|
1697
|
+
globalThis.fetch = originalFetch;
|
|
1698
|
+
originalFetch = null;
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
|
|
1702
|
+
// src/index.ts
|
|
1703
|
+
init_tracing();
|
|
1704
|
+
|
|
1705
|
+
// src/internals/conditional-recorder.ts
|
|
1706
|
+
var isWorkerContext = () => {
|
|
1707
|
+
if (typeof globalThis === "undefined") {
|
|
1708
|
+
return false;
|
|
1709
|
+
}
|
|
1710
|
+
return globalThis.__ELASTICDASH_WORKER__ === true;
|
|
1711
|
+
};
|
|
1712
|
+
var recordToolCallFn = null;
|
|
1713
|
+
var getRecordToolCall = async () => {
|
|
1714
|
+
if (!isWorkerContext()) {
|
|
1715
|
+
return null;
|
|
1716
|
+
}
|
|
1717
|
+
if (recordToolCallFn !== null) {
|
|
1718
|
+
return recordToolCallFn;
|
|
1719
|
+
}
|
|
1720
|
+
try {
|
|
1721
|
+
const { recordToolCall: recordToolCall2 } = await Promise.resolve().then(() => (init_tracing(), tracing_exports));
|
|
1722
|
+
recordToolCallFn = recordToolCall2;
|
|
1723
|
+
return recordToolCallFn;
|
|
1724
|
+
} catch (err) {
|
|
1725
|
+
console.warn("Failed to load recordToolCall from tracing module:", err);
|
|
1726
|
+
return null;
|
|
1727
|
+
}
|
|
1728
|
+
};
|
|
1729
|
+
var safeRecordToolCall = async (name, input, output) => {
|
|
1730
|
+
const recorder = await getRecordToolCall();
|
|
1731
|
+
if (recorder) {
|
|
1732
|
+
recorder(name, input, output);
|
|
1733
|
+
}
|
|
1734
|
+
};
|
|
1735
|
+
var isWorker = () => {
|
|
1736
|
+
return isWorkerContext();
|
|
1737
|
+
};
|
|
1738
|
+
|
|
1739
|
+
// src/index.ts
|
|
1740
|
+
init_recorder();
|
|
1741
|
+
|
|
1742
|
+
// src/capture/replay.ts
|
|
1743
|
+
var ReplayController = class {
|
|
1744
|
+
constructor(replayMode, checkpoint, history) {
|
|
1745
|
+
this.replayMode = replayMode;
|
|
1746
|
+
this.checkpoint = checkpoint;
|
|
1747
|
+
this.history = history;
|
|
1748
|
+
this.historyMap = new Map(history.map((e) => [e.id, e]));
|
|
1749
|
+
this.sideEffectMap = new Map(
|
|
1750
|
+
history.filter((e) => e.type === "side_effect").map((e) => [e.id, e])
|
|
1751
|
+
);
|
|
1752
|
+
}
|
|
1753
|
+
historyMap;
|
|
1754
|
+
/** Side effects keyed by their assigned sideEffectId, independent of main event IDs */
|
|
1755
|
+
sideEffectMap;
|
|
1756
|
+
shouldReplay(eventId) {
|
|
1757
|
+
return this.replayMode && eventId <= this.checkpoint;
|
|
1758
|
+
}
|
|
1759
|
+
getRecordedEvent(eventId) {
|
|
1760
|
+
return this.historyMap.get(eventId);
|
|
1761
|
+
}
|
|
1762
|
+
getRecordedResult(eventId) {
|
|
1763
|
+
return this.historyMap.get(eventId)?.output;
|
|
1764
|
+
}
|
|
1765
|
+
/** Returns true if the side effect with this sideEffectId has a recorded value to replay */
|
|
1766
|
+
shouldReplaySideEffect(n) {
|
|
1767
|
+
return this.replayMode && this.sideEffectMap.has(n);
|
|
1768
|
+
}
|
|
1769
|
+
getSideEffectResult(n) {
|
|
1770
|
+
return this.sideEffectMap.get(n)?.output;
|
|
1771
|
+
}
|
|
1772
|
+
getRecordedSideEffectEvent(n) {
|
|
1773
|
+
return this.sideEffectMap.get(n);
|
|
1774
|
+
}
|
|
1775
|
+
shouldReplaySideEffectOfType(n, expectedName) {
|
|
1776
|
+
if (!this.replayMode) return false;
|
|
1777
|
+
const event = this.sideEffectMap.get(n);
|
|
1778
|
+
return !!event && event.type === "side_effect" && event.name === expectedName;
|
|
1779
|
+
}
|
|
1780
|
+
getSideEffectResultOfType(n, expectedName) {
|
|
1781
|
+
const event = this.sideEffectMap.get(n);
|
|
1782
|
+
if (!event || event.type !== "side_effect" || event.name !== expectedName) return void 0;
|
|
1783
|
+
return event.output;
|
|
1784
|
+
}
|
|
1785
|
+
};
|
|
1786
|
+
|
|
1787
|
+
// src/interceptors/tool.ts
|
|
1788
|
+
init_recorder();
|
|
1789
|
+
init_context();
|
|
1790
|
+
init_side_effects();
|
|
1791
|
+
var TOOL_WRAPPER_ACTIVE_KEY2 = "__elasticdash_tool_wrapper_active__";
|
|
1792
|
+
function toTraceArgs(input) {
|
|
1793
|
+
if (input && typeof input === "object" && !Array.isArray(input)) {
|
|
1794
|
+
return input;
|
|
1795
|
+
}
|
|
1796
|
+
if (input === void 0) return void 0;
|
|
1797
|
+
return { value: input };
|
|
1798
|
+
}
|
|
1799
|
+
function isReadableStream(v) {
|
|
1800
|
+
return typeof v === "object" && v !== null && typeof v.getReader === "function" && typeof v.tee === "function";
|
|
1801
|
+
}
|
|
1802
|
+
function isAsyncIterable(v) {
|
|
1803
|
+
return typeof v === "object" && v !== null && Symbol.asyncIterator in v;
|
|
1804
|
+
}
|
|
1805
|
+
async function bufferReadableStream(stream) {
|
|
1806
|
+
const decoder = new TextDecoder();
|
|
1807
|
+
const reader = stream.getReader();
|
|
1808
|
+
let raw = "";
|
|
1809
|
+
try {
|
|
1810
|
+
for (; ; ) {
|
|
1811
|
+
const { done, value } = await reader.read();
|
|
1812
|
+
if (done) break;
|
|
1813
|
+
raw += decoder.decode(value, { stream: true });
|
|
1814
|
+
}
|
|
1815
|
+
} finally {
|
|
1816
|
+
reader.releaseLock();
|
|
1817
|
+
}
|
|
1818
|
+
return raw;
|
|
1819
|
+
}
|
|
1820
|
+
function reconstructStream(raw) {
|
|
1821
|
+
const encoder = new TextEncoder();
|
|
1822
|
+
return new ReadableStream({
|
|
1823
|
+
start(ctrl) {
|
|
1824
|
+
ctrl.enqueue(encoder.encode(raw));
|
|
1825
|
+
ctrl.close();
|
|
1826
|
+
}
|
|
1827
|
+
});
|
|
1828
|
+
}
|
|
1829
|
+
function wrapAsyncIterable(source, onComplete) {
|
|
1830
|
+
return {
|
|
1831
|
+
[Symbol.asyncIterator]() {
|
|
1832
|
+
const iter = source[Symbol.asyncIterator]();
|
|
1833
|
+
const collected = [];
|
|
1834
|
+
return {
|
|
1835
|
+
async next() {
|
|
1836
|
+
const result = await iter.next();
|
|
1837
|
+
if (!result.done) {
|
|
1838
|
+
collected.push(result.value);
|
|
1839
|
+
} else {
|
|
1840
|
+
onComplete(collected);
|
|
1841
|
+
}
|
|
1842
|
+
return result;
|
|
1843
|
+
},
|
|
1844
|
+
async return(value) {
|
|
1845
|
+
onComplete(collected);
|
|
1846
|
+
return iter.return ? iter.return(value) : { done: true, value: void 0 };
|
|
1847
|
+
}
|
|
1848
|
+
};
|
|
1849
|
+
}
|
|
1850
|
+
};
|
|
1851
|
+
}
|
|
1852
|
+
function wrapTool(name, fn) {
|
|
1853
|
+
return async (...args) => {
|
|
1854
|
+
const ctx = getCaptureContext();
|
|
1855
|
+
if (!ctx) return fn(...args);
|
|
1856
|
+
const trace = getCurrentTrace();
|
|
1857
|
+
const { recorder, replay } = ctx;
|
|
1858
|
+
const id = recorder.nextId();
|
|
1859
|
+
const input = args.length === 1 ? args[0] : args;
|
|
1860
|
+
if (replay.shouldReplay(id)) {
|
|
1861
|
+
const historical = replay.getRecordedEvent(id);
|
|
1862
|
+
if (historical) recorder.record(historical);
|
|
1863
|
+
if (historical?.streamed === true) {
|
|
1864
|
+
const raw = typeof historical.streamRaw === "string" ? historical.streamRaw : "";
|
|
1865
|
+
const stream = reconstructStream(raw);
|
|
1866
|
+
if (trace && typeof trace.recordToolCall === "function") {
|
|
1867
|
+
trace.recordToolCall({ name, args: toTraceArgs(input), result: stream, workflowEventId: id });
|
|
1868
|
+
}
|
|
1869
|
+
return stream;
|
|
1870
|
+
}
|
|
1871
|
+
const replayed = replay.getRecordedResult(id);
|
|
1872
|
+
if (trace && typeof trace.recordToolCall === "function") {
|
|
1873
|
+
trace.recordToolCall({ name, args: toTraceArgs(input), result: replayed, workflowEventId: id });
|
|
1874
|
+
}
|
|
1875
|
+
return replayed;
|
|
1876
|
+
}
|
|
1877
|
+
const g3 = globalThis;
|
|
1878
|
+
const prev = g3[TOOL_WRAPPER_ACTIVE_KEY2];
|
|
1879
|
+
g3[TOOL_WRAPPER_ACTIVE_KEY2] = true;
|
|
1880
|
+
const start = rawDateNow();
|
|
1881
|
+
try {
|
|
1882
|
+
const output = await fn(...args);
|
|
1883
|
+
if (isReadableStream(output)) {
|
|
1884
|
+
const [streamForCaller, streamForRecorder] = output.tee();
|
|
1885
|
+
bufferReadableStream(streamForRecorder).then((rawText) => {
|
|
1886
|
+
recorder.record({ id, type: "tool", name, input, output: null, streamed: true, streamRaw: rawText, timestamp: start, durationMs: rawDateNow() - start });
|
|
1887
|
+
}).catch(() => {
|
|
1888
|
+
recorder.record({ id, type: "tool", name, input, output: null, streamed: true, streamRaw: "", timestamp: start, durationMs: rawDateNow() - start });
|
|
1889
|
+
});
|
|
1890
|
+
const result = streamForCaller;
|
|
1891
|
+
if (trace && typeof trace.recordToolCall === "function") {
|
|
1892
|
+
trace.recordToolCall({ name, args: toTraceArgs(input), result, workflowEventId: id });
|
|
1893
|
+
}
|
|
1894
|
+
return result;
|
|
1895
|
+
}
|
|
1896
|
+
if (isAsyncIterable(output)) {
|
|
1897
|
+
const wrapped = wrapAsyncIterable(output, (chunks) => {
|
|
1898
|
+
const rawText = chunks.map((c) => typeof c === "string" ? c : JSON.stringify(c)).join("");
|
|
1899
|
+
recorder.record({ id, type: "tool", name, input, output: null, streamed: true, streamRaw: rawText, timestamp: start, durationMs: rawDateNow() - start });
|
|
1900
|
+
});
|
|
1901
|
+
if (trace && typeof trace.recordToolCall === "function") {
|
|
1902
|
+
trace.recordToolCall({ name, args: toTraceArgs(input), result: wrapped, workflowEventId: id });
|
|
1903
|
+
}
|
|
1904
|
+
return wrapped;
|
|
1905
|
+
}
|
|
1906
|
+
recorder.record({
|
|
1907
|
+
id,
|
|
1908
|
+
type: "tool",
|
|
1909
|
+
name,
|
|
1910
|
+
input,
|
|
1911
|
+
output,
|
|
1912
|
+
timestamp: start,
|
|
1913
|
+
durationMs: rawDateNow() - start
|
|
1914
|
+
});
|
|
1915
|
+
if (trace && typeof trace.recordToolCall === "function") {
|
|
1916
|
+
trace.recordToolCall({ name, args: toTraceArgs(input), result: output, workflowEventId: id });
|
|
1917
|
+
}
|
|
1918
|
+
return output;
|
|
1919
|
+
} catch (e) {
|
|
1920
|
+
recorder.record({
|
|
1921
|
+
id,
|
|
1922
|
+
type: "tool",
|
|
1923
|
+
name,
|
|
1924
|
+
input,
|
|
1925
|
+
output: { error: String(e) },
|
|
1926
|
+
timestamp: start,
|
|
1927
|
+
durationMs: rawDateNow() - start
|
|
1928
|
+
});
|
|
1929
|
+
if (trace && typeof trace.recordToolCall === "function") {
|
|
1930
|
+
trace.recordToolCall({ name, args: toTraceArgs(input), result: { error: String(e) }, workflowEventId: id });
|
|
1931
|
+
}
|
|
1932
|
+
throw e;
|
|
1933
|
+
} finally {
|
|
1934
|
+
if (prev === void 0) delete g3[TOOL_WRAPPER_ACTIVE_KEY2];
|
|
1935
|
+
else g3[TOOL_WRAPPER_ACTIVE_KEY2] = prev;
|
|
1936
|
+
}
|
|
1937
|
+
};
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
// src/interceptors/workflow-ai.ts
|
|
1941
|
+
init_recorder();
|
|
1942
|
+
function wrapAI(modelName, callFn) {
|
|
1943
|
+
return async (...args) => {
|
|
1944
|
+
const ctx = getCaptureContext();
|
|
1945
|
+
if (!ctx) return callFn(...args);
|
|
1946
|
+
const { recorder, replay } = ctx;
|
|
1947
|
+
const id = recorder.nextId();
|
|
1948
|
+
if (replay.shouldReplay(id)) {
|
|
1949
|
+
return replay.getRecordedResult(id);
|
|
1950
|
+
}
|
|
1951
|
+
const start = Date.now();
|
|
1952
|
+
const output = await callFn(...args);
|
|
1953
|
+
recorder.record({
|
|
1954
|
+
id,
|
|
1955
|
+
type: "ai",
|
|
1956
|
+
name: modelName,
|
|
1957
|
+
input: args.length === 1 ? args[0] : args,
|
|
1958
|
+
output,
|
|
1959
|
+
timestamp: start,
|
|
1960
|
+
durationMs: Date.now() - start
|
|
1961
|
+
});
|
|
1962
|
+
return output;
|
|
1963
|
+
};
|
|
1964
|
+
}
|
|
1965
|
+
|
|
1966
|
+
// src/interceptors/db.ts
|
|
1967
|
+
init_recorder();
|
|
1968
|
+
function wrapDB(client, methodNames, label) {
|
|
1969
|
+
const prefix = label ?? (client.constructor?.name ?? "db");
|
|
1970
|
+
for (const method of methodNames) {
|
|
1971
|
+
const original = client[method];
|
|
1972
|
+
if (typeof original !== "function") continue;
|
|
1973
|
+
client[method] = async (...args) => {
|
|
1974
|
+
const ctx = getCaptureContext();
|
|
1975
|
+
if (!ctx) return original.apply(client, args);
|
|
1976
|
+
const { recorder, replay } = ctx;
|
|
1977
|
+
const id = recorder.nextId();
|
|
1978
|
+
const name = `${prefix}.${method}`;
|
|
1979
|
+
if (replay.shouldReplay(id)) {
|
|
1980
|
+
return replay.getRecordedResult(id);
|
|
1981
|
+
}
|
|
1982
|
+
const start = Date.now();
|
|
1983
|
+
const output = await original.apply(client, args);
|
|
1984
|
+
recorder.record({
|
|
1985
|
+
id,
|
|
1986
|
+
type: "db",
|
|
1987
|
+
name,
|
|
1988
|
+
input: args.length === 1 ? args[0] : args,
|
|
1989
|
+
output,
|
|
1990
|
+
timestamp: start,
|
|
1991
|
+
durationMs: Date.now() - start
|
|
1992
|
+
});
|
|
1993
|
+
return output;
|
|
1994
|
+
};
|
|
1995
|
+
}
|
|
1996
|
+
return client;
|
|
1997
|
+
}
|
|
1998
|
+
function wrapPgClient(client) {
|
|
1999
|
+
return wrapDB(client, ["query"], "pg");
|
|
2000
|
+
}
|
|
2001
|
+
function wrapKnex(knex) {
|
|
2002
|
+
return wrapDB(knex, ["raw"], "knex");
|
|
2003
|
+
}
|
|
2004
|
+
function wrapMongoCollection(collection) {
|
|
2005
|
+
return wrapDB(
|
|
2006
|
+
collection,
|
|
2007
|
+
["find", "findOne", "insertOne", "updateOne", "deleteOne"],
|
|
2008
|
+
"mongo"
|
|
2009
|
+
);
|
|
2010
|
+
}
|
|
2011
|
+
function wrapRedisClient(client) {
|
|
2012
|
+
return wrapDB(
|
|
2013
|
+
client,
|
|
2014
|
+
["get", "set", "del", "hget", "hset"],
|
|
2015
|
+
"redis"
|
|
2016
|
+
);
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
// src/interceptors/db-auto.ts
|
|
2020
|
+
init_recorder();
|
|
2021
|
+
var appliedPatches = [];
|
|
2022
|
+
function wrapProtoMethod(proto, method, eventName) {
|
|
2023
|
+
const p = proto;
|
|
2024
|
+
if (typeof p[method] !== "function") return;
|
|
2025
|
+
const original = p[method];
|
|
2026
|
+
appliedPatches.push({ proto: p, method, original });
|
|
2027
|
+
p[method] = function(...args) {
|
|
2028
|
+
if (args.length > 0 && typeof args[args.length - 1] === "function") {
|
|
2029
|
+
return original.apply(this, args);
|
|
2030
|
+
}
|
|
2031
|
+
const ctx = getCaptureContext();
|
|
2032
|
+
if (!ctx) return original.apply(this, args);
|
|
2033
|
+
const { recorder, replay } = ctx;
|
|
2034
|
+
const id = recorder.nextId();
|
|
2035
|
+
if (replay.shouldReplay(id)) {
|
|
2036
|
+
const historicalEvent = replay.getRecordedEvent(id);
|
|
2037
|
+
if (historicalEvent) recorder.record(historicalEvent);
|
|
2038
|
+
return Promise.resolve(replay.getRecordedResult(id));
|
|
2039
|
+
}
|
|
2040
|
+
const start = Date.now();
|
|
2041
|
+
const input = args.length === 1 ? args[0] : args;
|
|
2042
|
+
let result;
|
|
2043
|
+
try {
|
|
2044
|
+
result = original.apply(this, args);
|
|
2045
|
+
} catch (err) {
|
|
2046
|
+
recorder.record({
|
|
2047
|
+
id,
|
|
2048
|
+
type: "db",
|
|
2049
|
+
name: eventName,
|
|
2050
|
+
input,
|
|
2051
|
+
output: { error: String(err) },
|
|
2052
|
+
timestamp: start,
|
|
2053
|
+
durationMs: Date.now() - start
|
|
2054
|
+
});
|
|
2055
|
+
throw err;
|
|
2056
|
+
}
|
|
2057
|
+
if (result != null && typeof result.then === "function") {
|
|
2058
|
+
return result.then((output) => {
|
|
2059
|
+
recorder.record({
|
|
2060
|
+
id,
|
|
2061
|
+
type: "db",
|
|
2062
|
+
name: eventName,
|
|
2063
|
+
input,
|
|
2064
|
+
output,
|
|
2065
|
+
timestamp: start,
|
|
2066
|
+
durationMs: Date.now() - start
|
|
2067
|
+
});
|
|
2068
|
+
return output;
|
|
2069
|
+
}).catch((err) => {
|
|
2070
|
+
recorder.record({
|
|
2071
|
+
id,
|
|
2072
|
+
type: "db",
|
|
2073
|
+
name: eventName,
|
|
2074
|
+
input,
|
|
2075
|
+
output: { error: String(err) },
|
|
2076
|
+
timestamp: start,
|
|
2077
|
+
durationMs: Date.now() - start
|
|
2078
|
+
});
|
|
2079
|
+
throw err;
|
|
2080
|
+
});
|
|
2081
|
+
}
|
|
2082
|
+
recorder.record({
|
|
2083
|
+
id,
|
|
2084
|
+
type: "db",
|
|
2085
|
+
name: eventName,
|
|
2086
|
+
input,
|
|
2087
|
+
output: result,
|
|
2088
|
+
timestamp: start,
|
|
2089
|
+
durationMs: Date.now() - start
|
|
2090
|
+
});
|
|
2091
|
+
return result;
|
|
2092
|
+
};
|
|
2093
|
+
}
|
|
2094
|
+
async function tryPatchPg() {
|
|
2095
|
+
const pgMod = await import("pg");
|
|
2096
|
+
const pg = pgMod.default ?? pgMod;
|
|
2097
|
+
const Client = pg.Client;
|
|
2098
|
+
if (Client?.prototype) {
|
|
2099
|
+
wrapProtoMethod(Client.prototype, "query", "pg.query");
|
|
2100
|
+
}
|
|
2101
|
+
}
|
|
2102
|
+
async function tryPatchMysql2() {
|
|
2103
|
+
const mod = await import("mysql2/promise");
|
|
2104
|
+
const mysql2 = mod.default ?? mod;
|
|
2105
|
+
const Connection = mysql2.Connection;
|
|
2106
|
+
if (Connection?.prototype) {
|
|
2107
|
+
wrapProtoMethod(Connection.prototype, "query", "mysql2.query");
|
|
2108
|
+
wrapProtoMethod(Connection.prototype, "execute", "mysql2.execute");
|
|
2109
|
+
}
|
|
2110
|
+
}
|
|
2111
|
+
async function tryPatchMongodb() {
|
|
2112
|
+
const mongMod = await import("mongodb");
|
|
2113
|
+
const Collection = mongMod.Collection ?? mongMod.default?.Collection;
|
|
2114
|
+
if (Collection?.prototype) {
|
|
2115
|
+
for (const method of ["find", "findOne", "insertOne", "updateOne", "deleteOne", "aggregate"]) {
|
|
2116
|
+
wrapProtoMethod(Collection.prototype, method, `mongodb.${method}`);
|
|
2117
|
+
}
|
|
2118
|
+
}
|
|
2119
|
+
}
|
|
2120
|
+
async function tryPatchIoredis() {
|
|
2121
|
+
const mod = await import("ioredis");
|
|
2122
|
+
const Redis = mod.default ?? mod;
|
|
2123
|
+
if (Redis?.prototype) {
|
|
2124
|
+
wrapProtoMethod(Redis.prototype, "call", "redis.call");
|
|
2125
|
+
}
|
|
2126
|
+
}
|
|
2127
|
+
async function installDBAutoInterceptor() {
|
|
2128
|
+
await Promise.allSettled([
|
|
2129
|
+
tryPatchPg(),
|
|
2130
|
+
tryPatchMysql2(),
|
|
2131
|
+
tryPatchMongodb(),
|
|
2132
|
+
tryPatchIoredis()
|
|
2133
|
+
]);
|
|
2134
|
+
}
|
|
2135
|
+
function uninstallDBAutoInterceptor() {
|
|
2136
|
+
for (const { proto, method, original } of appliedPatches) {
|
|
2137
|
+
proto[method] = original;
|
|
2138
|
+
}
|
|
2139
|
+
appliedPatches.length = 0;
|
|
2140
|
+
}
|
|
2141
|
+
|
|
2142
|
+
// src/interceptors/http.ts
|
|
2143
|
+
init_recorder();
|
|
2144
|
+
var AI_URL_PATTERNS = [
|
|
2145
|
+
/https?:\/\/api\.openai\.com\/v1\/((chat\/)?completions|embeddings)/,
|
|
2146
|
+
/https?:\/\/generativelanguage\.googleapis\.com\/.*\/models\/[^/:]+:(generateContent|streamGenerateContent)/,
|
|
2147
|
+
/https?:\/\/api\.x\.ai\/v1\/(chat\/)?completions/,
|
|
2148
|
+
/https?:\/\/api\.moonshot\.ai\/v1\/(chat\/)?completions/
|
|
2149
|
+
];
|
|
2150
|
+
function isAIProviderUrl(url) {
|
|
2151
|
+
return AI_URL_PATTERNS.some((p) => p.test(url));
|
|
2152
|
+
}
|
|
2153
|
+
function parseQuery(url) {
|
|
2154
|
+
try {
|
|
2155
|
+
const { searchParams } = new URL(url);
|
|
2156
|
+
if (searchParams.size === 0) return void 0;
|
|
2157
|
+
return Object.fromEntries(searchParams.entries());
|
|
2158
|
+
} catch {
|
|
2159
|
+
const qIdx = url.indexOf("?");
|
|
2160
|
+
if (qIdx === -1) return void 0;
|
|
2161
|
+
try {
|
|
2162
|
+
const params = new URLSearchParams(url.slice(qIdx + 1));
|
|
2163
|
+
if (![...params].length) return void 0;
|
|
2164
|
+
return Object.fromEntries(params.entries());
|
|
2165
|
+
} catch {
|
|
2166
|
+
return void 0;
|
|
2167
|
+
}
|
|
2168
|
+
}
|
|
2169
|
+
}
|
|
2170
|
+
function parseBody(body) {
|
|
2171
|
+
if (body == null) return void 0;
|
|
2172
|
+
if (typeof body === "string") {
|
|
2173
|
+
try {
|
|
2174
|
+
return JSON.parse(body);
|
|
2175
|
+
} catch {
|
|
2176
|
+
return body;
|
|
2177
|
+
}
|
|
2178
|
+
}
|
|
2179
|
+
if (body instanceof URLSearchParams) {
|
|
2180
|
+
return Object.fromEntries(body.entries());
|
|
2181
|
+
}
|
|
2182
|
+
return "[binary]";
|
|
2183
|
+
}
|
|
2184
|
+
function normalizeHeaders(headers) {
|
|
2185
|
+
if (!headers) return void 0;
|
|
2186
|
+
if (headers instanceof Headers) {
|
|
2187
|
+
const obj = {};
|
|
2188
|
+
headers.forEach((v, k) => {
|
|
2189
|
+
obj[k] = v;
|
|
2190
|
+
});
|
|
2191
|
+
return obj;
|
|
2192
|
+
}
|
|
2193
|
+
if (Array.isArray(headers)) return Object.fromEntries(headers);
|
|
2194
|
+
return headers;
|
|
2195
|
+
}
|
|
2196
|
+
function pickReplayResponseHeaders(headers) {
|
|
2197
|
+
if (!headers) return { "Content-Type": "application/json" };
|
|
2198
|
+
const out = {};
|
|
2199
|
+
for (const [key, value] of Object.entries(headers)) {
|
|
2200
|
+
if (typeof value === "string") out[key] = value;
|
|
2201
|
+
}
|
|
2202
|
+
if (!out["Content-Type"]) out["Content-Type"] = "application/json";
|
|
2203
|
+
return out;
|
|
2204
|
+
}
|
|
2205
|
+
function isStreamingContentType(headers) {
|
|
2206
|
+
const ct = headers.get("content-type") ?? "";
|
|
2207
|
+
return ct.includes("text/event-stream") || ct.includes("application/x-ndjson") || ct.includes("application/stream+json") || ct.includes("application/jsonl");
|
|
2208
|
+
}
|
|
2209
|
+
async function bufferStream(stream) {
|
|
2210
|
+
const decoder = new TextDecoder();
|
|
2211
|
+
const reader = stream.getReader();
|
|
2212
|
+
let raw = "";
|
|
2213
|
+
try {
|
|
2214
|
+
for (; ; ) {
|
|
2215
|
+
const { done, value } = await reader.read();
|
|
2216
|
+
if (done) break;
|
|
2217
|
+
raw += decoder.decode(value, { stream: true });
|
|
2218
|
+
}
|
|
2219
|
+
} finally {
|
|
2220
|
+
reader.releaseLock();
|
|
2221
|
+
}
|
|
2222
|
+
return raw;
|
|
2223
|
+
}
|
|
2224
|
+
function reconstructStream2(raw) {
|
|
2225
|
+
const encoder = new TextEncoder();
|
|
2226
|
+
return new ReadableStream({
|
|
2227
|
+
start(ctrl) {
|
|
2228
|
+
ctrl.enqueue(encoder.encode(raw));
|
|
2229
|
+
ctrl.close();
|
|
2230
|
+
}
|
|
2231
|
+
});
|
|
2232
|
+
}
|
|
2233
|
+
var originalFetch2;
|
|
2234
|
+
function interceptFetch() {
|
|
2235
|
+
if (originalFetch2) return;
|
|
2236
|
+
originalFetch2 = globalThis.fetch;
|
|
2237
|
+
globalThis.fetch = async (input, init) => {
|
|
2238
|
+
const ctx = getCaptureContext();
|
|
2239
|
+
if (!ctx) return originalFetch2(input, init);
|
|
2240
|
+
const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
|
|
2241
|
+
const method = (init?.method ?? (input instanceof Request ? input.method : "GET")).toUpperCase();
|
|
2242
|
+
const rawHeaders = init?.headers ?? (input instanceof Request ? input.headers : void 0);
|
|
2243
|
+
const rawBody = init?.body ?? (input instanceof Request ? input.body : void 0);
|
|
2244
|
+
if (isAIProviderUrl(url)) {
|
|
2245
|
+
return originalFetch2(input, init);
|
|
2246
|
+
}
|
|
2247
|
+
const { recorder, replay } = ctx;
|
|
2248
|
+
const id = recorder.nextId();
|
|
2249
|
+
if (replay.shouldReplay(id)) {
|
|
2250
|
+
const historicalEvent = replay.getRecordedEvent(id);
|
|
2251
|
+
const historicalInput = historicalEvent?.input;
|
|
2252
|
+
const historicalMethod = typeof historicalInput?.method === "string" ? historicalInput.method.toUpperCase() : "GET";
|
|
2253
|
+
const historicalUrl = typeof historicalInput?.url === "string" ? historicalInput.url : void 0;
|
|
2254
|
+
const isReplayMatch = !!historicalEvent && historicalEvent.type === "http" && historicalEvent.name === "fetch" && historicalMethod === method && historicalUrl === url;
|
|
2255
|
+
if (isReplayMatch && historicalEvent) {
|
|
2256
|
+
recorder.record(historicalEvent);
|
|
2257
|
+
const replayMeta = historicalInput?.__elasticdashResponse ?? {};
|
|
2258
|
+
const replayStatus = typeof replayMeta.status === "number" ? replayMeta.status : 200;
|
|
2259
|
+
const replayStatusText = typeof replayMeta.statusText === "string" ? replayMeta.statusText : "";
|
|
2260
|
+
const replayHeaders = pickReplayResponseHeaders(
|
|
2261
|
+
replayMeta.headers && typeof replayMeta.headers === "object" ? replayMeta.headers : void 0
|
|
2262
|
+
);
|
|
2263
|
+
if (historicalEvent.streamed === true) {
|
|
2264
|
+
const raw = typeof historicalEvent.streamRaw === "string" ? historicalEvent.streamRaw : "";
|
|
2265
|
+
return new Response(reconstructStream2(raw), {
|
|
2266
|
+
status: replayStatus,
|
|
2267
|
+
statusText: replayStatusText,
|
|
2268
|
+
headers: replayHeaders
|
|
2269
|
+
});
|
|
2270
|
+
}
|
|
2271
|
+
const historicalOutput = replay.getRecordedResult(id);
|
|
2272
|
+
const body2 = historicalOutput != null ? JSON.stringify(historicalOutput) : null;
|
|
2273
|
+
return new Response(body2, {
|
|
2274
|
+
status: replayStatus,
|
|
2275
|
+
statusText: replayStatusText,
|
|
2276
|
+
headers: replayHeaders
|
|
2277
|
+
});
|
|
2278
|
+
}
|
|
2279
|
+
}
|
|
2280
|
+
const query = parseQuery(url);
|
|
2281
|
+
const body = parseBody(rawBody);
|
|
2282
|
+
const headers = normalizeHeaders(rawHeaders);
|
|
2283
|
+
const start = Date.now();
|
|
2284
|
+
const res = await originalFetch2(input, init);
|
|
2285
|
+
const responseHeadersObj = {};
|
|
2286
|
+
res.headers.forEach((v, k) => {
|
|
2287
|
+
responseHeadersObj[k] = v;
|
|
2288
|
+
});
|
|
2289
|
+
const elasticdashResponse = {
|
|
2290
|
+
status: res.status,
|
|
2291
|
+
statusText: res.statusText,
|
|
2292
|
+
headers: responseHeadersObj,
|
|
2293
|
+
url: res.url
|
|
2294
|
+
};
|
|
2295
|
+
const baseInput = {
|
|
2296
|
+
url,
|
|
2297
|
+
method,
|
|
2298
|
+
...query ? { query } : {},
|
|
2299
|
+
...body !== void 0 ? { body } : {},
|
|
2300
|
+
...headers && Object.keys(headers).length > 0 ? { headers } : {},
|
|
2301
|
+
__elasticdashResponse: elasticdashResponse
|
|
2302
|
+
};
|
|
2303
|
+
if (isStreamingContentType(res.headers) && res.body) {
|
|
2304
|
+
const [streamForCaller, streamForRecorder] = res.body.tee();
|
|
2305
|
+
bufferStream(streamForRecorder).then((rawText) => {
|
|
2306
|
+
recorder.record({
|
|
2307
|
+
id,
|
|
2308
|
+
type: "http",
|
|
2309
|
+
name: "fetch",
|
|
2310
|
+
input: baseInput,
|
|
2311
|
+
output: null,
|
|
2312
|
+
streamed: true,
|
|
2313
|
+
streamRaw: rawText,
|
|
2314
|
+
timestamp: start,
|
|
2315
|
+
durationMs: Date.now() - start
|
|
2316
|
+
});
|
|
2317
|
+
}).catch(() => {
|
|
2318
|
+
recorder.record({
|
|
2319
|
+
id,
|
|
2320
|
+
type: "http",
|
|
2321
|
+
name: "fetch",
|
|
2322
|
+
input: baseInput,
|
|
2323
|
+
output: null,
|
|
2324
|
+
streamed: true,
|
|
2325
|
+
streamRaw: "",
|
|
2326
|
+
timestamp: start,
|
|
2327
|
+
durationMs: Date.now() - start
|
|
2328
|
+
});
|
|
2329
|
+
});
|
|
2330
|
+
return new Response(streamForCaller, {
|
|
2331
|
+
status: res.status,
|
|
2332
|
+
statusText: res.statusText,
|
|
2333
|
+
headers: res.headers
|
|
2334
|
+
});
|
|
2335
|
+
}
|
|
2336
|
+
let output = null;
|
|
2337
|
+
try {
|
|
2338
|
+
output = await res.clone().json();
|
|
2339
|
+
} catch {
|
|
2340
|
+
}
|
|
2341
|
+
recorder.record({
|
|
2342
|
+
id,
|
|
2343
|
+
type: "http",
|
|
2344
|
+
name: "fetch",
|
|
2345
|
+
input: baseInput,
|
|
2346
|
+
output,
|
|
2347
|
+
timestamp: start,
|
|
2348
|
+
durationMs: Date.now() - start
|
|
2349
|
+
});
|
|
2350
|
+
return res;
|
|
2351
|
+
};
|
|
2352
|
+
}
|
|
2353
|
+
function restoreFetch() {
|
|
2354
|
+
if (originalFetch2) {
|
|
2355
|
+
globalThis.fetch = originalFetch2;
|
|
2356
|
+
originalFetch2 = void 0;
|
|
2357
|
+
}
|
|
2358
|
+
}
|
|
2359
|
+
|
|
2360
|
+
// src/index.ts
|
|
2361
|
+
init_side_effects();
|
|
2362
|
+
|
|
2363
|
+
// src/workflow-runner.ts
|
|
2364
|
+
init_recorder();
|
|
2365
|
+
init_side_effects();
|
|
2366
|
+
async function runWorkflow(workflowFn, options = {}) {
|
|
2367
|
+
const {
|
|
2368
|
+
replayMode = false,
|
|
2369
|
+
checkpoint = 0,
|
|
2370
|
+
history = [],
|
|
2371
|
+
interceptHttp = true,
|
|
2372
|
+
interceptSideEffects = true
|
|
2373
|
+
} = options;
|
|
2374
|
+
const recorder = new TraceRecorder();
|
|
2375
|
+
const replay = new ReplayController(replayMode, checkpoint, history);
|
|
2376
|
+
setCaptureContext({ recorder, replay });
|
|
2377
|
+
if (interceptHttp) interceptFetch();
|
|
2378
|
+
if (interceptSideEffects) {
|
|
2379
|
+
interceptRandom();
|
|
2380
|
+
interceptDateNow();
|
|
2381
|
+
}
|
|
2382
|
+
try {
|
|
2383
|
+
const result = await workflowFn();
|
|
2384
|
+
return { result, trace: recorder.toTrace() };
|
|
2385
|
+
} finally {
|
|
2386
|
+
if (interceptHttp) restoreFetch();
|
|
2387
|
+
if (interceptSideEffects) {
|
|
2388
|
+
restoreRandom();
|
|
2389
|
+
restoreDateNow();
|
|
2390
|
+
}
|
|
2391
|
+
setCaptureContext(void 0);
|
|
2392
|
+
}
|
|
2393
|
+
}
|
|
2394
|
+
|
|
2395
|
+
// src/core/agent-state.ts
|
|
2396
|
+
function serializeAgentState(plan, trace) {
|
|
2397
|
+
const resumeFromTaskIndex = plan.tasks.findIndex(
|
|
2398
|
+
(t) => t.status !== "completed"
|
|
2399
|
+
);
|
|
2400
|
+
return {
|
|
2401
|
+
plan: JSON.parse(JSON.stringify(plan)),
|
|
2402
|
+
trace: JSON.parse(JSON.stringify(trace)),
|
|
2403
|
+
resumeFromTaskIndex: resumeFromTaskIndex === -1 ? plan.tasks.length : resumeFromTaskIndex
|
|
2404
|
+
};
|
|
2405
|
+
}
|
|
2406
|
+
function deserializeAgentState(raw) {
|
|
2407
|
+
if (!raw || typeof raw !== "object") {
|
|
2408
|
+
throw new Error("AgentState must be a non-null object");
|
|
2409
|
+
}
|
|
2410
|
+
const obj = raw;
|
|
2411
|
+
if (!obj.plan || typeof obj.plan !== "object") {
|
|
2412
|
+
throw new Error("AgentState.plan is required");
|
|
2413
|
+
}
|
|
2414
|
+
const plan = obj.plan;
|
|
2415
|
+
if (!Array.isArray(plan.tasks)) {
|
|
2416
|
+
throw new Error("AgentState.plan.tasks must be an array");
|
|
2417
|
+
}
|
|
2418
|
+
if (typeof plan.id !== "string") {
|
|
2419
|
+
throw new Error("AgentState.plan.id must be a string");
|
|
2420
|
+
}
|
|
2421
|
+
const trace = Array.isArray(obj.trace) ? obj.trace : [];
|
|
2422
|
+
const resumeFromTaskIndex = typeof obj.resumeFromTaskIndex === "number" ? obj.resumeFromTaskIndex : 0;
|
|
2423
|
+
for (let i = 0; i < resumeFromTaskIndex; i++) {
|
|
2424
|
+
const task = plan.tasks[i];
|
|
2425
|
+
if (!task) continue;
|
|
2426
|
+
if (task.status !== "completed") {
|
|
2427
|
+
throw new Error(
|
|
2428
|
+
`Task at index ${i} (id="${task.id}") has status "${task.status}" but must be "completed" before resumeFromTaskIndex=${resumeFromTaskIndex}`
|
|
2429
|
+
);
|
|
2430
|
+
}
|
|
2431
|
+
if (task.output === void 0) {
|
|
2432
|
+
throw new Error(
|
|
2433
|
+
`Task at index ${i} (id="${task.id}") is completed but has no output. Cannot resume safely.`
|
|
2434
|
+
);
|
|
2435
|
+
}
|
|
2436
|
+
}
|
|
2437
|
+
return { plan, trace, resumeFromTaskIndex };
|
|
2438
|
+
}
|
|
2439
|
+
function extractTaskOutputs(plan) {
|
|
2440
|
+
const outputs = {};
|
|
2441
|
+
for (const task of plan.tasks) {
|
|
2442
|
+
if (task.status === "completed" && task.output !== void 0) {
|
|
2443
|
+
outputs[task.id] = task.output;
|
|
2444
|
+
}
|
|
2445
|
+
}
|
|
2446
|
+
return outputs;
|
|
2447
|
+
}
|
|
2448
|
+
function resolveTaskInput(input, previousOutputs) {
|
|
2449
|
+
if (input === null || input === void 0) return input;
|
|
2450
|
+
if (Array.isArray(input)) {
|
|
2451
|
+
return input.map((item) => resolveTaskInput(item, previousOutputs));
|
|
2452
|
+
}
|
|
2453
|
+
if (typeof input === "object") {
|
|
2454
|
+
const obj = input;
|
|
2455
|
+
if (typeof obj["$ref"] === "string") {
|
|
2456
|
+
return resolveRef(obj["$ref"], previousOutputs);
|
|
2457
|
+
}
|
|
2458
|
+
const resolved = {};
|
|
2459
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
2460
|
+
resolved[k] = resolveTaskInput(v, previousOutputs);
|
|
2461
|
+
}
|
|
2462
|
+
return resolved;
|
|
2463
|
+
}
|
|
2464
|
+
return input;
|
|
2465
|
+
}
|
|
2466
|
+
function resolveRef(ref, previousOutputs) {
|
|
2467
|
+
const parts = ref.split(".");
|
|
2468
|
+
const taskId = parts[0];
|
|
2469
|
+
const pathParts = parts.slice(1);
|
|
2470
|
+
let current = previousOutputs[taskId];
|
|
2471
|
+
for (const part of pathParts) {
|
|
2472
|
+
if (part === "output") continue;
|
|
2473
|
+
if (current === null || current === void 0) return void 0;
|
|
2474
|
+
current = current[part];
|
|
2475
|
+
}
|
|
2476
|
+
return current;
|
|
2477
|
+
}
|
|
2478
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
2479
|
+
0 && (module.exports = {
|
|
2480
|
+
ReplayController,
|
|
2481
|
+
TraceRecorder,
|
|
2482
|
+
afterAll,
|
|
2483
|
+
afterEach,
|
|
2484
|
+
aiTest,
|
|
2485
|
+
beforeAll,
|
|
2486
|
+
beforeEach,
|
|
2487
|
+
clearRegistry,
|
|
2488
|
+
createTraceHandle,
|
|
2489
|
+
deserializeAgentState,
|
|
2490
|
+
expect,
|
|
2491
|
+
extractTaskOutputs,
|
|
2492
|
+
fetchCapturedTrace,
|
|
2493
|
+
getCaptureContext,
|
|
2494
|
+
getCurrentTrace,
|
|
2495
|
+
getRegistry,
|
|
2496
|
+
installAIInterceptor,
|
|
2497
|
+
installDBAutoInterceptor,
|
|
2498
|
+
interceptDateNow,
|
|
2499
|
+
interceptFetch,
|
|
2500
|
+
interceptRandom,
|
|
2501
|
+
isWorker,
|
|
2502
|
+
recordToolCall,
|
|
2503
|
+
registerMatchers,
|
|
2504
|
+
reportResults,
|
|
2505
|
+
resolveTaskInput,
|
|
2506
|
+
restoreDateNow,
|
|
2507
|
+
restoreFetch,
|
|
2508
|
+
restoreRandom,
|
|
2509
|
+
runFiles,
|
|
2510
|
+
runWorkflow,
|
|
2511
|
+
safeRecordToolCall,
|
|
2512
|
+
serializeAgentState,
|
|
2513
|
+
setCaptureContext,
|
|
2514
|
+
setCurrentTrace,
|
|
2515
|
+
startLLMProxy,
|
|
2516
|
+
startTraceSession,
|
|
2517
|
+
uninstallAIInterceptor,
|
|
2518
|
+
uninstallDBAutoInterceptor,
|
|
2519
|
+
wrapAI,
|
|
2520
|
+
wrapDB,
|
|
2521
|
+
wrapKnex,
|
|
2522
|
+
wrapMongoCollection,
|
|
2523
|
+
wrapPgClient,
|
|
2524
|
+
wrapRedisClient,
|
|
2525
|
+
wrapTool
|
|
2526
|
+
});
|