braintrust 3.23.0 → 3.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dev/dist/index.d.mts +6 -0
- package/dev/dist/index.d.ts +6 -0
- package/dev/dist/index.js +902 -588
- package/dev/dist/index.mjs +320 -6
- package/dist/apply-auto-instrumentation.js +211 -189
- package/dist/apply-auto-instrumentation.mjs +23 -1
- package/dist/auto-instrumentations/bundler/esbuild.cjs +40 -0
- package/dist/auto-instrumentations/bundler/esbuild.mjs +2 -2
- package/dist/auto-instrumentations/bundler/next.cjs +40 -0
- package/dist/auto-instrumentations/bundler/next.mjs +3 -3
- package/dist/auto-instrumentations/bundler/rollup.cjs +40 -0
- package/dist/auto-instrumentations/bundler/rollup.mjs +2 -2
- package/dist/auto-instrumentations/bundler/vite.cjs +40 -0
- package/dist/auto-instrumentations/bundler/vite.d.mts +1 -1
- package/dist/auto-instrumentations/bundler/vite.d.ts +1 -1
- package/dist/auto-instrumentations/bundler/vite.mjs +2 -2
- package/dist/auto-instrumentations/bundler/webpack-loader.cjs +40 -0
- package/dist/auto-instrumentations/bundler/webpack.cjs +40 -0
- package/dist/auto-instrumentations/bundler/webpack.mjs +3 -3
- package/dist/auto-instrumentations/{chunk-KIMLYPRW.mjs → chunk-BURMPO7L.mjs} +1 -1
- package/dist/auto-instrumentations/{chunk-YXLNSAMJ.mjs → chunk-F43DNLPD.mjs} +40 -0
- package/dist/auto-instrumentations/{chunk-EXY7QCJD.mjs → chunk-GSZHTAQW.mjs} +1 -1
- package/dist/auto-instrumentations/hook.mjs +150 -1
- package/dist/auto-instrumentations/index.cjs +40 -0
- package/dist/auto-instrumentations/index.mjs +1 -1
- package/dist/browser.d.mts +131 -9
- package/dist/browser.d.ts +131 -9
- package/dist/browser.js +703 -67
- package/dist/browser.mjs +703 -67
- package/dist/{chunk-CDIKAHDZ.js → chunk-7AUY2XWX.js} +21 -1
- package/dist/{chunk-36IPYKMG.mjs → chunk-7F6GCRHH.mjs} +20 -0
- package/dist/{chunk-FZWPFCVE.js → chunk-SU6EHKJV.js} +1085 -801
- package/dist/{chunk-IXL4PMY4.mjs → chunk-XE5FS7QY.mjs} +291 -7
- package/dist/cli.js +328 -10
- package/dist/edge-light.js +703 -67
- package/dist/edge-light.mjs +703 -67
- package/dist/index.d.mts +131 -9
- package/dist/index.d.ts +131 -9
- package/dist/index.js +891 -532
- package/dist/index.mjs +423 -64
- package/dist/instrumentation/index.d.mts +130 -9
- package/dist/instrumentation/index.d.ts +130 -9
- package/dist/instrumentation/index.js +601 -67
- package/dist/instrumentation/index.mjs +601 -67
- package/dist/vitest-evals-reporter.js +17 -16
- package/dist/vitest-evals-reporter.mjs +3 -2
- package/dist/workerd.js +703 -67
- package/dist/workerd.mjs +703 -67
- package/package.json +3 -3
- package/util/dist/index.js +4 -0
- package/util/dist/index.mjs +4 -0
package/dist/index.mjs
CHANGED
|
@@ -139,7 +139,7 @@ import {
|
|
|
139
139
|
wrapMastraAgent,
|
|
140
140
|
wrapTraced,
|
|
141
141
|
zodToJsonSchema
|
|
142
|
-
} from "./chunk-
|
|
142
|
+
} from "./chunk-XE5FS7QY.mjs";
|
|
143
143
|
import {
|
|
144
144
|
__export,
|
|
145
145
|
aiSDKChannels,
|
|
@@ -153,6 +153,7 @@ import {
|
|
|
153
153
|
googleADKChannels,
|
|
154
154
|
googleGenAIChannels,
|
|
155
155
|
groqChannels,
|
|
156
|
+
harnessAgentChannels,
|
|
156
157
|
huggingFaceChannels,
|
|
157
158
|
isomorph_default,
|
|
158
159
|
langSmithChannels,
|
|
@@ -163,7 +164,7 @@ import {
|
|
|
163
164
|
openRouterChannels,
|
|
164
165
|
piCodingAgentChannels,
|
|
165
166
|
strandsAgentSDKChannels
|
|
166
|
-
} from "./chunk-
|
|
167
|
+
} from "./chunk-7F6GCRHH.mjs";
|
|
167
168
|
|
|
168
169
|
// src/exports.ts
|
|
169
170
|
var exports_exports = {};
|
|
@@ -905,6 +906,26 @@ function wrapAISDK(aiSDK, options = {}) {
|
|
|
905
906
|
}
|
|
906
907
|
});
|
|
907
908
|
}
|
|
909
|
+
function isHarnessAgentInstance(instance) {
|
|
910
|
+
try {
|
|
911
|
+
const visited = /* @__PURE__ */ new Set();
|
|
912
|
+
let prototype = Object.getPrototypeOf(instance);
|
|
913
|
+
while (prototype !== null && !visited.has(prototype)) {
|
|
914
|
+
visited.add(prototype);
|
|
915
|
+
const constructor = Object.getOwnPropertyDescriptor(
|
|
916
|
+
prototype,
|
|
917
|
+
"constructor"
|
|
918
|
+
)?.value;
|
|
919
|
+
const constructorName = typeof constructor === "function" ? Object.getOwnPropertyDescriptor(constructor, "name")?.value : void 0;
|
|
920
|
+
if (constructorName === "HarnessAgent") {
|
|
921
|
+
return true;
|
|
922
|
+
}
|
|
923
|
+
prototype = Object.getPrototypeOf(prototype);
|
|
924
|
+
}
|
|
925
|
+
} catch {
|
|
926
|
+
}
|
|
927
|
+
return false;
|
|
928
|
+
}
|
|
908
929
|
var wrapAgentClass = (AgentClass, options = {}) => {
|
|
909
930
|
const typedAgentClass = AgentClass;
|
|
910
931
|
return new Proxy(typedAgentClass, {
|
|
@@ -914,9 +935,46 @@ var wrapAgentClass = (AgentClass, options = {}) => {
|
|
|
914
935
|
args,
|
|
915
936
|
newTarget
|
|
916
937
|
);
|
|
938
|
+
const harnessAgent = isHarnessAgentInstance(instance) ? instance : null;
|
|
917
939
|
return new Proxy(instance, {
|
|
918
940
|
get(instanceTarget, prop, instanceReceiver) {
|
|
919
941
|
const original = Reflect.get(instanceTarget, prop, instanceTarget);
|
|
942
|
+
if (harnessAgent && typeof original === "function") {
|
|
943
|
+
switch (prop) {
|
|
944
|
+
case "generate":
|
|
945
|
+
return wrapHarnessAgentGenerate(
|
|
946
|
+
original,
|
|
947
|
+
harnessAgent,
|
|
948
|
+
harnessAgentChannels.generate,
|
|
949
|
+
"HarnessAgent.generate",
|
|
950
|
+
options
|
|
951
|
+
);
|
|
952
|
+
case "stream":
|
|
953
|
+
return wrapHarnessAgentStream(
|
|
954
|
+
original,
|
|
955
|
+
harnessAgent,
|
|
956
|
+
harnessAgentChannels.stream,
|
|
957
|
+
"HarnessAgent.stream",
|
|
958
|
+
options
|
|
959
|
+
);
|
|
960
|
+
case "continueGenerate":
|
|
961
|
+
return wrapHarnessAgentGenerate(
|
|
962
|
+
original,
|
|
963
|
+
harnessAgent,
|
|
964
|
+
harnessAgentChannels.continueGenerate,
|
|
965
|
+
"HarnessAgent.continueGenerate",
|
|
966
|
+
options
|
|
967
|
+
);
|
|
968
|
+
case "continueStream":
|
|
969
|
+
return wrapHarnessAgentStream(
|
|
970
|
+
original,
|
|
971
|
+
harnessAgent,
|
|
972
|
+
harnessAgentChannels.continueStream,
|
|
973
|
+
"HarnessAgent.continueStream",
|
|
974
|
+
options
|
|
975
|
+
);
|
|
976
|
+
}
|
|
977
|
+
}
|
|
920
978
|
if (prop === "generate" && typeof original === "function" && instanceTarget.constructor.name !== "WorkflowAgent") {
|
|
921
979
|
return wrapAgentGenerate(original, instanceTarget, options);
|
|
922
980
|
}
|
|
@@ -932,6 +990,26 @@ var wrapAgentClass = (AgentClass, options = {}) => {
|
|
|
932
990
|
}
|
|
933
991
|
});
|
|
934
992
|
};
|
|
993
|
+
var wrapHarnessAgentGenerate = (generate, instance, channel, name, options) => makeGenerateTextWrapper(
|
|
994
|
+
channel,
|
|
995
|
+
name,
|
|
996
|
+
generate.bind(instance),
|
|
997
|
+
{
|
|
998
|
+
self: instance,
|
|
999
|
+
spanType: "task" /* TASK */
|
|
1000
|
+
},
|
|
1001
|
+
options
|
|
1002
|
+
);
|
|
1003
|
+
var wrapHarnessAgentStream = (stream, instance, channel, name, options) => makeStreamWrapper(
|
|
1004
|
+
channel,
|
|
1005
|
+
name,
|
|
1006
|
+
stream.bind(instance),
|
|
1007
|
+
{
|
|
1008
|
+
self: instance,
|
|
1009
|
+
spanType: "task" /* TASK */
|
|
1010
|
+
},
|
|
1011
|
+
options
|
|
1012
|
+
);
|
|
935
1013
|
var wrapAgentGenerate = (generate, instance, options = {}) => {
|
|
936
1014
|
const defaultName = `${instance.constructor.name}.generate`;
|
|
937
1015
|
return async (params) => makeGenerateTextWrapper(
|
|
@@ -1158,6 +1236,7 @@ var BraintrustLanguageModelWrapper = class {
|
|
|
1158
1236
|
this.supportsUrl = (url) => this.model.supportsUrl(url);
|
|
1159
1237
|
}
|
|
1160
1238
|
}
|
|
1239
|
+
model;
|
|
1161
1240
|
supportsUrl;
|
|
1162
1241
|
get specificationVersion() {
|
|
1163
1242
|
return this.model.specificationVersion;
|
|
@@ -1740,6 +1819,7 @@ function BraintrustMiddleware(config = {}) {
|
|
|
1740
1819
|
var EVE_TRACE_STATE_KEY = "braintrust.eve.tracing";
|
|
1741
1820
|
var MAX_EVE_CACHE_ENTRIES = 1e4;
|
|
1742
1821
|
var MAX_STORED_LLM_INPUTS = 100;
|
|
1822
|
+
var MAX_STORED_REASONING_BLOCKS = 100;
|
|
1743
1823
|
var MAX_STORED_SPAN_REFERENCES = 1e4;
|
|
1744
1824
|
var MAX_STORED_STEP_STARTS = 1e4;
|
|
1745
1825
|
function braintrustEveHook(options) {
|
|
@@ -1778,6 +1858,7 @@ var ResumedEveSpan = class {
|
|
|
1778
1858
|
this.reference = reference;
|
|
1779
1859
|
this.endTime = reference.endTime;
|
|
1780
1860
|
}
|
|
1861
|
+
reference;
|
|
1781
1862
|
endTime;
|
|
1782
1863
|
get rootSpanId() {
|
|
1783
1864
|
return this.reference.rootSpanId;
|
|
@@ -1810,6 +1891,7 @@ var EveBridge = class {
|
|
|
1810
1891
|
constructor(state) {
|
|
1811
1892
|
this.state = state;
|
|
1812
1893
|
}
|
|
1894
|
+
state;
|
|
1813
1895
|
eventQueuesBySession = /* @__PURE__ */ new Map();
|
|
1814
1896
|
completedToolKeys = new LRUCache({
|
|
1815
1897
|
max: MAX_EVE_CACHE_ENTRIES
|
|
@@ -1988,6 +2070,9 @@ var EveBridge = class {
|
|
|
1988
2070
|
case "step.started":
|
|
1989
2071
|
await this.handleStepStarted(event, ctx, hookMetadata);
|
|
1990
2072
|
return true;
|
|
2073
|
+
case "reasoning.completed":
|
|
2074
|
+
this.handleReasoningCompleted(event, ctx);
|
|
2075
|
+
return true;
|
|
1991
2076
|
case "message.completed":
|
|
1992
2077
|
this.handleMessageCompleted(event, ctx);
|
|
1993
2078
|
return true;
|
|
@@ -2079,6 +2164,7 @@ var EveBridge = class {
|
|
|
2079
2164
|
key,
|
|
2080
2165
|
metadata,
|
|
2081
2166
|
metrics: {},
|
|
2167
|
+
sessionId,
|
|
2082
2168
|
span,
|
|
2083
2169
|
stepsByIndex: /* @__PURE__ */ new Map(),
|
|
2084
2170
|
turnId: event.data.turnId
|
|
@@ -2109,6 +2195,12 @@ var EveBridge = class {
|
|
|
2109
2195
|
const endTime = eventTime(event);
|
|
2110
2196
|
existing.span.end(endTime === void 0 ? void 0 : { endTime });
|
|
2111
2197
|
this.markStepEnded(event.data.turnId, event.data.stepIndex);
|
|
2198
|
+
clearStoredEveReasoning(
|
|
2199
|
+
this.state,
|
|
2200
|
+
sessionId,
|
|
2201
|
+
event.data.turnId,
|
|
2202
|
+
event.data.stepIndex
|
|
2203
|
+
);
|
|
2112
2204
|
}
|
|
2113
2205
|
const stepOrdinal = this.stepOrdinal(event);
|
|
2114
2206
|
const metadata = { ...turn.metadata };
|
|
@@ -2118,6 +2210,13 @@ var EveBridge = class {
|
|
|
2118
2210
|
event.data.turnId,
|
|
2119
2211
|
event.data.stepIndex
|
|
2120
2212
|
);
|
|
2213
|
+
const reasoning = readStoredEveReasoning(
|
|
2214
|
+
this.state,
|
|
2215
|
+
sessionId,
|
|
2216
|
+
event.data.turnId,
|
|
2217
|
+
event.data.stepIndex
|
|
2218
|
+
);
|
|
2219
|
+
const output = mergeEveReasoning(void 0, reasoning);
|
|
2121
2220
|
const { rowId: eventId, spanId } = await generateEveIds(
|
|
2122
2221
|
"step",
|
|
2123
2222
|
sessionId,
|
|
@@ -2135,32 +2234,52 @@ var EveBridge = class {
|
|
|
2135
2234
|
spanId,
|
|
2136
2235
|
startTime: eventTime(event)
|
|
2137
2236
|
});
|
|
2138
|
-
span.log({
|
|
2237
|
+
span.log({
|
|
2238
|
+
...input !== void 0 ? { input } : {},
|
|
2239
|
+
metadata
|
|
2240
|
+
});
|
|
2139
2241
|
turn.stepsByIndex.set(event.data.stepIndex, {
|
|
2140
2242
|
...input !== void 0 ? { input } : {},
|
|
2141
2243
|
metadata,
|
|
2142
2244
|
metrics: {},
|
|
2245
|
+
...output !== void 0 ? { output } : {},
|
|
2246
|
+
reasoning,
|
|
2143
2247
|
span
|
|
2144
2248
|
});
|
|
2145
2249
|
}
|
|
2250
|
+
handleReasoningCompleted(event, ctx) {
|
|
2251
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2252
|
+
if (!sessionId) {
|
|
2253
|
+
return;
|
|
2254
|
+
}
|
|
2255
|
+
const reasoning = storeEveReasoning(this.state, sessionId, event);
|
|
2256
|
+
const step = this.stepForEvent(event, ctx);
|
|
2257
|
+
if (step) {
|
|
2258
|
+
step.reasoning = reasoning;
|
|
2259
|
+
step.output = mergeEveReasoning(step.output, reasoning);
|
|
2260
|
+
}
|
|
2261
|
+
}
|
|
2146
2262
|
handleMessageCompleted(event, ctx) {
|
|
2147
2263
|
const step = this.stepForEvent(event, ctx);
|
|
2148
2264
|
if (!step) {
|
|
2149
2265
|
return;
|
|
2150
2266
|
}
|
|
2151
|
-
const existingMessage =
|
|
2267
|
+
const existingMessage = eveOutputMessage(step.output);
|
|
2152
2268
|
const existingToolCalls = isObject(existingMessage) ? existingMessage.tool_calls : void 0;
|
|
2153
|
-
step.output =
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2269
|
+
step.output = mergeEveReasoning(
|
|
2270
|
+
[
|
|
2271
|
+
{
|
|
2272
|
+
finish_reason: normalizedFinishReason(event.data.finishReason),
|
|
2273
|
+
index: 0,
|
|
2274
|
+
message: {
|
|
2275
|
+
content: event.data.message,
|
|
2276
|
+
role: "assistant",
|
|
2277
|
+
...Array.isArray(existingToolCalls) ? { tool_calls: existingToolCalls } : {}
|
|
2278
|
+
}
|
|
2161
2279
|
}
|
|
2162
|
-
|
|
2163
|
-
|
|
2280
|
+
],
|
|
2281
|
+
step.reasoning
|
|
2282
|
+
);
|
|
2164
2283
|
const turn = this.turnForEvent(event, ctx);
|
|
2165
2284
|
if (turn && event.data.finishReason !== "tool-calls") {
|
|
2166
2285
|
turn.output = event.data.message;
|
|
@@ -2169,16 +2288,19 @@ var EveBridge = class {
|
|
|
2169
2288
|
handleResultCompleted(event, ctx) {
|
|
2170
2289
|
const step = this.stepForEvent(event, ctx);
|
|
2171
2290
|
if (step) {
|
|
2172
|
-
step.output =
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2291
|
+
step.output = mergeEveReasoning(
|
|
2292
|
+
[
|
|
2293
|
+
{
|
|
2294
|
+
finish_reason: "stop",
|
|
2295
|
+
index: 0,
|
|
2296
|
+
message: {
|
|
2297
|
+
content: event.data.result,
|
|
2298
|
+
role: "assistant"
|
|
2299
|
+
}
|
|
2179
2300
|
}
|
|
2180
|
-
|
|
2181
|
-
|
|
2301
|
+
],
|
|
2302
|
+
step.reasoning
|
|
2303
|
+
);
|
|
2182
2304
|
}
|
|
2183
2305
|
const turn = this.turnForEvent(event, ctx);
|
|
2184
2306
|
if (turn) {
|
|
@@ -2228,18 +2350,20 @@ var EveBridge = class {
|
|
|
2228
2350
|
type: "function"
|
|
2229
2351
|
});
|
|
2230
2352
|
}
|
|
2231
|
-
step.output =
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2353
|
+
step.output = mergeEveReasoning(
|
|
2354
|
+
[
|
|
2355
|
+
{
|
|
2356
|
+
finish_reason: "tool_calls",
|
|
2357
|
+
index: 0,
|
|
2358
|
+
message: {
|
|
2359
|
+
content: null,
|
|
2360
|
+
role: "assistant",
|
|
2361
|
+
tool_calls: [...toolCallsById.values()]
|
|
2362
|
+
}
|
|
2239
2363
|
}
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2364
|
+
],
|
|
2365
|
+
step.reasoning
|
|
2366
|
+
);
|
|
2243
2367
|
}
|
|
2244
2368
|
async handleActionResult(event, ctx, hookMetadata) {
|
|
2245
2369
|
if (isToolResult(event.data.result)) {
|
|
@@ -2417,6 +2541,7 @@ var EveBridge = class {
|
|
|
2417
2541
|
...costUsd !== void 0 ? { estimated_cost: costUsd } : {}
|
|
2418
2542
|
};
|
|
2419
2543
|
step.metrics = { ...step.metrics, ...metrics };
|
|
2544
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2420
2545
|
if (Array.isArray(step.output) && isObject(step.output[0])) {
|
|
2421
2546
|
const finishReason = step.output[0].finish_reason;
|
|
2422
2547
|
if (typeof finishReason !== "string") {
|
|
@@ -2441,6 +2566,14 @@ var EveBridge = class {
|
|
|
2441
2566
|
turn.stepsByIndex.delete(event.data.stepIndex);
|
|
2442
2567
|
}
|
|
2443
2568
|
this.markStepEnded(event.data.turnId, event.data.stepIndex);
|
|
2569
|
+
if (sessionId) {
|
|
2570
|
+
clearStoredEveReasoning(
|
|
2571
|
+
this.state,
|
|
2572
|
+
sessionId,
|
|
2573
|
+
event.data.turnId,
|
|
2574
|
+
event.data.stepIndex
|
|
2575
|
+
);
|
|
2576
|
+
}
|
|
2444
2577
|
}
|
|
2445
2578
|
handleStepFailed(event, ctx) {
|
|
2446
2579
|
const step = this.stepForEvent(event, ctx);
|
|
@@ -2458,6 +2591,15 @@ var EveBridge = class {
|
|
|
2458
2591
|
const turn = this.turnForEvent(event, ctx);
|
|
2459
2592
|
turn?.stepsByIndex.delete(event.data.stepIndex);
|
|
2460
2593
|
this.markStepEnded(event.data.turnId, event.data.stepIndex);
|
|
2594
|
+
const sessionId = sessionIdFromContext(ctx);
|
|
2595
|
+
if (sessionId) {
|
|
2596
|
+
clearStoredEveReasoning(
|
|
2597
|
+
this.state,
|
|
2598
|
+
sessionId,
|
|
2599
|
+
event.data.turnId,
|
|
2600
|
+
event.data.stepIndex
|
|
2601
|
+
);
|
|
2602
|
+
}
|
|
2461
2603
|
}
|
|
2462
2604
|
handleTurnCompleted(event, ctx) {
|
|
2463
2605
|
const turn = this.turnForEvent(event, ctx);
|
|
@@ -2555,6 +2697,7 @@ var EveBridge = class {
|
|
|
2555
2697
|
key,
|
|
2556
2698
|
metadata,
|
|
2557
2699
|
metrics: {},
|
|
2700
|
+
sessionId,
|
|
2558
2701
|
span,
|
|
2559
2702
|
stepsByIndex: /* @__PURE__ */ new Map(),
|
|
2560
2703
|
turnId: event.data.turnId
|
|
@@ -2817,6 +2960,9 @@ var EveBridge = class {
|
|
|
2817
2960
|
const normalized = normalizeEveTraceState(current);
|
|
2818
2961
|
return {
|
|
2819
2962
|
...normalized,
|
|
2963
|
+
reasoningBlocks: normalized.reasoningBlocks.filter(
|
|
2964
|
+
(entry) => !entry.key.startsWith(`${turn.sessionId}\0${turn.turnId}\0`)
|
|
2965
|
+
),
|
|
2820
2966
|
stepStarts: normalized.stepStarts.filter(
|
|
2821
2967
|
(entry) => entry.turnId !== turn.turnId
|
|
2822
2968
|
)
|
|
@@ -2856,6 +3002,7 @@ function emptyEveTraceState() {
|
|
|
2856
3002
|
return {
|
|
2857
3003
|
llmInputs: [],
|
|
2858
3004
|
metadata: {},
|
|
3005
|
+
reasoningBlocks: [],
|
|
2859
3006
|
spanReferences: [],
|
|
2860
3007
|
stepStarts: []
|
|
2861
3008
|
};
|
|
@@ -2906,6 +3053,21 @@ function normalizeEveTraceState(state) {
|
|
|
2906
3053
|
const input = entry["input"];
|
|
2907
3054
|
return typeof key === "string" && isCapturedModelInput(input) ? [{ input, key }] : [];
|
|
2908
3055
|
}).slice(-MAX_STORED_LLM_INPUTS) : [];
|
|
3056
|
+
const reasoningBlocks = Array.isArray(state["reasoningBlocks"]) ? state["reasoningBlocks"].flatMap((entry) => {
|
|
3057
|
+
if (!isObject(entry)) {
|
|
3058
|
+
return [];
|
|
3059
|
+
}
|
|
3060
|
+
const content = entry["content"];
|
|
3061
|
+
const eventAt = entry["eventAt"];
|
|
3062
|
+
const key = entry["key"];
|
|
3063
|
+
return typeof content === "string" && (eventAt === void 0 || typeof eventAt === "string") && typeof key === "string" ? [
|
|
3064
|
+
{
|
|
3065
|
+
content,
|
|
3066
|
+
...typeof eventAt === "string" ? { eventAt } : {},
|
|
3067
|
+
key
|
|
3068
|
+
}
|
|
3069
|
+
] : [];
|
|
3070
|
+
}).slice(-MAX_STORED_REASONING_BLOCKS) : [];
|
|
2909
3071
|
const stepStarts = Array.isArray(state["stepStarts"]) ? state["stepStarts"].flatMap((entry) => {
|
|
2910
3072
|
if (!isObject(entry)) {
|
|
2911
3073
|
return [];
|
|
@@ -2916,7 +3078,13 @@ function normalizeEveTraceState(state) {
|
|
|
2916
3078
|
const turnId = entry["turnId"];
|
|
2917
3079
|
return typeof ordinal === "number" && Number.isInteger(ordinal) && ordinal >= 0 && typeof open === "boolean" && typeof stepIndex === "number" && Number.isInteger(stepIndex) && typeof turnId === "string" ? [{ open, ordinal, stepIndex, turnId }] : [];
|
|
2918
3080
|
}).slice(-MAX_STORED_STEP_STARTS) : [];
|
|
2919
|
-
return {
|
|
3081
|
+
return {
|
|
3082
|
+
llmInputs,
|
|
3083
|
+
metadata: { ...metadata },
|
|
3084
|
+
reasoningBlocks,
|
|
3085
|
+
spanReferences,
|
|
3086
|
+
stepStarts
|
|
3087
|
+
};
|
|
2920
3088
|
}
|
|
2921
3089
|
function readEveTraceState(state) {
|
|
2922
3090
|
try {
|
|
@@ -2925,23 +3093,85 @@ function readEveTraceState(state) {
|
|
|
2925
3093
|
return emptyEveTraceState();
|
|
2926
3094
|
}
|
|
2927
3095
|
}
|
|
2928
|
-
function
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
3096
|
+
function storeEveReasoning(state, sessionId, event) {
|
|
3097
|
+
const eventAt = event.meta?.at;
|
|
3098
|
+
const key = llmInputKey(sessionId, event.data.turnId, event.data.stepIndex);
|
|
3099
|
+
let stored = [];
|
|
3100
|
+
state.update((current) => {
|
|
3101
|
+
const normalized = normalizeEveTraceState(current);
|
|
3102
|
+
const alreadyStored = normalized.reasoningBlocks.some(
|
|
3103
|
+
(entry) => entry.content === event.data.reasoning && entry.eventAt === eventAt && entry.key === key
|
|
3104
|
+
);
|
|
3105
|
+
const reasoningBlocks = alreadyStored ? normalized.reasoningBlocks : [
|
|
3106
|
+
...normalized.reasoningBlocks,
|
|
3107
|
+
{
|
|
3108
|
+
content: event.data.reasoning,
|
|
3109
|
+
...eventAt ? { eventAt } : {},
|
|
3110
|
+
key
|
|
3111
|
+
}
|
|
3112
|
+
].slice(-MAX_STORED_REASONING_BLOCKS);
|
|
3113
|
+
stored = reasoningBlocks.flatMap(
|
|
3114
|
+
(entry) => entry.key === key ? [
|
|
3115
|
+
{
|
|
3116
|
+
content: entry.content,
|
|
3117
|
+
...entry.eventAt ? { eventAt: entry.eventAt } : {}
|
|
3118
|
+
}
|
|
3119
|
+
] : []
|
|
3120
|
+
);
|
|
3121
|
+
return alreadyStored ? normalized : { ...normalized, reasoningBlocks };
|
|
3122
|
+
});
|
|
3123
|
+
return stored;
|
|
3124
|
+
}
|
|
3125
|
+
function readStoredEveReasoning(state, sessionId, turnId, stepIndex) {
|
|
3126
|
+
const key = llmInputKey(sessionId, turnId, stepIndex);
|
|
3127
|
+
return readEveTraceState(state).reasoningBlocks.flatMap(
|
|
3128
|
+
(entry) => entry.key === key ? [
|
|
3129
|
+
{
|
|
3130
|
+
content: entry.content,
|
|
3131
|
+
...entry.eventAt ? { eventAt: entry.eventAt } : {}
|
|
3132
|
+
}
|
|
3133
|
+
] : []
|
|
3134
|
+
);
|
|
3135
|
+
}
|
|
3136
|
+
function clearStoredEveReasoning(state, sessionId, turnId, stepIndex) {
|
|
3137
|
+
const key = llmInputKey(sessionId, turnId, stepIndex);
|
|
3138
|
+
state.update((current) => {
|
|
3139
|
+
const normalized = normalizeEveTraceState(current);
|
|
3140
|
+
return {
|
|
3141
|
+
...normalized,
|
|
3142
|
+
reasoningBlocks: normalized.reasoningBlocks.filter(
|
|
3143
|
+
(entry) => entry.key !== key
|
|
3144
|
+
)
|
|
3145
|
+
};
|
|
3146
|
+
});
|
|
3147
|
+
}
|
|
3148
|
+
function eveOutputMessage(output) {
|
|
3149
|
+
return Array.isArray(output) && isObject(output[0]) ? output[0]["message"] : void 0;
|
|
3150
|
+
}
|
|
3151
|
+
function mergeEveReasoning(output, reasoning) {
|
|
3152
|
+
if (reasoning.length === 0) {
|
|
3153
|
+
return output;
|
|
2943
3154
|
}
|
|
2944
|
-
const
|
|
3155
|
+
const choice = Array.isArray(output) && isObject(output[0]) ? output[0] : {};
|
|
3156
|
+
const message = isObject(choice["message"]) ? choice["message"] : {};
|
|
3157
|
+
return [
|
|
3158
|
+
{
|
|
3159
|
+
...choice,
|
|
3160
|
+
index: typeof choice["index"] === "number" ? choice["index"] : 0,
|
|
3161
|
+
message: {
|
|
3162
|
+
...message,
|
|
3163
|
+
content: "content" in message ? message["content"] : null,
|
|
3164
|
+
reasoning: reasoning.map((block) => ({ content: block.content })),
|
|
3165
|
+
role: typeof message["role"] === "string" ? message["role"] : "assistant"
|
|
3166
|
+
}
|
|
3167
|
+
}
|
|
3168
|
+
];
|
|
3169
|
+
}
|
|
3170
|
+
function captureEveModelInput(state, input) {
|
|
3171
|
+
const sessionId = input.session.id;
|
|
3172
|
+
const turnId = input.turn.id;
|
|
3173
|
+
const stepIndex = input.step.index;
|
|
3174
|
+
const captured = capturedModelInput(input.modelInput);
|
|
2945
3175
|
if (!captured) {
|
|
2946
3176
|
return;
|
|
2947
3177
|
}
|
|
@@ -2982,18 +3212,14 @@ function consumeCapturedEveModelInput(state, sessionId, turnId, stepIndex) {
|
|
|
2982
3212
|
}
|
|
2983
3213
|
}
|
|
2984
3214
|
function capturedModelInput(modelInput) {
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
if (
|
|
2990
|
-
|
|
2991
|
-
}
|
|
2992
|
-
|
|
2993
|
-
const value = [
|
|
2994
|
-
...instructions !== void 0 ? [{ content: instructions, role: "system" }] : [],
|
|
2995
|
-
...messages
|
|
2996
|
-
];
|
|
3215
|
+
const { instructions, messages } = modelInput;
|
|
3216
|
+
const value = [];
|
|
3217
|
+
if (typeof instructions === "string") {
|
|
3218
|
+
value.push({ content: instructions, role: "system" });
|
|
3219
|
+
} else if (instructions) {
|
|
3220
|
+
value.push(...instructions.map(capturedEveModelMessage));
|
|
3221
|
+
}
|
|
3222
|
+
value.push(...messages.map(capturedEveModelMessage));
|
|
2997
3223
|
try {
|
|
2998
3224
|
const cloned = JSON.parse(JSON.stringify(value));
|
|
2999
3225
|
if (!Array.isArray(cloned)) {
|
|
@@ -3004,8 +3230,123 @@ function capturedModelInput(modelInput) {
|
|
|
3004
3230
|
return void 0;
|
|
3005
3231
|
}
|
|
3006
3232
|
}
|
|
3233
|
+
function capturedEveModelMessage(message) {
|
|
3234
|
+
const { content, role } = message;
|
|
3235
|
+
if (typeof content === "string") {
|
|
3236
|
+
return { content, role };
|
|
3237
|
+
}
|
|
3238
|
+
return { content: content.map(capturedEveModelContentPart), role };
|
|
3239
|
+
}
|
|
3240
|
+
function capturedEveModelContentPart(part) {
|
|
3241
|
+
switch (part.type) {
|
|
3242
|
+
case "text":
|
|
3243
|
+
case "reasoning":
|
|
3244
|
+
return { text: part.text, type: part.type };
|
|
3245
|
+
case "image":
|
|
3246
|
+
return {
|
|
3247
|
+
image: part.image,
|
|
3248
|
+
...part.mediaType !== void 0 ? { mediaType: part.mediaType } : {},
|
|
3249
|
+
type: "image"
|
|
3250
|
+
};
|
|
3251
|
+
case "file":
|
|
3252
|
+
case "reasoning-file":
|
|
3253
|
+
return {
|
|
3254
|
+
data: part.data,
|
|
3255
|
+
...part.type === "file" && part.filename !== void 0 ? { filename: part.filename } : {},
|
|
3256
|
+
mediaType: part.mediaType,
|
|
3257
|
+
type: part.type
|
|
3258
|
+
};
|
|
3259
|
+
case "custom":
|
|
3260
|
+
return {
|
|
3261
|
+
..."kind" in part ? { kind: part.kind } : {},
|
|
3262
|
+
type: "custom"
|
|
3263
|
+
};
|
|
3264
|
+
case "tool-call":
|
|
3265
|
+
return {
|
|
3266
|
+
input: part.input,
|
|
3267
|
+
...part.providerExecuted !== void 0 ? { providerExecuted: part.providerExecuted } : {},
|
|
3268
|
+
toolCallId: part.toolCallId,
|
|
3269
|
+
toolName: part.toolName,
|
|
3270
|
+
type: "tool-call"
|
|
3271
|
+
};
|
|
3272
|
+
case "tool-result": {
|
|
3273
|
+
const output = part.output;
|
|
3274
|
+
let capturedOutput;
|
|
3275
|
+
switch (output.type) {
|
|
3276
|
+
case "text":
|
|
3277
|
+
case "error-text":
|
|
3278
|
+
capturedOutput = { type: output.type, value: output.value };
|
|
3279
|
+
break;
|
|
3280
|
+
case "json":
|
|
3281
|
+
case "error-json":
|
|
3282
|
+
capturedOutput = { type: output.type, value: output.value };
|
|
3283
|
+
break;
|
|
3284
|
+
case "execution-denied":
|
|
3285
|
+
capturedOutput = {
|
|
3286
|
+
...output.reason !== void 0 ? { reason: output.reason } : {},
|
|
3287
|
+
type: "execution-denied"
|
|
3288
|
+
};
|
|
3289
|
+
break;
|
|
3290
|
+
case "content":
|
|
3291
|
+
capturedOutput = {
|
|
3292
|
+
type: "content",
|
|
3293
|
+
value: output.value.map(capturedEveModelContentPart)
|
|
3294
|
+
};
|
|
3295
|
+
break;
|
|
3296
|
+
}
|
|
3297
|
+
return {
|
|
3298
|
+
output: capturedOutput,
|
|
3299
|
+
toolCallId: part.toolCallId,
|
|
3300
|
+
toolName: part.toolName,
|
|
3301
|
+
type: "tool-result"
|
|
3302
|
+
};
|
|
3303
|
+
}
|
|
3304
|
+
case "tool-approval-request":
|
|
3305
|
+
return {
|
|
3306
|
+
approvalId: part.approvalId,
|
|
3307
|
+
...part.isAutomatic !== void 0 ? { isAutomatic: part.isAutomatic } : {},
|
|
3308
|
+
...part.signature !== void 0 ? { signature: part.signature } : {},
|
|
3309
|
+
toolCallId: part.toolCallId,
|
|
3310
|
+
type: "tool-approval-request"
|
|
3311
|
+
};
|
|
3312
|
+
case "tool-approval-response":
|
|
3313
|
+
return {
|
|
3314
|
+
approvalId: part.approvalId,
|
|
3315
|
+
approved: part.approved,
|
|
3316
|
+
...part.providerExecuted !== void 0 ? { providerExecuted: part.providerExecuted } : {},
|
|
3317
|
+
...part.reason !== void 0 ? { reason: part.reason } : {},
|
|
3318
|
+
type: "tool-approval-response"
|
|
3319
|
+
};
|
|
3320
|
+
case "file-data":
|
|
3321
|
+
case "image-data":
|
|
3322
|
+
return {
|
|
3323
|
+
data: part.data,
|
|
3324
|
+
...part.type === "file-data" && part.filename !== void 0 ? { filename: part.filename } : {},
|
|
3325
|
+
mediaType: part.mediaType,
|
|
3326
|
+
type: part.type
|
|
3327
|
+
};
|
|
3328
|
+
case "file-url":
|
|
3329
|
+
case "image-url":
|
|
3330
|
+
return {
|
|
3331
|
+
...part.type === "file-url" && part.mediaType !== void 0 ? { mediaType: part.mediaType } : {},
|
|
3332
|
+
type: part.type,
|
|
3333
|
+
url: part.url
|
|
3334
|
+
};
|
|
3335
|
+
case "file-id":
|
|
3336
|
+
case "image-file-id":
|
|
3337
|
+
return { fileId: part.fileId, type: part.type };
|
|
3338
|
+
case "file-reference":
|
|
3339
|
+
case "image-file-reference":
|
|
3340
|
+
return {
|
|
3341
|
+
providerReference: part.providerReference,
|
|
3342
|
+
type: part.type
|
|
3343
|
+
};
|
|
3344
|
+
}
|
|
3345
|
+
}
|
|
3007
3346
|
function isCapturedModelInput(input) {
|
|
3008
|
-
return Array.isArray(input)
|
|
3347
|
+
return Array.isArray(input) && input.every(
|
|
3348
|
+
(message) => isObject(message) && (message["role"] === "system" || message["role"] === "user" || message["role"] === "assistant" || message["role"] === "tool") && (typeof message["content"] === "string" || Array.isArray(message["content"]) && message["content"].every(isObject))
|
|
3349
|
+
);
|
|
3009
3350
|
}
|
|
3010
3351
|
function llmInputKey(sessionId, turnId, stepIndex) {
|
|
3011
3352
|
return `${sessionId}\0${turnId}\0${stepIndex}`;
|
|
@@ -6142,6 +6483,8 @@ var BaseNode = class {
|
|
|
6142
6483
|
this.graph = graph;
|
|
6143
6484
|
this.id = id;
|
|
6144
6485
|
}
|
|
6486
|
+
graph;
|
|
6487
|
+
id;
|
|
6145
6488
|
__type = "node";
|
|
6146
6489
|
dependencies = [];
|
|
6147
6490
|
addDependency(dependency) {
|
|
@@ -6175,6 +6518,7 @@ var PromptNode = class extends BaseNode {
|
|
|
6175
6518
|
super(graph, id);
|
|
6176
6519
|
this.prompt = prompt;
|
|
6177
6520
|
}
|
|
6521
|
+
prompt;
|
|
6178
6522
|
async build(context) {
|
|
6179
6523
|
return {
|
|
6180
6524
|
type: "function",
|
|
@@ -6187,6 +6531,7 @@ var GateNode = class extends BaseNode {
|
|
|
6187
6531
|
super(graph, id);
|
|
6188
6532
|
this.condition = condition;
|
|
6189
6533
|
}
|
|
6534
|
+
condition;
|
|
6190
6535
|
async build(context) {
|
|
6191
6536
|
return {
|
|
6192
6537
|
type: "gate",
|
|
@@ -6211,6 +6556,7 @@ var PromptTemplateNode = class extends BaseNode {
|
|
|
6211
6556
|
super(graph, id);
|
|
6212
6557
|
this.prompt = prompt;
|
|
6213
6558
|
}
|
|
6559
|
+
prompt;
|
|
6214
6560
|
async build(context) {
|
|
6215
6561
|
return {
|
|
6216
6562
|
type: "prompt_template",
|
|
@@ -6223,6 +6569,7 @@ var LiteralNode = class extends BaseNode {
|
|
|
6223
6569
|
super(graph, id);
|
|
6224
6570
|
this.value = value;
|
|
6225
6571
|
}
|
|
6572
|
+
value;
|
|
6226
6573
|
async build(context) {
|
|
6227
6574
|
return {
|
|
6228
6575
|
type: "literal",
|
|
@@ -7316,6 +7663,10 @@ var SpanFetcher = class _SpanFetcher extends ObjectFetcher {
|
|
|
7316
7663
|
this._state = _state;
|
|
7317
7664
|
this.spanTypeFilter = spanTypeFilter;
|
|
7318
7665
|
}
|
|
7666
|
+
_objectId;
|
|
7667
|
+
rootSpanId;
|
|
7668
|
+
_state;
|
|
7669
|
+
spanTypeFilter;
|
|
7319
7670
|
static buildFilter(rootSpanId, spanTypeFilter, includeScorers = false) {
|
|
7320
7671
|
const children = [
|
|
7321
7672
|
// Base filter: root_span_id = 'value'
|
|
@@ -7766,6 +8117,8 @@ var EvalResultWithSummary = class {
|
|
|
7766
8117
|
this.summary = summary;
|
|
7767
8118
|
this.results = results;
|
|
7768
8119
|
}
|
|
8120
|
+
summary;
|
|
8121
|
+
results;
|
|
7769
8122
|
/**
|
|
7770
8123
|
* @deprecated Use `summary` instead.
|
|
7771
8124
|
*/
|
|
@@ -9184,6 +9537,7 @@ var ToolBuilder = class {
|
|
|
9184
9537
|
constructor(project) {
|
|
9185
9538
|
this.project = project;
|
|
9186
9539
|
}
|
|
9540
|
+
project;
|
|
9187
9541
|
taskCounter = 0;
|
|
9188
9542
|
// This type definition is just a catch all so that the implementation can be
|
|
9189
9543
|
// less specific than the two more specific declarations above.
|
|
@@ -9214,6 +9568,7 @@ var ScorerBuilder = class {
|
|
|
9214
9568
|
constructor(project) {
|
|
9215
9569
|
this.project = project;
|
|
9216
9570
|
}
|
|
9571
|
+
project;
|
|
9217
9572
|
taskCounter = 0;
|
|
9218
9573
|
create(opts) {
|
|
9219
9574
|
this.taskCounter++;
|
|
@@ -9273,6 +9628,7 @@ var ClassifierBuilder = class {
|
|
|
9273
9628
|
constructor(project) {
|
|
9274
9629
|
this.project = project;
|
|
9275
9630
|
}
|
|
9631
|
+
project;
|
|
9276
9632
|
taskCounter = 0;
|
|
9277
9633
|
create(opts) {
|
|
9278
9634
|
this.taskCounter++;
|
|
@@ -9308,6 +9664,7 @@ var CodeFunction = class {
|
|
|
9308
9664
|
throw new Error("parameters are required if return type is defined");
|
|
9309
9665
|
}
|
|
9310
9666
|
}
|
|
9667
|
+
project;
|
|
9311
9668
|
handler;
|
|
9312
9669
|
name;
|
|
9313
9670
|
slug;
|
|
@@ -9395,6 +9752,7 @@ var PromptBuilder = class {
|
|
|
9395
9752
|
constructor(project) {
|
|
9396
9753
|
this.project = project;
|
|
9397
9754
|
}
|
|
9755
|
+
project;
|
|
9398
9756
|
create(opts) {
|
|
9399
9757
|
const toolFunctions = [];
|
|
9400
9758
|
const rawTools = [];
|
|
@@ -9471,6 +9829,7 @@ var ParametersBuilder = class {
|
|
|
9471
9829
|
constructor(project) {
|
|
9472
9830
|
this.project = project;
|
|
9473
9831
|
}
|
|
9832
|
+
project;
|
|
9474
9833
|
create(opts) {
|
|
9475
9834
|
const slug = opts.slug ?? slugify(opts.name, { lower: true, strict: true });
|
|
9476
9835
|
const codeParameters = new CodeParameters(this.project, {
|