@botpress/evals 1.19.0-beta.9 → 2.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts +21 -9
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +103 -36
- package/dist/client.js.map +4 -4
- package/dist/definition.d.ts +9 -2
- package/dist/definition.d.ts.map +1 -1
- package/dist/definition.js.map +2 -2
- package/dist/graders/index.js +21 -74
- package/dist/graders/index.js.map +2 -2
- package/dist/graders/llm.d.ts +12 -12
- package/dist/graders/llm.d.ts.map +1 -1
- package/dist/graders/llm.js +30 -75
- package/dist/graders/llm.js.map +2 -2
- package/dist/graders/response.js +16 -65
- package/dist/graders/response.js.map +2 -2
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -0
- package/dist/index.js.map +3 -3
- package/dist/loader.js.map +2 -2
- package/dist/manifest.d.ts +13 -0
- package/dist/manifest.d.ts.map +1 -0
- package/dist/runner.d.ts +5 -1
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +316 -183
- package/dist/runner.js.map +4 -4
- package/dist/spans/index.d.ts +5 -0
- package/dist/spans/index.d.ts.map +1 -0
- package/dist/spans/index.js +725 -0
- package/dist/spans/index.js.map +7 -0
- package/dist/spans/local-span-source.d.ts +19 -0
- package/dist/spans/local-span-source.d.ts.map +1 -0
- package/dist/spans/local-span-source.js +543 -0
- package/dist/spans/local-span-source.js.map +7 -0
- package/dist/spans/span-source.d.ts +40 -0
- package/dist/spans/span-source.d.ts.map +1 -0
- package/dist/spans/span-source.js +1 -0
- package/dist/{sse-collector.d.ts → spans/sse-collector.d.ts} +44 -5
- package/dist/spans/sse-collector.d.ts.map +1 -0
- package/dist/{sse-collector.js → spans/sse-collector.js} +97 -20
- package/dist/spans/sse-collector.js.map +7 -0
- package/dist/spans/trace.d.ts.map +1 -0
- package/dist/spans/trace.js.map +7 -0
- package/dist/spans/vortex-span-source.d.ts +42 -0
- package/dist/spans/vortex-span-source.d.ts.map +1 -0
- package/dist/spans/vortex-span-source.js +184 -0
- package/dist/spans/vortex-span-source.js.map +7 -0
- package/dist/stores/eval-definition-loader.d.ts +4 -0
- package/dist/stores/eval-definition-loader.d.ts.map +1 -0
- package/dist/stores/eval-definition-loader.js +142 -0
- package/dist/stores/eval-definition-loader.js.map +7 -0
- package/dist/stores/eval-store.d.ts +61 -0
- package/dist/stores/eval-store.d.ts.map +1 -0
- package/dist/stores/eval-store.js +1 -0
- package/dist/stores/eval-store.js.map +7 -0
- package/dist/stores/index.d.ts +8 -0
- package/dist/stores/index.d.ts.map +1 -0
- package/dist/stores/index.js +1156 -0
- package/dist/stores/index.js.map +7 -0
- package/dist/stores/local-eval-store.d.ts +66 -0
- package/dist/stores/local-eval-store.d.ts.map +1 -0
- package/dist/stores/local-eval-store.js +595 -0
- package/dist/stores/local-eval-store.js.map +7 -0
- package/dist/stores/vortex-eval-store-entry.d.ts +6 -0
- package/dist/stores/vortex-eval-store-entry.d.ts.map +1 -0
- package/dist/stores/vortex-eval-store-entry.js +702 -0
- package/dist/stores/vortex-eval-store-entry.js.map +7 -0
- package/dist/stores/vortex-eval-store.d.ts +78 -0
- package/dist/stores/vortex-eval-store.d.ts.map +1 -0
- package/dist/stores/vortex-eval-store.js +562 -0
- package/dist/stores/vortex-eval-store.js.map +7 -0
- package/dist/transformer.d.ts +2 -2
- package/dist/transformer.d.ts.map +1 -1
- package/dist/transformer.js +1 -1
- package/dist/transformer.js.map +2 -2
- package/dist/types.d.ts +38 -33
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +2 -2
- package/package.json +66 -6
- package/dist/sse-collector.d.ts.map +0 -1
- package/dist/sse-collector.js.map +0 -7
- package/dist/trace.d.ts.map +0 -1
- /package/dist/{trace.js.map → spans/span-source.js.map} +0 -0
- /package/dist/{trace.d.ts → spans/trace.d.ts} +0 -0
- /package/dist/{trace.js → spans/trace.js} +0 -0
|
@@ -0,0 +1,725 @@
|
|
|
1
|
+
// ../analytics/src/errors.ts
|
|
2
|
+
var AdkError = class extends Error {
|
|
3
|
+
/**
|
|
4
|
+
* Marker for cross-module-instance checks — `instanceof` breaks when a
|
|
5
|
+
* package is duplicated in the dependency graph.
|
|
6
|
+
* @internal
|
|
7
|
+
*/
|
|
8
|
+
static __IS_ADK_BASE_ERROR = true;
|
|
9
|
+
/**
|
|
10
|
+
* Stable SCREAMING_SNAKE_CASE identifier, e.g. 'BOT_NOT_FOUND'. Subclasses
|
|
11
|
+
* pin `Code` to a literal union (`class EvalRunnerError extends
|
|
12
|
+
* AdkError<EvalErrorCode>`) so typos are compile errors and `code` narrows
|
|
13
|
+
* in switches.
|
|
14
|
+
*/
|
|
15
|
+
code;
|
|
16
|
+
/**
|
|
17
|
+
* True for user/environment conditions the developer can fix themselves
|
|
18
|
+
* (validation, auth, connectivity). False for internal bugs. Defaults to
|
|
19
|
+
* false so unclassified errors surface as bugs until someone triages them.
|
|
20
|
+
*/
|
|
21
|
+
expected;
|
|
22
|
+
/** Structured context. Sanitized before any telemetry leaves the machine. */
|
|
23
|
+
details;
|
|
24
|
+
/** Actionable next step shown to the user, e.g. "Run 'adk login' first." */
|
|
25
|
+
suggestion;
|
|
26
|
+
constructor(opts) {
|
|
27
|
+
super(opts.message, opts.cause !== void 0 ? { cause: opts.cause } : void 0);
|
|
28
|
+
this.name = this.constructor.name;
|
|
29
|
+
this.code = opts.code;
|
|
30
|
+
this.expected = opts.expected ?? false;
|
|
31
|
+
if (opts.details !== void 0) {
|
|
32
|
+
this.details = opts.details;
|
|
33
|
+
}
|
|
34
|
+
if (opts.suggestion !== void 0) {
|
|
35
|
+
this.suggestion = opts.suggestion;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
// src/errors.ts
|
|
41
|
+
var EvalRunnerError = class extends AdkError {
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
// src/spans/sse-collector.ts
|
|
45
|
+
function spanKey(span) {
|
|
46
|
+
return `${span.id.trace}:${span.id.span}`;
|
|
47
|
+
}
|
|
48
|
+
function spanData(span) {
|
|
49
|
+
return span.data && typeof span.data === "object" ? span.data : {};
|
|
50
|
+
}
|
|
51
|
+
var TURN_HANDLER_SPAN_NAMES = /* @__PURE__ */ new Set(["handler.conversation", "handler.event"]);
|
|
52
|
+
function isTurnHandlerSpan(span) {
|
|
53
|
+
return TURN_HANDLER_SPAN_NAMES.has(span.name);
|
|
54
|
+
}
|
|
55
|
+
var SSECollector = class _SSECollector {
|
|
56
|
+
devServerUrl;
|
|
57
|
+
headers;
|
|
58
|
+
/** All spans accumulated since connect(), keyed by spanKey for dedup. */
|
|
59
|
+
spanMap = /* @__PURE__ */ new Map();
|
|
60
|
+
/** Index into spanMap insertion order marking where the current turn started. */
|
|
61
|
+
turnStartKeys = /* @__PURE__ */ new Set();
|
|
62
|
+
/** Handler span IDs seen before the current turn (for startTurn filtering). */
|
|
63
|
+
seenHandlerKeys = /* @__PURE__ */ new Set();
|
|
64
|
+
/** Callbacks waiting for span updates. */
|
|
65
|
+
waiters = [];
|
|
66
|
+
/** AbortController for the active SSE connection. */
|
|
67
|
+
abortController = null;
|
|
68
|
+
/** Whether the background loop is running. */
|
|
69
|
+
connected = false;
|
|
70
|
+
/** Error from the SSE stream if it died unexpectedly. */
|
|
71
|
+
streamError = null;
|
|
72
|
+
/** Passive counters used only to explain timeout failures. */
|
|
73
|
+
updateEventCount = 0;
|
|
74
|
+
keepaliveEventCount = 0;
|
|
75
|
+
lastEvent = null;
|
|
76
|
+
/** Set while a settle phase is in flight; called by disconnect() to short-circuit. */
|
|
77
|
+
_cancelSettle = null;
|
|
78
|
+
constructor(devServerUrl, headers = {}) {
|
|
79
|
+
this.devServerUrl = devServerUrl;
|
|
80
|
+
this.headers = headers;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Open a streaming SSE connection filtered by conversationId or workflowId.
|
|
84
|
+
* Starts a background loop that parses SSE events and accumulates spans.
|
|
85
|
+
*/
|
|
86
|
+
async connect(filter) {
|
|
87
|
+
const attributeName = "conversationId" in filter ? "conversationId" : "workflowId";
|
|
88
|
+
const attributeValue = "conversationId" in filter ? filter.conversationId : filter.workflowId;
|
|
89
|
+
const url = `${this.devServerUrl}/api/traces/stream?attributeName=${encodeURIComponent(attributeName)}&attributeValue=${encodeURIComponent(attributeValue)}`;
|
|
90
|
+
this.abortController = new AbortController();
|
|
91
|
+
const myController = this.abortController;
|
|
92
|
+
const response = await fetch(url, {
|
|
93
|
+
headers: { ...this.headers, Accept: "text/event-stream" },
|
|
94
|
+
signal: myController.signal
|
|
95
|
+
});
|
|
96
|
+
if (!response.ok) {
|
|
97
|
+
throw new EvalRunnerError({
|
|
98
|
+
code: "SSE_CONNECT_FAILED",
|
|
99
|
+
message: `SSE connection failed: ${response.status} ${response.statusText}`,
|
|
100
|
+
expected: true,
|
|
101
|
+
suggestion: "Make sure the dev server is running (`adk dev`) and reachable."
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
if (!response.body) {
|
|
105
|
+
throw new EvalRunnerError({ code: "SSE_NO_BODY", message: "SSE response has no body" });
|
|
106
|
+
}
|
|
107
|
+
this.connected = true;
|
|
108
|
+
this._consumeStream(response.body).catch((err) => {
|
|
109
|
+
if (this.connected && this.abortController === myController) {
|
|
110
|
+
this.streamError = err instanceof Error ? err : new Error(String(err));
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
await this._waitForSnapshot();
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Re-point at a new conversation, keeping `spanMap` and `seenHandlerKeys` (so
|
|
117
|
+
* the earlier conversation stays visible and its completed handler can't
|
|
118
|
+
* satisfy the next turn). Resets the snapshot latch so connect() awaits the
|
|
119
|
+
* new conversation's snapshot.
|
|
120
|
+
*/
|
|
121
|
+
async repoint(filter) {
|
|
122
|
+
this.connected = false;
|
|
123
|
+
this.abortController?.abort();
|
|
124
|
+
this.abortController = null;
|
|
125
|
+
this._cancelSettle?.();
|
|
126
|
+
this._cancelSettle = null;
|
|
127
|
+
this.waiters = [];
|
|
128
|
+
this.snapshotReceived = false;
|
|
129
|
+
this.snapshotResolvers = [];
|
|
130
|
+
this.streamError = null;
|
|
131
|
+
await this.connect(filter);
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Mark the beginning of a new turn.
|
|
135
|
+
* Records which turn-handler span IDs (handler.conversation / handler.event)
|
|
136
|
+
* have been seen so far, so waitForTurnComplete() only resolves on NEW
|
|
137
|
+
* handler completions.
|
|
138
|
+
*/
|
|
139
|
+
startTurn() {
|
|
140
|
+
this.turnStartKeys = new Set(this.spanMap.keys());
|
|
141
|
+
for (const [key, span] of this.spanMap) {
|
|
142
|
+
if (isTurnHandlerSpan(span)) {
|
|
143
|
+
this.seenHandlerKeys.add(key);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
static SETTLE_MS = 100;
|
|
148
|
+
/**
|
|
149
|
+
* Wait for a turn-handler span (handler.conversation or handler.event) to
|
|
150
|
+
* reach terminal status (ok or error).
|
|
151
|
+
* Only considers spans NOT seen before the last startTurn() call.
|
|
152
|
+
*
|
|
153
|
+
* After the handler closes, waits for a quiet period (no new spans for SETTLE_MS)
|
|
154
|
+
* to ensure all child spans (tool calls, messages, state saves) have arrived.
|
|
155
|
+
* The runtime's HttpSpanExporter fires POSTs without awaiting them, so child
|
|
156
|
+
* spans may arrive slightly after the handler close event.
|
|
157
|
+
*/
|
|
158
|
+
async waitForTurnComplete(opts) {
|
|
159
|
+
const accept = opts.acceptSpanNames;
|
|
160
|
+
const settle = { quietMs: opts.settleQuietMs, maxMs: opts.settleMaxMs };
|
|
161
|
+
if (this._hasNewCompletedHandler(accept)) {
|
|
162
|
+
return this._settleAfterHandler(settle);
|
|
163
|
+
}
|
|
164
|
+
if (opts.abortSignal?.aborted) {
|
|
165
|
+
throw new Error("waitForTurnComplete aborted");
|
|
166
|
+
}
|
|
167
|
+
return new Promise((resolve, reject) => {
|
|
168
|
+
const timer = setTimeout(() => {
|
|
169
|
+
this._removeWaiter(waiter);
|
|
170
|
+
opts.abortSignal?.removeEventListener("abort", onAbort);
|
|
171
|
+
const expected = accept ? [...accept].join(" or ") : "handler.conversation or handler.event";
|
|
172
|
+
const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : `no ${expected} span completed`;
|
|
173
|
+
reject(
|
|
174
|
+
new Error(
|
|
175
|
+
`waitForTurnComplete timed out after ${opts.timeout}ms \u2014 ${reason}; diagnostics: ${this._formatTurnDiagnostics()}`
|
|
176
|
+
)
|
|
177
|
+
);
|
|
178
|
+
}, opts.timeout);
|
|
179
|
+
const onAbort = () => {
|
|
180
|
+
clearTimeout(timer);
|
|
181
|
+
this._removeWaiter(waiter);
|
|
182
|
+
reject(new Error("waitForTurnComplete aborted"));
|
|
183
|
+
};
|
|
184
|
+
opts.abortSignal?.addEventListener("abort", onAbort, { once: true });
|
|
185
|
+
const waiter = (_span) => {
|
|
186
|
+
if (this._hasNewCompletedHandler(accept)) {
|
|
187
|
+
clearTimeout(timer);
|
|
188
|
+
this._removeWaiter(waiter);
|
|
189
|
+
opts.abortSignal?.removeEventListener("abort", onAbort);
|
|
190
|
+
this._settleAfterHandler(settle).then(resolve, reject);
|
|
191
|
+
}
|
|
192
|
+
};
|
|
193
|
+
this.waiters.push(waiter);
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
static MAX_SETTLE_MS = 2e3;
|
|
197
|
+
/**
|
|
198
|
+
* After the handler span closes, wait for spans to stop arriving.
|
|
199
|
+
* Resolves when no new spans arrive for SETTLE_MS, or after MAX_SETTLE_MS hard cap.
|
|
200
|
+
*
|
|
201
|
+
* Exposes a cancellation hook via _cancelSettle so disconnect() can short-
|
|
202
|
+
* circuit an in-flight settle phase and avoid leaking timers for up to 2s.
|
|
203
|
+
*/
|
|
204
|
+
_settleAfterHandler(settle) {
|
|
205
|
+
const quietMs = settle?.quietMs ?? _SSECollector.SETTLE_MS;
|
|
206
|
+
const maxMs = settle?.maxMs ?? _SSECollector.MAX_SETTLE_MS;
|
|
207
|
+
return new Promise((resolve) => {
|
|
208
|
+
let resolved = false;
|
|
209
|
+
const done = () => {
|
|
210
|
+
if (resolved) return;
|
|
211
|
+
resolved = true;
|
|
212
|
+
clearTimeout(hardCap);
|
|
213
|
+
clearTimeout(settleTimer);
|
|
214
|
+
this._removeWaiter(settleWaiter);
|
|
215
|
+
this._cancelSettle = null;
|
|
216
|
+
resolve();
|
|
217
|
+
};
|
|
218
|
+
const hardCap = setTimeout(done, maxMs);
|
|
219
|
+
let settleTimer = setTimeout(done, quietMs);
|
|
220
|
+
const settleWaiter = () => {
|
|
221
|
+
if (resolved) return;
|
|
222
|
+
clearTimeout(settleTimer);
|
|
223
|
+
settleTimer = setTimeout(done, quietMs);
|
|
224
|
+
};
|
|
225
|
+
this.waiters.push(settleWaiter);
|
|
226
|
+
this._cancelSettle = done;
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* Wait for a handler.workflow span with matching workflow name.
|
|
231
|
+
* - signal 'entered': resolves when the span appears (any status)
|
|
232
|
+
* - signal 'completed': resolves when the span has status !== 'running'
|
|
233
|
+
*/
|
|
234
|
+
async waitForWorkflow(name, opts) {
|
|
235
|
+
if (this._matchesWorkflowCondition(name, opts.signal)) {
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
if (opts.abortSignal?.aborted) {
|
|
239
|
+
throw new Error(`waitForWorkflow("${name}", { signal: "${opts.signal}" }) aborted`);
|
|
240
|
+
}
|
|
241
|
+
return new Promise((resolve, reject) => {
|
|
242
|
+
const timer = setTimeout(() => {
|
|
243
|
+
this._removeWaiter(waiter);
|
|
244
|
+
opts.abortSignal?.removeEventListener("abort", onAbort);
|
|
245
|
+
const reason = this.streamError ? `SSE stream died: ${this.streamError.message}` : "span not found";
|
|
246
|
+
reject(
|
|
247
|
+
new Error(
|
|
248
|
+
`waitForWorkflow("${name}", { signal: "${opts.signal}" }) timed out after ${opts.timeout}ms \u2014 ${reason}`
|
|
249
|
+
)
|
|
250
|
+
);
|
|
251
|
+
}, opts.timeout);
|
|
252
|
+
const onAbort = () => {
|
|
253
|
+
clearTimeout(timer);
|
|
254
|
+
this._removeWaiter(waiter);
|
|
255
|
+
reject(new Error(`waitForWorkflow("${name}", { signal: "${opts.signal}" }) aborted`));
|
|
256
|
+
};
|
|
257
|
+
opts.abortSignal?.addEventListener("abort", onAbort, { once: true });
|
|
258
|
+
const waiter = (_span) => {
|
|
259
|
+
if (this._matchesWorkflowCondition(name, opts.signal)) {
|
|
260
|
+
clearTimeout(timer);
|
|
261
|
+
this._removeWaiter(waiter);
|
|
262
|
+
opts.abortSignal?.removeEventListener("abort", onAbort);
|
|
263
|
+
resolve();
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
this.waiters.push(waiter);
|
|
267
|
+
});
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Return spans accumulated since the last startTurn() call.
|
|
271
|
+
*/
|
|
272
|
+
getTurnSpans() {
|
|
273
|
+
const result = [];
|
|
274
|
+
for (const [key, span] of this.spanMap) {
|
|
275
|
+
if (!this.turnStartKeys.has(key)) {
|
|
276
|
+
result.push(span);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
return result;
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Return all spans accumulated since connect().
|
|
283
|
+
*/
|
|
284
|
+
getAllSpans() {
|
|
285
|
+
return Array.from(this.spanMap.values());
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Close the SSE connection and clean up.
|
|
289
|
+
*/
|
|
290
|
+
disconnect() {
|
|
291
|
+
this.connected = false;
|
|
292
|
+
this.abortController?.abort();
|
|
293
|
+
this.abortController = null;
|
|
294
|
+
this._cancelSettle?.();
|
|
295
|
+
this._cancelSettle = null;
|
|
296
|
+
this.waiters = [];
|
|
297
|
+
this.snapshotResolvers = [];
|
|
298
|
+
}
|
|
299
|
+
// ── Private ──────────────────────────────────────────────────
|
|
300
|
+
/** Whether we have received the initial snapshot. */
|
|
301
|
+
snapshotReceived = false;
|
|
302
|
+
snapshotResolvers = [];
|
|
303
|
+
static SNAPSHOT_TIMEOUT_MS = 1e4;
|
|
304
|
+
/**
|
|
305
|
+
* Returns a promise that resolves once the snapshot event has been processed,
|
|
306
|
+
* or rejects after a timeout to prevent connect() from hanging indefinitely.
|
|
307
|
+
*/
|
|
308
|
+
_waitForSnapshot() {
|
|
309
|
+
if (this.snapshotReceived) {
|
|
310
|
+
return Promise.resolve();
|
|
311
|
+
}
|
|
312
|
+
return new Promise((resolve, reject) => {
|
|
313
|
+
let settled = false;
|
|
314
|
+
const resolver = () => {
|
|
315
|
+
if (settled) return;
|
|
316
|
+
settled = true;
|
|
317
|
+
clearTimeout(timer);
|
|
318
|
+
resolve();
|
|
319
|
+
};
|
|
320
|
+
const timer = setTimeout(() => {
|
|
321
|
+
if (settled) return;
|
|
322
|
+
settled = true;
|
|
323
|
+
const idx = this.snapshotResolvers.indexOf(resolver);
|
|
324
|
+
if (idx !== -1) this.snapshotResolvers.splice(idx, 1);
|
|
325
|
+
reject(
|
|
326
|
+
new Error(
|
|
327
|
+
`SSE snapshot not received within ${_SSECollector.SNAPSHOT_TIMEOUT_MS}ms \u2014 is the dev server healthy?`
|
|
328
|
+
)
|
|
329
|
+
);
|
|
330
|
+
}, _SSECollector.SNAPSHOT_TIMEOUT_MS);
|
|
331
|
+
this.snapshotResolvers.push(resolver);
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Consume the ReadableStream body, parse SSE lines, and dispatch events.
|
|
336
|
+
*/
|
|
337
|
+
async _consumeStream(body) {
|
|
338
|
+
const decoder = new TextDecoder();
|
|
339
|
+
let buffer = "";
|
|
340
|
+
const reader = body.getReader();
|
|
341
|
+
try {
|
|
342
|
+
while (this.connected) {
|
|
343
|
+
const { done, value } = await reader.read();
|
|
344
|
+
if (done) break;
|
|
345
|
+
buffer += decoder.decode(value, { stream: true });
|
|
346
|
+
const parts = buffer.split("\n\n");
|
|
347
|
+
buffer = parts.pop() ?? "";
|
|
348
|
+
for (const part of parts) {
|
|
349
|
+
this._parseSSEBlock(part);
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
} finally {
|
|
353
|
+
reader.releaseLock();
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Parse a single SSE event block (text between \n\n boundaries).
|
|
358
|
+
* Extracts event name and data payload.
|
|
359
|
+
*/
|
|
360
|
+
_parseSSEBlock(block) {
|
|
361
|
+
const lines = block.split("\n");
|
|
362
|
+
let eventName = "";
|
|
363
|
+
let dataLine = "";
|
|
364
|
+
for (const line of lines) {
|
|
365
|
+
if (line.startsWith("event:")) {
|
|
366
|
+
eventName = line.slice("event:".length).trim();
|
|
367
|
+
} else if (line.startsWith("data:")) {
|
|
368
|
+
const value = line.slice("data:".length).trim();
|
|
369
|
+
dataLine = dataLine ? `${dataLine}
|
|
370
|
+
${value}` : value;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
if (!eventName || !dataLine) {
|
|
374
|
+
if (!this.snapshotReceived && block.includes("retry:")) {
|
|
375
|
+
}
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
try {
|
|
379
|
+
const payload = JSON.parse(dataLine);
|
|
380
|
+
this._handleEvent(eventName, payload);
|
|
381
|
+
} catch {
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* Dispatch a parsed SSE event to the appropriate handler.
|
|
386
|
+
*/
|
|
387
|
+
_handleEvent(eventName, payload) {
|
|
388
|
+
this._recordEvent(eventName);
|
|
389
|
+
if (eventName === "snapshot") {
|
|
390
|
+
const data = payload;
|
|
391
|
+
const spans = Array.isArray(data.spans) ? data.spans : [];
|
|
392
|
+
for (const span of spans) {
|
|
393
|
+
this._upsertSpan(span);
|
|
394
|
+
}
|
|
395
|
+
this.snapshotReceived = true;
|
|
396
|
+
for (const resolve of this.snapshotResolvers) {
|
|
397
|
+
resolve();
|
|
398
|
+
}
|
|
399
|
+
this.snapshotResolvers = [];
|
|
400
|
+
} else if (eventName === "update") {
|
|
401
|
+
const data = payload;
|
|
402
|
+
if (data.span) {
|
|
403
|
+
this._upsertSpan(data.span);
|
|
404
|
+
this._notifyWaiters(data.span);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
_recordEvent(eventName) {
|
|
409
|
+
this.lastEvent = { name: eventName, at: Date.now() };
|
|
410
|
+
if (eventName === "update") {
|
|
411
|
+
this.updateEventCount++;
|
|
412
|
+
} else if (eventName === "keepalive") {
|
|
413
|
+
this.keepaliveEventCount++;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
_formatTurnDiagnostics() {
|
|
417
|
+
const turnSpans = this.getTurnSpans();
|
|
418
|
+
const spanNames = /* @__PURE__ */ new Set();
|
|
419
|
+
const handlerStatuses = /* @__PURE__ */ new Set();
|
|
420
|
+
for (const span of turnSpans) {
|
|
421
|
+
spanNames.add(span.name);
|
|
422
|
+
if (isTurnHandlerSpan(span)) {
|
|
423
|
+
handlerStatuses.add(span.status);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
const lastEvent = this.lastEvent ? `${this.lastEvent.name} ${Date.now() - this.lastEvent.at}ms ago` : "none";
|
|
427
|
+
return [
|
|
428
|
+
`snapshotReceived=${this.snapshotReceived}`,
|
|
429
|
+
`totalUpdates=${this.updateEventCount}`,
|
|
430
|
+
`totalKeepalives=${this.keepaliveEventCount}`,
|
|
431
|
+
`lastEvent=${lastEvent}`,
|
|
432
|
+
`totalSpans=${this.spanMap.size}`,
|
|
433
|
+
`turnSpans=${turnSpans.length}`,
|
|
434
|
+
`turnSpanNames=${this._formatSet(spanNames)}`,
|
|
435
|
+
`turnHandlerStatuses=${this._formatSet(handlerStatuses)}`
|
|
436
|
+
].join(", ");
|
|
437
|
+
}
|
|
438
|
+
_formatSet(values, limit = 12) {
|
|
439
|
+
if (values.size === 0) {
|
|
440
|
+
return "none";
|
|
441
|
+
}
|
|
442
|
+
const items = Array.from(values);
|
|
443
|
+
const suffix = items.length > limit ? `|+${items.length - limit} more` : "";
|
|
444
|
+
return `${items.slice(0, limit).join("|")}${suffix}`;
|
|
445
|
+
}
|
|
446
|
+
/**
|
|
447
|
+
* Insert or replace a span in the map (dedup by trace+span ID).
|
|
448
|
+
*/
|
|
449
|
+
_upsertSpan(span) {
|
|
450
|
+
this.spanMap.set(spanKey(span), span);
|
|
451
|
+
}
|
|
452
|
+
/**
|
|
453
|
+
* Notify all waiters that a new span was received.
|
|
454
|
+
*/
|
|
455
|
+
_notifyWaiters(span) {
|
|
456
|
+
const current = [...this.waiters];
|
|
457
|
+
for (const waiter of current) {
|
|
458
|
+
waiter(span);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* Remove a waiter callback from the list.
|
|
463
|
+
*/
|
|
464
|
+
_removeWaiter(fn) {
|
|
465
|
+
const idx = this.waiters.indexOf(fn);
|
|
466
|
+
if (idx !== -1) {
|
|
467
|
+
this.waiters.splice(idx, 1);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
/**
|
|
471
|
+
* Check if there is a turn-handler span (handler.conversation / handler.event) that:
|
|
472
|
+
* 1. Was NOT seen before startTurn()
|
|
473
|
+
* 2. Has terminal status (ok or error)
|
|
474
|
+
*
|
|
475
|
+
* Marks the matched key as consumed so the same handler span cannot
|
|
476
|
+
* satisfy both the current turn's settle phase AND the next turn's
|
|
477
|
+
* waitForTurnComplete check (which would otherwise resolve instantly
|
|
478
|
+
* against stale data instead of waiting for the new turn's handler).
|
|
479
|
+
*/
|
|
480
|
+
_hasNewCompletedHandler(acceptSpanNames) {
|
|
481
|
+
for (const [key, span] of this.spanMap) {
|
|
482
|
+
if (isTurnHandlerSpan(span) && (!acceptSpanNames || acceptSpanNames.has(span.name)) && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
|
|
483
|
+
this.seenHandlerKeys.add(key);
|
|
484
|
+
return true;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
return false;
|
|
488
|
+
}
|
|
489
|
+
/**
|
|
490
|
+
* Check if a handler.workflow span matches the given condition.
|
|
491
|
+
*/
|
|
492
|
+
_matchesWorkflowCondition(name, signal) {
|
|
493
|
+
for (const span of this.spanMap.values()) {
|
|
494
|
+
if (span.name !== "handler.workflow") continue;
|
|
495
|
+
const d = spanData(span);
|
|
496
|
+
const wfName = d["workflow.name"] || d["workflowName"];
|
|
497
|
+
if (wfName !== name) continue;
|
|
498
|
+
if (signal === "entered") {
|
|
499
|
+
return true;
|
|
500
|
+
}
|
|
501
|
+
if (signal === "completed") {
|
|
502
|
+
return span.status === "ok" || span.status === "error";
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
return false;
|
|
506
|
+
}
|
|
507
|
+
};
|
|
508
|
+
|
|
509
|
+
// src/spans/local-span-source.ts
|
|
510
|
+
var LocalSpanSource = class {
|
|
511
|
+
collector;
|
|
512
|
+
constructor(devServerUrl, headers = {}) {
|
|
513
|
+
this.collector = new SSECollector(devServerUrl, headers);
|
|
514
|
+
}
|
|
515
|
+
async connect(filter) {
|
|
516
|
+
await this.collector.connect(filter);
|
|
517
|
+
}
|
|
518
|
+
async repoint(filter) {
|
|
519
|
+
await this.collector.repoint(filter);
|
|
520
|
+
}
|
|
521
|
+
startTurn() {
|
|
522
|
+
this.collector.startTurn();
|
|
523
|
+
}
|
|
524
|
+
async waitForTurnComplete(opts) {
|
|
525
|
+
await this.collector.waitForTurnComplete(opts);
|
|
526
|
+
}
|
|
527
|
+
async waitForWorkflow(name, opts) {
|
|
528
|
+
await this.collector.waitForWorkflow(name, opts);
|
|
529
|
+
}
|
|
530
|
+
getTurnSpans() {
|
|
531
|
+
return this.collector.getTurnSpans();
|
|
532
|
+
}
|
|
533
|
+
getAllSpans() {
|
|
534
|
+
return this.collector.getAllSpans();
|
|
535
|
+
}
|
|
536
|
+
disconnect() {
|
|
537
|
+
this.collector.disconnect();
|
|
538
|
+
}
|
|
539
|
+
};
|
|
540
|
+
|
|
541
|
+
// src/spans/vortex-span-source.ts
|
|
542
|
+
function spanKey2(span) {
|
|
543
|
+
return `${span.id.trace}:${span.id.span}`;
|
|
544
|
+
}
|
|
545
|
+
function spanData2(span) {
|
|
546
|
+
return span.data && typeof span.data === "object" ? span.data : {};
|
|
547
|
+
}
|
|
548
|
+
function normalizeSpan(span) {
|
|
549
|
+
if (span.status !== "running") return span;
|
|
550
|
+
const data = spanData2(span);
|
|
551
|
+
const autonomousStatus = data["autonomous.status"];
|
|
552
|
+
if (autonomousStatus === "generation_error" || autonomousStatus === "execution_error" || autonomousStatus === "invalid_code_error" || autonomousStatus === "exit_error" || autonomousStatus === "aborted") {
|
|
553
|
+
return { ...span, status: "error" };
|
|
554
|
+
}
|
|
555
|
+
if (autonomousStatus === "thinking_requested" || autonomousStatus === "callback_requested" || autonomousStatus === "exit_success") {
|
|
556
|
+
return { ...span, status: "ok" };
|
|
557
|
+
}
|
|
558
|
+
const { startedAt, endedAt } = span.timing;
|
|
559
|
+
if (typeof endedAt === "number" && Number.isFinite(endedAt) && endedAt > startedAt) {
|
|
560
|
+
return { ...span, status: "ok" };
|
|
561
|
+
}
|
|
562
|
+
return span;
|
|
563
|
+
}
|
|
564
|
+
function sleep(ms, signal) {
|
|
565
|
+
return new Promise((resolve, reject) => {
|
|
566
|
+
if (signal?.aborted) return reject(new Error("aborted"));
|
|
567
|
+
const onAbort = () => {
|
|
568
|
+
clearTimeout(timer);
|
|
569
|
+
reject(new Error("aborted"));
|
|
570
|
+
};
|
|
571
|
+
const timer = setTimeout(() => {
|
|
572
|
+
signal?.removeEventListener("abort", onAbort);
|
|
573
|
+
resolve();
|
|
574
|
+
}, ms);
|
|
575
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
var VortexSpanSource = class _VortexSpanSource {
|
|
579
|
+
config;
|
|
580
|
+
conversationId = "";
|
|
581
|
+
traceIds = /* @__PURE__ */ new Set();
|
|
582
|
+
spanMap = /* @__PURE__ */ new Map();
|
|
583
|
+
turnStartKeys = /* @__PURE__ */ new Set();
|
|
584
|
+
seenHandlerKeys = /* @__PURE__ */ new Set();
|
|
585
|
+
consecutiveFailures = 0;
|
|
586
|
+
constructor(config) {
|
|
587
|
+
this.config = {
|
|
588
|
+
url: config.url.replace(/\/$/, ""),
|
|
589
|
+
botId: config.botId,
|
|
590
|
+
pollIntervalMs: config.pollIntervalMs ?? 500,
|
|
591
|
+
...config.workspaceId ? { workspaceId: config.workspaceId } : {},
|
|
592
|
+
...config.token ? { token: config.token } : {}
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
async connect(filter) {
|
|
596
|
+
this.conversationId = filter.conversationId;
|
|
597
|
+
await this._pollOnce();
|
|
598
|
+
}
|
|
599
|
+
/**
|
|
600
|
+
* Switch polling to a new conversation, keeping `spanMap` and
|
|
601
|
+
* `seenHandlerKeys`. `traceIds` resets — they belonged to the prior conversation.
|
|
602
|
+
*/
|
|
603
|
+
async repoint(filter) {
|
|
604
|
+
this.conversationId = filter.conversationId;
|
|
605
|
+
this.traceIds = /* @__PURE__ */ new Set();
|
|
606
|
+
await this._pollOnce();
|
|
607
|
+
}
|
|
608
|
+
static TURN_HANDLER_SPAN_NAMES = /* @__PURE__ */ new Set(["handler.conversation", "handler.event"]);
|
|
609
|
+
startTurn() {
|
|
610
|
+
this.turnStartKeys = new Set(this.spanMap.keys());
|
|
611
|
+
for (const [key, span] of this.spanMap) {
|
|
612
|
+
if (_VortexSpanSource.TURN_HANDLER_SPAN_NAMES.has(span.name)) {
|
|
613
|
+
this.seenHandlerKeys.add(key);
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
static SETTLE_POLLS = 3;
|
|
618
|
+
async waitForTurnComplete(opts) {
|
|
619
|
+
const deadline = Date.now() + opts.timeout;
|
|
620
|
+
const maxPolls = opts.settleMaxMs ? Math.floor(opts.settleMaxMs / this.config.pollIntervalMs) : Infinity;
|
|
621
|
+
const settlePolls = Math.min(
|
|
622
|
+
maxPolls,
|
|
623
|
+
Math.max(
|
|
624
|
+
_VortexSpanSource.SETTLE_POLLS,
|
|
625
|
+
opts.settleQuietMs ? Math.ceil(opts.settleQuietMs / this.config.pollIntervalMs) : 0
|
|
626
|
+
)
|
|
627
|
+
);
|
|
628
|
+
while (Date.now() < deadline) {
|
|
629
|
+
if (opts.abortSignal?.aborted) throw new Error("waitForTurnComplete aborted");
|
|
630
|
+
await this._pollOnce();
|
|
631
|
+
if (this._hasNewCompletedHandler(opts.acceptSpanNames)) {
|
|
632
|
+
for (let i = 0; i < settlePolls; i++) {
|
|
633
|
+
await sleep(this.config.pollIntervalMs, opts.abortSignal).catch(() => {
|
|
634
|
+
});
|
|
635
|
+
await this._pollOnce();
|
|
636
|
+
}
|
|
637
|
+
return;
|
|
638
|
+
}
|
|
639
|
+
await sleep(this.config.pollIntervalMs, opts.abortSignal).catch(() => {
|
|
640
|
+
});
|
|
641
|
+
}
|
|
642
|
+
throw new Error(`waitForTurnComplete timed out after ${opts.timeout}ms`);
|
|
643
|
+
}
|
|
644
|
+
async waitForWorkflow(name, opts) {
|
|
645
|
+
const deadline = Date.now() + opts.timeout;
|
|
646
|
+
while (Date.now() < deadline) {
|
|
647
|
+
if (opts.abortSignal?.aborted) throw new Error(`waitForWorkflow("${name}") aborted`);
|
|
648
|
+
await this._pollOnce();
|
|
649
|
+
if (this._matchesWorkflowCondition(name, opts.signal)) return;
|
|
650
|
+
await sleep(this.config.pollIntervalMs, opts.abortSignal).catch(() => {
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
throw new Error(`waitForWorkflow("${name}", { signal: "${opts.signal}" }) timed out after ${opts.timeout}ms`);
|
|
654
|
+
}
|
|
655
|
+
getTurnSpans() {
|
|
656
|
+
const result = [];
|
|
657
|
+
for (const [key, span] of this.spanMap) {
|
|
658
|
+
if (!this.turnStartKeys.has(key)) {
|
|
659
|
+
result.push(span);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
return result;
|
|
663
|
+
}
|
|
664
|
+
getAllSpans() {
|
|
665
|
+
return Array.from(this.spanMap.values());
|
|
666
|
+
}
|
|
667
|
+
disconnect() {
|
|
668
|
+
}
|
|
669
|
+
async _pollOnce() {
|
|
670
|
+
const url = new URL(`${this.config.url}/v1/traces/bot/${this.config.botId}/spans/adk`);
|
|
671
|
+
url.searchParams.set("conversationId", this.conversationId);
|
|
672
|
+
for (const traceId of this.traceIds) {
|
|
673
|
+
url.searchParams.append("traceId", traceId);
|
|
674
|
+
}
|
|
675
|
+
const res = await fetch(url, {
|
|
676
|
+
headers: {
|
|
677
|
+
Accept: "application/json",
|
|
678
|
+
...this.config.token ? { Authorization: `Bearer ${this.config.token}` } : {}
|
|
679
|
+
}
|
|
680
|
+
});
|
|
681
|
+
if (!res.ok) {
|
|
682
|
+
this.consecutiveFailures++;
|
|
683
|
+
const body = await res.text().catch(() => "");
|
|
684
|
+
const msg = `Vortex poll failed: ${res.status} ${body.slice(0, 200)}`;
|
|
685
|
+
if (this.consecutiveFailures >= 5) {
|
|
686
|
+
throw new Error(`${msg} (${this.consecutiveFailures} consecutive failures)`);
|
|
687
|
+
}
|
|
688
|
+
console.warn(`[VortexSpanSource] ${msg}`);
|
|
689
|
+
return;
|
|
690
|
+
}
|
|
691
|
+
this.consecutiveFailures = 0;
|
|
692
|
+
const payload = await res.json();
|
|
693
|
+
const spans = Array.isArray(payload) ? payload : Array.isArray(payload.spans) ? payload.spans : [];
|
|
694
|
+
for (const rawSpan of spans) {
|
|
695
|
+
const span = normalizeSpan(rawSpan);
|
|
696
|
+
this.spanMap.set(spanKey2(span), span);
|
|
697
|
+
this.traceIds.add(span.id.trace);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
_hasNewCompletedHandler(acceptSpanNames) {
|
|
701
|
+
for (const [key, span] of this.spanMap) {
|
|
702
|
+
if (_VortexSpanSource.TURN_HANDLER_SPAN_NAMES.has(span.name) && (!acceptSpanNames || acceptSpanNames.has(span.name)) && !this.seenHandlerKeys.has(key) && (span.status === "ok" || span.status === "error")) {
|
|
703
|
+
this.seenHandlerKeys.add(key);
|
|
704
|
+
return true;
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
return false;
|
|
708
|
+
}
|
|
709
|
+
_matchesWorkflowCondition(name, signal) {
|
|
710
|
+
for (const span of this.spanMap.values()) {
|
|
711
|
+
if (span.name !== "handler.workflow") continue;
|
|
712
|
+
const d = spanData2(span);
|
|
713
|
+
const wfName = d["workflow.name"] || d["workflowName"];
|
|
714
|
+
if (wfName !== name) continue;
|
|
715
|
+
if (signal === "entered") return true;
|
|
716
|
+
if (signal === "completed") return span.status === "ok" || span.status === "error";
|
|
717
|
+
}
|
|
718
|
+
return false;
|
|
719
|
+
}
|
|
720
|
+
};
|
|
721
|
+
export {
|
|
722
|
+
LocalSpanSource,
|
|
723
|
+
VortexSpanSource
|
|
724
|
+
};
|
|
725
|
+
//# sourceMappingURL=index.js.map
|