@evalution/sdk 0.0.0-stage → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/adapter.d.ts +764 -0
- package/dist/adapter.js +53 -0
- package/dist/index.d.ts +487 -0
- package/dist/index.js +1070 -0
- package/dist/portkit-DV-XcagM.js +84 -0
- package/dist/trace-ingestor-Cu8xQW6B.d.ts +674 -0
- package/package.json +57 -4
- package/README.md +0 -3
package/dist/index.js
ADDED
|
@@ -0,0 +1,1070 @@
|
|
|
1
|
+
import { trace } from "@opentelemetry/api";
|
|
2
|
+
//#region src/brand.ts
|
|
3
|
+
function makeBrand(sym) {
|
|
4
|
+
return {
|
|
5
|
+
brand: (value) => Object.defineProperty(value, sym, { value: true }),
|
|
6
|
+
isBranded: (value) => typeof value === "object" && value !== null && value[sym] === true
|
|
7
|
+
};
|
|
8
|
+
}
|
|
9
|
+
//#endregion
|
|
10
|
+
//#region src/http-trace-sink.ts
|
|
11
|
+
/**
|
|
12
|
+
* Where an evalution server accepts {@link TraceSinkEvent}s, relative to its
|
|
13
|
+
* base URL.
|
|
14
|
+
*/
|
|
15
|
+
const TRACE_EVENTS_PATH = "/api/traces";
|
|
16
|
+
/**
|
|
17
|
+
* A {@link TraceSink} that sends what it's given to an evalution server,
|
|
18
|
+
* which records it through its own trace providers — so an app's traces show
|
|
19
|
+
* up in the playground without the app writing to a trace store itself.
|
|
20
|
+
*
|
|
21
|
+
* Calls return as soon as the event is queued, never waiting on the network,
|
|
22
|
+
* so tracing doesn't slow the app down. Queued events go out in order, batched
|
|
23
|
+
* into one request at a time, so a span's end never overtakes its start. A
|
|
24
|
+
* request that fails is logged and its events dropped: tracing never breaks
|
|
25
|
+
* the app. Await {@link flush} before a short-lived process exits to be sure
|
|
26
|
+
* everything was sent.
|
|
27
|
+
*/
|
|
28
|
+
var HttpTraceSink = class {
|
|
29
|
+
options;
|
|
30
|
+
target;
|
|
31
|
+
/** Set once it's known there's nowhere to send, so events aren't queued. */
|
|
32
|
+
disabled = false;
|
|
33
|
+
queue = [];
|
|
34
|
+
sending;
|
|
35
|
+
/** Whether the last request failed, so a down server is reported once. */
|
|
36
|
+
failing = false;
|
|
37
|
+
constructor(options) {
|
|
38
|
+
this.options = options;
|
|
39
|
+
}
|
|
40
|
+
resolveTarget() {
|
|
41
|
+
this.target ??= this.lookUpTarget().catch(() => void 0);
|
|
42
|
+
return this.target;
|
|
43
|
+
}
|
|
44
|
+
async lookUpTarget() {
|
|
45
|
+
const { endpoint, fetch: send } = this.options;
|
|
46
|
+
const target = typeof endpoint === "function" ? await endpoint() : { endpoint };
|
|
47
|
+
if (!target) return void 0;
|
|
48
|
+
return {
|
|
49
|
+
url: target.endpoint.replace(/\/+$/, "") + TRACE_EVENTS_PATH,
|
|
50
|
+
headers: target.headers,
|
|
51
|
+
fetch: target.fetch ?? send ?? ((...args) => globalThis.fetch(...args))
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
async recordSpanStart(span) {
|
|
55
|
+
this.enqueue({
|
|
56
|
+
type: "span-start",
|
|
57
|
+
span
|
|
58
|
+
});
|
|
59
|
+
return span;
|
|
60
|
+
}
|
|
61
|
+
async recordSpanEnd(span) {
|
|
62
|
+
this.enqueue({
|
|
63
|
+
type: "span-end",
|
|
64
|
+
span
|
|
65
|
+
});
|
|
66
|
+
return span;
|
|
67
|
+
}
|
|
68
|
+
async failTrace(traceId, errorMessage) {
|
|
69
|
+
this.enqueue({
|
|
70
|
+
type: "trace-fail",
|
|
71
|
+
traceId,
|
|
72
|
+
errorMessage
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
/** Resolves once every event recorded so far has been sent (or dropped). */
|
|
76
|
+
async flush() {
|
|
77
|
+
return this.sending ?? Promise.resolve();
|
|
78
|
+
}
|
|
79
|
+
enqueue(event) {
|
|
80
|
+
if (this.disabled) return;
|
|
81
|
+
this.queue.push(event);
|
|
82
|
+
this.sending ??= this.drain();
|
|
83
|
+
}
|
|
84
|
+
async drain() {
|
|
85
|
+
await new Promise((resolve) => setTimeout(resolve));
|
|
86
|
+
const target = await this.resolveTarget();
|
|
87
|
+
if (!target) {
|
|
88
|
+
this.disabled = true;
|
|
89
|
+
this.queue = [];
|
|
90
|
+
}
|
|
91
|
+
while (target && this.queue.length > 0) {
|
|
92
|
+
const events = this.queue;
|
|
93
|
+
this.queue = [];
|
|
94
|
+
await this.send(target, events);
|
|
95
|
+
}
|
|
96
|
+
this.sending = void 0;
|
|
97
|
+
}
|
|
98
|
+
async send(target, events) {
|
|
99
|
+
try {
|
|
100
|
+
const body = { events };
|
|
101
|
+
const res = await target.fetch(target.url, {
|
|
102
|
+
method: "POST",
|
|
103
|
+
headers: {
|
|
104
|
+
"content-type": "application/json",
|
|
105
|
+
...target.headers
|
|
106
|
+
},
|
|
107
|
+
body: JSON.stringify(body)
|
|
108
|
+
});
|
|
109
|
+
if (!res.ok) throw new Error(`${res.status} ${await res.text().catch(() => "")}`);
|
|
110
|
+
this.failing = false;
|
|
111
|
+
} catch (err) {
|
|
112
|
+
if (!this.failing) console.warn(`[evalution] Couldn't send traces to ${target.url}:`, err instanceof Error ? err.message : err);
|
|
113
|
+
this.failing = true;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
//#endregion
|
|
118
|
+
//#region src/trace-query.ts
|
|
119
|
+
const TEXT_COLUMNS = [
|
|
120
|
+
"name",
|
|
121
|
+
"status",
|
|
122
|
+
"model",
|
|
123
|
+
"environment",
|
|
124
|
+
"annotations",
|
|
125
|
+
"checks",
|
|
126
|
+
"version",
|
|
127
|
+
"variation"
|
|
128
|
+
];
|
|
129
|
+
const NUMBER_COLUMNS = [
|
|
130
|
+
"startTime",
|
|
131
|
+
"spanCount",
|
|
132
|
+
"duration",
|
|
133
|
+
"totalTokens",
|
|
134
|
+
"cost"
|
|
135
|
+
];
|
|
136
|
+
/** Whether `column` names a {@link TraceTextColumn}. */
|
|
137
|
+
function isTraceTextColumn(column) {
|
|
138
|
+
return TEXT_COLUMNS.includes(column);
|
|
139
|
+
}
|
|
140
|
+
/** Whether `column` names a {@link TraceNumberColumn}. */
|
|
141
|
+
function isTraceNumberColumn(column) {
|
|
142
|
+
return NUMBER_COLUMNS.includes(column);
|
|
143
|
+
}
|
|
144
|
+
const DATE_RESOLUTIONS = [
|
|
145
|
+
"hour",
|
|
146
|
+
"day",
|
|
147
|
+
"week",
|
|
148
|
+
"month"
|
|
149
|
+
];
|
|
150
|
+
/**
|
|
151
|
+
* The lower bounds of every bucket but the first, for the number columns
|
|
152
|
+
* grouped into buckets: `[1000, 5000]` makes the buckets `< 1000`,
|
|
153
|
+
* `1000 – 5000` and `≥ 5000`. Durations are in ms and costs in dollars.
|
|
154
|
+
*/
|
|
155
|
+
const TRACE_BUCKET_BOUNDS = {
|
|
156
|
+
spanCount: [
|
|
157
|
+
2,
|
|
158
|
+
6,
|
|
159
|
+
21,
|
|
160
|
+
101
|
|
161
|
+
],
|
|
162
|
+
duration: [
|
|
163
|
+
1e3,
|
|
164
|
+
5e3,
|
|
165
|
+
15e3,
|
|
166
|
+
6e4,
|
|
167
|
+
3e5
|
|
168
|
+
],
|
|
169
|
+
totalTokens: [
|
|
170
|
+
1e3,
|
|
171
|
+
1e4,
|
|
172
|
+
1e5,
|
|
173
|
+
1e6
|
|
174
|
+
],
|
|
175
|
+
cost: [
|
|
176
|
+
.001,
|
|
177
|
+
.01,
|
|
178
|
+
.1,
|
|
179
|
+
1,
|
|
180
|
+
10
|
|
181
|
+
]
|
|
182
|
+
};
|
|
183
|
+
/** A trace's value in a text column, or `null` if it has none. */
|
|
184
|
+
function traceTextValue(trace, column) {
|
|
185
|
+
switch (column) {
|
|
186
|
+
case "name": return trace.name;
|
|
187
|
+
case "status": return trace.status;
|
|
188
|
+
case "model": return trace.model ?? null;
|
|
189
|
+
case "environment": return trace.environment ?? null;
|
|
190
|
+
case "version": return trace.promptVersion ?? null;
|
|
191
|
+
case "variation": return trace.promptVariation ?? null;
|
|
192
|
+
case "annotations": {
|
|
193
|
+
const counts = trace.annotationCounts;
|
|
194
|
+
return counts.issue > 0 ? "issue" : counts.good > 0 ? "good" : counts.note > 0 ? "note" : null;
|
|
195
|
+
}
|
|
196
|
+
case "checks": {
|
|
197
|
+
const counts = trace.checkCounts;
|
|
198
|
+
if (!counts) return null;
|
|
199
|
+
return [
|
|
200
|
+
"fail",
|
|
201
|
+
"error",
|
|
202
|
+
"pass",
|
|
203
|
+
"scored",
|
|
204
|
+
"skipped"
|
|
205
|
+
].find((outcome) => counts[outcome] > 0) ?? null;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
/** A trace's value in a number column, or `undefined` if it has none. */
|
|
210
|
+
function traceNumberValue(trace, column) {
|
|
211
|
+
switch (column) {
|
|
212
|
+
case "startTime": return trace.startTime;
|
|
213
|
+
case "spanCount": return trace.spanCount;
|
|
214
|
+
case "duration": return trace.endTime !== void 0 ? trace.endTime - trace.startTime : void 0;
|
|
215
|
+
case "totalTokens": return trace.totalTokens;
|
|
216
|
+
case "cost": return trace.cost;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/** Whether `trace` meets `filter`. */
|
|
220
|
+
function matchesTraceFilter(trace, filter) {
|
|
221
|
+
switch (filter.op) {
|
|
222
|
+
case "in": return filter.values.includes(traceTextValue(trace, filter.column));
|
|
223
|
+
case "notIn": return !filter.values.includes(traceTextValue(trace, filter.column));
|
|
224
|
+
case "unset": return traceNumberValue(trace, filter.column) === void 0;
|
|
225
|
+
case "range": {
|
|
226
|
+
const value = traceNumberValue(trace, filter.column);
|
|
227
|
+
return value !== void 0 && (filter.min === void 0 || value >= filter.min) && (filter.max === void 0 || value < filter.max);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
/** Whether `trace` is one `query` lists. */
|
|
232
|
+
function matchesTraceQuery(trace, query) {
|
|
233
|
+
const search = query.search?.trim().toLowerCase();
|
|
234
|
+
if (search && !trace.name.toLowerCase().includes(search) && !trace.id.toLowerCase().includes(search)) return false;
|
|
235
|
+
return (query.filters ?? []).every((filter) => matchesTraceFilter(trace, filter));
|
|
236
|
+
}
|
|
237
|
+
/** Whether `query` lists every trace. */
|
|
238
|
+
function isEmptyTraceQuery(query) {
|
|
239
|
+
return !query?.search?.trim() && !query?.filters?.length;
|
|
240
|
+
}
|
|
241
|
+
/** The traces of `traces` that `query` lists, in the same order. */
|
|
242
|
+
function filterTraces(traces, query) {
|
|
243
|
+
if (!query || isEmptyTraceQuery(query)) return traces;
|
|
244
|
+
return traces.filter((trace) => matchesTraceQuery(trace, query));
|
|
245
|
+
}
|
|
246
|
+
const HOUR_MS = 36e5;
|
|
247
|
+
/** Each time zone's wall-clock formatter: slow to make, so made once. */
|
|
248
|
+
const wallClocks = /* @__PURE__ */ new Map();
|
|
249
|
+
function wallClock(timeZone) {
|
|
250
|
+
let format = wallClocks.get(timeZone);
|
|
251
|
+
if (!format) {
|
|
252
|
+
format = new Intl.DateTimeFormat("en-US", {
|
|
253
|
+
timeZone,
|
|
254
|
+
hourCycle: "h23",
|
|
255
|
+
year: "numeric",
|
|
256
|
+
month: "numeric",
|
|
257
|
+
day: "numeric",
|
|
258
|
+
hour: "numeric",
|
|
259
|
+
minute: "numeric",
|
|
260
|
+
second: "numeric"
|
|
261
|
+
});
|
|
262
|
+
wallClocks.set(timeZone, format);
|
|
263
|
+
}
|
|
264
|
+
return format;
|
|
265
|
+
}
|
|
266
|
+
/** How far `timeZone`'s wall clock is ahead of UTC at `ms`, in ms. */
|
|
267
|
+
function zoneOffset(ms, timeZone) {
|
|
268
|
+
if (timeZone === "UTC") return 0;
|
|
269
|
+
const parts = Object.fromEntries(wallClock(timeZone).formatToParts(ms).map((part) => [part.type, Number(part.value)]));
|
|
270
|
+
return Date.UTC(parts.year, parts.month - 1, parts.day, parts.hour, parts.minute, parts.second) - Math.floor(ms / 1e3) * 1e3;
|
|
271
|
+
}
|
|
272
|
+
/** The instant `timeZone`'s wall clock reads `wall` (a time written as if UTC). */
|
|
273
|
+
function fromWallTime(wall, timeZone) {
|
|
274
|
+
return wall - zoneOffset(wall - zoneOffset(wall, timeZone), timeZone);
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* The `[min, max)` of the hour, day, week or month `ms` falls in, on
|
|
278
|
+
* `timeZone`'s clock.
|
|
279
|
+
*/
|
|
280
|
+
function traceDateBucket(ms, resolution = "day", timeZone = "UTC") {
|
|
281
|
+
const offset = zoneOffset(ms, timeZone);
|
|
282
|
+
if (resolution === "hour") {
|
|
283
|
+
const min = Math.floor((ms + offset) / HOUR_MS) * HOUR_MS - offset;
|
|
284
|
+
return {
|
|
285
|
+
min,
|
|
286
|
+
max: min + HOUR_MS
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
const wall = new Date(ms + offset);
|
|
290
|
+
const year = wall.getUTCFullYear();
|
|
291
|
+
const month = wall.getUTCMonth();
|
|
292
|
+
const day = wall.getUTCDate();
|
|
293
|
+
let from;
|
|
294
|
+
let to;
|
|
295
|
+
if (resolution === "month") {
|
|
296
|
+
from = Date.UTC(year, month, 1);
|
|
297
|
+
to = Date.UTC(year, month + 1, 1);
|
|
298
|
+
} else if (resolution === "week") {
|
|
299
|
+
const sinceMonday = (wall.getUTCDay() + 6) % 7;
|
|
300
|
+
from = Date.UTC(year, month, day - sinceMonday);
|
|
301
|
+
to = Date.UTC(year, month, day - sinceMonday + 7);
|
|
302
|
+
} else {
|
|
303
|
+
from = Date.UTC(year, month, day);
|
|
304
|
+
to = Date.UTC(year, month, day + 1);
|
|
305
|
+
}
|
|
306
|
+
return {
|
|
307
|
+
min: fromWallTime(from, timeZone),
|
|
308
|
+
max: fromWallTime(to, timeZone)
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* The bucket of `column` that `value` falls in, as the filter listing it:
|
|
313
|
+
* see {@link TRACE_BUCKET_BOUNDS}.
|
|
314
|
+
*/
|
|
315
|
+
function traceBucketFilter(column, value) {
|
|
316
|
+
if (value === void 0) return {
|
|
317
|
+
column,
|
|
318
|
+
op: "unset"
|
|
319
|
+
};
|
|
320
|
+
const bounds = TRACE_BUCKET_BOUNDS[column];
|
|
321
|
+
return traceBucketAt(column, bounds.filter((bound) => value >= bound).length);
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* The `index`th bucket of `column`, counting from the one below every bound
|
|
325
|
+
* of {@link TRACE_BUCKET_BOUNDS}, as the filter listing it.
|
|
326
|
+
*/
|
|
327
|
+
function traceBucketAt(column, index) {
|
|
328
|
+
const bounds = TRACE_BUCKET_BOUNDS[column];
|
|
329
|
+
const min = index > 0 ? bounds[index - 1] : void 0;
|
|
330
|
+
const max = index < bounds.length ? bounds[index] : void 0;
|
|
331
|
+
return {
|
|
332
|
+
column,
|
|
333
|
+
op: "range",
|
|
334
|
+
...min !== void 0 && { min },
|
|
335
|
+
...max !== void 0 && { max }
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
/** The filter listing the group of `groupBy` that `trace` belongs to. */
|
|
339
|
+
function traceGroupFilter(trace, groupBy, timeZone) {
|
|
340
|
+
const { column } = groupBy;
|
|
341
|
+
if (isTraceTextColumn(column)) return {
|
|
342
|
+
column,
|
|
343
|
+
op: "in",
|
|
344
|
+
values: [traceTextValue(trace, column)]
|
|
345
|
+
};
|
|
346
|
+
if (column === "startTime") return {
|
|
347
|
+
column,
|
|
348
|
+
op: "range",
|
|
349
|
+
...traceDateBucket(trace.startTime, groupBy.resolution, timeZone)
|
|
350
|
+
};
|
|
351
|
+
return traceBucketFilter(column, traceNumberValue(trace, column));
|
|
352
|
+
}
|
|
353
|
+
/** {@link TraceGroup.key} for the group `filter` lists. */
|
|
354
|
+
function traceGroupKey(filter) {
|
|
355
|
+
switch (filter.op) {
|
|
356
|
+
case "in":
|
|
357
|
+
case "notIn": return `${filter.column}:${filter.op}:${JSON.stringify(filter.values)}`;
|
|
358
|
+
case "unset": return `${filter.column}:unset`;
|
|
359
|
+
case "range": return `${filter.column}:range:${filter.min ?? ""}:${filter.max ?? ""}`;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
/** The one value every item agrees on, else `undefined`. */
|
|
363
|
+
function shared(values) {
|
|
364
|
+
const first = values[0];
|
|
365
|
+
return values.every((v) => v === first) ? first : void 0;
|
|
366
|
+
}
|
|
367
|
+
/** The sum of the values that are set; `undefined` when none is. */
|
|
368
|
+
function sum(values) {
|
|
369
|
+
const set = values.filter((v) => v !== void 0);
|
|
370
|
+
return set.length > 0 ? set.reduce((a, b) => a + b, 0) : void 0;
|
|
371
|
+
}
|
|
372
|
+
/** `counts` added up; `undefined` when none is set. */
|
|
373
|
+
function sumCounts(counts) {
|
|
374
|
+
const set = counts.filter((c) => c !== void 0);
|
|
375
|
+
if (set.length === 0) return void 0;
|
|
376
|
+
const total = {
|
|
377
|
+
pass: 0,
|
|
378
|
+
fail: 0,
|
|
379
|
+
error: 0,
|
|
380
|
+
skipped: 0,
|
|
381
|
+
scored: 0
|
|
382
|
+
};
|
|
383
|
+
for (const c of set) for (const outcome of Object.keys(total)) total[outcome] += c[outcome];
|
|
384
|
+
return total;
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* What `traces` (non-empty) add up to. Takes summaries of groups as well as
|
|
388
|
+
* traces, so the groups of several listings can be added together.
|
|
389
|
+
*/
|
|
390
|
+
function summarizeTraces(traces) {
|
|
391
|
+
const running = traces.some((t) => t.status === "running");
|
|
392
|
+
const totalTokens = sum(traces.map((t) => t.totalTokens));
|
|
393
|
+
const cost = sum(traces.map((t) => t.cost));
|
|
394
|
+
const checkCounts = sumCounts(traces.map((t) => t.checkCounts));
|
|
395
|
+
const model = shared(traces.map((t) => t.model));
|
|
396
|
+
const environment = shared(traces.map((t) => t.environment));
|
|
397
|
+
const promptVersion = shared(traces.map((t) => t.promptVersion));
|
|
398
|
+
const promptVariation = shared(traces.map((t) => t.promptVariation));
|
|
399
|
+
return {
|
|
400
|
+
startTime: Math.min(...traces.map((t) => t.startTime)),
|
|
401
|
+
...!running && { endTime: Math.max(...traces.map((t) => t.endTime ?? t.startTime)) },
|
|
402
|
+
status: running ? "running" : traces.some((t) => t.status === "error") ? "error" : "ok",
|
|
403
|
+
spanCount: traces.reduce((n, t) => n + t.spanCount, 0),
|
|
404
|
+
...totalTokens !== void 0 && { totalTokens },
|
|
405
|
+
...cost !== void 0 && { cost },
|
|
406
|
+
...checkCounts && { checkCounts },
|
|
407
|
+
...model !== void 0 && { model },
|
|
408
|
+
...environment !== void 0 && { environment },
|
|
409
|
+
...promptVersion !== void 0 && { promptVersion },
|
|
410
|
+
...promptVariation !== void 0 && { promptVariation },
|
|
411
|
+
annotationCounts: {
|
|
412
|
+
issue: traces.reduce((n, t) => n + t.annotationCounts.issue, 0),
|
|
413
|
+
good: traces.reduce((n, t) => n + t.annotationCounts.good, 0),
|
|
414
|
+
note: traces.reduce((n, t) => n + t.annotationCounts.note, 0)
|
|
415
|
+
}
|
|
416
|
+
};
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* `traces` as the groups of `groupBy`, in the order their first traces come.
|
|
420
|
+
* Filter them first: this groups every trace it's given.
|
|
421
|
+
*/
|
|
422
|
+
function groupTraceSummaries(traces, groupBy, timeZone) {
|
|
423
|
+
const members = /* @__PURE__ */ new Map();
|
|
424
|
+
for (const trace of traces) {
|
|
425
|
+
const filter = traceGroupFilter(trace, groupBy, timeZone);
|
|
426
|
+
const key = traceGroupKey(filter);
|
|
427
|
+
const group = members.get(key);
|
|
428
|
+
if (group) group.traces.push(trace);
|
|
429
|
+
else members.set(key, {
|
|
430
|
+
filter,
|
|
431
|
+
traces: [trace]
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
return Array.from(members, ([key, group]) => ({
|
|
435
|
+
key,
|
|
436
|
+
filter: group.filter,
|
|
437
|
+
count: group.traces.length,
|
|
438
|
+
summary: summarizeTraces(group.traces)
|
|
439
|
+
}));
|
|
440
|
+
}
|
|
441
|
+
/**
|
|
442
|
+
* The groups of several listings as one: those with the same key added
|
|
443
|
+
* together, in the order their keys first come.
|
|
444
|
+
*/
|
|
445
|
+
function mergeTraceGroups(groups) {
|
|
446
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
447
|
+
for (const group of groups) {
|
|
448
|
+
const same = byKey.get(group.key);
|
|
449
|
+
if (same) same.push(group);
|
|
450
|
+
else byKey.set(group.key, [group]);
|
|
451
|
+
}
|
|
452
|
+
return Array.from(byKey.values(), (same) => same.length === 1 ? same[0] : {
|
|
453
|
+
key: same[0].key,
|
|
454
|
+
filter: same[0].filter,
|
|
455
|
+
count: same.reduce((n, group) => n + group.count, 0),
|
|
456
|
+
summary: summarizeTraces(same.map((group) => group.summary))
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
const optionalNumber = (value) => value === void 0 || typeof value === "number" && Number.isFinite(value);
|
|
460
|
+
/** `value` as a {@link TraceFilter}. Throws a `TypeError` if it isn't one. */
|
|
461
|
+
function parseTraceFilter(value) {
|
|
462
|
+
const filter = value;
|
|
463
|
+
if (filter && typeof filter === "object") {
|
|
464
|
+
const { column, op } = filter;
|
|
465
|
+
if (op === "in" || op === "notIn") {
|
|
466
|
+
const { values } = filter;
|
|
467
|
+
if (isTraceTextColumn(column) && Array.isArray(values) && values.every((v) => v === null || typeof v === "string")) return {
|
|
468
|
+
column,
|
|
469
|
+
op,
|
|
470
|
+
values
|
|
471
|
+
};
|
|
472
|
+
} else if (op === "unset") {
|
|
473
|
+
if (isTraceNumberColumn(column)) return {
|
|
474
|
+
column,
|
|
475
|
+
op
|
|
476
|
+
};
|
|
477
|
+
} else if (op === "range") {
|
|
478
|
+
const { min, max } = filter;
|
|
479
|
+
if (isTraceNumberColumn(column) && optionalNumber(min) && optionalNumber(max)) return {
|
|
480
|
+
column,
|
|
481
|
+
op,
|
|
482
|
+
...min !== void 0 && { min },
|
|
483
|
+
...max !== void 0 && { max }
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
throw new TypeError(`Not a trace filter: ${JSON.stringify(value)}`);
|
|
488
|
+
}
|
|
489
|
+
/** The query parameters a {@link TraceQuery} or {@link TraceGroupQuery} travels as. */
|
|
490
|
+
function traceQueryParams(query) {
|
|
491
|
+
const search = query.search?.trim();
|
|
492
|
+
return {
|
|
493
|
+
...search && { q: search },
|
|
494
|
+
...query.filters?.length && { filters: JSON.stringify(query.filters) },
|
|
495
|
+
...query.groupBy && {
|
|
496
|
+
groupBy: query.groupBy.column,
|
|
497
|
+
...query.groupBy.resolution && { resolution: query.groupBy.resolution }
|
|
498
|
+
},
|
|
499
|
+
...query.timeZone && { tz: query.timeZone }
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* The {@link TraceQuery} that {@link traceQueryParams} wrote into `params`.
|
|
504
|
+
* Throws a `TypeError` naming what's wrong with one that can't be read.
|
|
505
|
+
*/
|
|
506
|
+
function parseTraceQuery(params) {
|
|
507
|
+
const search = params.q?.trim();
|
|
508
|
+
let filters;
|
|
509
|
+
if (params.filters) {
|
|
510
|
+
let parsed;
|
|
511
|
+
try {
|
|
512
|
+
parsed = JSON.parse(params.filters);
|
|
513
|
+
} catch {
|
|
514
|
+
throw new TypeError("`filters` isn't JSON.");
|
|
515
|
+
}
|
|
516
|
+
if (!Array.isArray(parsed)) throw new TypeError("`filters` isn't an array.");
|
|
517
|
+
filters = parsed.map(parseTraceFilter);
|
|
518
|
+
}
|
|
519
|
+
return {
|
|
520
|
+
...search && { search },
|
|
521
|
+
...filters?.length && { filters }
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
/** {@link parseTraceQuery} for the parameters of a {@link TraceGroupQuery}. */
|
|
525
|
+
function parseTraceGroupQuery(params) {
|
|
526
|
+
const { groupBy: column, resolution, tz } = params;
|
|
527
|
+
if (!isTraceTextColumn(column) && !isTraceNumberColumn(column)) throw new TypeError("`groupBy` isn't a trace column.");
|
|
528
|
+
if (resolution !== void 0 && !DATE_RESOLUTIONS.includes(resolution)) throw new TypeError("`resolution` isn't hour, day, week or month.");
|
|
529
|
+
if (tz !== void 0) try {
|
|
530
|
+
new Intl.DateTimeFormat("en-US", { timeZone: tz });
|
|
531
|
+
} catch {
|
|
532
|
+
throw new TypeError("`tz` isn't a time zone.");
|
|
533
|
+
}
|
|
534
|
+
return {
|
|
535
|
+
...parseTraceQuery(params),
|
|
536
|
+
groupBy: {
|
|
537
|
+
column,
|
|
538
|
+
...resolution && { resolution }
|
|
539
|
+
},
|
|
540
|
+
...tz && { timeZone: tz }
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
//#endregion
|
|
544
|
+
//#region src/client.ts
|
|
545
|
+
/** The Evalution Cloud API, unless `EVALUTION_API_URL` says otherwise. */
|
|
546
|
+
const DEFAULT_API_URL = "https://api.evalut.io";
|
|
547
|
+
/** How much of an API key is safe to print: its kind and a few characters. */
|
|
548
|
+
const PRINTABLE_API_KEY_PREFIX = 15;
|
|
549
|
+
/**
|
|
550
|
+
* The kind of an API key, read from its prefix, or `undefined` for a string
|
|
551
|
+
* that isn't one.
|
|
552
|
+
*/
|
|
553
|
+
function apiKeyKind(key) {
|
|
554
|
+
if (key.startsWith("evl_ingest_")) return "ingest";
|
|
555
|
+
if (key.startsWith("evl_user_")) return "personal";
|
|
556
|
+
}
|
|
557
|
+
/** A listing's query as a URL's search, `?` included; empty if it has none. */
|
|
558
|
+
function listParams(query) {
|
|
559
|
+
const { search, filters, groupBy, timeZone, ...window } = query;
|
|
560
|
+
const params = new URLSearchParams(traceQueryParams({
|
|
561
|
+
search,
|
|
562
|
+
filters,
|
|
563
|
+
groupBy,
|
|
564
|
+
timeZone
|
|
565
|
+
}));
|
|
566
|
+
for (const [name, value] of Object.entries(window)) if (value !== void 0) params.set(name, String(value));
|
|
567
|
+
return params.size > 0 ? `?${params}` : "";
|
|
568
|
+
}
|
|
569
|
+
/**
|
|
570
|
+
* The client for Evalution's API. As a {@link TraceSink} it sends an app's
|
|
571
|
+
* traces, and it makes the read and control calls the local server needs.
|
|
572
|
+
*
|
|
573
|
+
* Where traces go is worked out once, on the first one sent:
|
|
574
|
+
*
|
|
575
|
+
* 1. the {@link EvalutionOptions.endpoint} option;
|
|
576
|
+
* 2. Evalution Cloud, when the API key is an ingest key;
|
|
577
|
+
* 3. the local instance `EVALUTION_ENDPOINT` names;
|
|
578
|
+
* 4. on Node, the instance portkit knows for this directory;
|
|
579
|
+
* 5. otherwise nowhere, and sending does nothing.
|
|
580
|
+
*
|
|
581
|
+
* A personal key never sends traces to the cloud, and no key is ever sent to
|
|
582
|
+
* a local instance. Tracing never breaks the app: see {@link HttpTraceSink},
|
|
583
|
+
* and await {@link flush} before a short-lived process exits. The read and
|
|
584
|
+
* control calls, by contrast, reject when they fail.
|
|
585
|
+
*/
|
|
586
|
+
var Evalution = class {
|
|
587
|
+
endpoint;
|
|
588
|
+
apiKey;
|
|
589
|
+
apiUrl;
|
|
590
|
+
project;
|
|
591
|
+
fetch;
|
|
592
|
+
sink;
|
|
593
|
+
/** The start of the API key, safe to show in messages, or `<none>`. */
|
|
594
|
+
get printableApiKey() {
|
|
595
|
+
return this.apiKey ? `${this.apiKey.slice(0, PRINTABLE_API_KEY_PREFIX)}…` : "<none>";
|
|
596
|
+
}
|
|
597
|
+
constructor(options = {}) {
|
|
598
|
+
const env = globalThis.process?.env ?? {};
|
|
599
|
+
this.endpoint = options.endpoint;
|
|
600
|
+
this.apiKey = options.apiKey ?? env.EVALUTION_API_KEY;
|
|
601
|
+
this.apiUrl = (options.apiUrl || env.EVALUTION_API_URL || "https://api.evalut.io").replace(/\/+$/, "");
|
|
602
|
+
this.project = options.project;
|
|
603
|
+
this.fetch = options.fetch ?? ((...args) => globalThis.fetch(...args));
|
|
604
|
+
this.sink = new HttpTraceSink({
|
|
605
|
+
endpoint: () => this.resolveTraceTarget(env.EVALUTION_ENDPOINT),
|
|
606
|
+
fetch: this.fetch
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
async resolveTraceTarget(envEndpoint) {
|
|
610
|
+
if (this.endpoint === false) return void 0;
|
|
611
|
+
if (this.endpoint) return { endpoint: this.endpoint };
|
|
612
|
+
if (this.apiKey) switch (apiKeyKind(this.apiKey)) {
|
|
613
|
+
case "ingest": return {
|
|
614
|
+
endpoint: this.apiUrl,
|
|
615
|
+
headers: { authorization: `Bearer ${this.apiKey}` }
|
|
616
|
+
};
|
|
617
|
+
case "personal": break;
|
|
618
|
+
default: console.warn("[evalution] Ignoring the API key: it isn't an evl_ingest_… or evl_user_… key.");
|
|
619
|
+
}
|
|
620
|
+
if (envEndpoint) return { endpoint: envEndpoint };
|
|
621
|
+
if (!globalThis.process?.versions?.node) return void 0;
|
|
622
|
+
return (await import("./portkit-DV-XcagM.js")).discoverPortkit();
|
|
623
|
+
}
|
|
624
|
+
recordSpanStart(span) {
|
|
625
|
+
return this.sink.recordSpanStart(span);
|
|
626
|
+
}
|
|
627
|
+
recordSpanEnd(span) {
|
|
628
|
+
return this.sink.recordSpanEnd(span);
|
|
629
|
+
}
|
|
630
|
+
failTrace(traceId, errorMessage) {
|
|
631
|
+
return this.sink.failTrace(traceId, errorMessage);
|
|
632
|
+
}
|
|
633
|
+
/** Resolves once every trace event recorded so far has been sent (or dropped). */
|
|
634
|
+
flush() {
|
|
635
|
+
return this.sink.flush();
|
|
636
|
+
}
|
|
637
|
+
/** The key's kind and whose it is. Takes either kind of key. */
|
|
638
|
+
whoami() {
|
|
639
|
+
return this.json("GET", "/cloud/v1/whoami", { key: "any" });
|
|
640
|
+
}
|
|
641
|
+
/** The most recent production traces of the project. */
|
|
642
|
+
listTraces(query = {}) {
|
|
643
|
+
return this.json("GET", `/api/traces${listParams(query)}`, { project: true });
|
|
644
|
+
}
|
|
645
|
+
/**
|
|
646
|
+
* The project's production traces as groups, one per value or bucket of
|
|
647
|
+
* a column, counting every trace in the window rather than the most recent.
|
|
648
|
+
*/
|
|
649
|
+
listTraceGroups(query) {
|
|
650
|
+
return this.json("GET", `/api/traces/groups${listParams(query)}`, { project: true });
|
|
651
|
+
}
|
|
652
|
+
/** One production trace with its spans, or `undefined` if there's none. */
|
|
653
|
+
async getTrace(id) {
|
|
654
|
+
const res = await this.request("GET", `/api/traces/${encodeURIComponent(id)}`, { project: true });
|
|
655
|
+
if (res.status === 404) return void 0;
|
|
656
|
+
return (await readOk(res)).json();
|
|
657
|
+
}
|
|
658
|
+
/** The projects the key's user is a member of. */
|
|
659
|
+
listProjects() {
|
|
660
|
+
return this.json("GET", "/cloud/v1/projects");
|
|
661
|
+
}
|
|
662
|
+
/** Creates a project in the user's organization. */
|
|
663
|
+
createProject(project) {
|
|
664
|
+
return this.json("POST", "/cloud/v1/projects", { body: project });
|
|
665
|
+
}
|
|
666
|
+
/** Records a running checkout of the project. */
|
|
667
|
+
registerInstance(instance) {
|
|
668
|
+
return this.json("PUT", "/cloud/v1/instances", {
|
|
669
|
+
project: true,
|
|
670
|
+
body: instance
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
/** Tells the cloud a registered instance is still running. */
|
|
674
|
+
async heartbeatInstance(id) {
|
|
675
|
+
await readOk(await this.instanceRequest("POST", id, "heartbeat"));
|
|
676
|
+
}
|
|
677
|
+
/** Tells the cloud a registered instance has stopped. */
|
|
678
|
+
async endInstanceSession(id) {
|
|
679
|
+
await readOk(await this.instanceRequest("DELETE", id, "session"));
|
|
680
|
+
}
|
|
681
|
+
/** Ensures a synced database exists, and returns how to reach it. */
|
|
682
|
+
getSyncDatabase(request) {
|
|
683
|
+
return this.json("POST", "/cloud/v1/sync/databases", {
|
|
684
|
+
project: true,
|
|
685
|
+
body: request
|
|
686
|
+
});
|
|
687
|
+
}
|
|
688
|
+
/** The project's synced databases. A dataset's has its id as the `key`; the others' `key` is empty. */
|
|
689
|
+
listSyncDatabases() {
|
|
690
|
+
return this.json("GET", "/cloud/v1/sync/databases", { project: true });
|
|
691
|
+
}
|
|
692
|
+
/** Deletes a dataset's synced database, for every member's machine. */
|
|
693
|
+
async deleteSyncDatabase(key) {
|
|
694
|
+
const path = `/cloud/v1/sync/databases/dataset/${encodeURIComponent(key.provider)}/${encodeURIComponent(key.key)}`;
|
|
695
|
+
await readOk(await this.request("DELETE", path, { project: true }));
|
|
696
|
+
}
|
|
697
|
+
instanceRequest(method, id, action) {
|
|
698
|
+
const path = `/cloud/v1/instances/${encodeURIComponent(id)}/${action}`;
|
|
699
|
+
return this.request(method, path, { project: true });
|
|
700
|
+
}
|
|
701
|
+
async json(method, path, options) {
|
|
702
|
+
return (await readOk(await this.request(method, path, options))).json();
|
|
703
|
+
}
|
|
704
|
+
request(method, path, { key = "personal", project, body } = {}) {
|
|
705
|
+
if (!this.apiKey) throw new Error("Evalution: set EVALUTION_API_KEY to use the cloud API.");
|
|
706
|
+
if (key === "personal" && apiKeyKind(this.apiKey) !== "personal") throw new Error(`Evalution: ${path} needs a personal (evl_user_…) key.`);
|
|
707
|
+
if (project && !this.project) throw new Error(`Evalution: ${path} needs a project id.`);
|
|
708
|
+
return this.fetch(this.apiUrl + path, {
|
|
709
|
+
method,
|
|
710
|
+
headers: {
|
|
711
|
+
authorization: `Bearer ${this.apiKey}`,
|
|
712
|
+
...project && { "evalution-project": this.project },
|
|
713
|
+
...body !== void 0 && { "content-type": "application/json" }
|
|
714
|
+
},
|
|
715
|
+
...body !== void 0 && { body: JSON.stringify(body) }
|
|
716
|
+
});
|
|
717
|
+
}
|
|
718
|
+
};
|
|
719
|
+
/** How {@link Evalution} rejects when the cloud refuses a call. */
|
|
720
|
+
var EvalutionApiError = class extends Error {
|
|
721
|
+
/** The HTTP status the cloud answered with. */
|
|
722
|
+
status;
|
|
723
|
+
/** The cloud's explanation, if it gave one. */
|
|
724
|
+
detail;
|
|
725
|
+
/** The cloud's whole answer, when it was JSON: a 410's `syncId`, say. */
|
|
726
|
+
body;
|
|
727
|
+
constructor(status, detail, body) {
|
|
728
|
+
super(`Evalution API: ${status} ${detail ?? ""}`.trim());
|
|
729
|
+
this.name = "EvalutionApiError";
|
|
730
|
+
this.status = status;
|
|
731
|
+
this.detail = detail;
|
|
732
|
+
this.body = body;
|
|
733
|
+
}
|
|
734
|
+
};
|
|
735
|
+
/** `res`, or a rejection saying why the cloud refused. */
|
|
736
|
+
async function readOk(res) {
|
|
737
|
+
if (res.ok) return res;
|
|
738
|
+
const text = await res.text().catch(() => "");
|
|
739
|
+
let detail = text || void 0;
|
|
740
|
+
let body;
|
|
741
|
+
try {
|
|
742
|
+
const parsed = JSON.parse(text);
|
|
743
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
744
|
+
body = parsed;
|
|
745
|
+
if (typeof body.error === "string") detail = body.error;
|
|
746
|
+
}
|
|
747
|
+
} catch {}
|
|
748
|
+
throw new EvalutionApiError(res.status, detail, body);
|
|
749
|
+
}
|
|
750
|
+
//#endregion
|
|
751
|
+
//#region src/evaluation-answers.ts
|
|
752
|
+
const isRecord = (v) => !!v && typeof v === "object" && !Array.isArray(v);
|
|
753
|
+
/**
|
|
754
|
+
* The per-question confidence a provider reported beside an evaluation's
|
|
755
|
+
* answers, keyed by question id.
|
|
756
|
+
*
|
|
757
|
+
* The AI SDK's answers carry no confidence of their own: a provider that
|
|
758
|
+
* computes one (TypeSafe does, for choice and score answers) reports it as
|
|
759
|
+
* `providerMetadata.<provider>.confidence[questionId]`. Read from whichever
|
|
760
|
+
* provider namespace has it, so this isn't tied to one provider.
|
|
761
|
+
*/
|
|
762
|
+
function evaluationConfidence(providerMetadata) {
|
|
763
|
+
const out = {};
|
|
764
|
+
if (!isRecord(providerMetadata)) return out;
|
|
765
|
+
for (const namespace of Object.values(providerMetadata)) {
|
|
766
|
+
if (!isRecord(namespace) || !isRecord(namespace.confidence)) continue;
|
|
767
|
+
for (const [id, value] of Object.entries(namespace.confidence)) if (typeof value === "number" && Number.isFinite(value)) out[id] = value;
|
|
768
|
+
}
|
|
769
|
+
return out;
|
|
770
|
+
}
|
|
771
|
+
/**
|
|
772
|
+
* An evaluation's answers as recorded on its span: each answer as the SDK
|
|
773
|
+
* returned it, plus its `confidence` where the provider reported one (see
|
|
774
|
+
* {@link evaluationConfidence}). Recorded together so a trace — from the
|
|
775
|
+
* playground or sent over OTLP — can show confidence without knowing which
|
|
776
|
+
* provider answered.
|
|
777
|
+
*/
|
|
778
|
+
function answersWithConfidence(answers, providerMetadata) {
|
|
779
|
+
if (!isRecord(answers)) return answers;
|
|
780
|
+
const confidence = evaluationConfidence(providerMetadata);
|
|
781
|
+
if (Object.keys(confidence).length === 0) return answers;
|
|
782
|
+
return Object.fromEntries(Object.entries(answers).map(([id, answer]) => [id, isRecord(answer) && id in confidence && !("confidence" in answer) ? {
|
|
783
|
+
...answer,
|
|
784
|
+
confidence: confidence[id]
|
|
785
|
+
} : answer]));
|
|
786
|
+
}
|
|
787
|
+
//#endregion
|
|
788
|
+
//#region src/prompt-tracer.ts
|
|
789
|
+
/**
|
|
790
|
+
* Attribute name a span can set to pick one of evalution's span-kind values
|
|
791
|
+
* (`'LLM'`, `'TOOL'`, `'AGENT'`, `'EMBEDDING'`, `'DEFAULT'`). Falls back to
|
|
792
|
+
* `'DEFAULT'` when absent or unrecognised.
|
|
793
|
+
*/
|
|
794
|
+
const SPAN_KIND_ATTRIBUTE = "evalution.span.type";
|
|
795
|
+
/**
|
|
796
|
+
* Attribute name a span can set to give a human-readable name to the prompt.
|
|
797
|
+
*/
|
|
798
|
+
const PROMPT_NAME_ATTRIBUTE = "gen_ai.prompt.name";
|
|
799
|
+
/**
|
|
800
|
+
* Attribute name a span can set to link itself to a specific prompt. The value
|
|
801
|
+
* is a globally-unique prompt ID unless {@link PROMPT_PROVIDER_ID_ATTRIBUTE} is
|
|
802
|
+
* also set, in which case it is scoped to that provider.
|
|
803
|
+
*/
|
|
804
|
+
const PROMPT_ID_ATTRIBUTE = "evalution.prompt.id";
|
|
805
|
+
/**
|
|
806
|
+
* Attribute name a span can set to scope its {@link PROMPT_ID_ATTRIBUTE} to a
|
|
807
|
+
* specific prompt provider. When absent, the prompt ID is treated as global.
|
|
808
|
+
*/
|
|
809
|
+
const PROMPT_PROVIDER_ID_ATTRIBUTE = "evalution.prompt.provider.id";
|
|
810
|
+
/**
|
|
811
|
+
* Attribute name a span can set to include an array of input parameters used
|
|
812
|
+
* to construct the final prompt.
|
|
813
|
+
*/
|
|
814
|
+
const PROMPT_INPUTS_ATTRIBUTE = "evalution.prompt.inputs";
|
|
815
|
+
/**
|
|
816
|
+
* Attribute name a span can set to record which version of its prompt ran —
|
|
817
|
+
* for a file-based prompt, the commit checked out, when nothing was
|
|
818
|
+
* uncommitted.
|
|
819
|
+
*/
|
|
820
|
+
const PROMPT_VERSION_ATTRIBUTE = "evalution.prompt.version";
|
|
821
|
+
/**
|
|
822
|
+
* Attribute name a span can set to record which variation of its prompt ran,
|
|
823
|
+
* on top of its {@link PROMPT_VERSION_ATTRIBUTE}.
|
|
824
|
+
*/
|
|
825
|
+
const PROMPT_VARIATION_ATTRIBUTE = "evalution.prompt.variation";
|
|
826
|
+
/**
|
|
827
|
+
* `identity` with any field it leaves unset filled from `fallback`.
|
|
828
|
+
*
|
|
829
|
+
* A config built by the `prompts()` helper carries an identity of its own —
|
|
830
|
+
* the one to keep, since its id is the global one — but only the playground
|
|
831
|
+
* knows what the run was launched with and against which version.
|
|
832
|
+
*/
|
|
833
|
+
function mergePromptIdentity(identity, fallback) {
|
|
834
|
+
if (!identity || !fallback) return identity ?? fallback;
|
|
835
|
+
const merged = { ...fallback };
|
|
836
|
+
for (const [key, value] of Object.entries(identity)) if (value !== void 0) merged[key] = value;
|
|
837
|
+
return merged;
|
|
838
|
+
}
|
|
839
|
+
/**
|
|
840
|
+
* Build the attributes that associate a span with a prompt.
|
|
841
|
+
*
|
|
842
|
+
* @param prompt - The prompt to attribute the span to.
|
|
843
|
+
* @param attributes - Attributes to merge in; these take precedence.
|
|
844
|
+
* @param includeInputs - Whether to stamp the run's inputs. Inputs belong on
|
|
845
|
+
* the **root span only**: they are identical on every span of a trace, so
|
|
846
|
+
* repeating them is invisible when the input is `["Ada"]` and wasteful when
|
|
847
|
+
* it is a fifty-message thread — multiplied by span count, then persisted.
|
|
848
|
+
*/
|
|
849
|
+
function getPromptSpanAttributes(prompt, attributes = {}, includeInputs = true) {
|
|
850
|
+
const inputs = includeInputs && (prompt.functionInputs || prompt.executeInputs) ? JSON.stringify({
|
|
851
|
+
functionInputs: prompt.functionInputs,
|
|
852
|
+
executeInputs: prompt.executeInputs,
|
|
853
|
+
resources: prompt.resources,
|
|
854
|
+
parameterDefinitions: prompt.parameterDefinitions,
|
|
855
|
+
executeParameterDefinitions: prompt.executeParameterDefinitions
|
|
856
|
+
}) : void 0;
|
|
857
|
+
return Object.fromEntries([
|
|
858
|
+
[SPAN_KIND_ATTRIBUTE, "LLM"],
|
|
859
|
+
[PROMPT_NAME_ATTRIBUTE, prompt.name],
|
|
860
|
+
[PROMPT_ID_ATTRIBUTE, prompt.id],
|
|
861
|
+
[PROMPT_INPUTS_ATTRIBUTE, inputs],
|
|
862
|
+
[PROMPT_VERSION_ATTRIBUTE, prompt.version],
|
|
863
|
+
[PROMPT_VARIATION_ATTRIBUTE, prompt.variation],
|
|
864
|
+
...Object.entries(attributes)
|
|
865
|
+
].filter(([, v]) => v !== void 0 && v !== null));
|
|
866
|
+
}
|
|
867
|
+
/**
|
|
868
|
+
* Wraps a {@link Tracer} so that every span it produces is tagged with the
|
|
869
|
+
* attributes that associate it with a prompt — the prompt's name
|
|
870
|
+
* ({@link PROMPT_NAME_ATTRIBUTE}), an optional global prompt ID
|
|
871
|
+
* ({@link PROMPT_ID_ATTRIBUTE}), and an `'LLM'` span kind
|
|
872
|
+
* ({@link SPAN_KIND_ATTRIBUTE}). Attributes set explicitly on an individual
|
|
873
|
+
* span take precedence over these defaults.
|
|
874
|
+
*
|
|
875
|
+
* This depends only on `@opentelemetry/api`, so it can be re-used by SDK
|
|
876
|
+
* adapter packages (e.g. `@evalution/vercel-ai-sdk`) without pulling in the
|
|
877
|
+
* rest of evalution.
|
|
878
|
+
*
|
|
879
|
+
* @param prompt - The prompt to attribute spans to. `name` is a human-readable
|
|
880
|
+
* name; the optional `id` is a globally-unique prompt ID used to resolve
|
|
881
|
+
* runtime traces back to the prompt.
|
|
882
|
+
* @param tracer - Tracer to wrap. Defaults to a tracer from the globally
|
|
883
|
+
* registered tracer provider.
|
|
884
|
+
* @returns A tracer that forwards to `tracer` while attaching the prompt
|
|
885
|
+
* attributes to each span it creates.
|
|
886
|
+
*/
|
|
887
|
+
function createTracerForPrompt(prompt, tracer) {
|
|
888
|
+
const inner = tracer ?? trace.getTracer("evalution");
|
|
889
|
+
let rootEmitted = false;
|
|
890
|
+
const withPromptAttributes = (options) => {
|
|
891
|
+
const attributes = getPromptSpanAttributes(prompt, options?.attributes, !rootEmitted);
|
|
892
|
+
rootEmitted = true;
|
|
893
|
+
return {
|
|
894
|
+
...options,
|
|
895
|
+
attributes
|
|
896
|
+
};
|
|
897
|
+
};
|
|
898
|
+
return {
|
|
899
|
+
startSpan(name, options, context) {
|
|
900
|
+
return inner.startSpan(name, withPromptAttributes(options), context);
|
|
901
|
+
},
|
|
902
|
+
startActiveSpan(name, ...rest) {
|
|
903
|
+
if (typeof rest[0] === "function") return inner.startActiveSpan(name, withPromptAttributes(), rest[0]);
|
|
904
|
+
if (typeof rest[1] === "function") return inner.startActiveSpan(name, withPromptAttributes(rest[0]), rest[1]);
|
|
905
|
+
return inner.startActiveSpan(name, withPromptAttributes(rest[0]), rest[1], rest[2]);
|
|
906
|
+
}
|
|
907
|
+
};
|
|
908
|
+
}
|
|
909
|
+
//#endregion
|
|
910
|
+
//#region src/trace-types.ts
|
|
911
|
+
/**
|
|
912
|
+
* Span attribute naming the eval check (by its id within the eval) that
|
|
913
|
+
* recorded the span while judging the trace — set on the check's `CHECK` span
|
|
914
|
+
* and on everything under it. Those spans aren't part of the run being
|
|
915
|
+
* judged, so the trace's rollups and chat view leave them out.
|
|
916
|
+
*/
|
|
917
|
+
const CHECK_ATTRIBUTE = "evalution.check";
|
|
918
|
+
/** Whether `span` was recorded by a check judging its trace, not by the run. */
|
|
919
|
+
function isCheckSpan(span) {
|
|
920
|
+
return span.attributes?.[CHECK_ATTRIBUTE] !== void 0;
|
|
921
|
+
}
|
|
922
|
+
/**
|
|
923
|
+
* The message list of a chat call, or `undefined` for any other input.
|
|
924
|
+
*
|
|
925
|
+
* A chat call's {@link LLMSpanDetails.input} is always a message list, and
|
|
926
|
+
* anything else is wrapped in an object, so being an array is what makes it
|
|
927
|
+
* one.
|
|
928
|
+
*/
|
|
929
|
+
function spanMessages(llm) {
|
|
930
|
+
return Array.isArray(llm?.input) ? llm.input : void 0;
|
|
931
|
+
}
|
|
932
|
+
/**
|
|
933
|
+
* The output of a turn that called tools: its text (if any) and the calls, as
|
|
934
|
+
* the one assistant message the next turn's input repeats. Without the calls
|
|
935
|
+
* such a turn would record no output at all, its text being empty.
|
|
936
|
+
*/
|
|
937
|
+
function toolCallingOutput(text, toolCalls) {
|
|
938
|
+
return [{
|
|
939
|
+
role: "assistant",
|
|
940
|
+
content: [...text ? [{
|
|
941
|
+
type: "text",
|
|
942
|
+
text
|
|
943
|
+
}] : [], ...toolCalls]
|
|
944
|
+
}];
|
|
945
|
+
}
|
|
946
|
+
/**
|
|
947
|
+
* An LLM span's output as a message list: text as the assistant message it
|
|
948
|
+
* is, a turn that called tools as the message it recorded. `undefined` for
|
|
949
|
+
* structured output, which can be an array too: every element must look like
|
|
950
|
+
* a message.
|
|
951
|
+
*/
|
|
952
|
+
function outputMessages(llm) {
|
|
953
|
+
const output = llm?.output;
|
|
954
|
+
if (typeof output === "string") return [{
|
|
955
|
+
role: "assistant",
|
|
956
|
+
content: output
|
|
957
|
+
}];
|
|
958
|
+
return Array.isArray(output) && output.length > 0 && output.every((m) => m && typeof m === "object" && typeof m.role === "string" && (typeof m.content === "string" || Array.isArray(m.content))) ? output : void 0;
|
|
959
|
+
}
|
|
960
|
+
/**
|
|
961
|
+
* The OpenTelemetry semantic-convention resource attribute naming the
|
|
962
|
+
* service that produced a span.
|
|
963
|
+
*/
|
|
964
|
+
const SERVICE_NAME_ATTRIBUTE = "service.name";
|
|
965
|
+
/**
|
|
966
|
+
* The OpenTelemetry semantic-convention resource attribute naming the
|
|
967
|
+
* deployment environment a span was produced in.
|
|
968
|
+
*/
|
|
969
|
+
const DEPLOYMENT_ENVIRONMENT_ATTRIBUTE = "deployment.environment.name";
|
|
970
|
+
/** {@link DEPLOYMENT_ENVIRONMENT_ATTRIBUTE}'s value on a run evalution itself made. */
|
|
971
|
+
const PLAYGROUND_ENVIRONMENT = "playground";
|
|
972
|
+
/**
|
|
973
|
+
* A tool result's output as the tool returned it. The AI SDK wraps a result
|
|
974
|
+
* in a typed envelope (`{ type: "json", value }`, `{ type: "error-text",
|
|
975
|
+
* value }`, `{ type: "execution-denied", reason }`, …); this unwraps it and
|
|
976
|
+
* notes whether it reports a failure. Anything else is kept as is.
|
|
977
|
+
*/
|
|
978
|
+
function unwrapToolResultOutput(output) {
|
|
979
|
+
if (!output || typeof output !== "object" || Array.isArray(output)) return { output };
|
|
980
|
+
const envelope = output;
|
|
981
|
+
const { type } = envelope;
|
|
982
|
+
if (type === "execution-denied") return {
|
|
983
|
+
output: output.reason,
|
|
984
|
+
isError: true
|
|
985
|
+
};
|
|
986
|
+
if (typeof type !== "string" || !("value" in envelope)) return { output };
|
|
987
|
+
return type.startsWith("error-") ? {
|
|
988
|
+
output: envelope.value,
|
|
989
|
+
isError: true
|
|
990
|
+
} : { output: envelope.value };
|
|
991
|
+
}
|
|
992
|
+
//#endregion
|
|
993
|
+
//#region src/trace-ingestor.ts
|
|
994
|
+
/**
|
|
995
|
+
* The resource of every run evalution itself makes — from the playground, an
|
|
996
|
+
* eval, or an MCP client — as opposed to one an app exported to it.
|
|
997
|
+
*/
|
|
998
|
+
const PLAYGROUND_RESOURCE = {
|
|
999
|
+
[SERVICE_NAME_ATTRIBUTE]: "evalution",
|
|
1000
|
+
[DEPLOYMENT_ENVIRONMENT_ATTRIBUTE]: PLAYGROUND_ENVIRONMENT
|
|
1001
|
+
};
|
|
1002
|
+
/**
|
|
1003
|
+
* Base class that holds the added sinks and fans each normalized record out
|
|
1004
|
+
* to all of them. Concrete ingestors translate their source into
|
|
1005
|
+
* {@link Span}/{@link Trace} records and call these protected helpers.
|
|
1006
|
+
*/
|
|
1007
|
+
var BaseTraceIngestor = class {
|
|
1008
|
+
sinks = [];
|
|
1009
|
+
resource;
|
|
1010
|
+
/** Recordings still under way, which {@link flush} waits out. */
|
|
1011
|
+
pending = /* @__PURE__ */ new Set();
|
|
1012
|
+
setResource(resource) {
|
|
1013
|
+
this.resource = { ...resource };
|
|
1014
|
+
}
|
|
1015
|
+
/** `span` with this ingestor's resource, if it has one, merged into its own. */
|
|
1016
|
+
withResource(span) {
|
|
1017
|
+
if (!this.resource) return span;
|
|
1018
|
+
return {
|
|
1019
|
+
...span,
|
|
1020
|
+
resource: {
|
|
1021
|
+
...span.resource,
|
|
1022
|
+
...this.resource
|
|
1023
|
+
}
|
|
1024
|
+
};
|
|
1025
|
+
}
|
|
1026
|
+
addSink(sink) {
|
|
1027
|
+
this.sinks.push(sink);
|
|
1028
|
+
}
|
|
1029
|
+
removeSink(sink) {
|
|
1030
|
+
const prevSinks = this.sinks;
|
|
1031
|
+
const newSinks = prevSinks.filter((s) => s !== sink);
|
|
1032
|
+
this.sinks = newSinks;
|
|
1033
|
+
return prevSinks.length > newSinks.length;
|
|
1034
|
+
}
|
|
1035
|
+
/**
|
|
1036
|
+
* Resolves once everything recorded so far has reached every sink and been
|
|
1037
|
+
* delivered by those that deliver in the background. Await it before a
|
|
1038
|
+
* short-lived process (a script, a serverless function) exits, so no trace
|
|
1039
|
+
* data is lost.
|
|
1040
|
+
*/
|
|
1041
|
+
async flush() {
|
|
1042
|
+
while (this.pending.size > 0) await Promise.allSettled(this.pending);
|
|
1043
|
+
await Promise.all(this.sinks.map((s) => s.flush?.()));
|
|
1044
|
+
}
|
|
1045
|
+
/** Registers `recording` for {@link flush} to wait out, and returns it. */
|
|
1046
|
+
track(recording) {
|
|
1047
|
+
this.pending.add(recording);
|
|
1048
|
+
const done = () => this.pending.delete(recording);
|
|
1049
|
+
recording.then(done, done);
|
|
1050
|
+
return recording;
|
|
1051
|
+
}
|
|
1052
|
+
async recordSpanStart(span) {
|
|
1053
|
+
const stamped = this.withResource(span);
|
|
1054
|
+
await this.track(Promise.all(this.sinks.map((s) => s.recordSpanStart(stamped))));
|
|
1055
|
+
}
|
|
1056
|
+
async recordSpanEnd(span) {
|
|
1057
|
+
const stamped = this.withResource(span);
|
|
1058
|
+
await this.track(Promise.all(this.sinks.map((s) => s.recordSpanEnd(stamped))));
|
|
1059
|
+
}
|
|
1060
|
+
async failTrace(traceId, msg) {
|
|
1061
|
+
await this.track(Promise.all(this.sinks.map((s) => s.failTrace(traceId, msg))));
|
|
1062
|
+
}
|
|
1063
|
+
};
|
|
1064
|
+
//#endregion
|
|
1065
|
+
//#region src/trace-sink.ts
|
|
1066
|
+
function isTraceSink(o) {
|
|
1067
|
+
return typeof o === "object" && o !== null && typeof o.recordSpanStart === "function" && typeof o.recordSpanEnd === "function" && typeof o.failTrace === "function";
|
|
1068
|
+
}
|
|
1069
|
+
//#endregion
|
|
1070
|
+
export { BaseTraceIngestor, CHECK_ATTRIBUTE, DEFAULT_API_URL, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, Evalution, EvalutionApiError, HttpTraceSink, PLAYGROUND_ENVIRONMENT, PLAYGROUND_RESOURCE, PROMPT_ID_ATTRIBUTE, PROMPT_INPUTS_ATTRIBUTE, PROMPT_NAME_ATTRIBUTE, PROMPT_PROVIDER_ID_ATTRIBUTE, PROMPT_VARIATION_ATTRIBUTE, PROMPT_VERSION_ATTRIBUTE, SERVICE_NAME_ATTRIBUTE, SPAN_KIND_ATTRIBUTE, TRACE_BUCKET_BOUNDS, TRACE_EVENTS_PATH, answersWithConfidence, apiKeyKind, createTracerForPrompt, evaluationConfidence, filterTraces, getPromptSpanAttributes, groupTraceSummaries, isCheckSpan, isEmptyTraceQuery, isTraceNumberColumn, isTraceSink, isTraceTextColumn, makeBrand, matchesTraceFilter, matchesTraceQuery, mergePromptIdentity, mergeTraceGroups, outputMessages, parseTraceGroupQuery, parseTraceQuery, spanMessages, summarizeTraces, toolCallingOutput, traceBucketAt, traceBucketFilter, traceDateBucket, traceGroupFilter, traceGroupKey, traceNumberValue, traceQueryParams, traceTextValue, unwrapToolResultOutput };
|