@evalution/sdk 0.0.0-stage → 2.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,1070 @@
1
+ import { trace } from "@opentelemetry/api";
2
+ //#region src/brand.ts
3
+ function makeBrand(sym) {
4
+ return {
5
+ brand: (value) => Object.defineProperty(value, sym, { value: true }),
6
+ isBranded: (value) => typeof value === "object" && value !== null && value[sym] === true
7
+ };
8
+ }
9
+ //#endregion
10
+ //#region src/http-trace-sink.ts
11
+ /**
12
+ * Where an evalution server accepts {@link TraceSinkEvent}s, relative to its
13
+ * base URL.
14
+ */
15
+ const TRACE_EVENTS_PATH = "/api/traces";
16
+ /**
17
+ * A {@link TraceSink} that sends what it's given to an evalution server,
18
+ * which records it through its own trace providers — so an app's traces show
19
+ * up in the playground without the app writing to a trace store itself.
20
+ *
21
+ * Calls return as soon as the event is queued, never waiting on the network,
22
+ * so tracing doesn't slow the app down. Queued events go out in order, batched
23
+ * into one request at a time, so a span's end never overtakes its start. A
24
+ * request that fails is logged and its events dropped: tracing never breaks
25
+ * the app. Await {@link flush} before a short-lived process exits to be sure
26
+ * everything was sent.
27
+ */
28
+ var HttpTraceSink = class {
29
+ options;
30
+ target;
31
+ /** Set once it's known there's nowhere to send, so events aren't queued. */
32
+ disabled = false;
33
+ queue = [];
34
+ sending;
35
+ /** Whether the last request failed, so a down server is reported once. */
36
+ failing = false;
37
+ constructor(options) {
38
+ this.options = options;
39
+ }
40
+ resolveTarget() {
41
+ this.target ??= this.lookUpTarget().catch(() => void 0);
42
+ return this.target;
43
+ }
44
+ async lookUpTarget() {
45
+ const { endpoint, fetch: send } = this.options;
46
+ const target = typeof endpoint === "function" ? await endpoint() : { endpoint };
47
+ if (!target) return void 0;
48
+ return {
49
+ url: target.endpoint.replace(/\/+$/, "") + TRACE_EVENTS_PATH,
50
+ headers: target.headers,
51
+ fetch: target.fetch ?? send ?? ((...args) => globalThis.fetch(...args))
52
+ };
53
+ }
54
+ async recordSpanStart(span) {
55
+ this.enqueue({
56
+ type: "span-start",
57
+ span
58
+ });
59
+ return span;
60
+ }
61
+ async recordSpanEnd(span) {
62
+ this.enqueue({
63
+ type: "span-end",
64
+ span
65
+ });
66
+ return span;
67
+ }
68
+ async failTrace(traceId, errorMessage) {
69
+ this.enqueue({
70
+ type: "trace-fail",
71
+ traceId,
72
+ errorMessage
73
+ });
74
+ }
75
+ /** Resolves once every event recorded so far has been sent (or dropped). */
76
+ async flush() {
77
+ return this.sending ?? Promise.resolve();
78
+ }
79
+ enqueue(event) {
80
+ if (this.disabled) return;
81
+ this.queue.push(event);
82
+ this.sending ??= this.drain();
83
+ }
84
+ async drain() {
85
+ await new Promise((resolve) => setTimeout(resolve));
86
+ const target = await this.resolveTarget();
87
+ if (!target) {
88
+ this.disabled = true;
89
+ this.queue = [];
90
+ }
91
+ while (target && this.queue.length > 0) {
92
+ const events = this.queue;
93
+ this.queue = [];
94
+ await this.send(target, events);
95
+ }
96
+ this.sending = void 0;
97
+ }
98
+ async send(target, events) {
99
+ try {
100
+ const body = { events };
101
+ const res = await target.fetch(target.url, {
102
+ method: "POST",
103
+ headers: {
104
+ "content-type": "application/json",
105
+ ...target.headers
106
+ },
107
+ body: JSON.stringify(body)
108
+ });
109
+ if (!res.ok) throw new Error(`${res.status} ${await res.text().catch(() => "")}`);
110
+ this.failing = false;
111
+ } catch (err) {
112
+ if (!this.failing) console.warn(`[evalution] Couldn't send traces to ${target.url}:`, err instanceof Error ? err.message : err);
113
+ this.failing = true;
114
+ }
115
+ }
116
+ };
117
+ //#endregion
118
+ //#region src/trace-query.ts
119
+ const TEXT_COLUMNS = [
120
+ "name",
121
+ "status",
122
+ "model",
123
+ "environment",
124
+ "annotations",
125
+ "checks",
126
+ "version",
127
+ "variation"
128
+ ];
129
+ const NUMBER_COLUMNS = [
130
+ "startTime",
131
+ "spanCount",
132
+ "duration",
133
+ "totalTokens",
134
+ "cost"
135
+ ];
136
+ /** Whether `column` names a {@link TraceTextColumn}. */
137
+ function isTraceTextColumn(column) {
138
+ return TEXT_COLUMNS.includes(column);
139
+ }
140
+ /** Whether `column` names a {@link TraceNumberColumn}. */
141
+ function isTraceNumberColumn(column) {
142
+ return NUMBER_COLUMNS.includes(column);
143
+ }
144
+ const DATE_RESOLUTIONS = [
145
+ "hour",
146
+ "day",
147
+ "week",
148
+ "month"
149
+ ];
150
+ /**
151
+ * The lower bounds of every bucket but the first, for the number columns
152
+ * grouped into buckets: `[1000, 5000]` makes the buckets `< 1000`,
153
+ * `1000 – 5000` and `≥ 5000`. Durations are in ms and costs in dollars.
154
+ */
155
+ const TRACE_BUCKET_BOUNDS = {
156
+ spanCount: [
157
+ 2,
158
+ 6,
159
+ 21,
160
+ 101
161
+ ],
162
+ duration: [
163
+ 1e3,
164
+ 5e3,
165
+ 15e3,
166
+ 6e4,
167
+ 3e5
168
+ ],
169
+ totalTokens: [
170
+ 1e3,
171
+ 1e4,
172
+ 1e5,
173
+ 1e6
174
+ ],
175
+ cost: [
176
+ .001,
177
+ .01,
178
+ .1,
179
+ 1,
180
+ 10
181
+ ]
182
+ };
183
+ /** A trace's value in a text column, or `null` if it has none. */
184
+ function traceTextValue(trace, column) {
185
+ switch (column) {
186
+ case "name": return trace.name;
187
+ case "status": return trace.status;
188
+ case "model": return trace.model ?? null;
189
+ case "environment": return trace.environment ?? null;
190
+ case "version": return trace.promptVersion ?? null;
191
+ case "variation": return trace.promptVariation ?? null;
192
+ case "annotations": {
193
+ const counts = trace.annotationCounts;
194
+ return counts.issue > 0 ? "issue" : counts.good > 0 ? "good" : counts.note > 0 ? "note" : null;
195
+ }
196
+ case "checks": {
197
+ const counts = trace.checkCounts;
198
+ if (!counts) return null;
199
+ return [
200
+ "fail",
201
+ "error",
202
+ "pass",
203
+ "scored",
204
+ "skipped"
205
+ ].find((outcome) => counts[outcome] > 0) ?? null;
206
+ }
207
+ }
208
+ }
209
+ /** A trace's value in a number column, or `undefined` if it has none. */
210
+ function traceNumberValue(trace, column) {
211
+ switch (column) {
212
+ case "startTime": return trace.startTime;
213
+ case "spanCount": return trace.spanCount;
214
+ case "duration": return trace.endTime !== void 0 ? trace.endTime - trace.startTime : void 0;
215
+ case "totalTokens": return trace.totalTokens;
216
+ case "cost": return trace.cost;
217
+ }
218
+ }
219
+ /** Whether `trace` meets `filter`. */
220
+ function matchesTraceFilter(trace, filter) {
221
+ switch (filter.op) {
222
+ case "in": return filter.values.includes(traceTextValue(trace, filter.column));
223
+ case "notIn": return !filter.values.includes(traceTextValue(trace, filter.column));
224
+ case "unset": return traceNumberValue(trace, filter.column) === void 0;
225
+ case "range": {
226
+ const value = traceNumberValue(trace, filter.column);
227
+ return value !== void 0 && (filter.min === void 0 || value >= filter.min) && (filter.max === void 0 || value < filter.max);
228
+ }
229
+ }
230
+ }
231
+ /** Whether `trace` is one `query` lists. */
232
+ function matchesTraceQuery(trace, query) {
233
+ const search = query.search?.trim().toLowerCase();
234
+ if (search && !trace.name.toLowerCase().includes(search) && !trace.id.toLowerCase().includes(search)) return false;
235
+ return (query.filters ?? []).every((filter) => matchesTraceFilter(trace, filter));
236
+ }
237
+ /** Whether `query` lists every trace. */
238
+ function isEmptyTraceQuery(query) {
239
+ return !query?.search?.trim() && !query?.filters?.length;
240
+ }
241
+ /** The traces of `traces` that `query` lists, in the same order. */
242
+ function filterTraces(traces, query) {
243
+ if (!query || isEmptyTraceQuery(query)) return traces;
244
+ return traces.filter((trace) => matchesTraceQuery(trace, query));
245
+ }
246
+ const HOUR_MS = 36e5;
247
+ /** Each time zone's wall-clock formatter: slow to make, so made once. */
248
+ const wallClocks = /* @__PURE__ */ new Map();
249
+ function wallClock(timeZone) {
250
+ let format = wallClocks.get(timeZone);
251
+ if (!format) {
252
+ format = new Intl.DateTimeFormat("en-US", {
253
+ timeZone,
254
+ hourCycle: "h23",
255
+ year: "numeric",
256
+ month: "numeric",
257
+ day: "numeric",
258
+ hour: "numeric",
259
+ minute: "numeric",
260
+ second: "numeric"
261
+ });
262
+ wallClocks.set(timeZone, format);
263
+ }
264
+ return format;
265
+ }
266
+ /** How far `timeZone`'s wall clock is ahead of UTC at `ms`, in ms. */
267
+ function zoneOffset(ms, timeZone) {
268
+ if (timeZone === "UTC") return 0;
269
+ const parts = Object.fromEntries(wallClock(timeZone).formatToParts(ms).map((part) => [part.type, Number(part.value)]));
270
+ return Date.UTC(parts.year, parts.month - 1, parts.day, parts.hour, parts.minute, parts.second) - Math.floor(ms / 1e3) * 1e3;
271
+ }
272
+ /** The instant `timeZone`'s wall clock reads `wall` (a time written as if UTC). */
273
+ function fromWallTime(wall, timeZone) {
274
+ return wall - zoneOffset(wall - zoneOffset(wall, timeZone), timeZone);
275
+ }
276
+ /**
277
+ * The `[min, max)` of the hour, day, week or month `ms` falls in, on
278
+ * `timeZone`'s clock.
279
+ */
280
+ function traceDateBucket(ms, resolution = "day", timeZone = "UTC") {
281
+ const offset = zoneOffset(ms, timeZone);
282
+ if (resolution === "hour") {
283
+ const min = Math.floor((ms + offset) / HOUR_MS) * HOUR_MS - offset;
284
+ return {
285
+ min,
286
+ max: min + HOUR_MS
287
+ };
288
+ }
289
+ const wall = new Date(ms + offset);
290
+ const year = wall.getUTCFullYear();
291
+ const month = wall.getUTCMonth();
292
+ const day = wall.getUTCDate();
293
+ let from;
294
+ let to;
295
+ if (resolution === "month") {
296
+ from = Date.UTC(year, month, 1);
297
+ to = Date.UTC(year, month + 1, 1);
298
+ } else if (resolution === "week") {
299
+ const sinceMonday = (wall.getUTCDay() + 6) % 7;
300
+ from = Date.UTC(year, month, day - sinceMonday);
301
+ to = Date.UTC(year, month, day - sinceMonday + 7);
302
+ } else {
303
+ from = Date.UTC(year, month, day);
304
+ to = Date.UTC(year, month, day + 1);
305
+ }
306
+ return {
307
+ min: fromWallTime(from, timeZone),
308
+ max: fromWallTime(to, timeZone)
309
+ };
310
+ }
311
+ /**
312
+ * The bucket of `column` that `value` falls in, as the filter listing it:
313
+ * see {@link TRACE_BUCKET_BOUNDS}.
314
+ */
315
+ function traceBucketFilter(column, value) {
316
+ if (value === void 0) return {
317
+ column,
318
+ op: "unset"
319
+ };
320
+ const bounds = TRACE_BUCKET_BOUNDS[column];
321
+ return traceBucketAt(column, bounds.filter((bound) => value >= bound).length);
322
+ }
323
+ /**
324
+ * The `index`th bucket of `column`, counting from the one below every bound
325
+ * of {@link TRACE_BUCKET_BOUNDS}, as the filter listing it.
326
+ */
327
+ function traceBucketAt(column, index) {
328
+ const bounds = TRACE_BUCKET_BOUNDS[column];
329
+ const min = index > 0 ? bounds[index - 1] : void 0;
330
+ const max = index < bounds.length ? bounds[index] : void 0;
331
+ return {
332
+ column,
333
+ op: "range",
334
+ ...min !== void 0 && { min },
335
+ ...max !== void 0 && { max }
336
+ };
337
+ }
338
+ /** The filter listing the group of `groupBy` that `trace` belongs to. */
339
+ function traceGroupFilter(trace, groupBy, timeZone) {
340
+ const { column } = groupBy;
341
+ if (isTraceTextColumn(column)) return {
342
+ column,
343
+ op: "in",
344
+ values: [traceTextValue(trace, column)]
345
+ };
346
+ if (column === "startTime") return {
347
+ column,
348
+ op: "range",
349
+ ...traceDateBucket(trace.startTime, groupBy.resolution, timeZone)
350
+ };
351
+ return traceBucketFilter(column, traceNumberValue(trace, column));
352
+ }
353
+ /** {@link TraceGroup.key} for the group `filter` lists. */
354
+ function traceGroupKey(filter) {
355
+ switch (filter.op) {
356
+ case "in":
357
+ case "notIn": return `${filter.column}:${filter.op}:${JSON.stringify(filter.values)}`;
358
+ case "unset": return `${filter.column}:unset`;
359
+ case "range": return `${filter.column}:range:${filter.min ?? ""}:${filter.max ?? ""}`;
360
+ }
361
+ }
362
+ /** The one value every item agrees on, else `undefined`. */
363
+ function shared(values) {
364
+ const first = values[0];
365
+ return values.every((v) => v === first) ? first : void 0;
366
+ }
367
+ /** The sum of the values that are set; `undefined` when none is. */
368
+ function sum(values) {
369
+ const set = values.filter((v) => v !== void 0);
370
+ return set.length > 0 ? set.reduce((a, b) => a + b, 0) : void 0;
371
+ }
372
+ /** `counts` added up; `undefined` when none is set. */
373
+ function sumCounts(counts) {
374
+ const set = counts.filter((c) => c !== void 0);
375
+ if (set.length === 0) return void 0;
376
+ const total = {
377
+ pass: 0,
378
+ fail: 0,
379
+ error: 0,
380
+ skipped: 0,
381
+ scored: 0
382
+ };
383
+ for (const c of set) for (const outcome of Object.keys(total)) total[outcome] += c[outcome];
384
+ return total;
385
+ }
386
+ /**
387
+ * What `traces` (non-empty) add up to. Takes summaries of groups as well as
388
+ * traces, so the groups of several listings can be added together.
389
+ */
390
+ function summarizeTraces(traces) {
391
+ const running = traces.some((t) => t.status === "running");
392
+ const totalTokens = sum(traces.map((t) => t.totalTokens));
393
+ const cost = sum(traces.map((t) => t.cost));
394
+ const checkCounts = sumCounts(traces.map((t) => t.checkCounts));
395
+ const model = shared(traces.map((t) => t.model));
396
+ const environment = shared(traces.map((t) => t.environment));
397
+ const promptVersion = shared(traces.map((t) => t.promptVersion));
398
+ const promptVariation = shared(traces.map((t) => t.promptVariation));
399
+ return {
400
+ startTime: Math.min(...traces.map((t) => t.startTime)),
401
+ ...!running && { endTime: Math.max(...traces.map((t) => t.endTime ?? t.startTime)) },
402
+ status: running ? "running" : traces.some((t) => t.status === "error") ? "error" : "ok",
403
+ spanCount: traces.reduce((n, t) => n + t.spanCount, 0),
404
+ ...totalTokens !== void 0 && { totalTokens },
405
+ ...cost !== void 0 && { cost },
406
+ ...checkCounts && { checkCounts },
407
+ ...model !== void 0 && { model },
408
+ ...environment !== void 0 && { environment },
409
+ ...promptVersion !== void 0 && { promptVersion },
410
+ ...promptVariation !== void 0 && { promptVariation },
411
+ annotationCounts: {
412
+ issue: traces.reduce((n, t) => n + t.annotationCounts.issue, 0),
413
+ good: traces.reduce((n, t) => n + t.annotationCounts.good, 0),
414
+ note: traces.reduce((n, t) => n + t.annotationCounts.note, 0)
415
+ }
416
+ };
417
+ }
418
+ /**
419
+ * `traces` as the groups of `groupBy`, in the order their first traces come.
420
+ * Filter them first: this groups every trace it's given.
421
+ */
422
+ function groupTraceSummaries(traces, groupBy, timeZone) {
423
+ const members = /* @__PURE__ */ new Map();
424
+ for (const trace of traces) {
425
+ const filter = traceGroupFilter(trace, groupBy, timeZone);
426
+ const key = traceGroupKey(filter);
427
+ const group = members.get(key);
428
+ if (group) group.traces.push(trace);
429
+ else members.set(key, {
430
+ filter,
431
+ traces: [trace]
432
+ });
433
+ }
434
+ return Array.from(members, ([key, group]) => ({
435
+ key,
436
+ filter: group.filter,
437
+ count: group.traces.length,
438
+ summary: summarizeTraces(group.traces)
439
+ }));
440
+ }
441
+ /**
442
+ * The groups of several listings as one: those with the same key added
443
+ * together, in the order their keys first come.
444
+ */
445
+ function mergeTraceGroups(groups) {
446
+ const byKey = /* @__PURE__ */ new Map();
447
+ for (const group of groups) {
448
+ const same = byKey.get(group.key);
449
+ if (same) same.push(group);
450
+ else byKey.set(group.key, [group]);
451
+ }
452
+ return Array.from(byKey.values(), (same) => same.length === 1 ? same[0] : {
453
+ key: same[0].key,
454
+ filter: same[0].filter,
455
+ count: same.reduce((n, group) => n + group.count, 0),
456
+ summary: summarizeTraces(same.map((group) => group.summary))
457
+ });
458
+ }
459
+ const optionalNumber = (value) => value === void 0 || typeof value === "number" && Number.isFinite(value);
460
+ /** `value` as a {@link TraceFilter}. Throws a `TypeError` if it isn't one. */
461
+ function parseTraceFilter(value) {
462
+ const filter = value;
463
+ if (filter && typeof filter === "object") {
464
+ const { column, op } = filter;
465
+ if (op === "in" || op === "notIn") {
466
+ const { values } = filter;
467
+ if (isTraceTextColumn(column) && Array.isArray(values) && values.every((v) => v === null || typeof v === "string")) return {
468
+ column,
469
+ op,
470
+ values
471
+ };
472
+ } else if (op === "unset") {
473
+ if (isTraceNumberColumn(column)) return {
474
+ column,
475
+ op
476
+ };
477
+ } else if (op === "range") {
478
+ const { min, max } = filter;
479
+ if (isTraceNumberColumn(column) && optionalNumber(min) && optionalNumber(max)) return {
480
+ column,
481
+ op,
482
+ ...min !== void 0 && { min },
483
+ ...max !== void 0 && { max }
484
+ };
485
+ }
486
+ }
487
+ throw new TypeError(`Not a trace filter: ${JSON.stringify(value)}`);
488
+ }
489
+ /** The query parameters a {@link TraceQuery} or {@link TraceGroupQuery} travels as. */
490
+ function traceQueryParams(query) {
491
+ const search = query.search?.trim();
492
+ return {
493
+ ...search && { q: search },
494
+ ...query.filters?.length && { filters: JSON.stringify(query.filters) },
495
+ ...query.groupBy && {
496
+ groupBy: query.groupBy.column,
497
+ ...query.groupBy.resolution && { resolution: query.groupBy.resolution }
498
+ },
499
+ ...query.timeZone && { tz: query.timeZone }
500
+ };
501
+ }
502
+ /**
503
+ * The {@link TraceQuery} that {@link traceQueryParams} wrote into `params`.
504
+ * Throws a `TypeError` naming what's wrong with one that can't be read.
505
+ */
506
+ function parseTraceQuery(params) {
507
+ const search = params.q?.trim();
508
+ let filters;
509
+ if (params.filters) {
510
+ let parsed;
511
+ try {
512
+ parsed = JSON.parse(params.filters);
513
+ } catch {
514
+ throw new TypeError("`filters` isn't JSON.");
515
+ }
516
+ if (!Array.isArray(parsed)) throw new TypeError("`filters` isn't an array.");
517
+ filters = parsed.map(parseTraceFilter);
518
+ }
519
+ return {
520
+ ...search && { search },
521
+ ...filters?.length && { filters }
522
+ };
523
+ }
524
+ /** {@link parseTraceQuery} for the parameters of a {@link TraceGroupQuery}. */
525
+ function parseTraceGroupQuery(params) {
526
+ const { groupBy: column, resolution, tz } = params;
527
+ if (!isTraceTextColumn(column) && !isTraceNumberColumn(column)) throw new TypeError("`groupBy` isn't a trace column.");
528
+ if (resolution !== void 0 && !DATE_RESOLUTIONS.includes(resolution)) throw new TypeError("`resolution` isn't hour, day, week or month.");
529
+ if (tz !== void 0) try {
530
+ new Intl.DateTimeFormat("en-US", { timeZone: tz });
531
+ } catch {
532
+ throw new TypeError("`tz` isn't a time zone.");
533
+ }
534
+ return {
535
+ ...parseTraceQuery(params),
536
+ groupBy: {
537
+ column,
538
+ ...resolution && { resolution }
539
+ },
540
+ ...tz && { timeZone: tz }
541
+ };
542
+ }
543
+ //#endregion
544
+ //#region src/client.ts
545
+ /** The Evalution Cloud API, unless `EVALUTION_API_URL` says otherwise. */
546
+ const DEFAULT_API_URL = "https://api.evalut.io";
547
+ /** How much of an API key is safe to print: its kind and a few characters. */
548
+ const PRINTABLE_API_KEY_PREFIX = 15;
549
+ /**
550
+ * The kind of an API key, read from its prefix, or `undefined` for a string
551
+ * that isn't one.
552
+ */
553
+ function apiKeyKind(key) {
554
+ if (key.startsWith("evl_ingest_")) return "ingest";
555
+ if (key.startsWith("evl_user_")) return "personal";
556
+ }
557
+ /** A listing's query as a URL's search, `?` included; empty if it has none. */
558
+ function listParams(query) {
559
+ const { search, filters, groupBy, timeZone, ...window } = query;
560
+ const params = new URLSearchParams(traceQueryParams({
561
+ search,
562
+ filters,
563
+ groupBy,
564
+ timeZone
565
+ }));
566
+ for (const [name, value] of Object.entries(window)) if (value !== void 0) params.set(name, String(value));
567
+ return params.size > 0 ? `?${params}` : "";
568
+ }
569
+ /**
570
+ * The client for Evalution's API. As a {@link TraceSink} it sends an app's
571
+ * traces, and it makes the read and control calls the local server needs.
572
+ *
573
+ * Where traces go is worked out once, on the first one sent:
574
+ *
575
+ * 1. the {@link EvalutionOptions.endpoint} option;
576
+ * 2. Evalution Cloud, when the API key is an ingest key;
577
+ * 3. the local instance `EVALUTION_ENDPOINT` names;
578
+ * 4. on Node, the instance portkit knows for this directory;
579
+ * 5. otherwise nowhere, and sending does nothing.
580
+ *
581
+ * A personal key never sends traces to the cloud, and no key is ever sent to
582
+ * a local instance. Tracing never breaks the app: see {@link HttpTraceSink},
583
+ * and await {@link flush} before a short-lived process exits. The read and
584
+ * control calls, by contrast, reject when they fail.
585
+ */
586
+ var Evalution = class {
587
+ endpoint;
588
+ apiKey;
589
+ apiUrl;
590
+ project;
591
+ fetch;
592
+ sink;
593
+ /** The start of the API key, safe to show in messages, or `<none>`. */
594
+ get printableApiKey() {
595
+ return this.apiKey ? `${this.apiKey.slice(0, PRINTABLE_API_KEY_PREFIX)}…` : "<none>";
596
+ }
597
+ constructor(options = {}) {
598
+ const env = globalThis.process?.env ?? {};
599
+ this.endpoint = options.endpoint;
600
+ this.apiKey = options.apiKey ?? env.EVALUTION_API_KEY;
601
+ this.apiUrl = (options.apiUrl || env.EVALUTION_API_URL || "https://api.evalut.io").replace(/\/+$/, "");
602
+ this.project = options.project;
603
+ this.fetch = options.fetch ?? ((...args) => globalThis.fetch(...args));
604
+ this.sink = new HttpTraceSink({
605
+ endpoint: () => this.resolveTraceTarget(env.EVALUTION_ENDPOINT),
606
+ fetch: this.fetch
607
+ });
608
+ }
609
+ async resolveTraceTarget(envEndpoint) {
610
+ if (this.endpoint === false) return void 0;
611
+ if (this.endpoint) return { endpoint: this.endpoint };
612
+ if (this.apiKey) switch (apiKeyKind(this.apiKey)) {
613
+ case "ingest": return {
614
+ endpoint: this.apiUrl,
615
+ headers: { authorization: `Bearer ${this.apiKey}` }
616
+ };
617
+ case "personal": break;
618
+ default: console.warn("[evalution] Ignoring the API key: it isn't an evl_ingest_… or evl_user_… key.");
619
+ }
620
+ if (envEndpoint) return { endpoint: envEndpoint };
621
+ if (!globalThis.process?.versions?.node) return void 0;
622
+ return (await import("./portkit-DV-XcagM.js")).discoverPortkit();
623
+ }
624
+ recordSpanStart(span) {
625
+ return this.sink.recordSpanStart(span);
626
+ }
627
+ recordSpanEnd(span) {
628
+ return this.sink.recordSpanEnd(span);
629
+ }
630
+ failTrace(traceId, errorMessage) {
631
+ return this.sink.failTrace(traceId, errorMessage);
632
+ }
633
+ /** Resolves once every trace event recorded so far has been sent (or dropped). */
634
+ flush() {
635
+ return this.sink.flush();
636
+ }
637
+ /** The key's kind and whose it is. Takes either kind of key. */
638
+ whoami() {
639
+ return this.json("GET", "/cloud/v1/whoami", { key: "any" });
640
+ }
641
+ /** The most recent production traces of the project. */
642
+ listTraces(query = {}) {
643
+ return this.json("GET", `/api/traces${listParams(query)}`, { project: true });
644
+ }
645
+ /**
646
+ * The project's production traces as groups, one per value or bucket of
647
+ * a column, counting every trace in the window rather than the most recent.
648
+ */
649
+ listTraceGroups(query) {
650
+ return this.json("GET", `/api/traces/groups${listParams(query)}`, { project: true });
651
+ }
652
+ /** One production trace with its spans, or `undefined` if there's none. */
653
+ async getTrace(id) {
654
+ const res = await this.request("GET", `/api/traces/${encodeURIComponent(id)}`, { project: true });
655
+ if (res.status === 404) return void 0;
656
+ return (await readOk(res)).json();
657
+ }
658
+ /** The projects the key's user is a member of. */
659
+ listProjects() {
660
+ return this.json("GET", "/cloud/v1/projects");
661
+ }
662
+ /** Creates a project in the user's organization. */
663
+ createProject(project) {
664
+ return this.json("POST", "/cloud/v1/projects", { body: project });
665
+ }
666
+ /** Records a running checkout of the project. */
667
+ registerInstance(instance) {
668
+ return this.json("PUT", "/cloud/v1/instances", {
669
+ project: true,
670
+ body: instance
671
+ });
672
+ }
673
+ /** Tells the cloud a registered instance is still running. */
674
+ async heartbeatInstance(id) {
675
+ await readOk(await this.instanceRequest("POST", id, "heartbeat"));
676
+ }
677
+ /** Tells the cloud a registered instance has stopped. */
678
+ async endInstanceSession(id) {
679
+ await readOk(await this.instanceRequest("DELETE", id, "session"));
680
+ }
681
+ /** Ensures a synced database exists, and returns how to reach it. */
682
+ getSyncDatabase(request) {
683
+ return this.json("POST", "/cloud/v1/sync/databases", {
684
+ project: true,
685
+ body: request
686
+ });
687
+ }
688
+ /** The project's synced databases. A dataset's has its id as the `key`; the others' `key` is empty. */
689
+ listSyncDatabases() {
690
+ return this.json("GET", "/cloud/v1/sync/databases", { project: true });
691
+ }
692
+ /** Deletes a dataset's synced database, for every member's machine. */
693
+ async deleteSyncDatabase(key) {
694
+ const path = `/cloud/v1/sync/databases/dataset/${encodeURIComponent(key.provider)}/${encodeURIComponent(key.key)}`;
695
+ await readOk(await this.request("DELETE", path, { project: true }));
696
+ }
697
+ instanceRequest(method, id, action) {
698
+ const path = `/cloud/v1/instances/${encodeURIComponent(id)}/${action}`;
699
+ return this.request(method, path, { project: true });
700
+ }
701
+ async json(method, path, options) {
702
+ return (await readOk(await this.request(method, path, options))).json();
703
+ }
704
+ request(method, path, { key = "personal", project, body } = {}) {
705
+ if (!this.apiKey) throw new Error("Evalution: set EVALUTION_API_KEY to use the cloud API.");
706
+ if (key === "personal" && apiKeyKind(this.apiKey) !== "personal") throw new Error(`Evalution: ${path} needs a personal (evl_user_…) key.`);
707
+ if (project && !this.project) throw new Error(`Evalution: ${path} needs a project id.`);
708
+ return this.fetch(this.apiUrl + path, {
709
+ method,
710
+ headers: {
711
+ authorization: `Bearer ${this.apiKey}`,
712
+ ...project && { "evalution-project": this.project },
713
+ ...body !== void 0 && { "content-type": "application/json" }
714
+ },
715
+ ...body !== void 0 && { body: JSON.stringify(body) }
716
+ });
717
+ }
718
+ };
719
+ /** How {@link Evalution} rejects when the cloud refuses a call. */
720
+ var EvalutionApiError = class extends Error {
721
+ /** The HTTP status the cloud answered with. */
722
+ status;
723
+ /** The cloud's explanation, if it gave one. */
724
+ detail;
725
+ /** The cloud's whole answer, when it was JSON: a 410's `syncId`, say. */
726
+ body;
727
+ constructor(status, detail, body) {
728
+ super(`Evalution API: ${status} ${detail ?? ""}`.trim());
729
+ this.name = "EvalutionApiError";
730
+ this.status = status;
731
+ this.detail = detail;
732
+ this.body = body;
733
+ }
734
+ };
735
+ /** `res`, or a rejection saying why the cloud refused. */
736
+ async function readOk(res) {
737
+ if (res.ok) return res;
738
+ const text = await res.text().catch(() => "");
739
+ let detail = text || void 0;
740
+ let body;
741
+ try {
742
+ const parsed = JSON.parse(text);
743
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
744
+ body = parsed;
745
+ if (typeof body.error === "string") detail = body.error;
746
+ }
747
+ } catch {}
748
+ throw new EvalutionApiError(res.status, detail, body);
749
+ }
750
+ //#endregion
751
+ //#region src/evaluation-answers.ts
752
+ const isRecord = (v) => !!v && typeof v === "object" && !Array.isArray(v);
753
+ /**
754
+ * The per-question confidence a provider reported beside an evaluation's
755
+ * answers, keyed by question id.
756
+ *
757
+ * The AI SDK's answers carry no confidence of their own: a provider that
758
+ * computes one (TypeSafe does, for choice and score answers) reports it as
759
+ * `providerMetadata.<provider>.confidence[questionId]`. Read from whichever
760
+ * provider namespace has it, so this isn't tied to one provider.
761
+ */
762
+ function evaluationConfidence(providerMetadata) {
763
+ const out = {};
764
+ if (!isRecord(providerMetadata)) return out;
765
+ for (const namespace of Object.values(providerMetadata)) {
766
+ if (!isRecord(namespace) || !isRecord(namespace.confidence)) continue;
767
+ for (const [id, value] of Object.entries(namespace.confidence)) if (typeof value === "number" && Number.isFinite(value)) out[id] = value;
768
+ }
769
+ return out;
770
+ }
771
+ /**
772
+ * An evaluation's answers as recorded on its span: each answer as the SDK
773
+ * returned it, plus its `confidence` where the provider reported one (see
774
+ * {@link evaluationConfidence}). Recorded together so a trace — from the
775
+ * playground or sent over OTLP — can show confidence without knowing which
776
+ * provider answered.
777
+ */
778
+ function answersWithConfidence(answers, providerMetadata) {
779
+ if (!isRecord(answers)) return answers;
780
+ const confidence = evaluationConfidence(providerMetadata);
781
+ if (Object.keys(confidence).length === 0) return answers;
782
+ return Object.fromEntries(Object.entries(answers).map(([id, answer]) => [id, isRecord(answer) && id in confidence && !("confidence" in answer) ? {
783
+ ...answer,
784
+ confidence: confidence[id]
785
+ } : answer]));
786
+ }
787
+ //#endregion
788
+ //#region src/prompt-tracer.ts
789
+ /**
790
+ * Attribute name a span can set to pick one of evalution's span-kind values
791
+ * (`'LLM'`, `'TOOL'`, `'AGENT'`, `'EMBEDDING'`, `'DEFAULT'`). Falls back to
792
+ * `'DEFAULT'` when absent or unrecognised.
793
+ */
794
+ const SPAN_KIND_ATTRIBUTE = "evalution.span.type";
795
+ /**
796
+ * Attribute name a span can set to give a human-readable name to the prompt.
797
+ */
798
+ const PROMPT_NAME_ATTRIBUTE = "gen_ai.prompt.name";
799
+ /**
800
+ * Attribute name a span can set to link itself to a specific prompt. The value
801
+ * is a globally-unique prompt ID unless {@link PROMPT_PROVIDER_ID_ATTRIBUTE} is
802
+ * also set, in which case it is scoped to that provider.
803
+ */
804
+ const PROMPT_ID_ATTRIBUTE = "evalution.prompt.id";
805
+ /**
806
+ * Attribute name a span can set to scope its {@link PROMPT_ID_ATTRIBUTE} to a
807
+ * specific prompt provider. When absent, the prompt ID is treated as global.
808
+ */
809
+ const PROMPT_PROVIDER_ID_ATTRIBUTE = "evalution.prompt.provider.id";
810
+ /**
811
+ * Attribute name a span can set to include an array of input parameters used
812
+ * to construct the final prompt.
813
+ */
814
+ const PROMPT_INPUTS_ATTRIBUTE = "evalution.prompt.inputs";
815
+ /**
816
+ * Attribute name a span can set to record which version of its prompt ran —
817
+ * for a file-based prompt, the commit checked out, when nothing was
818
+ * uncommitted.
819
+ */
820
+ const PROMPT_VERSION_ATTRIBUTE = "evalution.prompt.version";
821
+ /**
822
+ * Attribute name a span can set to record which variation of its prompt ran,
823
+ * on top of its {@link PROMPT_VERSION_ATTRIBUTE}.
824
+ */
825
+ const PROMPT_VARIATION_ATTRIBUTE = "evalution.prompt.variation";
826
+ /**
827
+ * `identity` with any field it leaves unset filled from `fallback`.
828
+ *
829
+ * A config built by the `prompts()` helper carries an identity of its own —
830
+ * the one to keep, since its id is the global one — but only the playground
831
+ * knows what the run was launched with and against which version.
832
+ */
833
+ function mergePromptIdentity(identity, fallback) {
834
+ if (!identity || !fallback) return identity ?? fallback;
835
+ const merged = { ...fallback };
836
+ for (const [key, value] of Object.entries(identity)) if (value !== void 0) merged[key] = value;
837
+ return merged;
838
+ }
839
+ /**
840
+ * Build the attributes that associate a span with a prompt.
841
+ *
842
+ * @param prompt - The prompt to attribute the span to.
843
+ * @param attributes - Attributes to merge in; these take precedence.
844
+ * @param includeInputs - Whether to stamp the run's inputs. Inputs belong on
845
+ * the **root span only**: they are identical on every span of a trace, so
846
+ * repeating them is invisible when the input is `["Ada"]` and wasteful when
847
+ * it is a fifty-message thread — multiplied by span count, then persisted.
848
+ */
849
+ function getPromptSpanAttributes(prompt, attributes = {}, includeInputs = true) {
850
+ const inputs = includeInputs && (prompt.functionInputs || prompt.executeInputs) ? JSON.stringify({
851
+ functionInputs: prompt.functionInputs,
852
+ executeInputs: prompt.executeInputs,
853
+ resources: prompt.resources,
854
+ parameterDefinitions: prompt.parameterDefinitions,
855
+ executeParameterDefinitions: prompt.executeParameterDefinitions
856
+ }) : void 0;
857
+ return Object.fromEntries([
858
+ [SPAN_KIND_ATTRIBUTE, "LLM"],
859
+ [PROMPT_NAME_ATTRIBUTE, prompt.name],
860
+ [PROMPT_ID_ATTRIBUTE, prompt.id],
861
+ [PROMPT_INPUTS_ATTRIBUTE, inputs],
862
+ [PROMPT_VERSION_ATTRIBUTE, prompt.version],
863
+ [PROMPT_VARIATION_ATTRIBUTE, prompt.variation],
864
+ ...Object.entries(attributes)
865
+ ].filter(([, v]) => v !== void 0 && v !== null));
866
+ }
867
+ /**
868
+ * Wraps a {@link Tracer} so that every span it produces is tagged with the
869
+ * attributes that associate it with a prompt — the prompt's name
870
+ * ({@link PROMPT_NAME_ATTRIBUTE}), an optional global prompt ID
871
+ * ({@link PROMPT_ID_ATTRIBUTE}), and an `'LLM'` span kind
872
+ * ({@link SPAN_KIND_ATTRIBUTE}). Attributes set explicitly on an individual
873
+ * span take precedence over these defaults.
874
+ *
875
+ * This depends only on `@opentelemetry/api`, so it can be re-used by SDK
876
+ * adapter packages (e.g. `@evalution/vercel-ai-sdk`) without pulling in the
877
+ * rest of evalution.
878
+ *
879
+ * @param prompt - The prompt to attribute spans to. `name` is a human-readable
880
+ * name; the optional `id` is a globally-unique prompt ID used to resolve
881
+ * runtime traces back to the prompt.
882
+ * @param tracer - Tracer to wrap. Defaults to a tracer from the globally
883
+ * registered tracer provider.
884
+ * @returns A tracer that forwards to `tracer` while attaching the prompt
885
+ * attributes to each span it creates.
886
+ */
887
+ function createTracerForPrompt(prompt, tracer) {
888
+ const inner = tracer ?? trace.getTracer("evalution");
889
+ let rootEmitted = false;
890
+ const withPromptAttributes = (options) => {
891
+ const attributes = getPromptSpanAttributes(prompt, options?.attributes, !rootEmitted);
892
+ rootEmitted = true;
893
+ return {
894
+ ...options,
895
+ attributes
896
+ };
897
+ };
898
+ return {
899
+ startSpan(name, options, context) {
900
+ return inner.startSpan(name, withPromptAttributes(options), context);
901
+ },
902
+ startActiveSpan(name, ...rest) {
903
+ if (typeof rest[0] === "function") return inner.startActiveSpan(name, withPromptAttributes(), rest[0]);
904
+ if (typeof rest[1] === "function") return inner.startActiveSpan(name, withPromptAttributes(rest[0]), rest[1]);
905
+ return inner.startActiveSpan(name, withPromptAttributes(rest[0]), rest[1], rest[2]);
906
+ }
907
+ };
908
+ }
909
+ //#endregion
910
+ //#region src/trace-types.ts
911
+ /**
912
+ * Span attribute naming the eval check (by its id within the eval) that
913
+ * recorded the span while judging the trace — set on the check's `CHECK` span
914
+ * and on everything under it. Those spans aren't part of the run being
915
+ * judged, so the trace's rollups and chat view leave them out.
916
+ */
917
+ const CHECK_ATTRIBUTE = "evalution.check";
918
+ /** Whether `span` was recorded by a check judging its trace, not by the run. */
919
+ function isCheckSpan(span) {
920
+ return span.attributes?.[CHECK_ATTRIBUTE] !== void 0;
921
+ }
922
+ /**
923
+ * The message list of a chat call, or `undefined` for any other input.
924
+ *
925
+ * A chat call's {@link LLMSpanDetails.input} is always a message list, and
926
+ * anything else is wrapped in an object, so being an array is what makes it
927
+ * one.
928
+ */
929
+ function spanMessages(llm) {
930
+ return Array.isArray(llm?.input) ? llm.input : void 0;
931
+ }
932
+ /**
933
+ * The output of a turn that called tools: its text (if any) and the calls, as
934
+ * the one assistant message the next turn's input repeats. Without the calls
935
+ * such a turn would record no output at all, its text being empty.
936
+ */
937
+ function toolCallingOutput(text, toolCalls) {
938
+ return [{
939
+ role: "assistant",
940
+ content: [...text ? [{
941
+ type: "text",
942
+ text
943
+ }] : [], ...toolCalls]
944
+ }];
945
+ }
946
+ /**
947
+ * An LLM span's output as a message list: text as the assistant message it
948
+ * is, a turn that called tools as the message it recorded. `undefined` for
949
+ * structured output, which can be an array too: every element must look like
950
+ * a message.
951
+ */
952
+ function outputMessages(llm) {
953
+ const output = llm?.output;
954
+ if (typeof output === "string") return [{
955
+ role: "assistant",
956
+ content: output
957
+ }];
958
+ return Array.isArray(output) && output.length > 0 && output.every((m) => m && typeof m === "object" && typeof m.role === "string" && (typeof m.content === "string" || Array.isArray(m.content))) ? output : void 0;
959
+ }
960
+ /**
961
+ * The OpenTelemetry semantic-convention resource attribute naming the
962
+ * service that produced a span.
963
+ */
964
+ const SERVICE_NAME_ATTRIBUTE = "service.name";
965
+ /**
966
+ * The OpenTelemetry semantic-convention resource attribute naming the
967
+ * deployment environment a span was produced in.
968
+ */
969
+ const DEPLOYMENT_ENVIRONMENT_ATTRIBUTE = "deployment.environment.name";
970
+ /** {@link DEPLOYMENT_ENVIRONMENT_ATTRIBUTE}'s value on a run evalution itself made. */
971
+ const PLAYGROUND_ENVIRONMENT = "playground";
972
+ /**
973
+ * A tool result's output as the tool returned it. The AI SDK wraps a result
974
+ * in a typed envelope (`{ type: "json", value }`, `{ type: "error-text",
975
+ * value }`, `{ type: "execution-denied", reason }`, …); this unwraps it and
976
+ * notes whether it reports a failure. Anything else is kept as is.
977
+ */
978
+ function unwrapToolResultOutput(output) {
979
+ if (!output || typeof output !== "object" || Array.isArray(output)) return { output };
980
+ const envelope = output;
981
+ const { type } = envelope;
982
+ if (type === "execution-denied") return {
983
+ output: output.reason,
984
+ isError: true
985
+ };
986
+ if (typeof type !== "string" || !("value" in envelope)) return { output };
987
+ return type.startsWith("error-") ? {
988
+ output: envelope.value,
989
+ isError: true
990
+ } : { output: envelope.value };
991
+ }
992
+ //#endregion
993
+ //#region src/trace-ingestor.ts
994
+ /**
995
+ * The resource of every run evalution itself makes — from the playground, an
996
+ * eval, or an MCP client — as opposed to one an app exported to it.
997
+ */
998
+ const PLAYGROUND_RESOURCE = {
999
+ [SERVICE_NAME_ATTRIBUTE]: "evalution",
1000
+ [DEPLOYMENT_ENVIRONMENT_ATTRIBUTE]: PLAYGROUND_ENVIRONMENT
1001
+ };
1002
+ /**
1003
+ * Base class that holds the added sinks and fans each normalized record out
1004
+ * to all of them. Concrete ingestors translate their source into
1005
+ * {@link Span}/{@link Trace} records and call these protected helpers.
1006
+ */
1007
+ var BaseTraceIngestor = class {
1008
+ sinks = [];
1009
+ resource;
1010
+ /** Recordings still under way, which {@link flush} waits out. */
1011
+ pending = /* @__PURE__ */ new Set();
1012
+ setResource(resource) {
1013
+ this.resource = { ...resource };
1014
+ }
1015
+ /** `span` with this ingestor's resource, if it has one, merged into its own. */
1016
+ withResource(span) {
1017
+ if (!this.resource) return span;
1018
+ return {
1019
+ ...span,
1020
+ resource: {
1021
+ ...span.resource,
1022
+ ...this.resource
1023
+ }
1024
+ };
1025
+ }
1026
+ addSink(sink) {
1027
+ this.sinks.push(sink);
1028
+ }
1029
+ removeSink(sink) {
1030
+ const prevSinks = this.sinks;
1031
+ const newSinks = prevSinks.filter((s) => s !== sink);
1032
+ this.sinks = newSinks;
1033
+ return prevSinks.length > newSinks.length;
1034
+ }
1035
+ /**
1036
+ * Resolves once everything recorded so far has reached every sink and been
1037
+ * delivered by those that deliver in the background. Await it before a
1038
+ * short-lived process (a script, a serverless function) exits, so no trace
1039
+ * data is lost.
1040
+ */
1041
+ async flush() {
1042
+ while (this.pending.size > 0) await Promise.allSettled(this.pending);
1043
+ await Promise.all(this.sinks.map((s) => s.flush?.()));
1044
+ }
1045
+ /** Registers `recording` for {@link flush} to wait out, and returns it. */
1046
+ track(recording) {
1047
+ this.pending.add(recording);
1048
+ const done = () => this.pending.delete(recording);
1049
+ recording.then(done, done);
1050
+ return recording;
1051
+ }
1052
+ async recordSpanStart(span) {
1053
+ const stamped = this.withResource(span);
1054
+ await this.track(Promise.all(this.sinks.map((s) => s.recordSpanStart(stamped))));
1055
+ }
1056
+ async recordSpanEnd(span) {
1057
+ const stamped = this.withResource(span);
1058
+ await this.track(Promise.all(this.sinks.map((s) => s.recordSpanEnd(stamped))));
1059
+ }
1060
+ async failTrace(traceId, msg) {
1061
+ await this.track(Promise.all(this.sinks.map((s) => s.failTrace(traceId, msg))));
1062
+ }
1063
+ };
1064
+ //#endregion
1065
+ //#region src/trace-sink.ts
1066
+ function isTraceSink(o) {
1067
+ return typeof o === "object" && o !== null && typeof o.recordSpanStart === "function" && typeof o.recordSpanEnd === "function" && typeof o.failTrace === "function";
1068
+ }
1069
+ //#endregion
1070
+ export { BaseTraceIngestor, CHECK_ATTRIBUTE, DEFAULT_API_URL, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, Evalution, EvalutionApiError, HttpTraceSink, PLAYGROUND_ENVIRONMENT, PLAYGROUND_RESOURCE, PROMPT_ID_ATTRIBUTE, PROMPT_INPUTS_ATTRIBUTE, PROMPT_NAME_ATTRIBUTE, PROMPT_PROVIDER_ID_ATTRIBUTE, PROMPT_VARIATION_ATTRIBUTE, PROMPT_VERSION_ATTRIBUTE, SERVICE_NAME_ATTRIBUTE, SPAN_KIND_ATTRIBUTE, TRACE_BUCKET_BOUNDS, TRACE_EVENTS_PATH, answersWithConfidence, apiKeyKind, createTracerForPrompt, evaluationConfidence, filterTraces, getPromptSpanAttributes, groupTraceSummaries, isCheckSpan, isEmptyTraceQuery, isTraceNumberColumn, isTraceSink, isTraceTextColumn, makeBrand, matchesTraceFilter, matchesTraceQuery, mergePromptIdentity, mergeTraceGroups, outputMessages, parseTraceGroupQuery, parseTraceQuery, spanMessages, summarizeTraces, toolCallingOutput, traceBucketAt, traceBucketFilter, traceDateBucket, traceGroupFilter, traceGroupKey, traceNumberValue, traceQueryParams, traceTextValue, unwrapToolResultOutput };