@evalution/sdk 0.0.0-stage → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,674 @@
1
+ import { AttributeValue, Attributes, Tracer } from "@opentelemetry/api";
2
+
3
+ //#region src/prompt-tracer.d.ts
4
+ interface PromptsHelperOptions {
5
+ /**
6
+ * Globally-unique stable identifier for this group of prompts.
7
+ */
8
+ readonly id: string;
9
+ }
10
+ type PromptsFactory<Args extends any[] = never[], Prompt = unknown> = (...args: Args) => Record<string, (...args: any[]) => Prompt>;
11
+ /**
12
+ * Intended to be used in the `satisfies` clause of a `prompts` export from an
13
+ * SDK adapter package.
14
+ */
15
+ type PromptsHelper = (opts: PromptsHelperOptions, factory: PromptsFactory<any[], any>) => PromptsFactory<any[], any>;
16
+ /**
17
+ * Attribute name a span can set to pick one of evalution's span-kind values
18
+ * (`'LLM'`, `'TOOL'`, `'AGENT'`, `'EMBEDDING'`, `'DEFAULT'`). Falls back to
19
+ * `'DEFAULT'` when absent or unrecognised.
20
+ */
21
+ declare const SPAN_KIND_ATTRIBUTE = "evalution.span.type";
22
+ /**
23
+ * Attribute name a span can set to give a human-readable name to the prompt.
24
+ */
25
+ declare const PROMPT_NAME_ATTRIBUTE = "gen_ai.prompt.name";
26
+ /**
27
+ * Attribute name a span can set to link itself to a specific prompt. The value
28
+ * is a globally-unique prompt ID unless {@link PROMPT_PROVIDER_ID_ATTRIBUTE} is
29
+ * also set, in which case it is scoped to that provider.
30
+ */
31
+ declare const PROMPT_ID_ATTRIBUTE = "evalution.prompt.id";
32
+ /**
33
+ * Attribute name a span can set to scope its {@link PROMPT_ID_ATTRIBUTE} to a
34
+ * specific prompt provider. When absent, the prompt ID is treated as global.
35
+ */
36
+ declare const PROMPT_PROVIDER_ID_ATTRIBUTE = "evalution.prompt.provider.id";
37
+ /**
38
+ * Attribute name a span can set to include an array of input parameters used
39
+ * to construct the final prompt.
40
+ */
41
+ declare const PROMPT_INPUTS_ATTRIBUTE = "evalution.prompt.inputs";
42
+ /**
43
+ * Attribute name a span can set to record which version of its prompt ran —
44
+ * for a file-based prompt, the commit checked out, when nothing was
45
+ * uncommitted.
46
+ */
47
+ declare const PROMPT_VERSION_ATTRIBUTE = "evalution.prompt.version";
48
+ /**
49
+ * Attribute name a span can set to record which variation of its prompt ran,
50
+ * on top of its {@link PROMPT_VERSION_ATTRIBUTE}.
51
+ */
52
+ declare const PROMPT_VARIATION_ATTRIBUTE = "evalution.prompt.variation";
53
+ interface PromptSpanInfo {
54
+ /** Globally-unique prompt id (e.g. `${groupId}#${name}`). */
55
+ id?: string;
56
+ /** Human-readable prompt name. */
57
+ name: string;
58
+ /**
59
+ * Positional arguments the prompt function was called with.
60
+ *
61
+ * Only the `prompts()` helper records these, because it is the only caller
62
+ * that sees the arguments and nothing else. A playground run records
63
+ * {@link functionInputs} instead: its arguments may include a live database
64
+ * handle, which is neither serializable nor the thing a replay needs.
65
+ */
66
+ functionParameters?: unknown[];
67
+ /**
68
+ * The unresolved inputs this run was launched with — the *recipe* rather
69
+ * than the resolution.
70
+ *
71
+ * This is what makes a past run replayable. A resolved value cannot be
72
+ * recorded faithfully (a live handle does not serialize) and would not be
73
+ * the right thing to record anyway: replaying a resource means running its
74
+ * `create()` again, which is the recipe, not the value it produced last
75
+ * time. Each entry is JSON-safe by construction.
76
+ */
77
+ functionInputs?: unknown[];
78
+ /** The unresolved execute-parameter inputs, keyed by parameter name. */
79
+ executeInputs?: Record<string, unknown>;
80
+ /**
81
+ * The run's named resource instances, by name, each with the receipt its
82
+ * `create()` produced. See `specs/resource-instances.md` §F.
83
+ */
84
+ resources?: Record<string, unknown>;
85
+ /**
86
+ * The prompt's parameter definitions as they stood when the run was
87
+ * launched, recorded beside the inputs.
88
+ *
89
+ * For a file-based prompt git is the version, and the playground has no
90
+ * business checking out old commits to replay one — so writing the shape
91
+ * down at run time is the cheap equivalent. A later replay diffs two known
92
+ * signatures instead of guessing whether the recorded inputs still fit.
93
+ */
94
+ parameterDefinitions?: unknown[];
95
+ /**
96
+ * The prompt's execute-parameter definitions, recorded for the same reason
97
+ * as {@link parameterDefinitions}: without them an entry of
98
+ * {@link executeInputs} has a name but no type.
99
+ */
100
+ executeParameterDefinitions?: unknown[];
101
+ /** The version the run executed against. See {@link PROMPT_VERSION_ATTRIBUTE}. */
102
+ version?: string;
103
+ /** The variation the run applied. See {@link PROMPT_VARIATION_ATTRIBUTE}. */
104
+ variation?: string;
105
+ }
106
+ /**
107
+ * `identity` with any field it leaves unset filled from `fallback`.
108
+ *
109
+ * A config built by the `prompts()` helper carries an identity of its own —
110
+ * the one to keep, since its id is the global one — but only the playground
111
+ * knows what the run was launched with and against which version.
112
+ */
113
+ declare function mergePromptIdentity(identity: PromptSpanInfo | undefined, fallback: PromptSpanInfo | undefined): PromptSpanInfo | undefined;
114
+ /**
115
+ * Build the attributes that associate a span with a prompt.
116
+ *
117
+ * @param prompt - The prompt to attribute the span to.
118
+ * @param attributes - Attributes to merge in; these take precedence.
119
+ * @param includeInputs - Whether to stamp the run's inputs. Inputs belong on
120
+ * the **root span only**: they are identical on every span of a trace, so
121
+ * repeating them is invisible when the input is `["Ada"]` and wasteful when
122
+ * it is a fifty-message thread — multiplied by span count, then persisted.
123
+ */
124
+ declare function getPromptSpanAttributes(prompt: PromptSpanInfo, attributes?: Attributes, includeInputs?: boolean): Record<string, AttributeValue>;
125
+ /**
126
+ * Wraps a {@link Tracer} so that every span it produces is tagged with the
127
+ * attributes that associate it with a prompt — the prompt's name
128
+ * ({@link PROMPT_NAME_ATTRIBUTE}), an optional global prompt ID
129
+ * ({@link PROMPT_ID_ATTRIBUTE}), and an `'LLM'` span kind
130
+ * ({@link SPAN_KIND_ATTRIBUTE}). Attributes set explicitly on an individual
131
+ * span take precedence over these defaults.
132
+ *
133
+ * This depends only on `@opentelemetry/api`, so it can be re-used by SDK
134
+ * adapter packages (e.g. `@evalution/vercel-ai-sdk`) without pulling in the
135
+ * rest of evalution.
136
+ *
137
+ * @param prompt - The prompt to attribute spans to. `name` is a human-readable
138
+ * name; the optional `id` is a globally-unique prompt ID used to resolve
139
+ * runtime traces back to the prompt.
140
+ * @param tracer - Tracer to wrap. Defaults to a tracer from the globally
141
+ * registered tracer provider.
142
+ * @returns A tracer that forwards to `tracer` while attaching the prompt
143
+ * attributes to each span it creates.
144
+ */
145
+ declare function createTracerForPrompt(prompt: PromptSpanInfo, tracer?: Tracer): Tracer;
146
+ //#endregion
147
+ //#region src/trace-types.d.ts
148
+ /**
149
+ * A reference to a prompt.
150
+ *
151
+ * `id` is interpreted as a globally-unique prompt ID unless `providerId` is
152
+ * present, in which case `id` is scoped to that specific prompt provider. This
153
+ * mirrors the OpenTelemetry attributes `evalution.prompt.id` (always present)
154
+ * and `evalution.prompt.provider.id` (optional, scoping).
155
+ */
156
+ interface PromptID {
157
+ /** The prompt ID — global unless {@link providerId} scopes it. */
158
+ id: string;
159
+ /** When set, {@link id} is scoped to this prompt provider. */
160
+ providerId?: string;
161
+ /** Positional arguments the prompt function was called with. */
162
+ functionParameters?: unknown[];
163
+ /**
164
+ * The unresolved inputs the run was launched with — the recipe, not the
165
+ * resolution.
166
+ *
167
+ * Replaying them is only exactly faithful for the `value` kind. A `resource`
168
+ * re-runs its `create()`, producing an *equivalent* value rather than the
169
+ * same one — a fresh handle, a newly seeded row — so a UI must present it as
170
+ * something that will be re-created, never as a restored value.
171
+ */
172
+ functionInputs?: unknown[];
173
+ /** The unresolved execute-parameter inputs, keyed by parameter name. */
174
+ executeInputs?: Record<string, unknown>;
175
+ /**
176
+ * The run's named resource instances, by name, with the receipt each
177
+ * `create()` produced. See `specs/resource-instances.md` §F.
178
+ */
179
+ resources?: Record<string, unknown>;
180
+ /**
181
+ * The prompt's parameter definitions as they stood when the run was
182
+ * launched, so a replay can diff them against today's signature rather than
183
+ * guess whether the recorded inputs still fit.
184
+ */
185
+ parameterDefinitions?: unknown[];
186
+ /**
187
+ * The prompt's execute-parameter definitions as they stood when the run was
188
+ * launched — the types of {@link executeInputs}, as
189
+ * {@link parameterDefinitions} is for {@link functionInputs}.
190
+ */
191
+ executeParameterDefinitions?: unknown[];
192
+ /** The version the run executed against. */
193
+ version?: string;
194
+ /** The variation applied on top of {@link version}, if any. */
195
+ variation?: string;
196
+ }
197
+ /**
198
+ * Classification of a {@link Span}. See also:
199
+ * - `lmnr.span.type` from https://laminar.sh/docs/tracing/otel
200
+ * - `mlflow.spanType` from https://mlflow.org/docs/latest/genai/tracing/opentelemetry/attribute-mapping/#translated-span-attributes
201
+ *
202
+ * Mapped from `gen_ai.operation.name`. `CHECK` is evalution's own: an eval
203
+ * check that traced something while judging the run — see
204
+ * {@link CHECK_ATTRIBUTE}.
205
+ */
206
+ type SpanKind = "LLM" | "TOOL" | "AGENT" | "EMBEDDING" | "CHECK" | "DEFAULT";
207
+ /**
208
+ * Span attribute naming the eval check (by its id within the eval) that
209
+ * recorded the span while judging the trace — set on the check's `CHECK` span
210
+ * and on everything under it. Those spans aren't part of the run being
211
+ * judged, so the trace's rollups and chat view leave them out.
212
+ */
213
+ declare const CHECK_ATTRIBUTE = "evalution.check";
214
+ /** Whether `span` was recorded by a check judging its trace, not by the run. */
215
+ declare function isCheckSpan(span: Pick<Span, "attributes">): boolean;
216
+ /** A text segment within a multi-part {@link SpanMessage} content. */
217
+ interface SpanTextPart {
218
+ type: "text";
219
+ text: string;
220
+ }
221
+ /** An image segment within a multi-part {@link SpanMessage} content. */
222
+ interface SpanImagePart {
223
+ type: "image";
224
+ /** A URL, or base64/data-URI image data. */
225
+ image: string;
226
+ /** MIME type (e.g. `image/png`), when known. */
227
+ mediaType?: string;
228
+ }
229
+ /**
230
+ * A tool call the assistant made, within a multi-part {@link SpanMessage}
231
+ * content. Matched to its {@link SpanToolResultPart} by `toolCallId`.
232
+ */
233
+ interface SpanToolCallPart {
234
+ type: "tool-call";
235
+ toolCallId: string;
236
+ toolName: string;
237
+ /** The arguments the model passed, as JSON. */
238
+ input: unknown;
239
+ }
240
+ /**
241
+ * A tool's result fed back to the model, within a multi-part
242
+ * {@link SpanMessage} content (usually a `tool` message).
243
+ */
244
+ interface SpanToolResultPart {
245
+ type: "tool-result";
246
+ toolCallId: string;
247
+ toolName: string;
248
+ /** What the tool returned, unwrapped from the SDK's output envelope. */
249
+ output: unknown;
250
+ /** Whether the result reports a failure (an error, or a denied call). */
251
+ isError?: boolean;
252
+ }
253
+ /**
254
+ * The reasoning (thinking) a model produced ahead of its answer, within a
255
+ * multi-part {@link SpanMessage} content — how a conversation's earlier
256
+ * assistant turns carry theirs. The turn a span records keeps its own in
257
+ * {@link LLMSpanDetails.reasoning}.
258
+ */
259
+ interface SpanReasoningPart {
260
+ type: "reasoning";
261
+ text: string;
262
+ }
263
+ type SpanMessageRole = "user" | "assistant" | "system" | "tool" | (string & {});
264
+ /** One segment of a multi-part {@link SpanMessage} content. */
265
+ type SpanContentPart = SpanTextPart | SpanImagePart | SpanToolCallPart | SpanToolResultPart | SpanReasoningPart;
266
+ /**
267
+ * A single message within an LLM span's input/output.
268
+ *
269
+ * `content` is a plain string for the common text-only case. It is an array
270
+ * of {@link SpanContentPart} when the message carries non-text content (e.g.
271
+ * an image, or a tool call or result) — the `string` form stays in the union so existing consumers and
272
+ * already-stored rows keep working unchanged.
273
+ */
274
+ interface SpanMessage {
275
+ role: SpanMessageRole;
276
+ content: string | SpanContentPart[];
277
+ }
278
+ /** LLM-specific attributes attached to `LLM` spans. */
279
+ interface LLMSpanDetails {
280
+ provider?: string;
281
+ model?: string;
282
+ /** Model parameters (temperature, max_tokens, …). */
283
+ modelParameters?: Record<string, unknown>;
284
+ /**
285
+ * Why the model stopped generating, as the provider or SDK reports it
286
+ * (`stop`, `length`, `tool-calls`, …). Not normalized across providers.
287
+ */
288
+ finishReason?: string;
289
+ /**
290
+ * What the model was given: a message list for a chat model, or any JSON
291
+ * for a model that takes something else (a System One call records
292
+ * `{ state, questions }`). Array-ness is the discriminant — see
293
+ * {@link spanMessages}.
294
+ */
295
+ input?: SpanMessage[] | {
296
+ [key: string]: unknown;
297
+ };
298
+ /**
299
+ * What it returned: text for a chat model, the assistant message of a turn
300
+ * that called tools (see `toolCallingOutput`), or JSON for structured
301
+ * output and for any model whose answer is data rather than prose.
302
+ */
303
+ output?: unknown;
304
+ /**
305
+ * The reasoning the model produced on the way to {@link output}, kept apart
306
+ * from it so an output stays the answer that checks and agents read.
307
+ */
308
+ reasoning?: string;
309
+ /** Input tokens, counting any read from or written to a prompt cache. */
310
+ promptTokens?: number;
311
+ completionTokens?: number;
312
+ totalTokens?: number;
313
+ /** How many of {@link promptTokens} were read from a prompt cache. */
314
+ cacheReadTokens?: number;
315
+ /** How many of {@link promptTokens} were written to a prompt cache. */
316
+ cacheWriteTokens?: number;
317
+ /**
318
+ * Dollar cost of the call, broken down by token type, if known. The total
319
+ * cost is `cost.prompt + cost.completion`; `cost.prompt` includes cache
320
+ * reads and writes, each at its own price.
321
+ */
322
+ cost?: {
323
+ prompt: number;
324
+ completion: number; /** The part of `prompt` spent on {@link cacheReadTokens}. */
325
+ cacheRead?: number; /** The part of `prompt` spent on {@link cacheWriteTokens}. */
326
+ cacheWrite?: number;
327
+ };
328
+ }
329
+ /**
330
+ * The message list of a chat call, or `undefined` for any other input.
331
+ *
332
+ * A chat call's {@link LLMSpanDetails.input} is always a message list, and
333
+ * anything else is wrapped in an object, so being an array is what makes it
334
+ * one.
335
+ */
336
+ declare function spanMessages(llm: LLMSpanDetails | undefined): SpanMessage[] | undefined;
337
+ /**
338
+ * The output of a turn that called tools: its text (if any) and the calls, as
339
+ * the one assistant message the next turn's input repeats. Without the calls
340
+ * such a turn would record no output at all, its text being empty.
341
+ */
342
+ declare function toolCallingOutput(text: string | undefined, toolCalls: SpanToolCallPart[]): SpanMessage[];
343
+ /**
344
+ * An LLM span's output as a message list: text as the assistant message it
345
+ * is, a turn that called tools as the message it recorded. `undefined` for
346
+ * structured output, which can be an array too: every element must look like
347
+ * a message.
348
+ */
349
+ declare function outputMessages(llm: LLMSpanDetails | undefined): SpanMessage[] | undefined;
350
+ interface ToolSpanDetails {
351
+ toolName: string;
352
+ /**
353
+ * The id the model gave this call, when the integration reports it — the
354
+ * same id as the {@link SpanToolCallPart} that requested it.
355
+ */
356
+ toolCallId?: string;
357
+ input: unknown;
358
+ output?: unknown;
359
+ }
360
+ /**
361
+ * A span in a {@link Trace}. Spans form a tree via {@link parentId}.
362
+ * Durations are derived from `startTime` and `endTime`; an in-progress span has
363
+ * no `endTime` yet.
364
+ */
365
+ interface Span {
366
+ id: string;
367
+ traceId: string;
368
+ /** `undefined` for the root span of the trace. */
369
+ parentId?: string;
370
+ name: string;
371
+ kind: SpanKind;
372
+ /** Start timestamp in milliseconds since epoch. */
373
+ startTime: number;
374
+ /** End timestamp in milliseconds since epoch, or `undefined` while running. */
375
+ endTime?: number;
376
+ status?: "ok" | "error";
377
+ /** Error message if `status` is `'error'`. */
378
+ errorMessage?: string;
379
+ /** Free-form attributes to show in the span's details pane. */
380
+ attributes?: Record<string, unknown>;
381
+ /**
382
+ * The OpenTelemetry resource that produced the span: attributes describing
383
+ * the process or service it ran in rather than the span itself
384
+ * (`service.name`, `deployment.environment.name`, …).
385
+ */
386
+ resource?: Record<string, unknown>;
387
+ /** LLM-specific details (present for `chat`/`completion`/`embedding` spans). */
388
+ llm?: LLMSpanDetails;
389
+ /**
390
+ * The prompt this span is attributed to, if any.
391
+ *
392
+ * Stored as emitted (`evalution.prompt.id` plus optional
393
+ * `evalution.prompt.provider.id`): a {@link PromptID} whose `id` is global
394
+ * unless `providerId` is set. The server resolves it against the prompt
395
+ * registry to a provider-scoped reference when a trace is served.
396
+ */
397
+ prompt?: PromptID;
398
+ tool?: ToolSpanDetails;
399
+ }
400
+ interface TraceBase {
401
+ id: string;
402
+ providerId?: string;
403
+ name: string;
404
+ /** Start timestamp (ms). */
405
+ startTime: number;
406
+ /** End timestamp (ms), or `undefined` while the trace is still running. */
407
+ endTime?: number;
408
+ status: "running" | "ok" | "error";
409
+ }
410
+ /**
411
+ * Top-level trace for a single invocation (e.g. one prompt execution).
412
+ */
413
+ interface Trace extends TraceBase {
414
+ /** Free-form attributes (e.g. prompt ID, function params). */
415
+ attributes?: Record<string, unknown>;
416
+ }
417
+ /** Compact trace entry for listings (sidebar / `GET /api/traces`). */
418
+ interface TraceSummary extends TraceBase {
419
+ providerId: string;
420
+ /** Number of spans currently associated with the trace. */
421
+ spanCount: number;
422
+ /**
423
+ * Sum of every LLM span's token usage (`llm.totalTokens`, falling back to
424
+ * `promptTokens + completionTokens` per span the way `SpanDetails` does).
425
+ * `undefined` if no span in the trace reports any token usage.
426
+ */
427
+ totalTokens?: number;
428
+ /** Sum of every LLM span's `llm.cost` (prompt + completion). `undefined` if no span reports a cost. */
429
+ cost?: number;
430
+ /** The model name, if every LLM span that reports one reports the same one. */
431
+ model?: string;
432
+ /** The prompt version the run executed against (`PromptID.version`). */
433
+ promptVersion?: string;
434
+ /** The prompt variation the run applied (`PromptID.variation`). */
435
+ promptVariation?: string;
436
+ /**
437
+ * The deployment environment the run happened in: its root span's
438
+ * resource's `deployment.environment.name` — `"playground"` for a run
439
+ * evalution itself made.
440
+ */
441
+ environment?: string;
442
+ /** How many annotations of each kind are attached to the trace (trace-level or on any of its spans). */
443
+ annotationCounts: Record<AnnotationKind, number>;
444
+ /**
445
+ * The eval run that produced the trace, if any. Trace providers never set
446
+ * it — the server fills it in when listing, from the eval providers' results
447
+ * and the runs in flight.
448
+ */
449
+ evalRun?: TraceEvalRun;
450
+ /**
451
+ * How the checks recorded against the trace break down, once it has any.
452
+ * Filled in by the server alongside {@link TraceSummary.evalRun}.
453
+ */
454
+ checkCounts?: EvalCounts;
455
+ }
456
+ /** How a run's, or a trace's, check results break down. */
457
+ interface EvalCounts {
458
+ pass: number;
459
+ fail: number;
460
+ error: number;
461
+ skipped: number;
462
+ /** Scores reported without a threshold, so neither pass nor fail. */
463
+ scored: number;
464
+ }
465
+ /** The eval run a trace came from — what the trace list groups by. */
466
+ interface TraceEvalRun {
467
+ /** The eval provider the run is recorded on. */
468
+ providerId: string;
469
+ evalId: string;
470
+ /** The eval's name. */
471
+ evalName: string;
472
+ runId: string;
473
+ /** When the run started (ms). */
474
+ startedAt: number;
475
+ /** The arm of the run that produced the trace. */
476
+ armId: string;
477
+ /** That arm's label: the version it ran, as the run view names it. */
478
+ armLabel: string;
479
+ }
480
+ /**
481
+ * The OpenTelemetry semantic-convention resource attribute naming the
482
+ * service that produced a span.
483
+ */
484
+ declare const SERVICE_NAME_ATTRIBUTE = "service.name";
485
+ /**
486
+ * The OpenTelemetry semantic-convention resource attribute naming the
487
+ * deployment environment a span was produced in.
488
+ */
489
+ declare const DEPLOYMENT_ENVIRONMENT_ATTRIBUTE = "deployment.environment.name";
490
+ /** {@link DEPLOYMENT_ENVIRONMENT_ATTRIBUTE}'s value on a run evalution itself made. */
491
+ declare const PLAYGROUND_ENVIRONMENT = "playground";
492
+ /** A trace together with all of its spans. */
493
+ interface TraceWithSpans {
494
+ trace: Trace;
495
+ spans: Span[];
496
+ }
497
+ /**
498
+ * Where an {@link Annotation} came from: the user, a coding agent we know by
499
+ * name, or any other agent (an MCP client that isn't Claude Code or Codex).
500
+ */
501
+ type AnnotationSource = "user" | "claude-code" | "codex" | "agent";
502
+ /** What kind of note an {@link Annotation} records. */
503
+ type AnnotationKind = "issue" | "good" | "note";
504
+ /**
505
+ * A note attached to a trace, or to one specific span within it — e.g. a
506
+ * reviewer flagging a bad tool call, or an agent leaving a note about a run
507
+ * it just replayed.
508
+ */
509
+ interface Annotation {
510
+ id: string;
511
+ traceId: string;
512
+ /** The span this annotation is attached to, or `undefined` for a trace-level annotation. */
513
+ spanId?: string;
514
+ kind: AnnotationKind;
515
+ note: string;
516
+ source: AnnotationSource;
517
+ /** Creation timestamp (ms). */
518
+ createdAt: number;
519
+ }
520
+ /** The fields of an {@link Annotation} that can be changed after it's made. */
521
+ type AnnotationChanges = Partial<Pick<Annotation, "kind" | "note">>;
522
+ /** The kind of change that occurred to a trace. */
523
+ type TraceChangeType = "add" | "update" | "remove";
524
+ /** Describes a single change emitted by `TraceProvider.watch`. */
525
+ interface TraceChangeEvent {
526
+ type: TraceChangeType;
527
+ traceId: string;
528
+ }
529
+ /** Real-time event pushed over the per-trace SSE subscription. */
530
+ type TraceStreamEvent = {
531
+ type: "span-start";
532
+ span: Span;
533
+ } | {
534
+ type: "span-end";
535
+ span: Span;
536
+ } | {
537
+ type: "span-update";
538
+ span: Span;
539
+ } | {
540
+ type: "trace-update";
541
+ trace: Trace;
542
+ } | {
543
+ type: "trace-end";
544
+ trace: Trace;
545
+ };
546
+ /** The change an annotation event describes. */
547
+ type AnnotationEventOp = "insert" | "update" | "delete";
548
+ /** The event a per-trace annotation subscription delivers. */
549
+ interface AnnotationEvent {
550
+ type: "annotation";
551
+ op: AnnotationEventOp;
552
+ annotation: Annotation;
553
+ }
554
+ /**
555
+ * Real-time event pushed over the per-trace SSE subscription, broadened
556
+ * beyond span/trace lifecycle to also carry annotation changes — so a client
557
+ * holding one `EventSource` open on a trace sees both without a second
558
+ * connection. See `specs/trace-workshopping.md` §C.2.
559
+ */
560
+ type TraceLiveEvent = TraceStreamEvent | AnnotationEvent;
561
+ /** Information about a registered trace provider, returned by `GET /api/trace-providers`. */
562
+ interface TraceProviderInfo {
563
+ id: string;
564
+ displayName?: string;
565
+ description?: string;
566
+ }
567
+ /**
568
+ * A tool result's output as the tool returned it. The AI SDK wraps a result
569
+ * in a typed envelope (`{ type: "json", value }`, `{ type: "error-text",
570
+ * value }`, `{ type: "execution-denied", reason }`, …); this unwraps it and
571
+ * notes whether it reports a failure. Anything else is kept as is.
572
+ */
573
+ declare function unwrapToolResultOutput(output: unknown): Pick<SpanToolResultPart, "output" | "isError">;
574
+ //#endregion
575
+ //#region src/trace-sink.d.ts
576
+ /**
577
+ * The write side of a `TraceProvider`. A {@link TraceIngestor} feeds
578
+ * normalized spans in; the sink owns root detection, trace creation, and
579
+ * event emission.
580
+ */
581
+ interface TraceSink {
582
+ /**
583
+ * Records a span start. Creates a `running` trace for a new root. Returns
584
+ * the stored (merged) span. Emits `span-start` (+ watch `add` for a new
585
+ * trace).
586
+ */
587
+ recordSpanStart(span: Span): Promise<Span>;
588
+ /**
589
+ * Records a span end. For a root span, finalizes the trace and emits
590
+ * `trace-end` (+ watch `update`). Returns the stored (merged) span.
591
+ */
592
+ recordSpanEnd(span: Span): Promise<Span>;
593
+ /**
594
+ * Finalizes a trace as `error` when execution fails before any span is
595
+ * produced (e.g. a bad model id throws before `onStart`).
596
+ */
597
+ failTrace(traceId: string, errorMessage: string): Promise<void>;
598
+ /**
599
+ * Optional: resolves once everything recorded so far has been delivered,
600
+ * for a sink that records in the background (e.g. `HttpTraceSink`).
601
+ */
602
+ flush?(): Promise<void>;
603
+ }
604
+ declare function isTraceSink(o: any): o is TraceSink;
605
+ //#endregion
606
+ //#region src/trace-ingestor.d.ts
607
+ /**
608
+ * A source of trace data. Translates some upstream signal (OTel spans, native
609
+ * AI SDK telemetry, an external collector) into normalized {@link Span}s and
610
+ * feeds the {@link TraceSink}(s) it is connected to.
611
+ */
612
+ interface TraceIngestor {
613
+ /**
614
+ * Add a sink. May be called multiple times; the ingestor writes to every
615
+ * added sink.
616
+ */
617
+ addSink(sink: TraceSink): void;
618
+ /**
619
+ * Remove a previously-added sink. Returns `true` if the given sink was found
620
+ * and removed, otherwise `false`.
621
+ */
622
+ removeSink(sink: TraceSink): boolean;
623
+ /**
624
+ * Optional: returns true if `other` is made redundant by this ingestor and
625
+ * should be excluded from the collected set. The decision may depend on
626
+ * this ingestor's own configuration. Used by the server to consolidate
627
+ * overlapping ingestors — e.g. `OTelTraceIngestor` reports any other
628
+ * `OTelTraceIngestor` redundant because OTel is a single process-global
629
+ * pipeline.
630
+ */
631
+ isRedundant?(other: TraceIngestor): boolean;
632
+ /**
633
+ * Optional: sets the resource attributes merged into every span this
634
+ * ingestor records from now on, over whatever resource the source itself
635
+ * reported. The server stamps {@link PLAYGROUND_RESOURCE} on the ingestors
636
+ * that record its own runs.
637
+ */
638
+ setResource?(resource: Record<string, unknown>): void;
639
+ }
640
+ /**
641
+ * The resource of every run evalution itself makes — from the playground, an
642
+ * eval, or an MCP client — as opposed to one an app exported to it.
643
+ */
644
+ declare const PLAYGROUND_RESOURCE: Readonly<Record<string, string>>;
645
+ /**
646
+ * Base class that holds the added sinks and fans each normalized record out
647
+ * to all of them. Concrete ingestors translate their source into
648
+ * {@link Span}/{@link Trace} records and call these protected helpers.
649
+ */
650
+ declare abstract class BaseTraceIngestor implements TraceIngestor {
651
+ protected sinks: TraceSink[];
652
+ private resource?;
653
+ /** Recordings still under way, which {@link flush} waits out. */
654
+ private pending;
655
+ setResource(resource: Record<string, unknown>): void;
656
+ /** `span` with this ingestor's resource, if it has one, merged into its own. */
657
+ private withResource;
658
+ addSink(sink: TraceSink): void;
659
+ removeSink(sink: TraceSink): boolean;
660
+ /**
661
+ * Resolves once everything recorded so far has reached every sink and been
662
+ * delivered by those that deliver in the background. Await it before a
663
+ * short-lived process (a script, a serverless function) exits, so no trace
664
+ * data is lost.
665
+ */
666
+ flush(): Promise<void>;
667
+ /** Registers `recording` for {@link flush} to wait out, and returns it. */
668
+ private track;
669
+ protected recordSpanStart(span: Span): Promise<void>;
670
+ protected recordSpanEnd(span: Span): Promise<void>;
671
+ protected failTrace(traceId: string, msg: string): Promise<void>;
672
+ }
673
+ //#endregion
674
+ export { PromptsHelperOptions as $, Trace as A, outputMessages as B, SpanMessage as C, SpanToolCallPart as D, SpanTextPart as E, TraceProviderInfo as F, PROMPT_INPUTS_ATTRIBUTE as G, toolCallingOutput as H, TraceStreamEvent as I, PROMPT_VARIATION_ATTRIBUTE as J, PROMPT_NAME_ATTRIBUTE as K, TraceSummary as L, TraceChangeType as M, TraceEvalRun as N, SpanToolResultPart as O, TraceLiveEvent as P, PromptsHelper as Q, TraceWithSpans as R, SpanKind as S, SpanReasoningPart as T, unwrapToolResultOutput as U, spanMessages as V, PROMPT_ID_ATTRIBUTE as W, PromptSpanInfo as X, PROMPT_VERSION_ATTRIBUTE as Y, PromptsFactory as Z, PromptID as _, isTraceSink as a, SpanContentPart as b, AnnotationEvent as c, AnnotationSource as d, SPAN_KIND_ATTRIBUTE as et, CHECK_ATTRIBUTE as f, PLAYGROUND_ENVIRONMENT as g, LLMSpanDetails as h, TraceSink as i, TraceChangeEvent as j, ToolSpanDetails as k, AnnotationEventOp as l, EvalCounts as m, PLAYGROUND_RESOURCE as n, getPromptSpanAttributes as nt, Annotation as o, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE as p, PROMPT_PROVIDER_ID_ATTRIBUTE as q, TraceIngestor as r, mergePromptIdentity as rt, AnnotationChanges as s, BaseTraceIngestor as t, createTracerForPrompt as tt, AnnotationKind as u, SERVICE_NAME_ATTRIBUTE as v, SpanMessageRole as w, SpanImagePart as x, Span as y, isCheckSpan as z };