@evalution/sdk 0.0.0-stage → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/adapter.d.ts +764 -0
- package/dist/adapter.js +53 -0
- package/dist/index.d.ts +487 -0
- package/dist/index.js +1070 -0
- package/dist/portkit-DV-XcagM.js +84 -0
- package/dist/trace-ingestor-Cu8xQW6B.d.ts +674 -0
- package/package.json +57 -4
- package/README.md +0 -3
|
@@ -0,0 +1,674 @@
|
|
|
1
|
+
import { AttributeValue, Attributes, Tracer } from "@opentelemetry/api";
|
|
2
|
+
|
|
3
|
+
//#region src/prompt-tracer.d.ts
|
|
4
|
+
interface PromptsHelperOptions {
|
|
5
|
+
/**
|
|
6
|
+
* Globally-unique stable identifier for this group of prompts.
|
|
7
|
+
*/
|
|
8
|
+
readonly id: string;
|
|
9
|
+
}
|
|
10
|
+
type PromptsFactory<Args extends any[] = never[], Prompt = unknown> = (...args: Args) => Record<string, (...args: any[]) => Prompt>;
|
|
11
|
+
/**
|
|
12
|
+
* Intended to be used in the `satisfies` clause of a `prompts` export from an
|
|
13
|
+
* SDK adapter package.
|
|
14
|
+
*/
|
|
15
|
+
type PromptsHelper = (opts: PromptsHelperOptions, factory: PromptsFactory<any[], any>) => PromptsFactory<any[], any>;
|
|
16
|
+
/**
|
|
17
|
+
* Attribute name a span can set to pick one of evalution's span-kind values
|
|
18
|
+
* (`'LLM'`, `'TOOL'`, `'AGENT'`, `'EMBEDDING'`, `'DEFAULT'`). Falls back to
|
|
19
|
+
* `'DEFAULT'` when absent or unrecognised.
|
|
20
|
+
*/
|
|
21
|
+
declare const SPAN_KIND_ATTRIBUTE = "evalution.span.type";
|
|
22
|
+
/**
|
|
23
|
+
* Attribute name a span can set to give a human-readable name to the prompt.
|
|
24
|
+
*/
|
|
25
|
+
declare const PROMPT_NAME_ATTRIBUTE = "gen_ai.prompt.name";
|
|
26
|
+
/**
|
|
27
|
+
* Attribute name a span can set to link itself to a specific prompt. The value
|
|
28
|
+
* is a globally-unique prompt ID unless {@link PROMPT_PROVIDER_ID_ATTRIBUTE} is
|
|
29
|
+
* also set, in which case it is scoped to that provider.
|
|
30
|
+
*/
|
|
31
|
+
declare const PROMPT_ID_ATTRIBUTE = "evalution.prompt.id";
|
|
32
|
+
/**
|
|
33
|
+
* Attribute name a span can set to scope its {@link PROMPT_ID_ATTRIBUTE} to a
|
|
34
|
+
* specific prompt provider. When absent, the prompt ID is treated as global.
|
|
35
|
+
*/
|
|
36
|
+
declare const PROMPT_PROVIDER_ID_ATTRIBUTE = "evalution.prompt.provider.id";
|
|
37
|
+
/**
|
|
38
|
+
* Attribute name a span can set to include an array of input parameters used
|
|
39
|
+
* to construct the final prompt.
|
|
40
|
+
*/
|
|
41
|
+
declare const PROMPT_INPUTS_ATTRIBUTE = "evalution.prompt.inputs";
|
|
42
|
+
/**
|
|
43
|
+
* Attribute name a span can set to record which version of its prompt ran —
|
|
44
|
+
* for a file-based prompt, the commit checked out, when nothing was
|
|
45
|
+
* uncommitted.
|
|
46
|
+
*/
|
|
47
|
+
declare const PROMPT_VERSION_ATTRIBUTE = "evalution.prompt.version";
|
|
48
|
+
/**
|
|
49
|
+
* Attribute name a span can set to record which variation of its prompt ran,
|
|
50
|
+
* on top of its {@link PROMPT_VERSION_ATTRIBUTE}.
|
|
51
|
+
*/
|
|
52
|
+
declare const PROMPT_VARIATION_ATTRIBUTE = "evalution.prompt.variation";
|
|
53
|
+
interface PromptSpanInfo {
|
|
54
|
+
/** Globally-unique prompt id (e.g. `${groupId}#${name}`). */
|
|
55
|
+
id?: string;
|
|
56
|
+
/** Human-readable prompt name. */
|
|
57
|
+
name: string;
|
|
58
|
+
/**
|
|
59
|
+
* Positional arguments the prompt function was called with.
|
|
60
|
+
*
|
|
61
|
+
* Only the `prompts()` helper records these, because it is the only caller
|
|
62
|
+
* that sees the arguments and nothing else. A playground run records
|
|
63
|
+
* {@link functionInputs} instead: its arguments may include a live database
|
|
64
|
+
* handle, which is neither serializable nor the thing a replay needs.
|
|
65
|
+
*/
|
|
66
|
+
functionParameters?: unknown[];
|
|
67
|
+
/**
|
|
68
|
+
* The unresolved inputs this run was launched with — the *recipe* rather
|
|
69
|
+
* than the resolution.
|
|
70
|
+
*
|
|
71
|
+
* This is what makes a past run replayable. A resolved value cannot be
|
|
72
|
+
* recorded faithfully (a live handle does not serialize) and would not be
|
|
73
|
+
* the right thing to record anyway: replaying a resource means running its
|
|
74
|
+
* `create()` again, which is the recipe, not the value it produced last
|
|
75
|
+
* time. Each entry is JSON-safe by construction.
|
|
76
|
+
*/
|
|
77
|
+
functionInputs?: unknown[];
|
|
78
|
+
/** The unresolved execute-parameter inputs, keyed by parameter name. */
|
|
79
|
+
executeInputs?: Record<string, unknown>;
|
|
80
|
+
/**
|
|
81
|
+
* The run's named resource instances, by name, each with the receipt its
|
|
82
|
+
* `create()` produced. See `specs/resource-instances.md` §F.
|
|
83
|
+
*/
|
|
84
|
+
resources?: Record<string, unknown>;
|
|
85
|
+
/**
|
|
86
|
+
* The prompt's parameter definitions as they stood when the run was
|
|
87
|
+
* launched, recorded beside the inputs.
|
|
88
|
+
*
|
|
89
|
+
* For a file-based prompt git is the version, and the playground has no
|
|
90
|
+
* business checking out old commits to replay one — so writing the shape
|
|
91
|
+
* down at run time is the cheap equivalent. A later replay diffs two known
|
|
92
|
+
* signatures instead of guessing whether the recorded inputs still fit.
|
|
93
|
+
*/
|
|
94
|
+
parameterDefinitions?: unknown[];
|
|
95
|
+
/**
|
|
96
|
+
* The prompt's execute-parameter definitions, recorded for the same reason
|
|
97
|
+
* as {@link parameterDefinitions}: without them an entry of
|
|
98
|
+
* {@link executeInputs} has a name but no type.
|
|
99
|
+
*/
|
|
100
|
+
executeParameterDefinitions?: unknown[];
|
|
101
|
+
/** The version the run executed against. See {@link PROMPT_VERSION_ATTRIBUTE}. */
|
|
102
|
+
version?: string;
|
|
103
|
+
/** The variation the run applied. See {@link PROMPT_VARIATION_ATTRIBUTE}. */
|
|
104
|
+
variation?: string;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* `identity` with any field it leaves unset filled from `fallback`.
|
|
108
|
+
*
|
|
109
|
+
* A config built by the `prompts()` helper carries an identity of its own —
|
|
110
|
+
* the one to keep, since its id is the global one — but only the playground
|
|
111
|
+
* knows what the run was launched with and against which version.
|
|
112
|
+
*/
|
|
113
|
+
declare function mergePromptIdentity(identity: PromptSpanInfo | undefined, fallback: PromptSpanInfo | undefined): PromptSpanInfo | undefined;
|
|
114
|
+
/**
|
|
115
|
+
* Build the attributes that associate a span with a prompt.
|
|
116
|
+
*
|
|
117
|
+
* @param prompt - The prompt to attribute the span to.
|
|
118
|
+
* @param attributes - Attributes to merge in; these take precedence.
|
|
119
|
+
* @param includeInputs - Whether to stamp the run's inputs. Inputs belong on
|
|
120
|
+
* the **root span only**: they are identical on every span of a trace, so
|
|
121
|
+
* repeating them is invisible when the input is `["Ada"]` and wasteful when
|
|
122
|
+
* it is a fifty-message thread — multiplied by span count, then persisted.
|
|
123
|
+
*/
|
|
124
|
+
declare function getPromptSpanAttributes(prompt: PromptSpanInfo, attributes?: Attributes, includeInputs?: boolean): Record<string, AttributeValue>;
|
|
125
|
+
/**
|
|
126
|
+
* Wraps a {@link Tracer} so that every span it produces is tagged with the
|
|
127
|
+
* attributes that associate it with a prompt — the prompt's name
|
|
128
|
+
* ({@link PROMPT_NAME_ATTRIBUTE}), an optional global prompt ID
|
|
129
|
+
* ({@link PROMPT_ID_ATTRIBUTE}), and an `'LLM'` span kind
|
|
130
|
+
* ({@link SPAN_KIND_ATTRIBUTE}). Attributes set explicitly on an individual
|
|
131
|
+
* span take precedence over these defaults.
|
|
132
|
+
*
|
|
133
|
+
* This depends only on `@opentelemetry/api`, so it can be re-used by SDK
|
|
134
|
+
* adapter packages (e.g. `@evalution/vercel-ai-sdk`) without pulling in the
|
|
135
|
+
* rest of evalution.
|
|
136
|
+
*
|
|
137
|
+
* @param prompt - The prompt to attribute spans to. `name` is a human-readable
|
|
138
|
+
* name; the optional `id` is a globally-unique prompt ID used to resolve
|
|
139
|
+
* runtime traces back to the prompt.
|
|
140
|
+
* @param tracer - Tracer to wrap. Defaults to a tracer from the globally
|
|
141
|
+
* registered tracer provider.
|
|
142
|
+
* @returns A tracer that forwards to `tracer` while attaching the prompt
|
|
143
|
+
* attributes to each span it creates.
|
|
144
|
+
*/
|
|
145
|
+
declare function createTracerForPrompt(prompt: PromptSpanInfo, tracer?: Tracer): Tracer;
|
|
146
|
+
//#endregion
|
|
147
|
+
//#region src/trace-types.d.ts
|
|
148
|
+
/**
|
|
149
|
+
* A reference to a prompt.
|
|
150
|
+
*
|
|
151
|
+
* `id` is interpreted as a globally-unique prompt ID unless `providerId` is
|
|
152
|
+
* present, in which case `id` is scoped to that specific prompt provider. This
|
|
153
|
+
* mirrors the OpenTelemetry attributes `evalution.prompt.id` (always present)
|
|
154
|
+
* and `evalution.prompt.provider.id` (optional, scoping).
|
|
155
|
+
*/
|
|
156
|
+
interface PromptID {
|
|
157
|
+
/** The prompt ID — global unless {@link providerId} scopes it. */
|
|
158
|
+
id: string;
|
|
159
|
+
/** When set, {@link id} is scoped to this prompt provider. */
|
|
160
|
+
providerId?: string;
|
|
161
|
+
/** Positional arguments the prompt function was called with. */
|
|
162
|
+
functionParameters?: unknown[];
|
|
163
|
+
/**
|
|
164
|
+
* The unresolved inputs the run was launched with — the recipe, not the
|
|
165
|
+
* resolution.
|
|
166
|
+
*
|
|
167
|
+
* Replaying them is only exactly faithful for the `value` kind. A `resource`
|
|
168
|
+
* re-runs its `create()`, producing an *equivalent* value rather than the
|
|
169
|
+
* same one — a fresh handle, a newly seeded row — so a UI must present it as
|
|
170
|
+
* something that will be re-created, never as a restored value.
|
|
171
|
+
*/
|
|
172
|
+
functionInputs?: unknown[];
|
|
173
|
+
/** The unresolved execute-parameter inputs, keyed by parameter name. */
|
|
174
|
+
executeInputs?: Record<string, unknown>;
|
|
175
|
+
/**
|
|
176
|
+
* The run's named resource instances, by name, with the receipt each
|
|
177
|
+
* `create()` produced. See `specs/resource-instances.md` §F.
|
|
178
|
+
*/
|
|
179
|
+
resources?: Record<string, unknown>;
|
|
180
|
+
/**
|
|
181
|
+
* The prompt's parameter definitions as they stood when the run was
|
|
182
|
+
* launched, so a replay can diff them against today's signature rather than
|
|
183
|
+
* guess whether the recorded inputs still fit.
|
|
184
|
+
*/
|
|
185
|
+
parameterDefinitions?: unknown[];
|
|
186
|
+
/**
|
|
187
|
+
* The prompt's execute-parameter definitions as they stood when the run was
|
|
188
|
+
* launched — the types of {@link executeInputs}, as
|
|
189
|
+
* {@link parameterDefinitions} is for {@link functionInputs}.
|
|
190
|
+
*/
|
|
191
|
+
executeParameterDefinitions?: unknown[];
|
|
192
|
+
/** The version the run executed against. */
|
|
193
|
+
version?: string;
|
|
194
|
+
/** The variation applied on top of {@link version}, if any. */
|
|
195
|
+
variation?: string;
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Classification of a {@link Span}. See also:
|
|
199
|
+
* - `lmnr.span.type` from https://laminar.sh/docs/tracing/otel
|
|
200
|
+
* - `mlflow.spanType` from https://mlflow.org/docs/latest/genai/tracing/opentelemetry/attribute-mapping/#translated-span-attributes
|
|
201
|
+
*
|
|
202
|
+
* Mapped from `gen_ai.operation.name`. `CHECK` is evalution's own: an eval
|
|
203
|
+
* check that traced something while judging the run — see
|
|
204
|
+
* {@link CHECK_ATTRIBUTE}.
|
|
205
|
+
*/
|
|
206
|
+
type SpanKind = "LLM" | "TOOL" | "AGENT" | "EMBEDDING" | "CHECK" | "DEFAULT";
|
|
207
|
+
/**
|
|
208
|
+
* Span attribute naming the eval check (by its id within the eval) that
|
|
209
|
+
* recorded the span while judging the trace — set on the check's `CHECK` span
|
|
210
|
+
* and on everything under it. Those spans aren't part of the run being
|
|
211
|
+
* judged, so the trace's rollups and chat view leave them out.
|
|
212
|
+
*/
|
|
213
|
+
declare const CHECK_ATTRIBUTE = "evalution.check";
|
|
214
|
+
/** Whether `span` was recorded by a check judging its trace, not by the run. */
|
|
215
|
+
declare function isCheckSpan(span: Pick<Span, "attributes">): boolean;
|
|
216
|
+
/** A text segment within a multi-part {@link SpanMessage} content. */
|
|
217
|
+
interface SpanTextPart {
|
|
218
|
+
type: "text";
|
|
219
|
+
text: string;
|
|
220
|
+
}
|
|
221
|
+
/** An image segment within a multi-part {@link SpanMessage} content. */
|
|
222
|
+
interface SpanImagePart {
|
|
223
|
+
type: "image";
|
|
224
|
+
/** A URL, or base64/data-URI image data. */
|
|
225
|
+
image: string;
|
|
226
|
+
/** MIME type (e.g. `image/png`), when known. */
|
|
227
|
+
mediaType?: string;
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* A tool call the assistant made, within a multi-part {@link SpanMessage}
|
|
231
|
+
* content. Matched to its {@link SpanToolResultPart} by `toolCallId`.
|
|
232
|
+
*/
|
|
233
|
+
interface SpanToolCallPart {
|
|
234
|
+
type: "tool-call";
|
|
235
|
+
toolCallId: string;
|
|
236
|
+
toolName: string;
|
|
237
|
+
/** The arguments the model passed, as JSON. */
|
|
238
|
+
input: unknown;
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* A tool's result fed back to the model, within a multi-part
|
|
242
|
+
* {@link SpanMessage} content (usually a `tool` message).
|
|
243
|
+
*/
|
|
244
|
+
interface SpanToolResultPart {
|
|
245
|
+
type: "tool-result";
|
|
246
|
+
toolCallId: string;
|
|
247
|
+
toolName: string;
|
|
248
|
+
/** What the tool returned, unwrapped from the SDK's output envelope. */
|
|
249
|
+
output: unknown;
|
|
250
|
+
/** Whether the result reports a failure (an error, or a denied call). */
|
|
251
|
+
isError?: boolean;
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* The reasoning (thinking) a model produced ahead of its answer, within a
|
|
255
|
+
* multi-part {@link SpanMessage} content — how a conversation's earlier
|
|
256
|
+
* assistant turns carry theirs. The turn a span records keeps its own in
|
|
257
|
+
* {@link LLMSpanDetails.reasoning}.
|
|
258
|
+
*/
|
|
259
|
+
interface SpanReasoningPart {
|
|
260
|
+
type: "reasoning";
|
|
261
|
+
text: string;
|
|
262
|
+
}
|
|
263
|
+
type SpanMessageRole = "user" | "assistant" | "system" | "tool" | (string & {});
|
|
264
|
+
/** One segment of a multi-part {@link SpanMessage} content. */
|
|
265
|
+
type SpanContentPart = SpanTextPart | SpanImagePart | SpanToolCallPart | SpanToolResultPart | SpanReasoningPart;
|
|
266
|
+
/**
|
|
267
|
+
* A single message within an LLM span's input/output.
|
|
268
|
+
*
|
|
269
|
+
* `content` is a plain string for the common text-only case. It is an array
|
|
270
|
+
* of {@link SpanContentPart} when the message carries non-text content (e.g.
|
|
271
|
+
* an image, or a tool call or result) — the `string` form stays in the union so existing consumers and
|
|
272
|
+
* already-stored rows keep working unchanged.
|
|
273
|
+
*/
|
|
274
|
+
interface SpanMessage {
|
|
275
|
+
role: SpanMessageRole;
|
|
276
|
+
content: string | SpanContentPart[];
|
|
277
|
+
}
|
|
278
|
+
/** LLM-specific attributes attached to `LLM` spans. */
|
|
279
|
+
interface LLMSpanDetails {
|
|
280
|
+
provider?: string;
|
|
281
|
+
model?: string;
|
|
282
|
+
/** Model parameters (temperature, max_tokens, …). */
|
|
283
|
+
modelParameters?: Record<string, unknown>;
|
|
284
|
+
/**
|
|
285
|
+
* Why the model stopped generating, as the provider or SDK reports it
|
|
286
|
+
* (`stop`, `length`, `tool-calls`, …). Not normalized across providers.
|
|
287
|
+
*/
|
|
288
|
+
finishReason?: string;
|
|
289
|
+
/**
|
|
290
|
+
* What the model was given: a message list for a chat model, or any JSON
|
|
291
|
+
* for a model that takes something else (a System One call records
|
|
292
|
+
* `{ state, questions }`). Array-ness is the discriminant — see
|
|
293
|
+
* {@link spanMessages}.
|
|
294
|
+
*/
|
|
295
|
+
input?: SpanMessage[] | {
|
|
296
|
+
[key: string]: unknown;
|
|
297
|
+
};
|
|
298
|
+
/**
|
|
299
|
+
* What it returned: text for a chat model, the assistant message of a turn
|
|
300
|
+
* that called tools (see `toolCallingOutput`), or JSON for structured
|
|
301
|
+
* output and for any model whose answer is data rather than prose.
|
|
302
|
+
*/
|
|
303
|
+
output?: unknown;
|
|
304
|
+
/**
|
|
305
|
+
* The reasoning the model produced on the way to {@link output}, kept apart
|
|
306
|
+
* from it so an output stays the answer that checks and agents read.
|
|
307
|
+
*/
|
|
308
|
+
reasoning?: string;
|
|
309
|
+
/** Input tokens, counting any read from or written to a prompt cache. */
|
|
310
|
+
promptTokens?: number;
|
|
311
|
+
completionTokens?: number;
|
|
312
|
+
totalTokens?: number;
|
|
313
|
+
/** How many of {@link promptTokens} were read from a prompt cache. */
|
|
314
|
+
cacheReadTokens?: number;
|
|
315
|
+
/** How many of {@link promptTokens} were written to a prompt cache. */
|
|
316
|
+
cacheWriteTokens?: number;
|
|
317
|
+
/**
|
|
318
|
+
* Dollar cost of the call, broken down by token type, if known. The total
|
|
319
|
+
* cost is `cost.prompt + cost.completion`; `cost.prompt` includes cache
|
|
320
|
+
* reads and writes, each at its own price.
|
|
321
|
+
*/
|
|
322
|
+
cost?: {
|
|
323
|
+
prompt: number;
|
|
324
|
+
completion: number; /** The part of `prompt` spent on {@link cacheReadTokens}. */
|
|
325
|
+
cacheRead?: number; /** The part of `prompt` spent on {@link cacheWriteTokens}. */
|
|
326
|
+
cacheWrite?: number;
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* The message list of a chat call, or `undefined` for any other input.
|
|
331
|
+
*
|
|
332
|
+
* A chat call's {@link LLMSpanDetails.input} is always a message list, and
|
|
333
|
+
* anything else is wrapped in an object, so being an array is what makes it
|
|
334
|
+
* one.
|
|
335
|
+
*/
|
|
336
|
+
declare function spanMessages(llm: LLMSpanDetails | undefined): SpanMessage[] | undefined;
|
|
337
|
+
/**
|
|
338
|
+
* The output of a turn that called tools: its text (if any) and the calls, as
|
|
339
|
+
* the one assistant message the next turn's input repeats. Without the calls
|
|
340
|
+
* such a turn would record no output at all, its text being empty.
|
|
341
|
+
*/
|
|
342
|
+
declare function toolCallingOutput(text: string | undefined, toolCalls: SpanToolCallPart[]): SpanMessage[];
|
|
343
|
+
/**
|
|
344
|
+
* An LLM span's output as a message list: text as the assistant message it
|
|
345
|
+
* is, a turn that called tools as the message it recorded. `undefined` for
|
|
346
|
+
* structured output, which can be an array too: every element must look like
|
|
347
|
+
* a message.
|
|
348
|
+
*/
|
|
349
|
+
declare function outputMessages(llm: LLMSpanDetails | undefined): SpanMessage[] | undefined;
|
|
350
|
+
interface ToolSpanDetails {
|
|
351
|
+
toolName: string;
|
|
352
|
+
/**
|
|
353
|
+
* The id the model gave this call, when the integration reports it — the
|
|
354
|
+
* same id as the {@link SpanToolCallPart} that requested it.
|
|
355
|
+
*/
|
|
356
|
+
toolCallId?: string;
|
|
357
|
+
input: unknown;
|
|
358
|
+
output?: unknown;
|
|
359
|
+
}
|
|
360
|
+
/**
|
|
361
|
+
* A span in a {@link Trace}. Spans form a tree via {@link parentId}.
|
|
362
|
+
* Durations are derived from `startTime` and `endTime`; an in-progress span has
|
|
363
|
+
* no `endTime` yet.
|
|
364
|
+
*/
|
|
365
|
+
interface Span {
|
|
366
|
+
id: string;
|
|
367
|
+
traceId: string;
|
|
368
|
+
/** `undefined` for the root span of the trace. */
|
|
369
|
+
parentId?: string;
|
|
370
|
+
name: string;
|
|
371
|
+
kind: SpanKind;
|
|
372
|
+
/** Start timestamp in milliseconds since epoch. */
|
|
373
|
+
startTime: number;
|
|
374
|
+
/** End timestamp in milliseconds since epoch, or `undefined` while running. */
|
|
375
|
+
endTime?: number;
|
|
376
|
+
status?: "ok" | "error";
|
|
377
|
+
/** Error message if `status` is `'error'`. */
|
|
378
|
+
errorMessage?: string;
|
|
379
|
+
/** Free-form attributes to show in the span's details pane. */
|
|
380
|
+
attributes?: Record<string, unknown>;
|
|
381
|
+
/**
|
|
382
|
+
* The OpenTelemetry resource that produced the span: attributes describing
|
|
383
|
+
* the process or service it ran in rather than the span itself
|
|
384
|
+
* (`service.name`, `deployment.environment.name`, …).
|
|
385
|
+
*/
|
|
386
|
+
resource?: Record<string, unknown>;
|
|
387
|
+
/** LLM-specific details (present for `chat`/`completion`/`embedding` spans). */
|
|
388
|
+
llm?: LLMSpanDetails;
|
|
389
|
+
/**
|
|
390
|
+
* The prompt this span is attributed to, if any.
|
|
391
|
+
*
|
|
392
|
+
* Stored as emitted (`evalution.prompt.id` plus optional
|
|
393
|
+
* `evalution.prompt.provider.id`): a {@link PromptID} whose `id` is global
|
|
394
|
+
* unless `providerId` is set. The server resolves it against the prompt
|
|
395
|
+
* registry to a provider-scoped reference when a trace is served.
|
|
396
|
+
*/
|
|
397
|
+
prompt?: PromptID;
|
|
398
|
+
tool?: ToolSpanDetails;
|
|
399
|
+
}
|
|
400
|
+
interface TraceBase {
|
|
401
|
+
id: string;
|
|
402
|
+
providerId?: string;
|
|
403
|
+
name: string;
|
|
404
|
+
/** Start timestamp (ms). */
|
|
405
|
+
startTime: number;
|
|
406
|
+
/** End timestamp (ms), or `undefined` while the trace is still running. */
|
|
407
|
+
endTime?: number;
|
|
408
|
+
status: "running" | "ok" | "error";
|
|
409
|
+
}
|
|
410
|
+
/**
|
|
411
|
+
* Top-level trace for a single invocation (e.g. one prompt execution).
|
|
412
|
+
*/
|
|
413
|
+
interface Trace extends TraceBase {
|
|
414
|
+
/** Free-form attributes (e.g. prompt ID, function params). */
|
|
415
|
+
attributes?: Record<string, unknown>;
|
|
416
|
+
}
|
|
417
|
+
/** Compact trace entry for listings (sidebar / `GET /api/traces`). */
|
|
418
|
+
interface TraceSummary extends TraceBase {
|
|
419
|
+
providerId: string;
|
|
420
|
+
/** Number of spans currently associated with the trace. */
|
|
421
|
+
spanCount: number;
|
|
422
|
+
/**
|
|
423
|
+
* Sum of every LLM span's token usage (`llm.totalTokens`, falling back to
|
|
424
|
+
* `promptTokens + completionTokens` per span the way `SpanDetails` does).
|
|
425
|
+
* `undefined` if no span in the trace reports any token usage.
|
|
426
|
+
*/
|
|
427
|
+
totalTokens?: number;
|
|
428
|
+
/** Sum of every LLM span's `llm.cost` (prompt + completion). `undefined` if no span reports a cost. */
|
|
429
|
+
cost?: number;
|
|
430
|
+
/** The model name, if every LLM span that reports one reports the same one. */
|
|
431
|
+
model?: string;
|
|
432
|
+
/** The prompt version the run executed against (`PromptID.version`). */
|
|
433
|
+
promptVersion?: string;
|
|
434
|
+
/** The prompt variation the run applied (`PromptID.variation`). */
|
|
435
|
+
promptVariation?: string;
|
|
436
|
+
/**
|
|
437
|
+
* The deployment environment the run happened in: its root span's
|
|
438
|
+
* resource's `deployment.environment.name` — `"playground"` for a run
|
|
439
|
+
* evalution itself made.
|
|
440
|
+
*/
|
|
441
|
+
environment?: string;
|
|
442
|
+
/** How many annotations of each kind are attached to the trace (trace-level or on any of its spans). */
|
|
443
|
+
annotationCounts: Record<AnnotationKind, number>;
|
|
444
|
+
/**
|
|
445
|
+
* The eval run that produced the trace, if any. Trace providers never set
|
|
446
|
+
* it — the server fills it in when listing, from the eval providers' results
|
|
447
|
+
* and the runs in flight.
|
|
448
|
+
*/
|
|
449
|
+
evalRun?: TraceEvalRun;
|
|
450
|
+
/**
|
|
451
|
+
* How the checks recorded against the trace break down, once it has any.
|
|
452
|
+
* Filled in by the server alongside {@link TraceSummary.evalRun}.
|
|
453
|
+
*/
|
|
454
|
+
checkCounts?: EvalCounts;
|
|
455
|
+
}
|
|
456
|
+
/** How a run's, or a trace's, check results break down. */
|
|
457
|
+
interface EvalCounts {
|
|
458
|
+
pass: number;
|
|
459
|
+
fail: number;
|
|
460
|
+
error: number;
|
|
461
|
+
skipped: number;
|
|
462
|
+
/** Scores reported without a threshold, so neither pass nor fail. */
|
|
463
|
+
scored: number;
|
|
464
|
+
}
|
|
465
|
+
/** The eval run a trace came from — what the trace list groups by. */
|
|
466
|
+
interface TraceEvalRun {
|
|
467
|
+
/** The eval provider the run is recorded on. */
|
|
468
|
+
providerId: string;
|
|
469
|
+
evalId: string;
|
|
470
|
+
/** The eval's name. */
|
|
471
|
+
evalName: string;
|
|
472
|
+
runId: string;
|
|
473
|
+
/** When the run started (ms). */
|
|
474
|
+
startedAt: number;
|
|
475
|
+
/** The arm of the run that produced the trace. */
|
|
476
|
+
armId: string;
|
|
477
|
+
/** That arm's label: the version it ran, as the run view names it. */
|
|
478
|
+
armLabel: string;
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* The OpenTelemetry semantic-convention resource attribute naming the
|
|
482
|
+
* service that produced a span.
|
|
483
|
+
*/
|
|
484
|
+
declare const SERVICE_NAME_ATTRIBUTE = "service.name";
|
|
485
|
+
/**
|
|
486
|
+
* The OpenTelemetry semantic-convention resource attribute naming the
|
|
487
|
+
* deployment environment a span was produced in.
|
|
488
|
+
*/
|
|
489
|
+
declare const DEPLOYMENT_ENVIRONMENT_ATTRIBUTE = "deployment.environment.name";
|
|
490
|
+
/** {@link DEPLOYMENT_ENVIRONMENT_ATTRIBUTE}'s value on a run evalution itself made. */
|
|
491
|
+
declare const PLAYGROUND_ENVIRONMENT = "playground";
|
|
492
|
+
/** A trace together with all of its spans. */
|
|
493
|
+
interface TraceWithSpans {
|
|
494
|
+
trace: Trace;
|
|
495
|
+
spans: Span[];
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* Where an {@link Annotation} came from: the user, a coding agent we know by
|
|
499
|
+
* name, or any other agent (an MCP client that isn't Claude Code or Codex).
|
|
500
|
+
*/
|
|
501
|
+
type AnnotationSource = "user" | "claude-code" | "codex" | "agent";
|
|
502
|
+
/** What kind of note an {@link Annotation} records. */
|
|
503
|
+
type AnnotationKind = "issue" | "good" | "note";
|
|
504
|
+
/**
|
|
505
|
+
* A note attached to a trace, or to one specific span within it — e.g. a
|
|
506
|
+
* reviewer flagging a bad tool call, or an agent leaving a note about a run
|
|
507
|
+
* it just replayed.
|
|
508
|
+
*/
|
|
509
|
+
interface Annotation {
|
|
510
|
+
id: string;
|
|
511
|
+
traceId: string;
|
|
512
|
+
/** The span this annotation is attached to, or `undefined` for a trace-level annotation. */
|
|
513
|
+
spanId?: string;
|
|
514
|
+
kind: AnnotationKind;
|
|
515
|
+
note: string;
|
|
516
|
+
source: AnnotationSource;
|
|
517
|
+
/** Creation timestamp (ms). */
|
|
518
|
+
createdAt: number;
|
|
519
|
+
}
|
|
520
|
+
/** The fields of an {@link Annotation} that can be changed after it's made. */
|
|
521
|
+
type AnnotationChanges = Partial<Pick<Annotation, "kind" | "note">>;
|
|
522
|
+
/** The kind of change that occurred to a trace. */
|
|
523
|
+
type TraceChangeType = "add" | "update" | "remove";
|
|
524
|
+
/** Describes a single change emitted by `TraceProvider.watch`. */
|
|
525
|
+
interface TraceChangeEvent {
|
|
526
|
+
type: TraceChangeType;
|
|
527
|
+
traceId: string;
|
|
528
|
+
}
|
|
529
|
+
/** Real-time event pushed over the per-trace SSE subscription. */
|
|
530
|
+
type TraceStreamEvent = {
|
|
531
|
+
type: "span-start";
|
|
532
|
+
span: Span;
|
|
533
|
+
} | {
|
|
534
|
+
type: "span-end";
|
|
535
|
+
span: Span;
|
|
536
|
+
} | {
|
|
537
|
+
type: "span-update";
|
|
538
|
+
span: Span;
|
|
539
|
+
} | {
|
|
540
|
+
type: "trace-update";
|
|
541
|
+
trace: Trace;
|
|
542
|
+
} | {
|
|
543
|
+
type: "trace-end";
|
|
544
|
+
trace: Trace;
|
|
545
|
+
};
|
|
546
|
+
/** The change an annotation event describes. */
|
|
547
|
+
type AnnotationEventOp = "insert" | "update" | "delete";
|
|
548
|
+
/** The event a per-trace annotation subscription delivers. */
|
|
549
|
+
interface AnnotationEvent {
|
|
550
|
+
type: "annotation";
|
|
551
|
+
op: AnnotationEventOp;
|
|
552
|
+
annotation: Annotation;
|
|
553
|
+
}
|
|
554
|
+
/**
|
|
555
|
+
* Real-time event pushed over the per-trace SSE subscription, broadened
|
|
556
|
+
* beyond span/trace lifecycle to also carry annotation changes — so a client
|
|
557
|
+
* holding one `EventSource` open on a trace sees both without a second
|
|
558
|
+
* connection. See `specs/trace-workshopping.md` §C.2.
|
|
559
|
+
*/
|
|
560
|
+
type TraceLiveEvent = TraceStreamEvent | AnnotationEvent;
|
|
561
|
+
/** Information about a registered trace provider, returned by `GET /api/trace-providers`. */
|
|
562
|
+
interface TraceProviderInfo {
|
|
563
|
+
id: string;
|
|
564
|
+
displayName?: string;
|
|
565
|
+
description?: string;
|
|
566
|
+
}
|
|
567
|
+
/**
|
|
568
|
+
* A tool result's output as the tool returned it. The AI SDK wraps a result
|
|
569
|
+
* in a typed envelope (`{ type: "json", value }`, `{ type: "error-text",
|
|
570
|
+
* value }`, `{ type: "execution-denied", reason }`, …); this unwraps it and
|
|
571
|
+
* notes whether it reports a failure. Anything else is kept as is.
|
|
572
|
+
*/
|
|
573
|
+
declare function unwrapToolResultOutput(output: unknown): Pick<SpanToolResultPart, "output" | "isError">;
|
|
574
|
+
//#endregion
|
|
575
|
+
//#region src/trace-sink.d.ts
|
|
576
|
+
/**
|
|
577
|
+
* The write side of a `TraceProvider`. A {@link TraceIngestor} feeds
|
|
578
|
+
* normalized spans in; the sink owns root detection, trace creation, and
|
|
579
|
+
* event emission.
|
|
580
|
+
*/
|
|
581
|
+
interface TraceSink {
|
|
582
|
+
/**
|
|
583
|
+
* Records a span start. Creates a `running` trace for a new root. Returns
|
|
584
|
+
* the stored (merged) span. Emits `span-start` (+ watch `add` for a new
|
|
585
|
+
* trace).
|
|
586
|
+
*/
|
|
587
|
+
recordSpanStart(span: Span): Promise<Span>;
|
|
588
|
+
/**
|
|
589
|
+
* Records a span end. For a root span, finalizes the trace and emits
|
|
590
|
+
* `trace-end` (+ watch `update`). Returns the stored (merged) span.
|
|
591
|
+
*/
|
|
592
|
+
recordSpanEnd(span: Span): Promise<Span>;
|
|
593
|
+
/**
|
|
594
|
+
* Finalizes a trace as `error` when execution fails before any span is
|
|
595
|
+
* produced (e.g. a bad model id throws before `onStart`).
|
|
596
|
+
*/
|
|
597
|
+
failTrace(traceId: string, errorMessage: string): Promise<void>;
|
|
598
|
+
/**
|
|
599
|
+
* Optional: resolves once everything recorded so far has been delivered,
|
|
600
|
+
* for a sink that records in the background (e.g. `HttpTraceSink`).
|
|
601
|
+
*/
|
|
602
|
+
flush?(): Promise<void>;
|
|
603
|
+
}
|
|
604
|
+
declare function isTraceSink(o: any): o is TraceSink;
|
|
605
|
+
//#endregion
|
|
606
|
+
//#region src/trace-ingestor.d.ts
|
|
607
|
+
/**
|
|
608
|
+
* A source of trace data. Translates some upstream signal (OTel spans, native
|
|
609
|
+
* AI SDK telemetry, an external collector) into normalized {@link Span}s and
|
|
610
|
+
* feeds the {@link TraceSink}(s) it is connected to.
|
|
611
|
+
*/
|
|
612
|
+
interface TraceIngestor {
|
|
613
|
+
/**
|
|
614
|
+
* Add a sink. May be called multiple times; the ingestor writes to every
|
|
615
|
+
* added sink.
|
|
616
|
+
*/
|
|
617
|
+
addSink(sink: TraceSink): void;
|
|
618
|
+
/**
|
|
619
|
+
* Remove a previously-added sink. Returns `true` if the given sink was found
|
|
620
|
+
* and removed, otherwise `false`.
|
|
621
|
+
*/
|
|
622
|
+
removeSink(sink: TraceSink): boolean;
|
|
623
|
+
/**
|
|
624
|
+
* Optional: returns true if `other` is made redundant by this ingestor and
|
|
625
|
+
* should be excluded from the collected set. The decision may depend on
|
|
626
|
+
* this ingestor's own configuration. Used by the server to consolidate
|
|
627
|
+
* overlapping ingestors — e.g. `OTelTraceIngestor` reports any other
|
|
628
|
+
* `OTelTraceIngestor` redundant because OTel is a single process-global
|
|
629
|
+
* pipeline.
|
|
630
|
+
*/
|
|
631
|
+
isRedundant?(other: TraceIngestor): boolean;
|
|
632
|
+
/**
|
|
633
|
+
* Optional: sets the resource attributes merged into every span this
|
|
634
|
+
* ingestor records from now on, over whatever resource the source itself
|
|
635
|
+
* reported. The server stamps {@link PLAYGROUND_RESOURCE} on the ingestors
|
|
636
|
+
* that record its own runs.
|
|
637
|
+
*/
|
|
638
|
+
setResource?(resource: Record<string, unknown>): void;
|
|
639
|
+
}
|
|
640
|
+
/**
|
|
641
|
+
* The resource of every run evalution itself makes — from the playground, an
|
|
642
|
+
* eval, or an MCP client — as opposed to one an app exported to it.
|
|
643
|
+
*/
|
|
644
|
+
declare const PLAYGROUND_RESOURCE: Readonly<Record<string, string>>;
|
|
645
|
+
/**
|
|
646
|
+
* Base class that holds the added sinks and fans each normalized record out
|
|
647
|
+
* to all of them. Concrete ingestors translate their source into
|
|
648
|
+
* {@link Span}/{@link Trace} records and call these protected helpers.
|
|
649
|
+
*/
|
|
650
|
+
declare abstract class BaseTraceIngestor implements TraceIngestor {
|
|
651
|
+
protected sinks: TraceSink[];
|
|
652
|
+
private resource?;
|
|
653
|
+
/** Recordings still under way, which {@link flush} waits out. */
|
|
654
|
+
private pending;
|
|
655
|
+
setResource(resource: Record<string, unknown>): void;
|
|
656
|
+
/** `span` with this ingestor's resource, if it has one, merged into its own. */
|
|
657
|
+
private withResource;
|
|
658
|
+
addSink(sink: TraceSink): void;
|
|
659
|
+
removeSink(sink: TraceSink): boolean;
|
|
660
|
+
/**
|
|
661
|
+
* Resolves once everything recorded so far has reached every sink and been
|
|
662
|
+
* delivered by those that deliver in the background. Await it before a
|
|
663
|
+
* short-lived process (a script, a serverless function) exits, so no trace
|
|
664
|
+
* data is lost.
|
|
665
|
+
*/
|
|
666
|
+
flush(): Promise<void>;
|
|
667
|
+
/** Registers `recording` for {@link flush} to wait out, and returns it. */
|
|
668
|
+
private track;
|
|
669
|
+
protected recordSpanStart(span: Span): Promise<void>;
|
|
670
|
+
protected recordSpanEnd(span: Span): Promise<void>;
|
|
671
|
+
protected failTrace(traceId: string, msg: string): Promise<void>;
|
|
672
|
+
}
|
|
673
|
+
//#endregion
|
|
674
|
+
export { PromptsHelperOptions as $, Trace as A, outputMessages as B, SpanMessage as C, SpanToolCallPart as D, SpanTextPart as E, TraceProviderInfo as F, PROMPT_INPUTS_ATTRIBUTE as G, toolCallingOutput as H, TraceStreamEvent as I, PROMPT_VARIATION_ATTRIBUTE as J, PROMPT_NAME_ATTRIBUTE as K, TraceSummary as L, TraceChangeType as M, TraceEvalRun as N, SpanToolResultPart as O, TraceLiveEvent as P, PromptsHelper as Q, TraceWithSpans as R, SpanKind as S, SpanReasoningPart as T, unwrapToolResultOutput as U, spanMessages as V, PROMPT_ID_ATTRIBUTE as W, PromptSpanInfo as X, PROMPT_VERSION_ATTRIBUTE as Y, PromptsFactory as Z, PromptID as _, isTraceSink as a, SpanContentPart as b, AnnotationEvent as c, AnnotationSource as d, SPAN_KIND_ATTRIBUTE as et, CHECK_ATTRIBUTE as f, PLAYGROUND_ENVIRONMENT as g, LLMSpanDetails as h, TraceSink as i, TraceChangeEvent as j, ToolSpanDetails as k, AnnotationEventOp as l, EvalCounts as m, PLAYGROUND_RESOURCE as n, getPromptSpanAttributes as nt, Annotation as o, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE as p, PROMPT_PROVIDER_ID_ATTRIBUTE as q, TraceIngestor as r, mergePromptIdentity as rt, AnnotationChanges as s, BaseTraceIngestor as t, createTracerForPrompt as tt, AnnotationKind as u, SERVICE_NAME_ATTRIBUTE as v, SpanMessageRole as w, SpanImagePart as x, Span as y, isCheckSpan as z };
|