@evalution/sdk 0.0.0-stage → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/adapter.d.ts +764 -0
- package/dist/adapter.js +53 -0
- package/dist/index.d.ts +267 -0
- package/dist/index.js +588 -0
- package/dist/portkit-DV-XcagM.js +84 -0
- package/dist/trace-ingestor-Cu8xQW6B.d.ts +674 -0
- package/package.json +45 -4
- package/README.md +0 -3
package/dist/index.js
ADDED
|
@@ -0,0 +1,588 @@
|
|
|
1
|
+
import { trace } from "@opentelemetry/api";
|
|
2
|
+
//#region src/brand.ts
|
|
3
|
+
function makeBrand(sym) {
|
|
4
|
+
return {
|
|
5
|
+
brand: (value) => Object.defineProperty(value, sym, { value: true }),
|
|
6
|
+
isBranded: (value) => typeof value === "object" && value !== null && value[sym] === true
|
|
7
|
+
};
|
|
8
|
+
}
|
|
9
|
+
//#endregion
|
|
10
|
+
//#region src/http-trace-sink.ts
|
|
11
|
+
/**
|
|
12
|
+
* Where an evalution server accepts {@link TraceSinkEvent}s, relative to its
|
|
13
|
+
* base URL.
|
|
14
|
+
*/
|
|
15
|
+
const TRACE_EVENTS_PATH = "/api/traces";
|
|
16
|
+
/**
|
|
17
|
+
* A {@link TraceSink} that sends what it's given to an evalution server,
|
|
18
|
+
* which records it through its own trace providers — so an app's traces show
|
|
19
|
+
* up in the playground without the app writing to a trace store itself.
|
|
20
|
+
*
|
|
21
|
+
* Calls return as soon as the event is queued, never waiting on the network,
|
|
22
|
+
* so tracing doesn't slow the app down. Queued events go out in order, batched
|
|
23
|
+
* into one request at a time, so a span's end never overtakes its start. A
|
|
24
|
+
* request that fails is logged and its events dropped: tracing never breaks
|
|
25
|
+
* the app. Await {@link flush} before a short-lived process exits to be sure
|
|
26
|
+
* everything was sent.
|
|
27
|
+
*/
|
|
28
|
+
var HttpTraceSink = class {
|
|
29
|
+
options;
|
|
30
|
+
target;
|
|
31
|
+
/** Set once it's known there's nowhere to send, so events aren't queued. */
|
|
32
|
+
disabled = false;
|
|
33
|
+
queue = [];
|
|
34
|
+
sending;
|
|
35
|
+
/** Whether the last request failed, so a down server is reported once. */
|
|
36
|
+
failing = false;
|
|
37
|
+
constructor(options) {
|
|
38
|
+
this.options = options;
|
|
39
|
+
}
|
|
40
|
+
resolveTarget() {
|
|
41
|
+
this.target ??= this.lookUpTarget().catch(() => void 0);
|
|
42
|
+
return this.target;
|
|
43
|
+
}
|
|
44
|
+
async lookUpTarget() {
|
|
45
|
+
const { endpoint, fetch: send } = this.options;
|
|
46
|
+
const target = typeof endpoint === "function" ? await endpoint() : { endpoint };
|
|
47
|
+
if (!target) return void 0;
|
|
48
|
+
return {
|
|
49
|
+
url: target.endpoint.replace(/\/+$/, "") + TRACE_EVENTS_PATH,
|
|
50
|
+
headers: target.headers,
|
|
51
|
+
fetch: target.fetch ?? send ?? ((...args) => globalThis.fetch(...args))
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
async recordSpanStart(span) {
|
|
55
|
+
this.enqueue({
|
|
56
|
+
type: "span-start",
|
|
57
|
+
span
|
|
58
|
+
});
|
|
59
|
+
return span;
|
|
60
|
+
}
|
|
61
|
+
async recordSpanEnd(span) {
|
|
62
|
+
this.enqueue({
|
|
63
|
+
type: "span-end",
|
|
64
|
+
span
|
|
65
|
+
});
|
|
66
|
+
return span;
|
|
67
|
+
}
|
|
68
|
+
async failTrace(traceId, errorMessage) {
|
|
69
|
+
this.enqueue({
|
|
70
|
+
type: "trace-fail",
|
|
71
|
+
traceId,
|
|
72
|
+
errorMessage
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
/** Resolves once every event recorded so far has been sent (or dropped). */
|
|
76
|
+
async flush() {
|
|
77
|
+
return this.sending ?? Promise.resolve();
|
|
78
|
+
}
|
|
79
|
+
enqueue(event) {
|
|
80
|
+
if (this.disabled) return;
|
|
81
|
+
this.queue.push(event);
|
|
82
|
+
this.sending ??= this.drain();
|
|
83
|
+
}
|
|
84
|
+
async drain() {
|
|
85
|
+
await new Promise((resolve) => setTimeout(resolve));
|
|
86
|
+
const target = await this.resolveTarget();
|
|
87
|
+
if (!target) {
|
|
88
|
+
this.disabled = true;
|
|
89
|
+
this.queue = [];
|
|
90
|
+
}
|
|
91
|
+
while (target && this.queue.length > 0) {
|
|
92
|
+
const events = this.queue;
|
|
93
|
+
this.queue = [];
|
|
94
|
+
await this.send(target, events);
|
|
95
|
+
}
|
|
96
|
+
this.sending = void 0;
|
|
97
|
+
}
|
|
98
|
+
async send(target, events) {
|
|
99
|
+
try {
|
|
100
|
+
const body = { events };
|
|
101
|
+
const res = await target.fetch(target.url, {
|
|
102
|
+
method: "POST",
|
|
103
|
+
headers: {
|
|
104
|
+
"content-type": "application/json",
|
|
105
|
+
...target.headers
|
|
106
|
+
},
|
|
107
|
+
body: JSON.stringify(body)
|
|
108
|
+
});
|
|
109
|
+
if (!res.ok) throw new Error(`${res.status} ${await res.text().catch(() => "")}`);
|
|
110
|
+
this.failing = false;
|
|
111
|
+
} catch (err) {
|
|
112
|
+
if (!this.failing) console.warn(`[evalution] Couldn't send traces to ${target.url}:`, err instanceof Error ? err.message : err);
|
|
113
|
+
this.failing = true;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
};
|
|
117
|
+
//#endregion
|
|
118
|
+
//#region src/client.ts
|
|
119
|
+
/** The Evalution Cloud API, unless `EVALUTION_API_URL` says otherwise. */
|
|
120
|
+
const DEFAULT_API_URL = "https://api.evalut.io";
|
|
121
|
+
/**
|
|
122
|
+
* The kind of an API key, read from its prefix, or `undefined` for a string
|
|
123
|
+
* that isn't one.
|
|
124
|
+
*/
|
|
125
|
+
function apiKeyKind(key) {
|
|
126
|
+
if (key.startsWith("evl_ingest_")) return "ingest";
|
|
127
|
+
if (key.startsWith("evl_user_")) return "personal";
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* The client for Evalution's API. As a {@link TraceSink} it sends an app's
|
|
131
|
+
* traces, and it makes the read and control calls the local server needs.
|
|
132
|
+
*
|
|
133
|
+
* Where traces go is worked out once, on the first one sent:
|
|
134
|
+
*
|
|
135
|
+
* 1. the {@link EvalutionOptions.endpoint} option;
|
|
136
|
+
* 2. the cloud, when the API key is an ingest key;
|
|
137
|
+
* 3. the local instance `EVALUTION_ENDPOINT` names;
|
|
138
|
+
* 4. on Node, the instance portkit knows for this directory;
|
|
139
|
+
* 5. otherwise nowhere, and sending does nothing.
|
|
140
|
+
*
|
|
141
|
+
* A personal key never sends traces to the cloud, and no key is ever sent to
|
|
142
|
+
* a local instance. Tracing never breaks the app: see {@link HttpTraceSink},
|
|
143
|
+
* and await {@link flush} before a short-lived process exits. The read and
|
|
144
|
+
* control calls, by contrast, reject when they fail.
|
|
145
|
+
*/
|
|
146
|
+
var Evalution = class {
|
|
147
|
+
endpoint;
|
|
148
|
+
apiKey;
|
|
149
|
+
apiUrl;
|
|
150
|
+
project;
|
|
151
|
+
fetch;
|
|
152
|
+
sink;
|
|
153
|
+
constructor(options = {}) {
|
|
154
|
+
const env = globalThis.process?.env ?? {};
|
|
155
|
+
this.endpoint = options.endpoint;
|
|
156
|
+
this.apiKey = options.apiKey ?? env.EVALUTION_API_KEY;
|
|
157
|
+
this.apiUrl = (options.apiUrl || env.EVALUTION_API_URL || "https://api.evalut.io").replace(/\/+$/, "");
|
|
158
|
+
this.project = options.project;
|
|
159
|
+
this.fetch = options.fetch ?? ((...args) => globalThis.fetch(...args));
|
|
160
|
+
this.sink = new HttpTraceSink({
|
|
161
|
+
endpoint: () => this.resolveTraceTarget(env.EVALUTION_ENDPOINT),
|
|
162
|
+
fetch: this.fetch
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
async resolveTraceTarget(envEndpoint) {
|
|
166
|
+
if (this.endpoint === false) return void 0;
|
|
167
|
+
if (this.endpoint) return { endpoint: this.endpoint };
|
|
168
|
+
if (this.apiKey) switch (apiKeyKind(this.apiKey)) {
|
|
169
|
+
case "ingest": return {
|
|
170
|
+
endpoint: this.apiUrl,
|
|
171
|
+
headers: { authorization: `Bearer ${this.apiKey}` }
|
|
172
|
+
};
|
|
173
|
+
case "personal": break;
|
|
174
|
+
default: console.warn("[evalution] Ignoring the API key: it isn't an evl_ingest_… or evl_user_… key.");
|
|
175
|
+
}
|
|
176
|
+
if (envEndpoint) return { endpoint: envEndpoint };
|
|
177
|
+
if (!globalThis.process?.versions?.node) return void 0;
|
|
178
|
+
return (await import("./portkit-DV-XcagM.js")).discoverPortkit();
|
|
179
|
+
}
|
|
180
|
+
recordSpanStart(span) {
|
|
181
|
+
return this.sink.recordSpanStart(span);
|
|
182
|
+
}
|
|
183
|
+
recordSpanEnd(span) {
|
|
184
|
+
return this.sink.recordSpanEnd(span);
|
|
185
|
+
}
|
|
186
|
+
failTrace(traceId, errorMessage) {
|
|
187
|
+
return this.sink.failTrace(traceId, errorMessage);
|
|
188
|
+
}
|
|
189
|
+
/** Resolves once every trace event recorded so far has been sent (or dropped). */
|
|
190
|
+
flush() {
|
|
191
|
+
return this.sink.flush();
|
|
192
|
+
}
|
|
193
|
+
/** The key's kind and whose it is. Takes either kind of key. */
|
|
194
|
+
whoami() {
|
|
195
|
+
return this.json("GET", "/cloud/v1/whoami", { key: "any" });
|
|
196
|
+
}
|
|
197
|
+
/** The most recent production traces of the project. */
|
|
198
|
+
listTraces(query = {}) {
|
|
199
|
+
const params = new URLSearchParams();
|
|
200
|
+
for (const [name, value] of Object.entries(query)) if (value !== void 0) params.set(name, String(value));
|
|
201
|
+
const search = params.size > 0 ? `?${params}` : "";
|
|
202
|
+
return this.json("GET", `/api/traces${search}`, { project: true });
|
|
203
|
+
}
|
|
204
|
+
/** One production trace with its spans, or `undefined` if there's none. */
|
|
205
|
+
async getTrace(id) {
|
|
206
|
+
const res = await this.request("GET", `/api/traces/${encodeURIComponent(id)}`, { project: true });
|
|
207
|
+
if (res.status === 404) return void 0;
|
|
208
|
+
return (await readOk(res)).json();
|
|
209
|
+
}
|
|
210
|
+
/** The projects the key's user is a member of. */
|
|
211
|
+
listProjects() {
|
|
212
|
+
return this.json("GET", "/cloud/v1/projects");
|
|
213
|
+
}
|
|
214
|
+
/** Creates a project in the user's organization. */
|
|
215
|
+
createProject(project) {
|
|
216
|
+
return this.json("POST", "/cloud/v1/projects", { body: project });
|
|
217
|
+
}
|
|
218
|
+
/** Records a running checkout of the project. */
|
|
219
|
+
registerInstance(instance) {
|
|
220
|
+
return this.json("PUT", "/cloud/v1/instances", {
|
|
221
|
+
project: true,
|
|
222
|
+
body: instance
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
/** Tells the cloud a registered instance is still running. */
|
|
226
|
+
async heartbeatInstance(id) {
|
|
227
|
+
await readOk(await this.instanceRequest("POST", id, "heartbeat"));
|
|
228
|
+
}
|
|
229
|
+
/** Tells the cloud a registered instance has stopped. */
|
|
230
|
+
async endInstanceSession(id) {
|
|
231
|
+
await readOk(await this.instanceRequest("DELETE", id, "session"));
|
|
232
|
+
}
|
|
233
|
+
/** Ensures a synced database exists, and returns how to reach it. */
|
|
234
|
+
getSyncDatabase(key) {
|
|
235
|
+
return this.json("POST", "/cloud/v1/sync/databases", {
|
|
236
|
+
project: true,
|
|
237
|
+
body: key
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
instanceRequest(method, id, action) {
|
|
241
|
+
const path = `/cloud/v1/instances/${encodeURIComponent(id)}/${action}`;
|
|
242
|
+
return this.request(method, path, { project: true });
|
|
243
|
+
}
|
|
244
|
+
async json(method, path, options) {
|
|
245
|
+
return (await readOk(await this.request(method, path, options))).json();
|
|
246
|
+
}
|
|
247
|
+
request(method, path, { key = "personal", project, body } = {}) {
|
|
248
|
+
if (!this.apiKey) throw new Error("Evalution: set EVALUTION_API_KEY to use the cloud API.");
|
|
249
|
+
if (key === "personal" && apiKeyKind(this.apiKey) !== "personal") throw new Error(`Evalution: ${path} needs a personal (evl_user_…) key.`);
|
|
250
|
+
if (project && !this.project) throw new Error(`Evalution: ${path} needs a project id.`);
|
|
251
|
+
return this.fetch(this.apiUrl + path, {
|
|
252
|
+
method,
|
|
253
|
+
headers: {
|
|
254
|
+
authorization: `Bearer ${this.apiKey}`,
|
|
255
|
+
...project && { "evalution-project": this.project },
|
|
256
|
+
...body !== void 0 && { "content-type": "application/json" }
|
|
257
|
+
},
|
|
258
|
+
...body !== void 0 && { body: JSON.stringify(body) }
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
};
|
|
262
|
+
/** `res`, or a rejection saying why the cloud refused. */
|
|
263
|
+
async function readOk(res) {
|
|
264
|
+
if (res.ok) return res;
|
|
265
|
+
const detail = await res.text().catch(() => "");
|
|
266
|
+
throw new Error(`Evalution API: ${res.status} ${detail}`.trim());
|
|
267
|
+
}
|
|
268
|
+
//#endregion
|
|
269
|
+
//#region src/evaluation-answers.ts
|
|
270
|
+
const isRecord = (v) => !!v && typeof v === "object" && !Array.isArray(v);
|
|
271
|
+
/**
|
|
272
|
+
* The per-question confidence a provider reported beside an evaluation's
|
|
273
|
+
* answers, keyed by question id.
|
|
274
|
+
*
|
|
275
|
+
* The AI SDK's answers carry no confidence of their own: a provider that
|
|
276
|
+
* computes one (TypeSafe does, for choice and score answers) reports it as
|
|
277
|
+
* `providerMetadata.<provider>.confidence[questionId]`. Read from whichever
|
|
278
|
+
* provider namespace has it, so this isn't tied to one provider.
|
|
279
|
+
*/
|
|
280
|
+
function evaluationConfidence(providerMetadata) {
|
|
281
|
+
const out = {};
|
|
282
|
+
if (!isRecord(providerMetadata)) return out;
|
|
283
|
+
for (const namespace of Object.values(providerMetadata)) {
|
|
284
|
+
if (!isRecord(namespace) || !isRecord(namespace.confidence)) continue;
|
|
285
|
+
for (const [id, value] of Object.entries(namespace.confidence)) if (typeof value === "number" && Number.isFinite(value)) out[id] = value;
|
|
286
|
+
}
|
|
287
|
+
return out;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* An evaluation's answers as recorded on its span: each answer as the SDK
|
|
291
|
+
* returned it, plus its `confidence` where the provider reported one (see
|
|
292
|
+
* {@link evaluationConfidence}). Recorded together so a trace — from the
|
|
293
|
+
* playground or sent over OTLP — can show confidence without knowing which
|
|
294
|
+
* provider answered.
|
|
295
|
+
*/
|
|
296
|
+
function answersWithConfidence(answers, providerMetadata) {
|
|
297
|
+
if (!isRecord(answers)) return answers;
|
|
298
|
+
const confidence = evaluationConfidence(providerMetadata);
|
|
299
|
+
if (Object.keys(confidence).length === 0) return answers;
|
|
300
|
+
return Object.fromEntries(Object.entries(answers).map(([id, answer]) => [id, isRecord(answer) && id in confidence && !("confidence" in answer) ? {
|
|
301
|
+
...answer,
|
|
302
|
+
confidence: confidence[id]
|
|
303
|
+
} : answer]));
|
|
304
|
+
}
|
|
305
|
+
//#endregion
|
|
306
|
+
//#region src/prompt-tracer.ts
|
|
307
|
+
/**
|
|
308
|
+
* Attribute name a span can set to pick one of evalution's span-kind values
|
|
309
|
+
* (`'LLM'`, `'TOOL'`, `'AGENT'`, `'EMBEDDING'`, `'DEFAULT'`). Falls back to
|
|
310
|
+
* `'DEFAULT'` when absent or unrecognised.
|
|
311
|
+
*/
|
|
312
|
+
const SPAN_KIND_ATTRIBUTE = "evalution.span.type";
|
|
313
|
+
/**
|
|
314
|
+
* Attribute name a span can set to give a human-readable name to the prompt.
|
|
315
|
+
*/
|
|
316
|
+
const PROMPT_NAME_ATTRIBUTE = "gen_ai.prompt.name";
|
|
317
|
+
/**
|
|
318
|
+
* Attribute name a span can set to link itself to a specific prompt. The value
|
|
319
|
+
* is a globally-unique prompt ID unless {@link PROMPT_PROVIDER_ID_ATTRIBUTE} is
|
|
320
|
+
* also set, in which case it is scoped to that provider.
|
|
321
|
+
*/
|
|
322
|
+
const PROMPT_ID_ATTRIBUTE = "evalution.prompt.id";
|
|
323
|
+
/**
|
|
324
|
+
* Attribute name a span can set to scope its {@link PROMPT_ID_ATTRIBUTE} to a
|
|
325
|
+
* specific prompt provider. When absent, the prompt ID is treated as global.
|
|
326
|
+
*/
|
|
327
|
+
const PROMPT_PROVIDER_ID_ATTRIBUTE = "evalution.prompt.provider.id";
|
|
328
|
+
/**
|
|
329
|
+
* Attribute name a span can set to include an array of input parameters used
|
|
330
|
+
* to construct the final prompt.
|
|
331
|
+
*/
|
|
332
|
+
const PROMPT_INPUTS_ATTRIBUTE = "evalution.prompt.inputs";
|
|
333
|
+
/**
|
|
334
|
+
* Attribute name a span can set to record which version of its prompt ran —
|
|
335
|
+
* for a file-based prompt, the commit checked out, when nothing was
|
|
336
|
+
* uncommitted.
|
|
337
|
+
*/
|
|
338
|
+
const PROMPT_VERSION_ATTRIBUTE = "evalution.prompt.version";
|
|
339
|
+
/**
|
|
340
|
+
* Attribute name a span can set to record which variation of its prompt ran,
|
|
341
|
+
* on top of its {@link PROMPT_VERSION_ATTRIBUTE}.
|
|
342
|
+
*/
|
|
343
|
+
const PROMPT_VARIATION_ATTRIBUTE = "evalution.prompt.variation";
|
|
344
|
+
/**
|
|
345
|
+
* `identity` with any field it leaves unset filled from `fallback`.
|
|
346
|
+
*
|
|
347
|
+
* A config built by the `prompts()` helper carries an identity of its own —
|
|
348
|
+
* the one to keep, since its id is the global one — but only the playground
|
|
349
|
+
* knows what the run was launched with and against which version.
|
|
350
|
+
*/
|
|
351
|
+
function mergePromptIdentity(identity, fallback) {
|
|
352
|
+
if (!identity || !fallback) return identity ?? fallback;
|
|
353
|
+
const merged = { ...fallback };
|
|
354
|
+
for (const [key, value] of Object.entries(identity)) if (value !== void 0) merged[key] = value;
|
|
355
|
+
return merged;
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* Build the attributes that associate a span with a prompt.
|
|
359
|
+
*
|
|
360
|
+
* @param prompt - The prompt to attribute the span to.
|
|
361
|
+
* @param attributes - Attributes to merge in; these take precedence.
|
|
362
|
+
* @param includeInputs - Whether to stamp the run's inputs. Inputs belong on
|
|
363
|
+
* the **root span only**: they are identical on every span of a trace, so
|
|
364
|
+
* repeating them is invisible when the input is `["Ada"]` and wasteful when
|
|
365
|
+
* it is a fifty-message thread — multiplied by span count, then persisted.
|
|
366
|
+
*/
|
|
367
|
+
function getPromptSpanAttributes(prompt, attributes = {}, includeInputs = true) {
|
|
368
|
+
const inputs = includeInputs && (prompt.functionInputs || prompt.executeInputs) ? JSON.stringify({
|
|
369
|
+
functionInputs: prompt.functionInputs,
|
|
370
|
+
executeInputs: prompt.executeInputs,
|
|
371
|
+
resources: prompt.resources,
|
|
372
|
+
parameterDefinitions: prompt.parameterDefinitions,
|
|
373
|
+
executeParameterDefinitions: prompt.executeParameterDefinitions
|
|
374
|
+
}) : void 0;
|
|
375
|
+
return Object.fromEntries([
|
|
376
|
+
[SPAN_KIND_ATTRIBUTE, "LLM"],
|
|
377
|
+
[PROMPT_NAME_ATTRIBUTE, prompt.name],
|
|
378
|
+
[PROMPT_ID_ATTRIBUTE, prompt.id],
|
|
379
|
+
[PROMPT_INPUTS_ATTRIBUTE, inputs],
|
|
380
|
+
[PROMPT_VERSION_ATTRIBUTE, prompt.version],
|
|
381
|
+
[PROMPT_VARIATION_ATTRIBUTE, prompt.variation],
|
|
382
|
+
...Object.entries(attributes)
|
|
383
|
+
].filter(([, v]) => v !== void 0 && v !== null));
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* Wraps a {@link Tracer} so that every span it produces is tagged with the
|
|
387
|
+
* attributes that associate it with a prompt — the prompt's name
|
|
388
|
+
* ({@link PROMPT_NAME_ATTRIBUTE}), an optional global prompt ID
|
|
389
|
+
* ({@link PROMPT_ID_ATTRIBUTE}), and an `'LLM'` span kind
|
|
390
|
+
* ({@link SPAN_KIND_ATTRIBUTE}). Attributes set explicitly on an individual
|
|
391
|
+
* span take precedence over these defaults.
|
|
392
|
+
*
|
|
393
|
+
* This depends only on `@opentelemetry/api`, so it can be re-used by SDK
|
|
394
|
+
* adapter packages (e.g. `@evalution/vercel-ai-sdk`) without pulling in the
|
|
395
|
+
* rest of evalution.
|
|
396
|
+
*
|
|
397
|
+
* @param prompt - The prompt to attribute spans to. `name` is a human-readable
|
|
398
|
+
* name; the optional `id` is a globally-unique prompt ID used to resolve
|
|
399
|
+
* runtime traces back to the prompt.
|
|
400
|
+
* @param tracer - Tracer to wrap. Defaults to a tracer from the globally
|
|
401
|
+
* registered tracer provider.
|
|
402
|
+
* @returns A tracer that forwards to `tracer` while attaching the prompt
|
|
403
|
+
* attributes to each span it creates.
|
|
404
|
+
*/
|
|
405
|
+
function createTracerForPrompt(prompt, tracer) {
|
|
406
|
+
const inner = tracer ?? trace.getTracer("evalution");
|
|
407
|
+
let rootEmitted = false;
|
|
408
|
+
const withPromptAttributes = (options) => {
|
|
409
|
+
const attributes = getPromptSpanAttributes(prompt, options?.attributes, !rootEmitted);
|
|
410
|
+
rootEmitted = true;
|
|
411
|
+
return {
|
|
412
|
+
...options,
|
|
413
|
+
attributes
|
|
414
|
+
};
|
|
415
|
+
};
|
|
416
|
+
return {
|
|
417
|
+
startSpan(name, options, context) {
|
|
418
|
+
return inner.startSpan(name, withPromptAttributes(options), context);
|
|
419
|
+
},
|
|
420
|
+
startActiveSpan(name, ...rest) {
|
|
421
|
+
if (typeof rest[0] === "function") return inner.startActiveSpan(name, withPromptAttributes(), rest[0]);
|
|
422
|
+
if (typeof rest[1] === "function") return inner.startActiveSpan(name, withPromptAttributes(rest[0]), rest[1]);
|
|
423
|
+
return inner.startActiveSpan(name, withPromptAttributes(rest[0]), rest[1], rest[2]);
|
|
424
|
+
}
|
|
425
|
+
};
|
|
426
|
+
}
|
|
427
|
+
//#endregion
|
|
428
|
+
//#region src/trace-types.ts
|
|
429
|
+
/**
|
|
430
|
+
* Span attribute naming the eval check (by its id within the eval) that
|
|
431
|
+
* recorded the span while judging the trace — set on the check's `CHECK` span
|
|
432
|
+
* and on everything under it. Those spans aren't part of the run being
|
|
433
|
+
* judged, so the trace's rollups and chat view leave them out.
|
|
434
|
+
*/
|
|
435
|
+
const CHECK_ATTRIBUTE = "evalution.check";
|
|
436
|
+
/** Whether `span` was recorded by a check judging its trace, not by the run. */
|
|
437
|
+
function isCheckSpan(span) {
|
|
438
|
+
return span.attributes?.[CHECK_ATTRIBUTE] !== void 0;
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* The message list of a chat call, or `undefined` for any other input.
|
|
442
|
+
*
|
|
443
|
+
* A chat call's {@link LLMSpanDetails.input} is always a message list, and
|
|
444
|
+
* anything else is wrapped in an object, so being an array is what makes it
|
|
445
|
+
* one.
|
|
446
|
+
*/
|
|
447
|
+
function spanMessages(llm) {
|
|
448
|
+
return Array.isArray(llm?.input) ? llm.input : void 0;
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* The output of a turn that called tools: its text (if any) and the calls, as
|
|
452
|
+
* the one assistant message the next turn's input repeats. Without the calls
|
|
453
|
+
* such a turn would record no output at all, its text being empty.
|
|
454
|
+
*/
|
|
455
|
+
function toolCallingOutput(text, toolCalls) {
|
|
456
|
+
return [{
|
|
457
|
+
role: "assistant",
|
|
458
|
+
content: [...text ? [{
|
|
459
|
+
type: "text",
|
|
460
|
+
text
|
|
461
|
+
}] : [], ...toolCalls]
|
|
462
|
+
}];
|
|
463
|
+
}
|
|
464
|
+
/**
|
|
465
|
+
* An LLM span's output as a message list: text as the assistant message it
|
|
466
|
+
* is, a turn that called tools as the message it recorded. `undefined` for
|
|
467
|
+
* structured output, which can be an array too: every element must look like
|
|
468
|
+
* a message.
|
|
469
|
+
*/
|
|
470
|
+
function outputMessages(llm) {
|
|
471
|
+
const output = llm?.output;
|
|
472
|
+
if (typeof output === "string") return [{
|
|
473
|
+
role: "assistant",
|
|
474
|
+
content: output
|
|
475
|
+
}];
|
|
476
|
+
return Array.isArray(output) && output.length > 0 && output.every((m) => m && typeof m === "object" && typeof m.role === "string" && (typeof m.content === "string" || Array.isArray(m.content))) ? output : void 0;
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* The OpenTelemetry semantic-convention resource attribute naming the
|
|
480
|
+
* service that produced a span.
|
|
481
|
+
*/
|
|
482
|
+
const SERVICE_NAME_ATTRIBUTE = "service.name";
|
|
483
|
+
/**
|
|
484
|
+
* The OpenTelemetry semantic-convention resource attribute naming the
|
|
485
|
+
* deployment environment a span was produced in.
|
|
486
|
+
*/
|
|
487
|
+
const DEPLOYMENT_ENVIRONMENT_ATTRIBUTE = "deployment.environment.name";
|
|
488
|
+
/** {@link DEPLOYMENT_ENVIRONMENT_ATTRIBUTE}'s value on a run evalution itself made. */
|
|
489
|
+
const PLAYGROUND_ENVIRONMENT = "playground";
|
|
490
|
+
/**
|
|
491
|
+
* A tool result's output as the tool returned it. The AI SDK wraps a result
|
|
492
|
+
* in a typed envelope (`{ type: "json", value }`, `{ type: "error-text",
|
|
493
|
+
* value }`, `{ type: "execution-denied", reason }`, …); this unwraps it and
|
|
494
|
+
* notes whether it reports a failure. Anything else is kept as is.
|
|
495
|
+
*/
|
|
496
|
+
function unwrapToolResultOutput(output) {
|
|
497
|
+
if (!output || typeof output !== "object" || Array.isArray(output)) return { output };
|
|
498
|
+
const envelope = output;
|
|
499
|
+
const { type } = envelope;
|
|
500
|
+
if (type === "execution-denied") return {
|
|
501
|
+
output: output.reason,
|
|
502
|
+
isError: true
|
|
503
|
+
};
|
|
504
|
+
if (typeof type !== "string" || !("value" in envelope)) return { output };
|
|
505
|
+
return type.startsWith("error-") ? {
|
|
506
|
+
output: envelope.value,
|
|
507
|
+
isError: true
|
|
508
|
+
} : { output: envelope.value };
|
|
509
|
+
}
|
|
510
|
+
//#endregion
|
|
511
|
+
//#region src/trace-ingestor.ts
|
|
512
|
+
/**
|
|
513
|
+
* The resource of every run evalution itself makes — from the playground, an
|
|
514
|
+
* eval, or an MCP client — as opposed to one an app exported to it.
|
|
515
|
+
*/
|
|
516
|
+
const PLAYGROUND_RESOURCE = {
|
|
517
|
+
[SERVICE_NAME_ATTRIBUTE]: "evalution",
|
|
518
|
+
[DEPLOYMENT_ENVIRONMENT_ATTRIBUTE]: PLAYGROUND_ENVIRONMENT
|
|
519
|
+
};
|
|
520
|
+
/**
|
|
521
|
+
* Base class that holds the added sinks and fans each normalized record out
|
|
522
|
+
* to all of them. Concrete ingestors translate their source into
|
|
523
|
+
* {@link Span}/{@link Trace} records and call these protected helpers.
|
|
524
|
+
*/
|
|
525
|
+
var BaseTraceIngestor = class {
|
|
526
|
+
sinks = [];
|
|
527
|
+
resource;
|
|
528
|
+
/** Recordings still under way, which {@link flush} waits out. */
|
|
529
|
+
pending = /* @__PURE__ */ new Set();
|
|
530
|
+
setResource(resource) {
|
|
531
|
+
this.resource = { ...resource };
|
|
532
|
+
}
|
|
533
|
+
/** `span` with this ingestor's resource, if it has one, merged into its own. */
|
|
534
|
+
withResource(span) {
|
|
535
|
+
if (!this.resource) return span;
|
|
536
|
+
return {
|
|
537
|
+
...span,
|
|
538
|
+
resource: {
|
|
539
|
+
...span.resource,
|
|
540
|
+
...this.resource
|
|
541
|
+
}
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
addSink(sink) {
|
|
545
|
+
this.sinks.push(sink);
|
|
546
|
+
}
|
|
547
|
+
removeSink(sink) {
|
|
548
|
+
const prevSinks = this.sinks;
|
|
549
|
+
const newSinks = prevSinks.filter((s) => s !== sink);
|
|
550
|
+
this.sinks = newSinks;
|
|
551
|
+
return prevSinks.length > newSinks.length;
|
|
552
|
+
}
|
|
553
|
+
/**
|
|
554
|
+
* Resolves once everything recorded so far has reached every sink and been
|
|
555
|
+
* delivered by those that deliver in the background. Await it before a
|
|
556
|
+
* short-lived process (a script, a serverless function) exits, so no trace
|
|
557
|
+
* data is lost.
|
|
558
|
+
*/
|
|
559
|
+
async flush() {
|
|
560
|
+
while (this.pending.size > 0) await Promise.allSettled(this.pending);
|
|
561
|
+
await Promise.all(this.sinks.map((s) => s.flush?.()));
|
|
562
|
+
}
|
|
563
|
+
/** Registers `recording` for {@link flush} to wait out, and returns it. */
|
|
564
|
+
track(recording) {
|
|
565
|
+
this.pending.add(recording);
|
|
566
|
+
const done = () => this.pending.delete(recording);
|
|
567
|
+
recording.then(done, done);
|
|
568
|
+
return recording;
|
|
569
|
+
}
|
|
570
|
+
async recordSpanStart(span) {
|
|
571
|
+
const stamped = this.withResource(span);
|
|
572
|
+
await this.track(Promise.all(this.sinks.map((s) => s.recordSpanStart(stamped))));
|
|
573
|
+
}
|
|
574
|
+
async recordSpanEnd(span) {
|
|
575
|
+
const stamped = this.withResource(span);
|
|
576
|
+
await this.track(Promise.all(this.sinks.map((s) => s.recordSpanEnd(stamped))));
|
|
577
|
+
}
|
|
578
|
+
async failTrace(traceId, msg) {
|
|
579
|
+
await this.track(Promise.all(this.sinks.map((s) => s.failTrace(traceId, msg))));
|
|
580
|
+
}
|
|
581
|
+
};
|
|
582
|
+
//#endregion
|
|
583
|
+
//#region src/trace-sink.ts
|
|
584
|
+
function isTraceSink(o) {
|
|
585
|
+
return typeof o === "object" && o !== null && typeof o.recordSpanStart === "function" && typeof o.recordSpanEnd === "function" && typeof o.failTrace === "function";
|
|
586
|
+
}
|
|
587
|
+
//#endregion
|
|
588
|
+
export { BaseTraceIngestor, CHECK_ATTRIBUTE, DEFAULT_API_URL, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, Evalution, HttpTraceSink, PLAYGROUND_ENVIRONMENT, PLAYGROUND_RESOURCE, PROMPT_ID_ATTRIBUTE, PROMPT_INPUTS_ATTRIBUTE, PROMPT_NAME_ATTRIBUTE, PROMPT_PROVIDER_ID_ATTRIBUTE, PROMPT_VARIATION_ATTRIBUTE, PROMPT_VERSION_ATTRIBUTE, SERVICE_NAME_ATTRIBUTE, SPAN_KIND_ATTRIBUTE, TRACE_EVENTS_PATH, answersWithConfidence, apiKeyKind, createTracerForPrompt, evaluationConfidence, getPromptSpanAttributes, isCheckSpan, isTraceSink, makeBrand, mergePromptIdentity, outputMessages, spanMessages, toolCallingOutput, unwrapToolResultOutput };
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
//#region src/portkit.ts
|
|
2
|
+
const SERVICES_URL = "https://portkit.localhost/services";
|
|
3
|
+
/** How long to wait for portkit to answer; Caddy may be starting. */
|
|
4
|
+
const DISCOVERY_TIMEOUT_MS = 1e3;
|
|
5
|
+
/**
|
|
6
|
+
* How long to wait for a response when the caller sets no limit. Just over the
|
|
7
|
+
* 60 s portkit holds a request while an on-demand service starts.
|
|
8
|
+
*/
|
|
9
|
+
const REQUEST_TIMEOUT_MS = 65e3;
|
|
10
|
+
/**
|
|
11
|
+
* Asks portkit for the evalution instance serving this directory or one of
|
|
12
|
+
* its parents. It never contacts the service itself, which would start one
|
|
13
|
+
* that's on demand.
|
|
14
|
+
*
|
|
15
|
+
* Resolves to `undefined` when portkit isn't installed — without its CA
|
|
16
|
+
* certificate no request is made — or has no such service, or doesn't answer
|
|
17
|
+
* in time.
|
|
18
|
+
*/
|
|
19
|
+
async function discoverPortkit({ env = process.env, cwd = process.cwd(), servicesUrl = SERVICES_URL } = {}) {
|
|
20
|
+
try {
|
|
21
|
+
const send = localTlsFetch(await readPortkitCa(env));
|
|
22
|
+
const res = await send(`${servicesUrl}?${new URLSearchParams({
|
|
23
|
+
label: "kind=evalution",
|
|
24
|
+
cwd
|
|
25
|
+
})}`, { signal: AbortSignal.timeout(DISCOVERY_TIMEOUT_MS) });
|
|
26
|
+
if (!res.ok) return void 0;
|
|
27
|
+
const { services } = await res.json();
|
|
28
|
+
const endpoint = services?.[0]?.url;
|
|
29
|
+
return endpoint ? {
|
|
30
|
+
endpoint,
|
|
31
|
+
fetch: send
|
|
32
|
+
} : void 0;
|
|
33
|
+
} catch {
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
async function readPortkitCa(env) {
|
|
38
|
+
const [{ readFile }, { homedir }, { join }] = await Promise.all([
|
|
39
|
+
import("node:fs/promises"),
|
|
40
|
+
import("node:os"),
|
|
41
|
+
import("node:path")
|
|
42
|
+
]);
|
|
43
|
+
return readFile(join(env.PORTKIT_HOME || (process.platform === "darwin" ? join(homedir(), "Library", "Application Support", "portkit") : join(homedir(), ".local", "share", "portkit")), "data", "caddy", "pki", "authorities", "local", "root.crt"), "utf8");
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* A `fetch` for portkit's `*.localhost` names. It connects to loopback whatever
|
|
47
|
+
* the name, since not every system resolves `*.localhost`, and trusts `ca` for
|
|
48
|
+
* these connections only.
|
|
49
|
+
*
|
|
50
|
+
* Takes what this package sends: a URL, and a string body.
|
|
51
|
+
*/
|
|
52
|
+
function localTlsFetch(ca) {
|
|
53
|
+
return async (input, init) => {
|
|
54
|
+
const { request } = await import("node:https");
|
|
55
|
+
const url = new URL(String(input));
|
|
56
|
+
return new Promise((resolve, reject) => {
|
|
57
|
+
const req = request({
|
|
58
|
+
host: "127.0.0.1",
|
|
59
|
+
port: url.port || 443,
|
|
60
|
+
servername: url.hostname,
|
|
61
|
+
path: url.pathname + url.search,
|
|
62
|
+
method: init?.method ?? "GET",
|
|
63
|
+
headers: {
|
|
64
|
+
...Object.fromEntries(new Headers(init?.headers)),
|
|
65
|
+
host: url.host
|
|
66
|
+
},
|
|
67
|
+
ca,
|
|
68
|
+
signal: init?.signal ?? AbortSignal.timeout(REQUEST_TIMEOUT_MS)
|
|
69
|
+
}, (res) => {
|
|
70
|
+
const chunks = [];
|
|
71
|
+
res.on("data", (chunk) => chunks.push(chunk));
|
|
72
|
+
res.on("error", reject);
|
|
73
|
+
res.on("end", () => {
|
|
74
|
+
const body = chunks.length > 0 ? Buffer.concat(chunks) : null;
|
|
75
|
+
resolve(new Response(body, { status: res.statusCode }));
|
|
76
|
+
});
|
|
77
|
+
});
|
|
78
|
+
req.on("error", reject);
|
|
79
|
+
req.end(init?.body);
|
|
80
|
+
});
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
//#endregion
|
|
84
|
+
export { discoverPortkit };
|