@evalution/sdk 0.0.0-stage → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,588 @@
1
+ import { trace } from "@opentelemetry/api";
2
+ //#region src/brand.ts
3
+ function makeBrand(sym) {
4
+ return {
5
+ brand: (value) => Object.defineProperty(value, sym, { value: true }),
6
+ isBranded: (value) => typeof value === "object" && value !== null && value[sym] === true
7
+ };
8
+ }
9
+ //#endregion
10
+ //#region src/http-trace-sink.ts
11
+ /**
12
+ * Where an evalution server accepts {@link TraceSinkEvent}s, relative to its
13
+ * base URL.
14
+ */
15
+ const TRACE_EVENTS_PATH = "/api/traces";
16
+ /**
17
+ * A {@link TraceSink} that sends what it's given to an evalution server,
18
+ * which records it through its own trace providers — so an app's traces show
19
+ * up in the playground without the app writing to a trace store itself.
20
+ *
21
+ * Calls return as soon as the event is queued, never waiting on the network,
22
+ * so tracing doesn't slow the app down. Queued events go out in order, batched
23
+ * into one request at a time, so a span's end never overtakes its start. A
24
+ * request that fails is logged and its events dropped: tracing never breaks
25
+ * the app. Await {@link flush} before a short-lived process exits to be sure
26
+ * everything was sent.
27
+ */
28
+ var HttpTraceSink = class {
29
+ options;
30
+ target;
31
+ /** Set once it's known there's nowhere to send, so events aren't queued. */
32
+ disabled = false;
33
+ queue = [];
34
+ sending;
35
+ /** Whether the last request failed, so a down server is reported once. */
36
+ failing = false;
37
+ constructor(options) {
38
+ this.options = options;
39
+ }
40
+ resolveTarget() {
41
+ this.target ??= this.lookUpTarget().catch(() => void 0);
42
+ return this.target;
43
+ }
44
+ async lookUpTarget() {
45
+ const { endpoint, fetch: send } = this.options;
46
+ const target = typeof endpoint === "function" ? await endpoint() : { endpoint };
47
+ if (!target) return void 0;
48
+ return {
49
+ url: target.endpoint.replace(/\/+$/, "") + TRACE_EVENTS_PATH,
50
+ headers: target.headers,
51
+ fetch: target.fetch ?? send ?? ((...args) => globalThis.fetch(...args))
52
+ };
53
+ }
54
+ async recordSpanStart(span) {
55
+ this.enqueue({
56
+ type: "span-start",
57
+ span
58
+ });
59
+ return span;
60
+ }
61
+ async recordSpanEnd(span) {
62
+ this.enqueue({
63
+ type: "span-end",
64
+ span
65
+ });
66
+ return span;
67
+ }
68
+ async failTrace(traceId, errorMessage) {
69
+ this.enqueue({
70
+ type: "trace-fail",
71
+ traceId,
72
+ errorMessage
73
+ });
74
+ }
75
+ /** Resolves once every event recorded so far has been sent (or dropped). */
76
+ async flush() {
77
+ return this.sending ?? Promise.resolve();
78
+ }
79
+ enqueue(event) {
80
+ if (this.disabled) return;
81
+ this.queue.push(event);
82
+ this.sending ??= this.drain();
83
+ }
84
+ async drain() {
85
+ await new Promise((resolve) => setTimeout(resolve));
86
+ const target = await this.resolveTarget();
87
+ if (!target) {
88
+ this.disabled = true;
89
+ this.queue = [];
90
+ }
91
+ while (target && this.queue.length > 0) {
92
+ const events = this.queue;
93
+ this.queue = [];
94
+ await this.send(target, events);
95
+ }
96
+ this.sending = void 0;
97
+ }
98
+ async send(target, events) {
99
+ try {
100
+ const body = { events };
101
+ const res = await target.fetch(target.url, {
102
+ method: "POST",
103
+ headers: {
104
+ "content-type": "application/json",
105
+ ...target.headers
106
+ },
107
+ body: JSON.stringify(body)
108
+ });
109
+ if (!res.ok) throw new Error(`${res.status} ${await res.text().catch(() => "")}`);
110
+ this.failing = false;
111
+ } catch (err) {
112
+ if (!this.failing) console.warn(`[evalution] Couldn't send traces to ${target.url}:`, err instanceof Error ? err.message : err);
113
+ this.failing = true;
114
+ }
115
+ }
116
+ };
117
+ //#endregion
118
+ //#region src/client.ts
119
+ /** The Evalution Cloud API, unless `EVALUTION_API_URL` says otherwise. */
120
+ const DEFAULT_API_URL = "https://api.evalut.io";
121
+ /**
122
+ * The kind of an API key, read from its prefix, or `undefined` for a string
123
+ * that isn't one.
124
+ */
125
+ function apiKeyKind(key) {
126
+ if (key.startsWith("evl_ingest_")) return "ingest";
127
+ if (key.startsWith("evl_user_")) return "personal";
128
+ }
129
+ /**
130
+ * The client for Evalution's API. As a {@link TraceSink} it sends an app's
131
+ * traces, and it makes the read and control calls the local server needs.
132
+ *
133
+ * Where traces go is worked out once, on the first one sent:
134
+ *
135
+ * 1. the {@link EvalutionOptions.endpoint} option;
136
+ * 2. the cloud, when the API key is an ingest key;
137
+ * 3. the local instance `EVALUTION_ENDPOINT` names;
138
+ * 4. on Node, the instance portkit knows for this directory;
139
+ * 5. otherwise nowhere, and sending does nothing.
140
+ *
141
+ * A personal key never sends traces to the cloud, and no key is ever sent to
142
+ * a local instance. Tracing never breaks the app: see {@link HttpTraceSink},
143
+ * and await {@link flush} before a short-lived process exits. The read and
144
+ * control calls, by contrast, reject when they fail.
145
+ */
146
+ var Evalution = class {
147
+ endpoint;
148
+ apiKey;
149
+ apiUrl;
150
+ project;
151
+ fetch;
152
+ sink;
153
+ constructor(options = {}) {
154
+ const env = globalThis.process?.env ?? {};
155
+ this.endpoint = options.endpoint;
156
+ this.apiKey = options.apiKey ?? env.EVALUTION_API_KEY;
157
+ this.apiUrl = (options.apiUrl || env.EVALUTION_API_URL || "https://api.evalut.io").replace(/\/+$/, "");
158
+ this.project = options.project;
159
+ this.fetch = options.fetch ?? ((...args) => globalThis.fetch(...args));
160
+ this.sink = new HttpTraceSink({
161
+ endpoint: () => this.resolveTraceTarget(env.EVALUTION_ENDPOINT),
162
+ fetch: this.fetch
163
+ });
164
+ }
165
+ async resolveTraceTarget(envEndpoint) {
166
+ if (this.endpoint === false) return void 0;
167
+ if (this.endpoint) return { endpoint: this.endpoint };
168
+ if (this.apiKey) switch (apiKeyKind(this.apiKey)) {
169
+ case "ingest": return {
170
+ endpoint: this.apiUrl,
171
+ headers: { authorization: `Bearer ${this.apiKey}` }
172
+ };
173
+ case "personal": break;
174
+ default: console.warn("[evalution] Ignoring the API key: it isn't an evl_ingest_… or evl_user_… key.");
175
+ }
176
+ if (envEndpoint) return { endpoint: envEndpoint };
177
+ if (!globalThis.process?.versions?.node) return void 0;
178
+ return (await import("./portkit-DV-XcagM.js")).discoverPortkit();
179
+ }
180
+ recordSpanStart(span) {
181
+ return this.sink.recordSpanStart(span);
182
+ }
183
+ recordSpanEnd(span) {
184
+ return this.sink.recordSpanEnd(span);
185
+ }
186
+ failTrace(traceId, errorMessage) {
187
+ return this.sink.failTrace(traceId, errorMessage);
188
+ }
189
+ /** Resolves once every trace event recorded so far has been sent (or dropped). */
190
+ flush() {
191
+ return this.sink.flush();
192
+ }
193
+ /** The key's kind and whose it is. Takes either kind of key. */
194
+ whoami() {
195
+ return this.json("GET", "/cloud/v1/whoami", { key: "any" });
196
+ }
197
+ /** The most recent production traces of the project. */
198
+ listTraces(query = {}) {
199
+ const params = new URLSearchParams();
200
+ for (const [name, value] of Object.entries(query)) if (value !== void 0) params.set(name, String(value));
201
+ const search = params.size > 0 ? `?${params}` : "";
202
+ return this.json("GET", `/api/traces${search}`, { project: true });
203
+ }
204
+ /** One production trace with its spans, or `undefined` if there's none. */
205
+ async getTrace(id) {
206
+ const res = await this.request("GET", `/api/traces/${encodeURIComponent(id)}`, { project: true });
207
+ if (res.status === 404) return void 0;
208
+ return (await readOk(res)).json();
209
+ }
210
+ /** The projects the key's user is a member of. */
211
+ listProjects() {
212
+ return this.json("GET", "/cloud/v1/projects");
213
+ }
214
+ /** Creates a project in the user's organization. */
215
+ createProject(project) {
216
+ return this.json("POST", "/cloud/v1/projects", { body: project });
217
+ }
218
+ /** Records a running checkout of the project. */
219
+ registerInstance(instance) {
220
+ return this.json("PUT", "/cloud/v1/instances", {
221
+ project: true,
222
+ body: instance
223
+ });
224
+ }
225
+ /** Tells the cloud a registered instance is still running. */
226
+ async heartbeatInstance(id) {
227
+ await readOk(await this.instanceRequest("POST", id, "heartbeat"));
228
+ }
229
+ /** Tells the cloud a registered instance has stopped. */
230
+ async endInstanceSession(id) {
231
+ await readOk(await this.instanceRequest("DELETE", id, "session"));
232
+ }
233
+ /** Ensures a synced database exists, and returns how to reach it. */
234
+ getSyncDatabase(key) {
235
+ return this.json("POST", "/cloud/v1/sync/databases", {
236
+ project: true,
237
+ body: key
238
+ });
239
+ }
240
+ instanceRequest(method, id, action) {
241
+ const path = `/cloud/v1/instances/${encodeURIComponent(id)}/${action}`;
242
+ return this.request(method, path, { project: true });
243
+ }
244
+ async json(method, path, options) {
245
+ return (await readOk(await this.request(method, path, options))).json();
246
+ }
247
+ request(method, path, { key = "personal", project, body } = {}) {
248
+ if (!this.apiKey) throw new Error("Evalution: set EVALUTION_API_KEY to use the cloud API.");
249
+ if (key === "personal" && apiKeyKind(this.apiKey) !== "personal") throw new Error(`Evalution: ${path} needs a personal (evl_user_…) key.`);
250
+ if (project && !this.project) throw new Error(`Evalution: ${path} needs a project id.`);
251
+ return this.fetch(this.apiUrl + path, {
252
+ method,
253
+ headers: {
254
+ authorization: `Bearer ${this.apiKey}`,
255
+ ...project && { "evalution-project": this.project },
256
+ ...body !== void 0 && { "content-type": "application/json" }
257
+ },
258
+ ...body !== void 0 && { body: JSON.stringify(body) }
259
+ });
260
+ }
261
+ };
262
+ /** `res`, or a rejection saying why the cloud refused. */
263
+ async function readOk(res) {
264
+ if (res.ok) return res;
265
+ const detail = await res.text().catch(() => "");
266
+ throw new Error(`Evalution API: ${res.status} ${detail}`.trim());
267
+ }
268
+ //#endregion
269
+ //#region src/evaluation-answers.ts
270
+ const isRecord = (v) => !!v && typeof v === "object" && !Array.isArray(v);
271
+ /**
272
+ * The per-question confidence a provider reported beside an evaluation's
273
+ * answers, keyed by question id.
274
+ *
275
+ * The AI SDK's answers carry no confidence of their own: a provider that
276
+ * computes one (TypeSafe does, for choice and score answers) reports it as
277
+ * `providerMetadata.<provider>.confidence[questionId]`. Read from whichever
278
+ * provider namespace has it, so this isn't tied to one provider.
279
+ */
280
+ function evaluationConfidence(providerMetadata) {
281
+ const out = {};
282
+ if (!isRecord(providerMetadata)) return out;
283
+ for (const namespace of Object.values(providerMetadata)) {
284
+ if (!isRecord(namespace) || !isRecord(namespace.confidence)) continue;
285
+ for (const [id, value] of Object.entries(namespace.confidence)) if (typeof value === "number" && Number.isFinite(value)) out[id] = value;
286
+ }
287
+ return out;
288
+ }
289
+ /**
290
+ * An evaluation's answers as recorded on its span: each answer as the SDK
291
+ * returned it, plus its `confidence` where the provider reported one (see
292
+ * {@link evaluationConfidence}). Recorded together so a trace — from the
293
+ * playground or sent over OTLP — can show confidence without knowing which
294
+ * provider answered.
295
+ */
296
+ function answersWithConfidence(answers, providerMetadata) {
297
+ if (!isRecord(answers)) return answers;
298
+ const confidence = evaluationConfidence(providerMetadata);
299
+ if (Object.keys(confidence).length === 0) return answers;
300
+ return Object.fromEntries(Object.entries(answers).map(([id, answer]) => [id, isRecord(answer) && id in confidence && !("confidence" in answer) ? {
301
+ ...answer,
302
+ confidence: confidence[id]
303
+ } : answer]));
304
+ }
305
+ //#endregion
306
+ //#region src/prompt-tracer.ts
307
+ /**
308
+ * Attribute name a span can set to pick one of evalution's span-kind values
309
+ * (`'LLM'`, `'TOOL'`, `'AGENT'`, `'EMBEDDING'`, `'DEFAULT'`). Falls back to
310
+ * `'DEFAULT'` when absent or unrecognised.
311
+ */
312
+ const SPAN_KIND_ATTRIBUTE = "evalution.span.type";
313
+ /**
314
+ * Attribute name a span can set to give a human-readable name to the prompt.
315
+ */
316
+ const PROMPT_NAME_ATTRIBUTE = "gen_ai.prompt.name";
317
+ /**
318
+ * Attribute name a span can set to link itself to a specific prompt. The value
319
+ * is a globally-unique prompt ID unless {@link PROMPT_PROVIDER_ID_ATTRIBUTE} is
320
+ * also set, in which case it is scoped to that provider.
321
+ */
322
+ const PROMPT_ID_ATTRIBUTE = "evalution.prompt.id";
323
+ /**
324
+ * Attribute name a span can set to scope its {@link PROMPT_ID_ATTRIBUTE} to a
325
+ * specific prompt provider. When absent, the prompt ID is treated as global.
326
+ */
327
+ const PROMPT_PROVIDER_ID_ATTRIBUTE = "evalution.prompt.provider.id";
328
+ /**
329
+ * Attribute name a span can set to include an array of input parameters used
330
+ * to construct the final prompt.
331
+ */
332
+ const PROMPT_INPUTS_ATTRIBUTE = "evalution.prompt.inputs";
333
+ /**
334
+ * Attribute name a span can set to record which version of its prompt ran —
335
+ * for a file-based prompt, the commit checked out, when nothing was
336
+ * uncommitted.
337
+ */
338
+ const PROMPT_VERSION_ATTRIBUTE = "evalution.prompt.version";
339
+ /**
340
+ * Attribute name a span can set to record which variation of its prompt ran,
341
+ * on top of its {@link PROMPT_VERSION_ATTRIBUTE}.
342
+ */
343
+ const PROMPT_VARIATION_ATTRIBUTE = "evalution.prompt.variation";
344
+ /**
345
+ * `identity` with any field it leaves unset filled from `fallback`.
346
+ *
347
+ * A config built by the `prompts()` helper carries an identity of its own —
348
+ * the one to keep, since its id is the global one — but only the playground
349
+ * knows what the run was launched with and against which version.
350
+ */
351
+ function mergePromptIdentity(identity, fallback) {
352
+ if (!identity || !fallback) return identity ?? fallback;
353
+ const merged = { ...fallback };
354
+ for (const [key, value] of Object.entries(identity)) if (value !== void 0) merged[key] = value;
355
+ return merged;
356
+ }
357
+ /**
358
+ * Build the attributes that associate a span with a prompt.
359
+ *
360
+ * @param prompt - The prompt to attribute the span to.
361
+ * @param attributes - Attributes to merge in; these take precedence.
362
+ * @param includeInputs - Whether to stamp the run's inputs. Inputs belong on
363
+ * the **root span only**: they are identical on every span of a trace, so
364
+ * repeating them is invisible when the input is `["Ada"]` and wasteful when
365
+ * it is a fifty-message thread — multiplied by span count, then persisted.
366
+ */
367
+ function getPromptSpanAttributes(prompt, attributes = {}, includeInputs = true) {
368
+ const inputs = includeInputs && (prompt.functionInputs || prompt.executeInputs) ? JSON.stringify({
369
+ functionInputs: prompt.functionInputs,
370
+ executeInputs: prompt.executeInputs,
371
+ resources: prompt.resources,
372
+ parameterDefinitions: prompt.parameterDefinitions,
373
+ executeParameterDefinitions: prompt.executeParameterDefinitions
374
+ }) : void 0;
375
+ return Object.fromEntries([
376
+ [SPAN_KIND_ATTRIBUTE, "LLM"],
377
+ [PROMPT_NAME_ATTRIBUTE, prompt.name],
378
+ [PROMPT_ID_ATTRIBUTE, prompt.id],
379
+ [PROMPT_INPUTS_ATTRIBUTE, inputs],
380
+ [PROMPT_VERSION_ATTRIBUTE, prompt.version],
381
+ [PROMPT_VARIATION_ATTRIBUTE, prompt.variation],
382
+ ...Object.entries(attributes)
383
+ ].filter(([, v]) => v !== void 0 && v !== null));
384
+ }
385
+ /**
386
+ * Wraps a {@link Tracer} so that every span it produces is tagged with the
387
+ * attributes that associate it with a prompt — the prompt's name
388
+ * ({@link PROMPT_NAME_ATTRIBUTE}), an optional global prompt ID
389
+ * ({@link PROMPT_ID_ATTRIBUTE}), and an `'LLM'` span kind
390
+ * ({@link SPAN_KIND_ATTRIBUTE}). Attributes set explicitly on an individual
391
+ * span take precedence over these defaults.
392
+ *
393
+ * This depends only on `@opentelemetry/api`, so it can be re-used by SDK
394
+ * adapter packages (e.g. `@evalution/vercel-ai-sdk`) without pulling in the
395
+ * rest of evalution.
396
+ *
397
+ * @param prompt - The prompt to attribute spans to. `name` is a human-readable
398
+ * name; the optional `id` is a globally-unique prompt ID used to resolve
399
+ * runtime traces back to the prompt.
400
+ * @param tracer - Tracer to wrap. Defaults to a tracer from the globally
401
+ * registered tracer provider.
402
+ * @returns A tracer that forwards to `tracer` while attaching the prompt
403
+ * attributes to each span it creates.
404
+ */
405
+ function createTracerForPrompt(prompt, tracer) {
406
+ const inner = tracer ?? trace.getTracer("evalution");
407
+ let rootEmitted = false;
408
+ const withPromptAttributes = (options) => {
409
+ const attributes = getPromptSpanAttributes(prompt, options?.attributes, !rootEmitted);
410
+ rootEmitted = true;
411
+ return {
412
+ ...options,
413
+ attributes
414
+ };
415
+ };
416
+ return {
417
+ startSpan(name, options, context) {
418
+ return inner.startSpan(name, withPromptAttributes(options), context);
419
+ },
420
+ startActiveSpan(name, ...rest) {
421
+ if (typeof rest[0] === "function") return inner.startActiveSpan(name, withPromptAttributes(), rest[0]);
422
+ if (typeof rest[1] === "function") return inner.startActiveSpan(name, withPromptAttributes(rest[0]), rest[1]);
423
+ return inner.startActiveSpan(name, withPromptAttributes(rest[0]), rest[1], rest[2]);
424
+ }
425
+ };
426
+ }
427
+ //#endregion
428
+ //#region src/trace-types.ts
429
+ /**
430
+ * Span attribute naming the eval check (by its id within the eval) that
431
+ * recorded the span while judging the trace — set on the check's `CHECK` span
432
+ * and on everything under it. Those spans aren't part of the run being
433
+ * judged, so the trace's rollups and chat view leave them out.
434
+ */
435
+ const CHECK_ATTRIBUTE = "evalution.check";
436
+ /** Whether `span` was recorded by a check judging its trace, not by the run. */
437
+ function isCheckSpan(span) {
438
+ return span.attributes?.[CHECK_ATTRIBUTE] !== void 0;
439
+ }
440
+ /**
441
+ * The message list of a chat call, or `undefined` for any other input.
442
+ *
443
+ * A chat call's {@link LLMSpanDetails.input} is always a message list, and
444
+ * anything else is wrapped in an object, so being an array is what makes it
445
+ * one.
446
+ */
447
+ function spanMessages(llm) {
448
+ return Array.isArray(llm?.input) ? llm.input : void 0;
449
+ }
450
+ /**
451
+ * The output of a turn that called tools: its text (if any) and the calls, as
452
+ * the one assistant message the next turn's input repeats. Without the calls
453
+ * such a turn would record no output at all, its text being empty.
454
+ */
455
+ function toolCallingOutput(text, toolCalls) {
456
+ return [{
457
+ role: "assistant",
458
+ content: [...text ? [{
459
+ type: "text",
460
+ text
461
+ }] : [], ...toolCalls]
462
+ }];
463
+ }
464
+ /**
465
+ * An LLM span's output as a message list: text as the assistant message it
466
+ * is, a turn that called tools as the message it recorded. `undefined` for
467
+ * structured output, which can be an array too: every element must look like
468
+ * a message.
469
+ */
470
+ function outputMessages(llm) {
471
+ const output = llm?.output;
472
+ if (typeof output === "string") return [{
473
+ role: "assistant",
474
+ content: output
475
+ }];
476
+ return Array.isArray(output) && output.length > 0 && output.every((m) => m && typeof m === "object" && typeof m.role === "string" && (typeof m.content === "string" || Array.isArray(m.content))) ? output : void 0;
477
+ }
478
+ /**
479
+ * The OpenTelemetry semantic-convention resource attribute naming the
480
+ * service that produced a span.
481
+ */
482
+ const SERVICE_NAME_ATTRIBUTE = "service.name";
483
+ /**
484
+ * The OpenTelemetry semantic-convention resource attribute naming the
485
+ * deployment environment a span was produced in.
486
+ */
487
+ const DEPLOYMENT_ENVIRONMENT_ATTRIBUTE = "deployment.environment.name";
488
+ /** {@link DEPLOYMENT_ENVIRONMENT_ATTRIBUTE}'s value on a run evalution itself made. */
489
+ const PLAYGROUND_ENVIRONMENT = "playground";
490
+ /**
491
+ * A tool result's output as the tool returned it. The AI SDK wraps a result
492
+ * in a typed envelope (`{ type: "json", value }`, `{ type: "error-text",
493
+ * value }`, `{ type: "execution-denied", reason }`, …); this unwraps it and
494
+ * notes whether it reports a failure. Anything else is kept as is.
495
+ */
496
+ function unwrapToolResultOutput(output) {
497
+ if (!output || typeof output !== "object" || Array.isArray(output)) return { output };
498
+ const envelope = output;
499
+ const { type } = envelope;
500
+ if (type === "execution-denied") return {
501
+ output: output.reason,
502
+ isError: true
503
+ };
504
+ if (typeof type !== "string" || !("value" in envelope)) return { output };
505
+ return type.startsWith("error-") ? {
506
+ output: envelope.value,
507
+ isError: true
508
+ } : { output: envelope.value };
509
+ }
510
+ //#endregion
511
+ //#region src/trace-ingestor.ts
512
+ /**
513
+ * The resource of every run evalution itself makes — from the playground, an
514
+ * eval, or an MCP client — as opposed to one an app exported to it.
515
+ */
516
+ const PLAYGROUND_RESOURCE = {
517
+ [SERVICE_NAME_ATTRIBUTE]: "evalution",
518
+ [DEPLOYMENT_ENVIRONMENT_ATTRIBUTE]: PLAYGROUND_ENVIRONMENT
519
+ };
520
+ /**
521
+ * Base class that holds the added sinks and fans each normalized record out
522
+ * to all of them. Concrete ingestors translate their source into
523
+ * {@link Span}/{@link Trace} records and call these protected helpers.
524
+ */
525
+ var BaseTraceIngestor = class {
526
+ sinks = [];
527
+ resource;
528
+ /** Recordings still under way, which {@link flush} waits out. */
529
+ pending = /* @__PURE__ */ new Set();
530
+ setResource(resource) {
531
+ this.resource = { ...resource };
532
+ }
533
+ /** `span` with this ingestor's resource, if it has one, merged into its own. */
534
+ withResource(span) {
535
+ if (!this.resource) return span;
536
+ return {
537
+ ...span,
538
+ resource: {
539
+ ...span.resource,
540
+ ...this.resource
541
+ }
542
+ };
543
+ }
544
+ addSink(sink) {
545
+ this.sinks.push(sink);
546
+ }
547
+ removeSink(sink) {
548
+ const prevSinks = this.sinks;
549
+ const newSinks = prevSinks.filter((s) => s !== sink);
550
+ this.sinks = newSinks;
551
+ return prevSinks.length > newSinks.length;
552
+ }
553
+ /**
554
+ * Resolves once everything recorded so far has reached every sink and been
555
+ * delivered by those that deliver in the background. Await it before a
556
+ * short-lived process (a script, a serverless function) exits, so no trace
557
+ * data is lost.
558
+ */
559
+ async flush() {
560
+ while (this.pending.size > 0) await Promise.allSettled(this.pending);
561
+ await Promise.all(this.sinks.map((s) => s.flush?.()));
562
+ }
563
+ /** Registers `recording` for {@link flush} to wait out, and returns it. */
564
+ track(recording) {
565
+ this.pending.add(recording);
566
+ const done = () => this.pending.delete(recording);
567
+ recording.then(done, done);
568
+ return recording;
569
+ }
570
+ async recordSpanStart(span) {
571
+ const stamped = this.withResource(span);
572
+ await this.track(Promise.all(this.sinks.map((s) => s.recordSpanStart(stamped))));
573
+ }
574
+ async recordSpanEnd(span) {
575
+ const stamped = this.withResource(span);
576
+ await this.track(Promise.all(this.sinks.map((s) => s.recordSpanEnd(stamped))));
577
+ }
578
+ async failTrace(traceId, msg) {
579
+ await this.track(Promise.all(this.sinks.map((s) => s.failTrace(traceId, msg))));
580
+ }
581
+ };
582
+ //#endregion
583
+ //#region src/trace-sink.ts
584
+ function isTraceSink(o) {
585
+ return typeof o === "object" && o !== null && typeof o.recordSpanStart === "function" && typeof o.recordSpanEnd === "function" && typeof o.failTrace === "function";
586
+ }
587
+ //#endregion
588
+ export { BaseTraceIngestor, CHECK_ATTRIBUTE, DEFAULT_API_URL, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, Evalution, HttpTraceSink, PLAYGROUND_ENVIRONMENT, PLAYGROUND_RESOURCE, PROMPT_ID_ATTRIBUTE, PROMPT_INPUTS_ATTRIBUTE, PROMPT_NAME_ATTRIBUTE, PROMPT_PROVIDER_ID_ATTRIBUTE, PROMPT_VARIATION_ATTRIBUTE, PROMPT_VERSION_ATTRIBUTE, SERVICE_NAME_ATTRIBUTE, SPAN_KIND_ATTRIBUTE, TRACE_EVENTS_PATH, answersWithConfidence, apiKeyKind, createTracerForPrompt, evaluationConfidence, getPromptSpanAttributes, isCheckSpan, isTraceSink, makeBrand, mergePromptIdentity, outputMessages, spanMessages, toolCallingOutput, unwrapToolResultOutput };
@@ -0,0 +1,84 @@
1
+ //#region src/portkit.ts
2
+ const SERVICES_URL = "https://portkit.localhost/services";
3
+ /** How long to wait for portkit to answer; Caddy may be starting. */
4
+ const DISCOVERY_TIMEOUT_MS = 1e3;
5
+ /**
6
+ * How long to wait for a response when the caller sets no limit. Just over the
7
+ * 60 s portkit holds a request while an on-demand service starts.
8
+ */
9
+ const REQUEST_TIMEOUT_MS = 65e3;
10
+ /**
11
+ * Asks portkit for the evalution instance serving this directory or one of
12
+ * its parents. It never contacts the service itself, which would start one
13
+ * that's on demand.
14
+ *
15
+ * Resolves to `undefined` when portkit isn't installed — without its CA
16
+ * certificate no request is made — or has no such service, or doesn't answer
17
+ * in time.
18
+ */
19
+ async function discoverPortkit({ env = process.env, cwd = process.cwd(), servicesUrl = SERVICES_URL } = {}) {
20
+ try {
21
+ const send = localTlsFetch(await readPortkitCa(env));
22
+ const res = await send(`${servicesUrl}?${new URLSearchParams({
23
+ label: "kind=evalution",
24
+ cwd
25
+ })}`, { signal: AbortSignal.timeout(DISCOVERY_TIMEOUT_MS) });
26
+ if (!res.ok) return void 0;
27
+ const { services } = await res.json();
28
+ const endpoint = services?.[0]?.url;
29
+ return endpoint ? {
30
+ endpoint,
31
+ fetch: send
32
+ } : void 0;
33
+ } catch {
34
+ return;
35
+ }
36
+ }
37
+ async function readPortkitCa(env) {
38
+ const [{ readFile }, { homedir }, { join }] = await Promise.all([
39
+ import("node:fs/promises"),
40
+ import("node:os"),
41
+ import("node:path")
42
+ ]);
43
+ return readFile(join(env.PORTKIT_HOME || (process.platform === "darwin" ? join(homedir(), "Library", "Application Support", "portkit") : join(homedir(), ".local", "share", "portkit")), "data", "caddy", "pki", "authorities", "local", "root.crt"), "utf8");
44
+ }
45
+ /**
46
+ * A `fetch` for portkit's `*.localhost` names. It connects to loopback whatever
47
+ * the name, since not every system resolves `*.localhost`, and trusts `ca` for
48
+ * these connections only.
49
+ *
50
+ * Takes what this package sends: a URL, and a string body.
51
+ */
52
+ function localTlsFetch(ca) {
53
+ return async (input, init) => {
54
+ const { request } = await import("node:https");
55
+ const url = new URL(String(input));
56
+ return new Promise((resolve, reject) => {
57
+ const req = request({
58
+ host: "127.0.0.1",
59
+ port: url.port || 443,
60
+ servername: url.hostname,
61
+ path: url.pathname + url.search,
62
+ method: init?.method ?? "GET",
63
+ headers: {
64
+ ...Object.fromEntries(new Headers(init?.headers)),
65
+ host: url.host
66
+ },
67
+ ca,
68
+ signal: init?.signal ?? AbortSignal.timeout(REQUEST_TIMEOUT_MS)
69
+ }, (res) => {
70
+ const chunks = [];
71
+ res.on("data", (chunk) => chunks.push(chunk));
72
+ res.on("error", reject);
73
+ res.on("end", () => {
74
+ const body = chunks.length > 0 ? Buffer.concat(chunks) : null;
75
+ resolve(new Response(body, { status: res.statusCode }));
76
+ });
77
+ });
78
+ req.on("error", reject);
79
+ req.end(init?.body);
80
+ });
81
+ };
82
+ }
83
+ //#endregion
84
+ export { discoverPortkit };