@evalution/sdk 0.0.0-stage → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,53 @@
1
+ import fs from "node:fs";
2
+ import path from "node:path";
3
+ //#region src/sdk-adapter.ts
4
+ /**
5
+ * Walk up the directory tree from both `rootDir` and `process.cwd()` looking
6
+ * for `node_modules/<packageName>/<dtsRelPath>`.
7
+ */
8
+ function findPackageDts(packageName, dtsRelPath, rootDir) {
9
+ const seen = /* @__PURE__ */ new Set();
10
+ for (const start of [rootDir, process.cwd()]) {
11
+ let dir = start;
12
+ while (!seen.has(dir)) {
13
+ seen.add(dir);
14
+ const candidate = path.join(dir, "node_modules", packageName, dtsRelPath);
15
+ try {
16
+ fs.accessSync(candidate);
17
+ return candidate;
18
+ } catch {}
19
+ const parent = path.dirname(dir);
20
+ if (parent === dir) break;
21
+ dir = parent;
22
+ }
23
+ }
24
+ return null;
25
+ }
26
+ /**
27
+ * Whether `err` is Node's "package not installed" error for `packageName`
28
+ * itself, as opposed to a resolution failure *inside* that package (a broken
29
+ * transitive dependency), which is a real error worth surfacing.
30
+ *
31
+ * Used to tell an uninstalled optional peer dependency apart from a genuine
32
+ * failure when lazily importing one.
33
+ */
34
+ function isMissingPackage(err, packageName) {
35
+ for (let e = err; e; e = e.cause) if (e.code === "ERR_MODULE_NOT_FOUND" && e.message?.includes(`'${packageName}'`)) return true;
36
+ return false;
37
+ }
38
+ /**
39
+ * Throws unless `updates` are in the style the adapter produces. For use at
40
+ * the top of {@link SDKAdapter.denormalizeUpdates}; narrows `updates` to that
41
+ * style's update type.
42
+ */
43
+ function assertUpdateStyle(updates, style) {
44
+ if (updates.style !== style) throw new Error(`Cannot apply "${String(updates.style)}" updates to a "${style}" prompt`);
45
+ }
46
+ /**
47
+ * Message shown when an optional peer dependency an adapter needs is missing.
48
+ */
49
+ function missingPackageMessage(packageName) {
50
+ return `The \`${packageName}\` package isn't installed in this project. Run \`npm install ${packageName}\` to execute these prompts.`;
51
+ }
52
+ //#endregion
53
+ export { assertUpdateStyle, findPackageDts, isMissingPackage, missingPackageMessage };
@@ -0,0 +1,267 @@
1
+ import { $ as PromptsHelperOptions, A as Trace, B as outputMessages, C as SpanMessage, D as SpanToolCallPart, E as SpanTextPart, F as TraceProviderInfo, G as PROMPT_INPUTS_ATTRIBUTE, H as toolCallingOutput, I as TraceStreamEvent, J as PROMPT_VARIATION_ATTRIBUTE, K as PROMPT_NAME_ATTRIBUTE, L as TraceSummary, M as TraceChangeType, N as TraceEvalRun, O as SpanToolResultPart, P as TraceLiveEvent, Q as PromptsHelper, R as TraceWithSpans, S as SpanKind, T as SpanReasoningPart, U as unwrapToolResultOutput, V as spanMessages, W as PROMPT_ID_ATTRIBUTE, X as PromptSpanInfo, Y as PROMPT_VERSION_ATTRIBUTE, Z as PromptsFactory, _ as PromptID, a as isTraceSink, b as SpanContentPart, c as AnnotationEvent, d as AnnotationSource, et as SPAN_KIND_ATTRIBUTE, f as CHECK_ATTRIBUTE, g as PLAYGROUND_ENVIRONMENT, h as LLMSpanDetails, i as TraceSink, j as TraceChangeEvent, k as ToolSpanDetails, l as AnnotationEventOp, m as EvalCounts, n as PLAYGROUND_RESOURCE, nt as getPromptSpanAttributes, o as Annotation, p as DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, q as PROMPT_PROVIDER_ID_ATTRIBUTE, r as TraceIngestor, rt as mergePromptIdentity, s as AnnotationChanges, t as BaseTraceIngestor, tt as createTracerForPrompt, u as AnnotationKind, v as SERVICE_NAME_ATTRIBUTE, w as SpanMessageRole, x as SpanImagePart, y as Span, z as isCheckSpan } from "./trace-ingestor-Cu8xQW6B.js";
2
+
3
+ //#region src/brand.d.ts
4
+ declare function makeBrand<T extends object>(sym: symbol): {
5
+ brand: <U extends T>(value: U) => U;
6
+ isBranded: (value: any) => value is T;
7
+ };
8
+ //#endregion
9
+ //#region src/client.d.ts
10
+ /** The Evalution Cloud API, unless `EVALUTION_API_URL` says otherwise. */
11
+ declare const DEFAULT_API_URL = "https://api.evalut.io";
12
+ /**
13
+ * What an API key can do: an `ingest` key writes one project's production
14
+ * traces and nothing else, and a `personal` key reads and manages a user's
15
+ * projects but can't write production traces.
16
+ */
17
+ type ApiKeyKind = "ingest" | "personal";
18
+ /**
19
+ * The kind of an API key, read from its prefix, or `undefined` for a string
20
+ * that isn't one.
21
+ */
22
+ declare function apiKeyKind(key: string): ApiKeyKind | undefined;
23
+ /** Options for {@link Evalution}. Each overrides what the environment says. */
24
+ interface EvalutionOptions {
25
+ /**
26
+ * The base URL of a local Evalution instance to send traces to, without
27
+ * authentication.
28
+ * - If this is set, traces always go to this local instance, or nowhere if
29
+ * this is `false`.
30
+ * - Otherwise, if an ingest API key is present, traces always go to the cloud.
31
+ * - Otherwise, this defaults to `EVALUTION_ENDPOINT`.
32
+ */
33
+ endpoint?: string | false;
34
+ /**
35
+ * API key for Evalution Cloud.
36
+ * - An ingest key sends traces to the cloud.
37
+ * - A personal key makes the read and control calls available, and traces go
38
+ * to a local instance.
39
+ *
40
+ * Defaults to `EVALUTION_API_KEY`.
41
+ */
42
+ apiKey?: string;
43
+ /** The cloud's base URL. Defaults to `EVALUTION_API_URL`, then {@link DEFAULT_API_URL}. */
44
+ apiUrl?: string;
45
+ /** The project id the read and control calls act on: the config's `cloud.project`. */
46
+ project?: string;
47
+ /** The `fetch` to send with. Defaults to the global one. */
48
+ fetch?: typeof fetch;
49
+ }
50
+ /** Which traces {@link Evalution.listTraces} returns. */
51
+ interface ListTracesQuery {
52
+ /** Only traces from this deployment environment. */
53
+ environment?: string;
54
+ /** Only traces started at or after this time (ms). */
55
+ from?: number;
56
+ /** Only traces started before this time (ms). */
57
+ to?: number;
58
+ /** At most this many, most recent first. */
59
+ limit?: number;
60
+ }
61
+ /** A project, as the cloud describes it. */
62
+ interface CloudProject {
63
+ /** The `prj_…` id committed in a repo's config. */
64
+ id: string;
65
+ slug: string;
66
+ name: string;
67
+ }
68
+ /** What {@link Evalution.whoami} resolves a key to. */
69
+ type CloudIdentity = {
70
+ kind: "personal";
71
+ userId: string;
72
+ } | {
73
+ kind: "ingest";
74
+ projectId: string;
75
+ environment: string;
76
+ };
77
+ /** One checkout of a project, as {@link Evalution.registerInstance} reports it. */
78
+ interface InstanceRegistration {
79
+ /** Identifies this checkout: the same folder is always the same instance. */
80
+ instanceKey: string;
81
+ /** How to show this instance in a list, e.g. `hostname:dir`. */
82
+ label: string;
83
+ /** The instance's own loopback URL. */
84
+ localUrl: string;
85
+ /** Its portkit URL, if it has one. */
86
+ portkitUrl?: string;
87
+ /** How portkit runs it, if it does. */
88
+ portkitMode?: string;
89
+ }
90
+ /** The cloud's answer to {@link Evalution.registerInstance}. */
91
+ interface RegisteredInstance {
92
+ id: string;
93
+ /** The project's shareable `*.local.evalut.io` URL. */
94
+ projectUrl: string;
95
+ }
96
+ /** Names one synced database of a project. */
97
+ interface SyncDatabaseKey {
98
+ kind: "traces" | "evals" | "dataset";
99
+ /** The dataset's id, for `kind: "dataset"`. */
100
+ key?: string;
101
+ }
102
+ /** Where a synced database lives, and a short-lived token for it. */
103
+ interface SyncDatabaseCredentials {
104
+ url: string;
105
+ token: string;
106
+ /** When {@link token} expires (ms). */
107
+ expiresAt: number;
108
+ }
109
+ /**
110
+ * The client for Evalution's API. As a {@link TraceSink} it sends an app's
111
+ * traces, and it makes the read and control calls the local server needs.
112
+ *
113
+ * Where traces go is worked out once, on the first one sent:
114
+ *
115
+ * 1. the {@link EvalutionOptions.endpoint} option;
116
+ * 2. the cloud, when the API key is an ingest key;
117
+ * 3. the local instance `EVALUTION_ENDPOINT` names;
118
+ * 4. on Node, the instance portkit knows for this directory;
119
+ * 5. otherwise nowhere, and sending does nothing.
120
+ *
121
+ * A personal key never sends traces to the cloud, and no key is ever sent to
122
+ * a local instance. Tracing never breaks the app: see {@link HttpTraceSink},
123
+ * and await {@link flush} before a short-lived process exits. The read and
124
+ * control calls, by contrast, reject when they fail.
125
+ */
126
+ declare class Evalution implements TraceSink {
127
+ private readonly endpoint?;
128
+ private readonly apiKey?;
129
+ private readonly apiUrl;
130
+ private readonly project?;
131
+ private readonly fetch;
132
+ private readonly sink;
133
+ constructor(options?: EvalutionOptions);
134
+ private resolveTraceTarget;
135
+ recordSpanStart(span: Span): Promise<Span>;
136
+ recordSpanEnd(span: Span): Promise<Span>;
137
+ failTrace(traceId: string, errorMessage: string): Promise<void>;
138
+ /** Resolves once every trace event recorded so far has been sent (or dropped). */
139
+ flush(): Promise<void>;
140
+ /** The key's kind and whose it is. Takes either kind of key. */
141
+ whoami(): Promise<CloudIdentity>;
142
+ /** The most recent production traces of the project. */
143
+ listTraces(query?: ListTracesQuery): Promise<TraceSummary[]>;
144
+ /** One production trace with its spans, or `undefined` if there's none. */
145
+ getTrace(id: string): Promise<TraceWithSpans | undefined>;
146
+ /** The projects the key's user is a member of. */
147
+ listProjects(): Promise<CloudProject[]>;
148
+ /** Creates a project in the user's organization. */
149
+ createProject(project: {
150
+ name: string;
151
+ slug?: string;
152
+ }): Promise<CloudProject>;
153
+ /** Records a running checkout of the project. */
154
+ registerInstance(instance: InstanceRegistration): Promise<RegisteredInstance>;
155
+ /** Tells the cloud a registered instance is still running. */
156
+ heartbeatInstance(id: string): Promise<void>;
157
+ /** Tells the cloud a registered instance has stopped. */
158
+ endInstanceSession(id: string): Promise<void>;
159
+ /** Ensures a synced database exists, and returns how to reach it. */
160
+ getSyncDatabase(key: SyncDatabaseKey): Promise<SyncDatabaseCredentials>;
161
+ private instanceRequest;
162
+ private json;
163
+ private request;
164
+ }
165
+ //#endregion
166
+ //#region src/evaluation-answers.d.ts
167
+ /**
168
+ * The per-question confidence a provider reported beside an evaluation's
169
+ * answers, keyed by question id.
170
+ *
171
+ * The AI SDK's answers carry no confidence of their own: a provider that
172
+ * computes one (TypeSafe does, for choice and score answers) reports it as
173
+ * `providerMetadata.<provider>.confidence[questionId]`. Read from whichever
174
+ * provider namespace has it, so this isn't tied to one provider.
175
+ */
176
+ declare function evaluationConfidence(providerMetadata: unknown): Record<string, number>;
177
+ /**
178
+ * An evaluation's answers as recorded on its span: each answer as the SDK
179
+ * returned it, plus its `confidence` where the provider reported one (see
180
+ * {@link evaluationConfidence}). Recorded together so a trace — from the
181
+ * playground or sent over OTLP — can show confidence without knowing which
182
+ * provider answered.
183
+ */
184
+ declare function answersWithConfidence(answers: unknown, providerMetadata: unknown): unknown;
185
+ //#endregion
186
+ //#region src/http-trace-sink.d.ts
187
+ /**
188
+ * Where an evalution server accepts {@link TraceSinkEvent}s, relative to its
189
+ * base URL.
190
+ */
191
+ declare const TRACE_EVENTS_PATH = "/api/traces";
192
+ /** One {@link TraceSink} call, as {@link HttpTraceSink} sends it. */
193
+ type TraceSinkEvent = {
194
+ type: "span-start";
195
+ span: Span;
196
+ } | {
197
+ type: "span-end";
198
+ span: Span;
199
+ } | {
200
+ type: "trace-fail";
201
+ traceId: string;
202
+ errorMessage: string;
203
+ };
204
+ /** The body of a `POST` to {@link TRACE_EVENTS_PATH}. */
205
+ interface TraceEventsRequest {
206
+ /** The events to record, in the order they happened. */
207
+ events: TraceSinkEvent[];
208
+ }
209
+ /** Where an {@link HttpTraceSink} sends, once it's known. */
210
+ interface HttpTraceTarget {
211
+ /** The server's base URL. */
212
+ endpoint: string;
213
+ /** Sent with every request, e.g. `authorization`. */
214
+ headers?: Record<string, string>;
215
+ /** Overrides {@link HttpTraceSinkOptions.fetch}. */
216
+ fetch?: typeof fetch;
217
+ }
218
+ /** Options for {@link HttpTraceSink}. */
219
+ interface HttpTraceSinkOptions {
220
+ /**
221
+ * The evalution server's base URL, e.g. `http://localhost:3000`. Events are
222
+ * posted to {@link TRACE_EVENTS_PATH} under it.
223
+ *
224
+ * May instead be a function that finds out where to send, which is called
225
+ * once, when the first event is recorded. Events wait in the queue until it
226
+ * settles; if it finds nowhere, or throws, they and all later ones are
227
+ * dropped.
228
+ */
229
+ endpoint: string | (() => Promise<HttpTraceTarget | undefined>);
230
+ /** The `fetch` to send with. Defaults to the global one. */
231
+ fetch?: typeof fetch;
232
+ }
233
+ /**
234
+ * A {@link TraceSink} that sends what it's given to an evalution server,
235
+ * which records it through its own trace providers — so an app's traces show
236
+ * up in the playground without the app writing to a trace store itself.
237
+ *
238
+ * Calls return as soon as the event is queued, never waiting on the network,
239
+ * so tracing doesn't slow the app down. Queued events go out in order, batched
240
+ * into one request at a time, so a span's end never overtakes its start. A
241
+ * request that fails is logged and its events dropped: tracing never breaks
242
+ * the app. Await {@link flush} before a short-lived process exits to be sure
243
+ * everything was sent.
244
+ */
245
+ declare class HttpTraceSink implements TraceSink {
246
+ private readonly options;
247
+ private target?;
248
+ /** Set once it's known there's nowhere to send, so events aren't queued. */
249
+ private disabled;
250
+ private queue;
251
+ private sending?;
252
+ /** Whether the last request failed, so a down server is reported once. */
253
+ private failing;
254
+ constructor(options: HttpTraceSinkOptions);
255
+ private resolveTarget;
256
+ private lookUpTarget;
257
+ recordSpanStart(span: Span): Promise<Span>;
258
+ recordSpanEnd(span: Span): Promise<Span>;
259
+ failTrace(traceId: string, errorMessage: string): Promise<void>;
260
+ /** Resolves once every event recorded so far has been sent (or dropped). */
261
+ flush(): Promise<void>;
262
+ private enqueue;
263
+ private drain;
264
+ private send;
265
+ }
266
+ //#endregion
267
+ export { Annotation, AnnotationChanges, AnnotationEvent, AnnotationEventOp, AnnotationKind, AnnotationSource, ApiKeyKind, BaseTraceIngestor, CHECK_ATTRIBUTE, CloudIdentity, CloudProject, DEFAULT_API_URL, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, EvalCounts, Evalution, EvalutionOptions, HttpTraceSink, HttpTraceSinkOptions, HttpTraceTarget, InstanceRegistration, LLMSpanDetails, ListTracesQuery, PLAYGROUND_ENVIRONMENT, PLAYGROUND_RESOURCE, PROMPT_ID_ATTRIBUTE, PROMPT_INPUTS_ATTRIBUTE, PROMPT_NAME_ATTRIBUTE, PROMPT_PROVIDER_ID_ATTRIBUTE, PROMPT_VARIATION_ATTRIBUTE, PROMPT_VERSION_ATTRIBUTE, PromptID, PromptSpanInfo, PromptsFactory, PromptsHelper, PromptsHelperOptions, RegisteredInstance, SERVICE_NAME_ATTRIBUTE, SPAN_KIND_ATTRIBUTE, Span, SpanContentPart, SpanImagePart, SpanKind, SpanMessage, SpanMessageRole, SpanReasoningPart, SpanTextPart, SpanToolCallPart, SpanToolResultPart, SyncDatabaseCredentials, SyncDatabaseKey, TRACE_EVENTS_PATH, ToolSpanDetails, Trace, TraceChangeEvent, TraceChangeType, TraceEvalRun, TraceEventsRequest, TraceIngestor, TraceLiveEvent, TraceProviderInfo, TraceSink, TraceSinkEvent, TraceStreamEvent, TraceSummary, TraceWithSpans, answersWithConfidence, apiKeyKind, createTracerForPrompt, evaluationConfidence, getPromptSpanAttributes, isCheckSpan, isTraceSink, makeBrand, mergePromptIdentity, outputMessages, spanMessages, toolCallingOutput, unwrapToolResultOutput };