@evalution/sdk 0.0.0-stage → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/adapter.d.ts +764 -0
- package/dist/adapter.js +53 -0
- package/dist/index.d.ts +267 -0
- package/dist/index.js +588 -0
- package/dist/portkit-DV-XcagM.js +84 -0
- package/dist/trace-ingestor-Cu8xQW6B.d.ts +674 -0
- package/package.json +45 -4
- package/README.md +0 -3
package/dist/adapter.js
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
//#region src/sdk-adapter.ts
|
|
4
|
+
/**
|
|
5
|
+
* Walk up the directory tree from both `rootDir` and `process.cwd()` looking
|
|
6
|
+
* for `node_modules/<packageName>/<dtsRelPath>`.
|
|
7
|
+
*/
|
|
8
|
+
function findPackageDts(packageName, dtsRelPath, rootDir) {
|
|
9
|
+
const seen = /* @__PURE__ */ new Set();
|
|
10
|
+
for (const start of [rootDir, process.cwd()]) {
|
|
11
|
+
let dir = start;
|
|
12
|
+
while (!seen.has(dir)) {
|
|
13
|
+
seen.add(dir);
|
|
14
|
+
const candidate = path.join(dir, "node_modules", packageName, dtsRelPath);
|
|
15
|
+
try {
|
|
16
|
+
fs.accessSync(candidate);
|
|
17
|
+
return candidate;
|
|
18
|
+
} catch {}
|
|
19
|
+
const parent = path.dirname(dir);
|
|
20
|
+
if (parent === dir) break;
|
|
21
|
+
dir = parent;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Whether `err` is Node's "package not installed" error for `packageName`
|
|
28
|
+
* itself, as opposed to a resolution failure *inside* that package (a broken
|
|
29
|
+
* transitive dependency), which is a real error worth surfacing.
|
|
30
|
+
*
|
|
31
|
+
* Used to tell an uninstalled optional peer dependency apart from a genuine
|
|
32
|
+
* failure when lazily importing one.
|
|
33
|
+
*/
|
|
34
|
+
function isMissingPackage(err, packageName) {
|
|
35
|
+
for (let e = err; e; e = e.cause) if (e.code === "ERR_MODULE_NOT_FOUND" && e.message?.includes(`'${packageName}'`)) return true;
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Throws unless `updates` are in the style the adapter produces. For use at
|
|
40
|
+
* the top of {@link SDKAdapter.denormalizeUpdates}; narrows `updates` to that
|
|
41
|
+
* style's update type.
|
|
42
|
+
*/
|
|
43
|
+
function assertUpdateStyle(updates, style) {
|
|
44
|
+
if (updates.style !== style) throw new Error(`Cannot apply "${String(updates.style)}" updates to a "${style}" prompt`);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Message shown when an optional peer dependency an adapter needs is missing.
|
|
48
|
+
*/
|
|
49
|
+
function missingPackageMessage(packageName) {
|
|
50
|
+
return `The \`${packageName}\` package isn't installed in this project. Run \`npm install ${packageName}\` to execute these prompts.`;
|
|
51
|
+
}
|
|
52
|
+
//#endregion
|
|
53
|
+
export { assertUpdateStyle, findPackageDts, isMissingPackage, missingPackageMessage };
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import { $ as PromptsHelperOptions, A as Trace, B as outputMessages, C as SpanMessage, D as SpanToolCallPart, E as SpanTextPart, F as TraceProviderInfo, G as PROMPT_INPUTS_ATTRIBUTE, H as toolCallingOutput, I as TraceStreamEvent, J as PROMPT_VARIATION_ATTRIBUTE, K as PROMPT_NAME_ATTRIBUTE, L as TraceSummary, M as TraceChangeType, N as TraceEvalRun, O as SpanToolResultPart, P as TraceLiveEvent, Q as PromptsHelper, R as TraceWithSpans, S as SpanKind, T as SpanReasoningPart, U as unwrapToolResultOutput, V as spanMessages, W as PROMPT_ID_ATTRIBUTE, X as PromptSpanInfo, Y as PROMPT_VERSION_ATTRIBUTE, Z as PromptsFactory, _ as PromptID, a as isTraceSink, b as SpanContentPart, c as AnnotationEvent, d as AnnotationSource, et as SPAN_KIND_ATTRIBUTE, f as CHECK_ATTRIBUTE, g as PLAYGROUND_ENVIRONMENT, h as LLMSpanDetails, i as TraceSink, j as TraceChangeEvent, k as ToolSpanDetails, l as AnnotationEventOp, m as EvalCounts, n as PLAYGROUND_RESOURCE, nt as getPromptSpanAttributes, o as Annotation, p as DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, q as PROMPT_PROVIDER_ID_ATTRIBUTE, r as TraceIngestor, rt as mergePromptIdentity, s as AnnotationChanges, t as BaseTraceIngestor, tt as createTracerForPrompt, u as AnnotationKind, v as SERVICE_NAME_ATTRIBUTE, w as SpanMessageRole, x as SpanImagePart, y as Span, z as isCheckSpan } from "./trace-ingestor-Cu8xQW6B.js";
|
|
2
|
+
|
|
3
|
+
//#region src/brand.d.ts
|
|
4
|
+
declare function makeBrand<T extends object>(sym: symbol): {
|
|
5
|
+
brand: <U extends T>(value: U) => U;
|
|
6
|
+
isBranded: (value: any) => value is T;
|
|
7
|
+
};
|
|
8
|
+
//#endregion
|
|
9
|
+
//#region src/client.d.ts
|
|
10
|
+
/** The Evalution Cloud API, unless `EVALUTION_API_URL` says otherwise. */
|
|
11
|
+
declare const DEFAULT_API_URL = "https://api.evalut.io";
|
|
12
|
+
/**
|
|
13
|
+
* What an API key can do: an `ingest` key writes one project's production
|
|
14
|
+
* traces and nothing else, and a `personal` key reads and manages a user's
|
|
15
|
+
* projects but can't write production traces.
|
|
16
|
+
*/
|
|
17
|
+
type ApiKeyKind = "ingest" | "personal";
|
|
18
|
+
/**
|
|
19
|
+
* The kind of an API key, read from its prefix, or `undefined` for a string
|
|
20
|
+
* that isn't one.
|
|
21
|
+
*/
|
|
22
|
+
declare function apiKeyKind(key: string): ApiKeyKind | undefined;
|
|
23
|
+
/** Options for {@link Evalution}. Each overrides what the environment says. */
|
|
24
|
+
interface EvalutionOptions {
|
|
25
|
+
/**
|
|
26
|
+
* The base URL of a local Evalution instance to send traces to, without
|
|
27
|
+
* authentication.
|
|
28
|
+
* - If this is set, traces always go to this local instance, or nowhere if
|
|
29
|
+
* this is `false`.
|
|
30
|
+
* - Otherwise, if an ingest API key is present, traces always go to the cloud.
|
|
31
|
+
* - Otherwise, this defaults to `EVALUTION_ENDPOINT`.
|
|
32
|
+
*/
|
|
33
|
+
endpoint?: string | false;
|
|
34
|
+
/**
|
|
35
|
+
* API key for Evalution Cloud.
|
|
36
|
+
* - An ingest key sends traces to the cloud.
|
|
37
|
+
* - A personal key makes the read and control calls available, and traces go
|
|
38
|
+
* to a local instance.
|
|
39
|
+
*
|
|
40
|
+
* Defaults to `EVALUTION_API_KEY`.
|
|
41
|
+
*/
|
|
42
|
+
apiKey?: string;
|
|
43
|
+
/** The cloud's base URL. Defaults to `EVALUTION_API_URL`, then {@link DEFAULT_API_URL}. */
|
|
44
|
+
apiUrl?: string;
|
|
45
|
+
/** The project id the read and control calls act on: the config's `cloud.project`. */
|
|
46
|
+
project?: string;
|
|
47
|
+
/** The `fetch` to send with. Defaults to the global one. */
|
|
48
|
+
fetch?: typeof fetch;
|
|
49
|
+
}
|
|
50
|
+
/** Which traces {@link Evalution.listTraces} returns. */
|
|
51
|
+
interface ListTracesQuery {
|
|
52
|
+
/** Only traces from this deployment environment. */
|
|
53
|
+
environment?: string;
|
|
54
|
+
/** Only traces started at or after this time (ms). */
|
|
55
|
+
from?: number;
|
|
56
|
+
/** Only traces started before this time (ms). */
|
|
57
|
+
to?: number;
|
|
58
|
+
/** At most this many, most recent first. */
|
|
59
|
+
limit?: number;
|
|
60
|
+
}
|
|
61
|
+
/** A project, as the cloud describes it. */
|
|
62
|
+
interface CloudProject {
|
|
63
|
+
/** The `prj_…` id committed in a repo's config. */
|
|
64
|
+
id: string;
|
|
65
|
+
slug: string;
|
|
66
|
+
name: string;
|
|
67
|
+
}
|
|
68
|
+
/** What {@link Evalution.whoami} resolves a key to. */
|
|
69
|
+
type CloudIdentity = {
|
|
70
|
+
kind: "personal";
|
|
71
|
+
userId: string;
|
|
72
|
+
} | {
|
|
73
|
+
kind: "ingest";
|
|
74
|
+
projectId: string;
|
|
75
|
+
environment: string;
|
|
76
|
+
};
|
|
77
|
+
/** One checkout of a project, as {@link Evalution.registerInstance} reports it. */
|
|
78
|
+
interface InstanceRegistration {
|
|
79
|
+
/** Identifies this checkout: the same folder is always the same instance. */
|
|
80
|
+
instanceKey: string;
|
|
81
|
+
/** How to show this instance in a list, e.g. `hostname:dir`. */
|
|
82
|
+
label: string;
|
|
83
|
+
/** The instance's own loopback URL. */
|
|
84
|
+
localUrl: string;
|
|
85
|
+
/** Its portkit URL, if it has one. */
|
|
86
|
+
portkitUrl?: string;
|
|
87
|
+
/** How portkit runs it, if it does. */
|
|
88
|
+
portkitMode?: string;
|
|
89
|
+
}
|
|
90
|
+
/** The cloud's answer to {@link Evalution.registerInstance}. */
|
|
91
|
+
interface RegisteredInstance {
|
|
92
|
+
id: string;
|
|
93
|
+
/** The project's shareable `*.local.evalut.io` URL. */
|
|
94
|
+
projectUrl: string;
|
|
95
|
+
}
|
|
96
|
+
/** Names one synced database of a project. */
|
|
97
|
+
interface SyncDatabaseKey {
|
|
98
|
+
kind: "traces" | "evals" | "dataset";
|
|
99
|
+
/** The dataset's id, for `kind: "dataset"`. */
|
|
100
|
+
key?: string;
|
|
101
|
+
}
|
|
102
|
+
/** Where a synced database lives, and a short-lived token for it. */
|
|
103
|
+
interface SyncDatabaseCredentials {
|
|
104
|
+
url: string;
|
|
105
|
+
token: string;
|
|
106
|
+
/** When {@link token} expires (ms). */
|
|
107
|
+
expiresAt: number;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* The client for Evalution's API. As a {@link TraceSink} it sends an app's
|
|
111
|
+
* traces, and it makes the read and control calls the local server needs.
|
|
112
|
+
*
|
|
113
|
+
* Where traces go is worked out once, on the first one sent:
|
|
114
|
+
*
|
|
115
|
+
* 1. the {@link EvalutionOptions.endpoint} option;
|
|
116
|
+
* 2. the cloud, when the API key is an ingest key;
|
|
117
|
+
* 3. the local instance `EVALUTION_ENDPOINT` names;
|
|
118
|
+
* 4. on Node, the instance portkit knows for this directory;
|
|
119
|
+
* 5. otherwise nowhere, and sending does nothing.
|
|
120
|
+
*
|
|
121
|
+
* A personal key never sends traces to the cloud, and no key is ever sent to
|
|
122
|
+
* a local instance. Tracing never breaks the app: see {@link HttpTraceSink},
|
|
123
|
+
* and await {@link flush} before a short-lived process exits. The read and
|
|
124
|
+
* control calls, by contrast, reject when they fail.
|
|
125
|
+
*/
|
|
126
|
+
declare class Evalution implements TraceSink {
|
|
127
|
+
private readonly endpoint?;
|
|
128
|
+
private readonly apiKey?;
|
|
129
|
+
private readonly apiUrl;
|
|
130
|
+
private readonly project?;
|
|
131
|
+
private readonly fetch;
|
|
132
|
+
private readonly sink;
|
|
133
|
+
constructor(options?: EvalutionOptions);
|
|
134
|
+
private resolveTraceTarget;
|
|
135
|
+
recordSpanStart(span: Span): Promise<Span>;
|
|
136
|
+
recordSpanEnd(span: Span): Promise<Span>;
|
|
137
|
+
failTrace(traceId: string, errorMessage: string): Promise<void>;
|
|
138
|
+
/** Resolves once every trace event recorded so far has been sent (or dropped). */
|
|
139
|
+
flush(): Promise<void>;
|
|
140
|
+
/** The key's kind and whose it is. Takes either kind of key. */
|
|
141
|
+
whoami(): Promise<CloudIdentity>;
|
|
142
|
+
/** The most recent production traces of the project. */
|
|
143
|
+
listTraces(query?: ListTracesQuery): Promise<TraceSummary[]>;
|
|
144
|
+
/** One production trace with its spans, or `undefined` if there's none. */
|
|
145
|
+
getTrace(id: string): Promise<TraceWithSpans | undefined>;
|
|
146
|
+
/** The projects the key's user is a member of. */
|
|
147
|
+
listProjects(): Promise<CloudProject[]>;
|
|
148
|
+
/** Creates a project in the user's organization. */
|
|
149
|
+
createProject(project: {
|
|
150
|
+
name: string;
|
|
151
|
+
slug?: string;
|
|
152
|
+
}): Promise<CloudProject>;
|
|
153
|
+
/** Records a running checkout of the project. */
|
|
154
|
+
registerInstance(instance: InstanceRegistration): Promise<RegisteredInstance>;
|
|
155
|
+
/** Tells the cloud a registered instance is still running. */
|
|
156
|
+
heartbeatInstance(id: string): Promise<void>;
|
|
157
|
+
/** Tells the cloud a registered instance has stopped. */
|
|
158
|
+
endInstanceSession(id: string): Promise<void>;
|
|
159
|
+
/** Ensures a synced database exists, and returns how to reach it. */
|
|
160
|
+
getSyncDatabase(key: SyncDatabaseKey): Promise<SyncDatabaseCredentials>;
|
|
161
|
+
private instanceRequest;
|
|
162
|
+
private json;
|
|
163
|
+
private request;
|
|
164
|
+
}
|
|
165
|
+
//#endregion
|
|
166
|
+
//#region src/evaluation-answers.d.ts
|
|
167
|
+
/**
|
|
168
|
+
* The per-question confidence a provider reported beside an evaluation's
|
|
169
|
+
* answers, keyed by question id.
|
|
170
|
+
*
|
|
171
|
+
* The AI SDK's answers carry no confidence of their own: a provider that
|
|
172
|
+
* computes one (TypeSafe does, for choice and score answers) reports it as
|
|
173
|
+
* `providerMetadata.<provider>.confidence[questionId]`. Read from whichever
|
|
174
|
+
* provider namespace has it, so this isn't tied to one provider.
|
|
175
|
+
*/
|
|
176
|
+
declare function evaluationConfidence(providerMetadata: unknown): Record<string, number>;
|
|
177
|
+
/**
|
|
178
|
+
* An evaluation's answers as recorded on its span: each answer as the SDK
|
|
179
|
+
* returned it, plus its `confidence` where the provider reported one (see
|
|
180
|
+
* {@link evaluationConfidence}). Recorded together so a trace — from the
|
|
181
|
+
* playground or sent over OTLP — can show confidence without knowing which
|
|
182
|
+
* provider answered.
|
|
183
|
+
*/
|
|
184
|
+
declare function answersWithConfidence(answers: unknown, providerMetadata: unknown): unknown;
|
|
185
|
+
//#endregion
|
|
186
|
+
//#region src/http-trace-sink.d.ts
|
|
187
|
+
/**
|
|
188
|
+
* Where an evalution server accepts {@link TraceSinkEvent}s, relative to its
|
|
189
|
+
* base URL.
|
|
190
|
+
*/
|
|
191
|
+
declare const TRACE_EVENTS_PATH = "/api/traces";
|
|
192
|
+
/** One {@link TraceSink} call, as {@link HttpTraceSink} sends it. */
|
|
193
|
+
type TraceSinkEvent = {
|
|
194
|
+
type: "span-start";
|
|
195
|
+
span: Span;
|
|
196
|
+
} | {
|
|
197
|
+
type: "span-end";
|
|
198
|
+
span: Span;
|
|
199
|
+
} | {
|
|
200
|
+
type: "trace-fail";
|
|
201
|
+
traceId: string;
|
|
202
|
+
errorMessage: string;
|
|
203
|
+
};
|
|
204
|
+
/** The body of a `POST` to {@link TRACE_EVENTS_PATH}. */
|
|
205
|
+
interface TraceEventsRequest {
|
|
206
|
+
/** The events to record, in the order they happened. */
|
|
207
|
+
events: TraceSinkEvent[];
|
|
208
|
+
}
|
|
209
|
+
/** Where an {@link HttpTraceSink} sends, once it's known. */
|
|
210
|
+
interface HttpTraceTarget {
|
|
211
|
+
/** The server's base URL. */
|
|
212
|
+
endpoint: string;
|
|
213
|
+
/** Sent with every request, e.g. `authorization`. */
|
|
214
|
+
headers?: Record<string, string>;
|
|
215
|
+
/** Overrides {@link HttpTraceSinkOptions.fetch}. */
|
|
216
|
+
fetch?: typeof fetch;
|
|
217
|
+
}
|
|
218
|
+
/** Options for {@link HttpTraceSink}. */
|
|
219
|
+
interface HttpTraceSinkOptions {
|
|
220
|
+
/**
|
|
221
|
+
* The evalution server's base URL, e.g. `http://localhost:3000`. Events are
|
|
222
|
+
* posted to {@link TRACE_EVENTS_PATH} under it.
|
|
223
|
+
*
|
|
224
|
+
* May instead be a function that finds out where to send, which is called
|
|
225
|
+
* once, when the first event is recorded. Events wait in the queue until it
|
|
226
|
+
* settles; if it finds nowhere, or throws, they and all later ones are
|
|
227
|
+
* dropped.
|
|
228
|
+
*/
|
|
229
|
+
endpoint: string | (() => Promise<HttpTraceTarget | undefined>);
|
|
230
|
+
/** The `fetch` to send with. Defaults to the global one. */
|
|
231
|
+
fetch?: typeof fetch;
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* A {@link TraceSink} that sends what it's given to an evalution server,
|
|
235
|
+
* which records it through its own trace providers — so an app's traces show
|
|
236
|
+
* up in the playground without the app writing to a trace store itself.
|
|
237
|
+
*
|
|
238
|
+
* Calls return as soon as the event is queued, never waiting on the network,
|
|
239
|
+
* so tracing doesn't slow the app down. Queued events go out in order, batched
|
|
240
|
+
* into one request at a time, so a span's end never overtakes its start. A
|
|
241
|
+
* request that fails is logged and its events dropped: tracing never breaks
|
|
242
|
+
* the app. Await {@link flush} before a short-lived process exits to be sure
|
|
243
|
+
* everything was sent.
|
|
244
|
+
*/
|
|
245
|
+
declare class HttpTraceSink implements TraceSink {
|
|
246
|
+
private readonly options;
|
|
247
|
+
private target?;
|
|
248
|
+
/** Set once it's known there's nowhere to send, so events aren't queued. */
|
|
249
|
+
private disabled;
|
|
250
|
+
private queue;
|
|
251
|
+
private sending?;
|
|
252
|
+
/** Whether the last request failed, so a down server is reported once. */
|
|
253
|
+
private failing;
|
|
254
|
+
constructor(options: HttpTraceSinkOptions);
|
|
255
|
+
private resolveTarget;
|
|
256
|
+
private lookUpTarget;
|
|
257
|
+
recordSpanStart(span: Span): Promise<Span>;
|
|
258
|
+
recordSpanEnd(span: Span): Promise<Span>;
|
|
259
|
+
failTrace(traceId: string, errorMessage: string): Promise<void>;
|
|
260
|
+
/** Resolves once every event recorded so far has been sent (or dropped). */
|
|
261
|
+
flush(): Promise<void>;
|
|
262
|
+
private enqueue;
|
|
263
|
+
private drain;
|
|
264
|
+
private send;
|
|
265
|
+
}
|
|
266
|
+
//#endregion
|
|
267
|
+
export { Annotation, AnnotationChanges, AnnotationEvent, AnnotationEventOp, AnnotationKind, AnnotationSource, ApiKeyKind, BaseTraceIngestor, CHECK_ATTRIBUTE, CloudIdentity, CloudProject, DEFAULT_API_URL, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, EvalCounts, Evalution, EvalutionOptions, HttpTraceSink, HttpTraceSinkOptions, HttpTraceTarget, InstanceRegistration, LLMSpanDetails, ListTracesQuery, PLAYGROUND_ENVIRONMENT, PLAYGROUND_RESOURCE, PROMPT_ID_ATTRIBUTE, PROMPT_INPUTS_ATTRIBUTE, PROMPT_NAME_ATTRIBUTE, PROMPT_PROVIDER_ID_ATTRIBUTE, PROMPT_VARIATION_ATTRIBUTE, PROMPT_VERSION_ATTRIBUTE, PromptID, PromptSpanInfo, PromptsFactory, PromptsHelper, PromptsHelperOptions, RegisteredInstance, SERVICE_NAME_ATTRIBUTE, SPAN_KIND_ATTRIBUTE, Span, SpanContentPart, SpanImagePart, SpanKind, SpanMessage, SpanMessageRole, SpanReasoningPart, SpanTextPart, SpanToolCallPart, SpanToolResultPart, SyncDatabaseCredentials, SyncDatabaseKey, TRACE_EVENTS_PATH, ToolSpanDetails, Trace, TraceChangeEvent, TraceChangeType, TraceEvalRun, TraceEventsRequest, TraceIngestor, TraceLiveEvent, TraceProviderInfo, TraceSink, TraceSinkEvent, TraceStreamEvent, TraceSummary, TraceWithSpans, answersWithConfidence, apiKeyKind, createTracerForPrompt, evaluationConfidence, getPromptSpanAttributes, isCheckSpan, isTraceSink, makeBrand, mergePromptIdentity, outputMessages, spanMessages, toolCallingOutput, unwrapToolResultOutput };
|