@evalution/sdk 0.0.0-stage → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/adapter.d.ts +764 -0
- package/dist/adapter.js +53 -0
- package/dist/index.d.ts +487 -0
- package/dist/index.js +1070 -0
- package/dist/portkit-DV-XcagM.js +84 -0
- package/dist/trace-ingestor-Cu8xQW6B.d.ts +674 -0
- package/package.json +57 -4
- package/README.md +0 -3
package/dist/adapter.js
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
//#region src/sdk-adapter.ts
|
|
4
|
+
/**
|
|
5
|
+
* Walk up the directory tree from both `rootDir` and `process.cwd()` looking
|
|
6
|
+
* for `node_modules/<packageName>/<dtsRelPath>`.
|
|
7
|
+
*/
|
|
8
|
+
function findPackageDts(packageName, dtsRelPath, rootDir) {
|
|
9
|
+
const seen = /* @__PURE__ */ new Set();
|
|
10
|
+
for (const start of [rootDir, process.cwd()]) {
|
|
11
|
+
let dir = start;
|
|
12
|
+
while (!seen.has(dir)) {
|
|
13
|
+
seen.add(dir);
|
|
14
|
+
const candidate = path.join(dir, "node_modules", packageName, dtsRelPath);
|
|
15
|
+
try {
|
|
16
|
+
fs.accessSync(candidate);
|
|
17
|
+
return candidate;
|
|
18
|
+
} catch {}
|
|
19
|
+
const parent = path.dirname(dir);
|
|
20
|
+
if (parent === dir) break;
|
|
21
|
+
dir = parent;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Whether `err` is Node's "package not installed" error for `packageName`
|
|
28
|
+
* itself, as opposed to a resolution failure *inside* that package (a broken
|
|
29
|
+
* transitive dependency), which is a real error worth surfacing.
|
|
30
|
+
*
|
|
31
|
+
* Used to tell an uninstalled optional peer dependency apart from a genuine
|
|
32
|
+
* failure when lazily importing one.
|
|
33
|
+
*/
|
|
34
|
+
function isMissingPackage(err, packageName) {
|
|
35
|
+
for (let e = err; e; e = e.cause) if (e.code === "ERR_MODULE_NOT_FOUND" && e.message?.includes(`'${packageName}'`)) return true;
|
|
36
|
+
return false;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Throws unless `updates` are in the style the adapter produces. For use at
|
|
40
|
+
* the top of {@link SDKAdapter.denormalizeUpdates}; narrows `updates` to that
|
|
41
|
+
* style's update type.
|
|
42
|
+
*/
|
|
43
|
+
function assertUpdateStyle(updates, style) {
|
|
44
|
+
if (updates.style !== style) throw new Error(`Cannot apply "${String(updates.style)}" updates to a "${style}" prompt`);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Message shown when an optional peer dependency an adapter needs is missing.
|
|
48
|
+
*/
|
|
49
|
+
function missingPackageMessage(packageName) {
|
|
50
|
+
return `The \`${packageName}\` package isn't installed in this project. Run \`npm install ${packageName}\` to execute these prompts.`;
|
|
51
|
+
}
|
|
52
|
+
//#endregion
|
|
53
|
+
export { assertUpdateStyle, findPackageDts, isMissingPackage, missingPackageMessage };
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,487 @@
|
|
|
1
|
+
import { $ as PromptsHelperOptions, A as Trace, B as outputMessages, C as SpanMessage, D as SpanToolCallPart, E as SpanTextPart, F as TraceProviderInfo, G as PROMPT_INPUTS_ATTRIBUTE, H as toolCallingOutput, I as TraceStreamEvent, J as PROMPT_VARIATION_ATTRIBUTE, K as PROMPT_NAME_ATTRIBUTE, L as TraceSummary, M as TraceChangeType, N as TraceEvalRun, O as SpanToolResultPart, P as TraceLiveEvent, Q as PromptsHelper, R as TraceWithSpans, S as SpanKind, T as SpanReasoningPart, U as unwrapToolResultOutput, V as spanMessages, W as PROMPT_ID_ATTRIBUTE, X as PromptSpanInfo, Y as PROMPT_VERSION_ATTRIBUTE, Z as PromptsFactory, _ as PromptID, a as isTraceSink, b as SpanContentPart, c as AnnotationEvent, d as AnnotationSource, et as SPAN_KIND_ATTRIBUTE, f as CHECK_ATTRIBUTE, g as PLAYGROUND_ENVIRONMENT, h as LLMSpanDetails, i as TraceSink, j as TraceChangeEvent, k as ToolSpanDetails, l as AnnotationEventOp, m as EvalCounts, n as PLAYGROUND_RESOURCE, nt as getPromptSpanAttributes, o as Annotation, p as DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, q as PROMPT_PROVIDER_ID_ATTRIBUTE, r as TraceIngestor, rt as mergePromptIdentity, s as AnnotationChanges, t as BaseTraceIngestor, tt as createTracerForPrompt, u as AnnotationKind, v as SERVICE_NAME_ATTRIBUTE, w as SpanMessageRole, x as SpanImagePart, y as Span, z as isCheckSpan } from "./trace-ingestor-Cu8xQW6B.js";
|
|
2
|
+
|
|
3
|
+
//#region src/brand.d.ts
|
|
4
|
+
declare function makeBrand<T extends object>(sym: symbol): {
|
|
5
|
+
brand: <U extends T>(value: U) => U;
|
|
6
|
+
isBranded: (value: any) => value is T;
|
|
7
|
+
};
|
|
8
|
+
//#endregion
|
|
9
|
+
//#region src/trace-query.d.ts
|
|
10
|
+
/** The trace-list columns whose value is text, or unset. */
|
|
11
|
+
type TraceTextColumn = "name" | "status" | "model" | "environment" | "annotations" | "checks" | "version" | "variation";
|
|
12
|
+
/** The trace-list columns whose value is a number, or unset. */
|
|
13
|
+
type TraceNumberColumn = "startTime" | "spanCount" | "duration" | "totalTokens" | "cost";
|
|
14
|
+
/** A trace-list column a listing can be filtered or grouped by. */
|
|
15
|
+
type TraceQueryColumn = TraceTextColumn | TraceNumberColumn;
|
|
16
|
+
/** Whether `column` names a {@link TraceTextColumn}. */
|
|
17
|
+
declare function isTraceTextColumn(column: unknown): column is TraceTextColumn;
|
|
18
|
+
/** Whether `column` names a {@link TraceNumberColumn}. */
|
|
19
|
+
declare function isTraceNumberColumn(column: unknown): column is TraceNumberColumn;
|
|
20
|
+
/**
|
|
21
|
+
* One condition on a trace's column. A trace is listed only if it meets every
|
|
22
|
+
* filter of the query.
|
|
23
|
+
*
|
|
24
|
+
* - `in` / `notIn`: the column's value is, or isn't, one of `values`, where
|
|
25
|
+
* `null` stands for a trace with no value.
|
|
26
|
+
* - `range`: the column's value is in `[min, max)`. A trace with no value
|
|
27
|
+
* never is.
|
|
28
|
+
* - `unset`: the trace has no value: no cost reported, say, or no duration
|
|
29
|
+
* because it's still running.
|
|
30
|
+
*/
|
|
31
|
+
type TraceFilter = {
|
|
32
|
+
column: TraceTextColumn;
|
|
33
|
+
op: "in" | "notIn";
|
|
34
|
+
values: (string | null)[];
|
|
35
|
+
} | {
|
|
36
|
+
column: TraceNumberColumn;
|
|
37
|
+
op: "range";
|
|
38
|
+
min?: number;
|
|
39
|
+
max?: number;
|
|
40
|
+
} | {
|
|
41
|
+
column: TraceNumberColumn;
|
|
42
|
+
op: "unset";
|
|
43
|
+
};
|
|
44
|
+
/** Which traces a listing holds. */
|
|
45
|
+
interface TraceQuery {
|
|
46
|
+
/**
|
|
47
|
+
* Only traces whose name or id contains this text, whatever its case.
|
|
48
|
+
* Blank matches everything.
|
|
49
|
+
*/
|
|
50
|
+
search?: string;
|
|
51
|
+
/** Only traces that meet every one of these. */
|
|
52
|
+
filters?: TraceFilter[];
|
|
53
|
+
}
|
|
54
|
+
/** How finely grouping by a trace's start time divides it. */
|
|
55
|
+
type TraceDateResolution = "hour" | "day" | "week" | "month";
|
|
56
|
+
/**
|
|
57
|
+
* The column a listing is grouped by. Text columns group traces with the
|
|
58
|
+
* same value; `startTime` groups by the hour, day, week (from Monday) or
|
|
59
|
+
* month a trace started in; the other number columns group into the buckets
|
|
60
|
+
* of {@link TRACE_BUCKET_BOUNDS}, with the traces that have no value in a
|
|
61
|
+
* group of their own.
|
|
62
|
+
*/
|
|
63
|
+
interface TraceGroupBy {
|
|
64
|
+
column: TraceQueryColumn;
|
|
65
|
+
/** For `startTime`. Defaults to `"day"`. */
|
|
66
|
+
resolution?: TraceDateResolution;
|
|
67
|
+
}
|
|
68
|
+
/** A {@link TraceQuery} whose matches are returned as groups. */
|
|
69
|
+
interface TraceGroupQuery extends TraceQuery {
|
|
70
|
+
groupBy: TraceGroupBy;
|
|
71
|
+
/**
|
|
72
|
+
* The IANA time zone days, weeks and months begin in, e.g.
|
|
73
|
+
* `"Europe/Berlin"`. Defaults to UTC.
|
|
74
|
+
*/
|
|
75
|
+
timeZone?: string;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* What a group of traces adds up to: spanning from the first trace's start to
|
|
79
|
+
* the last one's end, with counts summed, and anything the traces don't all
|
|
80
|
+
* agree on left unset.
|
|
81
|
+
*/
|
|
82
|
+
type TraceGroupSummary = Omit<TraceSummary, "id" | "providerId" | "name" | "evalRun">;
|
|
83
|
+
/** The traces of a listing that share a value, or a bucket, of a column. */
|
|
84
|
+
interface TraceGroup {
|
|
85
|
+
/** Names the group among those of one listing, the same from any provider. */
|
|
86
|
+
key: string;
|
|
87
|
+
/**
|
|
88
|
+
* What the group's traces have in common. Added to the query's filters, it
|
|
89
|
+
* lists exactly them.
|
|
90
|
+
*/
|
|
91
|
+
filter: TraceFilter;
|
|
92
|
+
/** How many traces the group holds. */
|
|
93
|
+
count: number;
|
|
94
|
+
summary: TraceGroupSummary;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* The lower bounds of every bucket but the first, for the number columns
|
|
98
|
+
* grouped into buckets: `[1000, 5000]` makes the buckets `< 1000`,
|
|
99
|
+
* `1000 – 5000` and `≥ 5000`. Durations are in ms and costs in dollars.
|
|
100
|
+
*/
|
|
101
|
+
declare const TRACE_BUCKET_BOUNDS: Record<Exclude<TraceNumberColumn, "startTime">, readonly number[]>;
|
|
102
|
+
/** A trace's value in a text column, or `null` if it has none. */
|
|
103
|
+
declare function traceTextValue(trace: TraceGroupMember, column: TraceTextColumn): string | null;
|
|
104
|
+
/** A trace's value in a number column, or `undefined` if it has none. */
|
|
105
|
+
declare function traceNumberValue(trace: TraceGroupMember, column: TraceNumberColumn): number | undefined;
|
|
106
|
+
/** What of a trace filtering and grouping read. */
|
|
107
|
+
type TraceGroupMember = Pick<TraceSummary, "name"> & TraceGroupSummary;
|
|
108
|
+
/** Whether `trace` meets `filter`. */
|
|
109
|
+
declare function matchesTraceFilter(trace: TraceSummary, filter: TraceFilter): boolean;
|
|
110
|
+
/** Whether `trace` is one `query` lists. */
|
|
111
|
+
declare function matchesTraceQuery(trace: TraceSummary, query: TraceQuery): boolean;
|
|
112
|
+
/** Whether `query` lists every trace. */
|
|
113
|
+
declare function isEmptyTraceQuery(query: TraceQuery | undefined): boolean;
|
|
114
|
+
/** The traces of `traces` that `query` lists, in the same order. */
|
|
115
|
+
declare function filterTraces<T extends TraceSummary>(traces: T[], query: TraceQuery | undefined): T[];
|
|
116
|
+
/**
|
|
117
|
+
* The `[min, max)` of the hour, day, week or month `ms` falls in, on
|
|
118
|
+
* `timeZone`'s clock.
|
|
119
|
+
*/
|
|
120
|
+
declare function traceDateBucket(ms: number, resolution?: TraceDateResolution, timeZone?: string): {
|
|
121
|
+
min: number;
|
|
122
|
+
max: number;
|
|
123
|
+
};
|
|
124
|
+
/**
|
|
125
|
+
* The bucket of `column` that `value` falls in, as the filter listing it:
|
|
126
|
+
* see {@link TRACE_BUCKET_BOUNDS}.
|
|
127
|
+
*/
|
|
128
|
+
declare function traceBucketFilter(column: Exclude<TraceNumberColumn, "startTime">, value: number | undefined): TraceFilter;
|
|
129
|
+
/**
|
|
130
|
+
* The `index`th bucket of `column`, counting from the one below every bound
|
|
131
|
+
* of {@link TRACE_BUCKET_BOUNDS}, as the filter listing it.
|
|
132
|
+
*/
|
|
133
|
+
declare function traceBucketAt(column: Exclude<TraceNumberColumn, "startTime">, index: number): TraceFilter;
|
|
134
|
+
/** The filter listing the group of `groupBy` that `trace` belongs to. */
|
|
135
|
+
declare function traceGroupFilter(trace: TraceSummary, groupBy: TraceGroupBy, timeZone?: string): TraceFilter;
|
|
136
|
+
/** {@link TraceGroup.key} for the group `filter` lists. */
|
|
137
|
+
declare function traceGroupKey(filter: TraceFilter): string;
|
|
138
|
+
/**
|
|
139
|
+
* What `traces` (non-empty) add up to. Takes summaries of groups as well as
|
|
140
|
+
* traces, so the groups of several listings can be added together.
|
|
141
|
+
*/
|
|
142
|
+
declare function summarizeTraces(traces: readonly TraceGroupSummary[]): TraceGroupSummary;
|
|
143
|
+
/**
|
|
144
|
+
* `traces` as the groups of `groupBy`, in the order their first traces come.
|
|
145
|
+
* Filter them first: this groups every trace it's given.
|
|
146
|
+
*/
|
|
147
|
+
declare function groupTraceSummaries(traces: TraceSummary[], groupBy: TraceGroupBy, timeZone?: string): TraceGroup[];
|
|
148
|
+
/**
|
|
149
|
+
* The groups of several listings as one: those with the same key added
|
|
150
|
+
* together, in the order their keys first come.
|
|
151
|
+
*/
|
|
152
|
+
declare function mergeTraceGroups(groups: TraceGroup[]): TraceGroup[];
|
|
153
|
+
/** The query parameters a {@link TraceQuery} or {@link TraceGroupQuery} travels as. */
|
|
154
|
+
declare function traceQueryParams(query: TraceQuery & Partial<TraceGroupQuery>): Record<string, string>;
|
|
155
|
+
/**
|
|
156
|
+
* The {@link TraceQuery} that {@link traceQueryParams} wrote into `params`.
|
|
157
|
+
* Throws a `TypeError` naming what's wrong with one that can't be read.
|
|
158
|
+
*/
|
|
159
|
+
declare function parseTraceQuery(params: Record<string, string | undefined>): TraceQuery;
|
|
160
|
+
/** {@link parseTraceQuery} for the parameters of a {@link TraceGroupQuery}. */
|
|
161
|
+
declare function parseTraceGroupQuery(params: Record<string, string | undefined>): TraceGroupQuery;
|
|
162
|
+
//#endregion
|
|
163
|
+
//#region src/client.d.ts
|
|
164
|
+
/** The Evalution Cloud API, unless `EVALUTION_API_URL` says otherwise. */
|
|
165
|
+
declare const DEFAULT_API_URL = "https://api.evalut.io";
|
|
166
|
+
/**
|
|
167
|
+
* What an API key can do: an `ingest` key writes one project's production
|
|
168
|
+
* traces and nothing else, and a `personal` key reads and manages a user's
|
|
169
|
+
* projects but can't write production traces.
|
|
170
|
+
*/
|
|
171
|
+
type ApiKeyKind = "ingest" | "personal";
|
|
172
|
+
/**
|
|
173
|
+
* The kind of an API key, read from its prefix, or `undefined` for a string
|
|
174
|
+
* that isn't one.
|
|
175
|
+
*/
|
|
176
|
+
declare function apiKeyKind(key: string): ApiKeyKind | undefined;
|
|
177
|
+
/** Options for {@link Evalution}. Each overrides what the environment says. */
|
|
178
|
+
interface EvalutionOptions {
|
|
179
|
+
/**
|
|
180
|
+
* The base URL of a local Evalution instance to send traces to, without
|
|
181
|
+
* authentication.
|
|
182
|
+
* - If this is set, traces always go to this local instance, or nowhere if
|
|
183
|
+
* this is `false`.
|
|
184
|
+
* - Otherwise, if an ingest API key is present, traces always go to the cloud.
|
|
185
|
+
* - Otherwise, this defaults to `EVALUTION_ENDPOINT`.
|
|
186
|
+
*/
|
|
187
|
+
endpoint?: string | false;
|
|
188
|
+
/**
|
|
189
|
+
* API key for Evalution Cloud.
|
|
190
|
+
* - An ingest key sends traces to the cloud.
|
|
191
|
+
* - A personal key makes the read and control calls available, and traces go
|
|
192
|
+
* to a local instance.
|
|
193
|
+
*
|
|
194
|
+
* Defaults to `EVALUTION_API_KEY`.
|
|
195
|
+
*/
|
|
196
|
+
apiKey?: string;
|
|
197
|
+
/** The cloud's base URL. Defaults to `EVALUTION_API_URL`, then {@link DEFAULT_API_URL}. */
|
|
198
|
+
apiUrl?: string;
|
|
199
|
+
/** The project id the read and control calls act on: the config's `cloud.project`. */
|
|
200
|
+
project?: string;
|
|
201
|
+
/** The `fetch` to send with. Defaults to the global one. */
|
|
202
|
+
fetch?: typeof fetch;
|
|
203
|
+
}
|
|
204
|
+
/** Which traces {@link Evalution.listTraces} returns. */
|
|
205
|
+
interface ListTracesQuery {
|
|
206
|
+
/** Only traces from this deployment environment. */
|
|
207
|
+
environment?: string;
|
|
208
|
+
/** Only traces started at or after this time (ms). */
|
|
209
|
+
from?: number;
|
|
210
|
+
/** Only traces started before this time (ms). */
|
|
211
|
+
to?: number;
|
|
212
|
+
/** At most this many, most recent first. */
|
|
213
|
+
limit?: number;
|
|
214
|
+
/** Only traces whose name or id contains this text. */
|
|
215
|
+
search?: string;
|
|
216
|
+
/** Only traces that meet every one of these. */
|
|
217
|
+
filters?: TraceFilter[];
|
|
218
|
+
}
|
|
219
|
+
/** Which traces {@link Evalution.listTraceGroups} groups, and how. */
|
|
220
|
+
interface ListTraceGroupsQuery extends Omit<ListTracesQuery, "limit">, Pick<TraceGroupQuery, "groupBy" | "timeZone"> {}
|
|
221
|
+
/** A project, as the cloud describes it. */
|
|
222
|
+
interface CloudProject {
|
|
223
|
+
/** The `prj_…` id committed in a repo's config. */
|
|
224
|
+
id: string;
|
|
225
|
+
slug: string;
|
|
226
|
+
name: string;
|
|
227
|
+
/** The slug of the organization it belongs to. */
|
|
228
|
+
orgSlug: string;
|
|
229
|
+
}
|
|
230
|
+
/** What {@link Evalution.whoami} resolves a key to. */
|
|
231
|
+
type CloudIdentity = {
|
|
232
|
+
kind: "personal";
|
|
233
|
+
userId: string;
|
|
234
|
+
} | {
|
|
235
|
+
kind: "ingest";
|
|
236
|
+
projectId: string;
|
|
237
|
+
environment: string;
|
|
238
|
+
};
|
|
239
|
+
/** One checkout of a project, as {@link Evalution.registerInstance} reports it. */
|
|
240
|
+
interface InstanceRegistration {
|
|
241
|
+
/** Identifies this checkout: the same folder is always the same instance. */
|
|
242
|
+
instanceKey: string;
|
|
243
|
+
/** How to show this instance in a list, e.g. `hostname:dir`. */
|
|
244
|
+
label: string;
|
|
245
|
+
/** The instance's own loopback URL. */
|
|
246
|
+
localUrl: string;
|
|
247
|
+
/** Its portkit URL, if it has one. */
|
|
248
|
+
portkitUrl?: string;
|
|
249
|
+
/** How portkit runs it, if it does. */
|
|
250
|
+
portkitMode?: string;
|
|
251
|
+
}
|
|
252
|
+
/** The cloud's answer to {@link Evalution.registerInstance}. */
|
|
253
|
+
interface RegisteredInstance {
|
|
254
|
+
id: string;
|
|
255
|
+
/** The project's shareable URL, `https://l.evalut.io/<org>/<project>`. */
|
|
256
|
+
projectUrl: string;
|
|
257
|
+
/**
|
|
258
|
+
* Where to open the browser on this machine: it remembers this instance
|
|
259
|
+
* there, so the project URL goes straight to it, then opens it.
|
|
260
|
+
*/
|
|
261
|
+
openUrl?: string;
|
|
262
|
+
}
|
|
263
|
+
/** Names one synced database of a project. */
|
|
264
|
+
interface SyncDatabaseKey {
|
|
265
|
+
kind: "traces" | "evals" | "dataset";
|
|
266
|
+
/**
|
|
267
|
+
* The id of the local provider whose database it is. Each provider of a
|
|
268
|
+
* kind syncs its own databases.
|
|
269
|
+
*/
|
|
270
|
+
provider: string;
|
|
271
|
+
/** The dataset's id, for `kind: "dataset"`. */
|
|
272
|
+
key?: string;
|
|
273
|
+
}
|
|
274
|
+
/** What {@link Evalution.getSyncDatabase} asks for. */
|
|
275
|
+
interface SyncDatabaseRequest extends SyncDatabaseKey {
|
|
276
|
+
/**
|
|
277
|
+
* How many migrations this client's schema for the kind has. The cloud
|
|
278
|
+
* refuses a client whose schema is older than one that synced before, with
|
|
279
|
+
* a 409.
|
|
280
|
+
*/
|
|
281
|
+
schemaVersion?: number;
|
|
282
|
+
/**
|
|
283
|
+
* Set when creating a dataset. Without it, the cloud answers 410 for a
|
|
284
|
+
* dataset that was deleted, so a machine that still has a copy can't
|
|
285
|
+
* bring it back. The 410's {@link EvalutionApiError.body} has the deleted
|
|
286
|
+
* database's `syncId`: a copy that synced with it should go too.
|
|
287
|
+
*/
|
|
288
|
+
create?: boolean;
|
|
289
|
+
}
|
|
290
|
+
/** Where a synced database lives, and a short-lived token for it. */
|
|
291
|
+
interface SyncDatabaseCredentials {
|
|
292
|
+
url: string;
|
|
293
|
+
token: string;
|
|
294
|
+
/** When {@link token} expires (ms). */
|
|
295
|
+
expiresAt: number;
|
|
296
|
+
/**
|
|
297
|
+
* Names this one cloud database: a new one, made after a dataset was
|
|
298
|
+
* deleted, say, has a new id. A local file records the ids it has synced
|
|
299
|
+
* with, so it can tell whether it holds this database's rows.
|
|
300
|
+
*/
|
|
301
|
+
syncId: string;
|
|
302
|
+
/** The linked project's id. */
|
|
303
|
+
projectId: string;
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* The client for Evalution's API. As a {@link TraceSink} it sends an app's
|
|
307
|
+
* traces, and it makes the read and control calls the local server needs.
|
|
308
|
+
*
|
|
309
|
+
* Where traces go is worked out once, on the first one sent:
|
|
310
|
+
*
|
|
311
|
+
* 1. the {@link EvalutionOptions.endpoint} option;
|
|
312
|
+
* 2. Evalution Cloud, when the API key is an ingest key;
|
|
313
|
+
* 3. the local instance `EVALUTION_ENDPOINT` names;
|
|
314
|
+
* 4. on Node, the instance portkit knows for this directory;
|
|
315
|
+
* 5. otherwise nowhere, and sending does nothing.
|
|
316
|
+
*
|
|
317
|
+
* A personal key never sends traces to the cloud, and no key is ever sent to
|
|
318
|
+
* a local instance. Tracing never breaks the app: see {@link HttpTraceSink},
|
|
319
|
+
* and await {@link flush} before a short-lived process exits. The read and
|
|
320
|
+
* control calls, by contrast, reject when they fail.
|
|
321
|
+
*/
|
|
322
|
+
declare class Evalution implements TraceSink {
|
|
323
|
+
private readonly endpoint?;
|
|
324
|
+
private readonly apiKey?;
|
|
325
|
+
private readonly apiUrl;
|
|
326
|
+
private readonly project?;
|
|
327
|
+
private readonly fetch;
|
|
328
|
+
private readonly sink;
|
|
329
|
+
/** The start of the API key, safe to show in messages, or `<none>`. */
|
|
330
|
+
get printableApiKey(): string;
|
|
331
|
+
constructor(options?: EvalutionOptions);
|
|
332
|
+
private resolveTraceTarget;
|
|
333
|
+
recordSpanStart(span: Span): Promise<Span>;
|
|
334
|
+
recordSpanEnd(span: Span): Promise<Span>;
|
|
335
|
+
failTrace(traceId: string, errorMessage: string): Promise<void>;
|
|
336
|
+
/** Resolves once every trace event recorded so far has been sent (or dropped). */
|
|
337
|
+
flush(): Promise<void>;
|
|
338
|
+
/** The key's kind and whose it is. Takes either kind of key. */
|
|
339
|
+
whoami(): Promise<CloudIdentity>;
|
|
340
|
+
/** The most recent production traces of the project. */
|
|
341
|
+
listTraces(query?: ListTracesQuery): Promise<TraceSummary[]>;
|
|
342
|
+
/**
|
|
343
|
+
* The project's production traces as groups, one per value or bucket of
|
|
344
|
+
* a column, counting every trace in the window rather than the most recent.
|
|
345
|
+
*/
|
|
346
|
+
listTraceGroups(query: ListTraceGroupsQuery): Promise<TraceGroup[]>;
|
|
347
|
+
/** One production trace with its spans, or `undefined` if there's none. */
|
|
348
|
+
getTrace(id: string): Promise<TraceWithSpans | undefined>;
|
|
349
|
+
/** The projects the key's user is a member of. */
|
|
350
|
+
listProjects(): Promise<CloudProject[]>;
|
|
351
|
+
/** Creates a project in the user's organization. */
|
|
352
|
+
createProject(project: {
|
|
353
|
+
name: string;
|
|
354
|
+
slug?: string;
|
|
355
|
+
}): Promise<CloudProject>;
|
|
356
|
+
/** Records a running checkout of the project. */
|
|
357
|
+
registerInstance(instance: InstanceRegistration): Promise<RegisteredInstance>;
|
|
358
|
+
/** Tells the cloud a registered instance is still running. */
|
|
359
|
+
heartbeatInstance(id: string): Promise<void>;
|
|
360
|
+
/** Tells the cloud a registered instance has stopped. */
|
|
361
|
+
endInstanceSession(id: string): Promise<void>;
|
|
362
|
+
/** Ensures a synced database exists, and returns how to reach it. */
|
|
363
|
+
getSyncDatabase(request: SyncDatabaseRequest): Promise<SyncDatabaseCredentials>;
|
|
364
|
+
/** The project's synced databases. A dataset's has its id as the `key`; the others' `key` is empty. */
|
|
365
|
+
listSyncDatabases(): Promise<Required<SyncDatabaseKey>[]>;
|
|
366
|
+
/** Deletes a dataset's synced database, for every member's machine. */
|
|
367
|
+
deleteSyncDatabase(key: SyncDatabaseKey & {
|
|
368
|
+
kind: "dataset";
|
|
369
|
+
key: string;
|
|
370
|
+
}): Promise<void>;
|
|
371
|
+
private instanceRequest;
|
|
372
|
+
private json;
|
|
373
|
+
private request;
|
|
374
|
+
}
|
|
375
|
+
/** How {@link Evalution} rejects when the cloud refuses a call. */
|
|
376
|
+
declare class EvalutionApiError extends Error {
|
|
377
|
+
/** The HTTP status the cloud answered with. */
|
|
378
|
+
readonly status: number;
|
|
379
|
+
/** The cloud's explanation, if it gave one. */
|
|
380
|
+
readonly detail?: string;
|
|
381
|
+
/** The cloud's whole answer, when it was JSON: a 410's `syncId`, say. */
|
|
382
|
+
readonly body?: Record<string, unknown>;
|
|
383
|
+
constructor(status: number, detail?: string, body?: Record<string, unknown>);
|
|
384
|
+
}
|
|
385
|
+
//#endregion
|
|
386
|
+
//#region src/evaluation-answers.d.ts
|
|
387
|
+
/**
|
|
388
|
+
* The per-question confidence a provider reported beside an evaluation's
|
|
389
|
+
* answers, keyed by question id.
|
|
390
|
+
*
|
|
391
|
+
* The AI SDK's answers carry no confidence of their own: a provider that
|
|
392
|
+
* computes one (TypeSafe does, for choice and score answers) reports it as
|
|
393
|
+
* `providerMetadata.<provider>.confidence[questionId]`. Read from whichever
|
|
394
|
+
* provider namespace has it, so this isn't tied to one provider.
|
|
395
|
+
*/
|
|
396
|
+
declare function evaluationConfidence(providerMetadata: unknown): Record<string, number>;
|
|
397
|
+
/**
|
|
398
|
+
* An evaluation's answers as recorded on its span: each answer as the SDK
|
|
399
|
+
* returned it, plus its `confidence` where the provider reported one (see
|
|
400
|
+
* {@link evaluationConfidence}). Recorded together so a trace — from the
|
|
401
|
+
* playground or sent over OTLP — can show confidence without knowing which
|
|
402
|
+
* provider answered.
|
|
403
|
+
*/
|
|
404
|
+
declare function answersWithConfidence(answers: unknown, providerMetadata: unknown): unknown;
|
|
405
|
+
//#endregion
|
|
406
|
+
//#region src/http-trace-sink.d.ts
|
|
407
|
+
/**
|
|
408
|
+
* Where an evalution server accepts {@link TraceSinkEvent}s, relative to its
|
|
409
|
+
* base URL.
|
|
410
|
+
*/
|
|
411
|
+
declare const TRACE_EVENTS_PATH = "/api/traces";
|
|
412
|
+
/** One {@link TraceSink} call, as {@link HttpTraceSink} sends it. */
|
|
413
|
+
type TraceSinkEvent = {
|
|
414
|
+
type: "span-start";
|
|
415
|
+
span: Span;
|
|
416
|
+
} | {
|
|
417
|
+
type: "span-end";
|
|
418
|
+
span: Span;
|
|
419
|
+
} | {
|
|
420
|
+
type: "trace-fail";
|
|
421
|
+
traceId: string;
|
|
422
|
+
errorMessage: string;
|
|
423
|
+
};
|
|
424
|
+
/** The body of a `POST` to {@link TRACE_EVENTS_PATH}. */
|
|
425
|
+
interface TraceEventsRequest {
|
|
426
|
+
/** The events to record, in the order they happened. */
|
|
427
|
+
events: TraceSinkEvent[];
|
|
428
|
+
}
|
|
429
|
+
/** Where an {@link HttpTraceSink} sends, once it's known. */
|
|
430
|
+
interface HttpTraceTarget {
|
|
431
|
+
/** The server's base URL. */
|
|
432
|
+
endpoint: string;
|
|
433
|
+
/** Sent with every request, e.g. `authorization`. */
|
|
434
|
+
headers?: Record<string, string>;
|
|
435
|
+
/** Overrides {@link HttpTraceSinkOptions.fetch}. */
|
|
436
|
+
fetch?: typeof fetch;
|
|
437
|
+
}
|
|
438
|
+
/** Options for {@link HttpTraceSink}. */
|
|
439
|
+
interface HttpTraceSinkOptions {
|
|
440
|
+
/**
|
|
441
|
+
* The evalution server's base URL, e.g. `http://localhost:3000`. Events are
|
|
442
|
+
* posted to {@link TRACE_EVENTS_PATH} under it.
|
|
443
|
+
*
|
|
444
|
+
* May instead be a function that finds out where to send, which is called
|
|
445
|
+
* once, when the first event is recorded. Events wait in the queue until it
|
|
446
|
+
* settles; if it finds nowhere, or throws, they and all later ones are
|
|
447
|
+
* dropped.
|
|
448
|
+
*/
|
|
449
|
+
endpoint: string | (() => Promise<HttpTraceTarget | undefined>);
|
|
450
|
+
/** The `fetch` to send with. Defaults to the global one. */
|
|
451
|
+
fetch?: typeof fetch;
|
|
452
|
+
}
|
|
453
|
+
/**
|
|
454
|
+
* A {@link TraceSink} that sends what it's given to an evalution server,
|
|
455
|
+
* which records it through its own trace providers — so an app's traces show
|
|
456
|
+
* up in the playground without the app writing to a trace store itself.
|
|
457
|
+
*
|
|
458
|
+
* Calls return as soon as the event is queued, never waiting on the network,
|
|
459
|
+
* so tracing doesn't slow the app down. Queued events go out in order, batched
|
|
460
|
+
* into one request at a time, so a span's end never overtakes its start. A
|
|
461
|
+
* request that fails is logged and its events dropped: tracing never breaks
|
|
462
|
+
* the app. Await {@link flush} before a short-lived process exits to be sure
|
|
463
|
+
* everything was sent.
|
|
464
|
+
*/
|
|
465
|
+
declare class HttpTraceSink implements TraceSink {
|
|
466
|
+
private readonly options;
|
|
467
|
+
private target?;
|
|
468
|
+
/** Set once it's known there's nowhere to send, so events aren't queued. */
|
|
469
|
+
private disabled;
|
|
470
|
+
private queue;
|
|
471
|
+
private sending?;
|
|
472
|
+
/** Whether the last request failed, so a down server is reported once. */
|
|
473
|
+
private failing;
|
|
474
|
+
constructor(options: HttpTraceSinkOptions);
|
|
475
|
+
private resolveTarget;
|
|
476
|
+
private lookUpTarget;
|
|
477
|
+
recordSpanStart(span: Span): Promise<Span>;
|
|
478
|
+
recordSpanEnd(span: Span): Promise<Span>;
|
|
479
|
+
failTrace(traceId: string, errorMessage: string): Promise<void>;
|
|
480
|
+
/** Resolves once every event recorded so far has been sent (or dropped). */
|
|
481
|
+
flush(): Promise<void>;
|
|
482
|
+
private enqueue;
|
|
483
|
+
private drain;
|
|
484
|
+
private send;
|
|
485
|
+
}
|
|
486
|
+
//#endregion
|
|
487
|
+
export { Annotation, AnnotationChanges, AnnotationEvent, AnnotationEventOp, AnnotationKind, AnnotationSource, ApiKeyKind, BaseTraceIngestor, CHECK_ATTRIBUTE, CloudIdentity, CloudProject, DEFAULT_API_URL, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, EvalCounts, Evalution, EvalutionApiError, EvalutionOptions, HttpTraceSink, HttpTraceSinkOptions, HttpTraceTarget, InstanceRegistration, LLMSpanDetails, ListTraceGroupsQuery, ListTracesQuery, PLAYGROUND_ENVIRONMENT, PLAYGROUND_RESOURCE, PROMPT_ID_ATTRIBUTE, PROMPT_INPUTS_ATTRIBUTE, PROMPT_NAME_ATTRIBUTE, PROMPT_PROVIDER_ID_ATTRIBUTE, PROMPT_VARIATION_ATTRIBUTE, PROMPT_VERSION_ATTRIBUTE, PromptID, PromptSpanInfo, PromptsFactory, PromptsHelper, PromptsHelperOptions, RegisteredInstance, SERVICE_NAME_ATTRIBUTE, SPAN_KIND_ATTRIBUTE, Span, SpanContentPart, SpanImagePart, SpanKind, SpanMessage, SpanMessageRole, SpanReasoningPart, SpanTextPart, SpanToolCallPart, SpanToolResultPart, SyncDatabaseCredentials, SyncDatabaseKey, SyncDatabaseRequest, TRACE_BUCKET_BOUNDS, TRACE_EVENTS_PATH, ToolSpanDetails, Trace, TraceChangeEvent, TraceChangeType, TraceDateResolution, TraceEvalRun, TraceEventsRequest, TraceFilter, TraceGroup, TraceGroupBy, TraceGroupQuery, TraceGroupSummary, TraceIngestor, TraceLiveEvent, TraceNumberColumn, TraceProviderInfo, TraceQuery, TraceQueryColumn, TraceSink, TraceSinkEvent, TraceStreamEvent, TraceSummary, TraceTextColumn, TraceWithSpans, answersWithConfidence, apiKeyKind, createTracerForPrompt, evaluationConfidence, filterTraces, getPromptSpanAttributes, groupTraceSummaries, isCheckSpan, isEmptyTraceQuery, isTraceNumberColumn, isTraceSink, isTraceTextColumn, makeBrand, matchesTraceFilter, matchesTraceQuery, mergePromptIdentity, mergeTraceGroups, outputMessages, parseTraceGroupQuery, parseTraceQuery, spanMessages, summarizeTraces, toolCallingOutput, traceBucketAt, traceBucketFilter, traceDateBucket, traceGroupFilter, traceGroupKey, traceNumberValue, traceQueryParams, traceTextValue, unwrapToolResultOutput };
|