@evalution/sdk 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +224 -4
- package/dist/index.js +492 -10
- package/package.json +15 -3
package/dist/index.d.ts
CHANGED
|
@@ -6,6 +6,160 @@ declare function makeBrand<T extends object>(sym: symbol): {
|
|
|
6
6
|
isBranded: (value: any) => value is T;
|
|
7
7
|
};
|
|
8
8
|
//#endregion
|
|
9
|
+
//#region src/trace-query.d.ts
|
|
10
|
+
/** The trace-list columns whose value is text, or unset. */
|
|
11
|
+
type TraceTextColumn = "name" | "status" | "model" | "environment" | "annotations" | "checks" | "version" | "variation";
|
|
12
|
+
/** The trace-list columns whose value is a number, or unset. */
|
|
13
|
+
type TraceNumberColumn = "startTime" | "spanCount" | "duration" | "totalTokens" | "cost";
|
|
14
|
+
/** A trace-list column a listing can be filtered or grouped by. */
|
|
15
|
+
type TraceQueryColumn = TraceTextColumn | TraceNumberColumn;
|
|
16
|
+
/** Whether `column` names a {@link TraceTextColumn}. */
|
|
17
|
+
declare function isTraceTextColumn(column: unknown): column is TraceTextColumn;
|
|
18
|
+
/** Whether `column` names a {@link TraceNumberColumn}. */
|
|
19
|
+
declare function isTraceNumberColumn(column: unknown): column is TraceNumberColumn;
|
|
20
|
+
/**
|
|
21
|
+
* One condition on a trace's column. A trace is listed only if it meets every
|
|
22
|
+
* filter of the query.
|
|
23
|
+
*
|
|
24
|
+
* - `in` / `notIn`: the column's value is, or isn't, one of `values`, where
|
|
25
|
+
* `null` stands for a trace with no value.
|
|
26
|
+
* - `range`: the column's value is in `[min, max)`. A trace with no value
|
|
27
|
+
* never is.
|
|
28
|
+
* - `unset`: the trace has no value: no cost reported, say, or no duration
|
|
29
|
+
* because it's still running.
|
|
30
|
+
*/
|
|
31
|
+
type TraceFilter = {
|
|
32
|
+
column: TraceTextColumn;
|
|
33
|
+
op: "in" | "notIn";
|
|
34
|
+
values: (string | null)[];
|
|
35
|
+
} | {
|
|
36
|
+
column: TraceNumberColumn;
|
|
37
|
+
op: "range";
|
|
38
|
+
min?: number;
|
|
39
|
+
max?: number;
|
|
40
|
+
} | {
|
|
41
|
+
column: TraceNumberColumn;
|
|
42
|
+
op: "unset";
|
|
43
|
+
};
|
|
44
|
+
/** Which traces a listing holds. */
|
|
45
|
+
interface TraceQuery {
|
|
46
|
+
/**
|
|
47
|
+
* Only traces whose name or id contains this text, whatever its case.
|
|
48
|
+
* Blank matches everything.
|
|
49
|
+
*/
|
|
50
|
+
search?: string;
|
|
51
|
+
/** Only traces that meet every one of these. */
|
|
52
|
+
filters?: TraceFilter[];
|
|
53
|
+
}
|
|
54
|
+
/** How finely grouping by a trace's start time divides it. */
|
|
55
|
+
type TraceDateResolution = "hour" | "day" | "week" | "month";
|
|
56
|
+
/**
|
|
57
|
+
* The column a listing is grouped by. Text columns group traces with the
|
|
58
|
+
* same value; `startTime` groups by the hour, day, week (from Monday) or
|
|
59
|
+
* month a trace started in; the other number columns group into the buckets
|
|
60
|
+
* of {@link TRACE_BUCKET_BOUNDS}, with the traces that have no value in a
|
|
61
|
+
* group of their own.
|
|
62
|
+
*/
|
|
63
|
+
interface TraceGroupBy {
|
|
64
|
+
column: TraceQueryColumn;
|
|
65
|
+
/** For `startTime`. Defaults to `"day"`. */
|
|
66
|
+
resolution?: TraceDateResolution;
|
|
67
|
+
}
|
|
68
|
+
/** A {@link TraceQuery} whose matches are returned as groups. */
|
|
69
|
+
interface TraceGroupQuery extends TraceQuery {
|
|
70
|
+
groupBy: TraceGroupBy;
|
|
71
|
+
/**
|
|
72
|
+
* The IANA time zone days, weeks and months begin in, e.g.
|
|
73
|
+
* `"Europe/Berlin"`. Defaults to UTC.
|
|
74
|
+
*/
|
|
75
|
+
timeZone?: string;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* What a group of traces adds up to: spanning from the first trace's start to
|
|
79
|
+
* the last one's end, with counts summed, and anything the traces don't all
|
|
80
|
+
* agree on left unset.
|
|
81
|
+
*/
|
|
82
|
+
type TraceGroupSummary = Omit<TraceSummary, "id" | "providerId" | "name" | "evalRun">;
|
|
83
|
+
/** The traces of a listing that share a value, or a bucket, of a column. */
|
|
84
|
+
interface TraceGroup {
|
|
85
|
+
/** Names the group among those of one listing, the same from any provider. */
|
|
86
|
+
key: string;
|
|
87
|
+
/**
|
|
88
|
+
* What the group's traces have in common. Added to the query's filters, it
|
|
89
|
+
* lists exactly them.
|
|
90
|
+
*/
|
|
91
|
+
filter: TraceFilter;
|
|
92
|
+
/** How many traces the group holds. */
|
|
93
|
+
count: number;
|
|
94
|
+
summary: TraceGroupSummary;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* The lower bounds of every bucket but the first, for the number columns
|
|
98
|
+
* grouped into buckets: `[1000, 5000]` makes the buckets `< 1000`,
|
|
99
|
+
* `1000 – 5000` and `≥ 5000`. Durations are in ms and costs in dollars.
|
|
100
|
+
*/
|
|
101
|
+
declare const TRACE_BUCKET_BOUNDS: Record<Exclude<TraceNumberColumn, "startTime">, readonly number[]>;
|
|
102
|
+
/** A trace's value in a text column, or `null` if it has none. */
|
|
103
|
+
declare function traceTextValue(trace: TraceGroupMember, column: TraceTextColumn): string | null;
|
|
104
|
+
/** A trace's value in a number column, or `undefined` if it has none. */
|
|
105
|
+
declare function traceNumberValue(trace: TraceGroupMember, column: TraceNumberColumn): number | undefined;
|
|
106
|
+
/** What of a trace filtering and grouping read. */
|
|
107
|
+
type TraceGroupMember = Pick<TraceSummary, "name"> & TraceGroupSummary;
|
|
108
|
+
/** Whether `trace` meets `filter`. */
|
|
109
|
+
declare function matchesTraceFilter(trace: TraceSummary, filter: TraceFilter): boolean;
|
|
110
|
+
/** Whether `trace` is one `query` lists. */
|
|
111
|
+
declare function matchesTraceQuery(trace: TraceSummary, query: TraceQuery): boolean;
|
|
112
|
+
/** Whether `query` lists every trace. */
|
|
113
|
+
declare function isEmptyTraceQuery(query: TraceQuery | undefined): boolean;
|
|
114
|
+
/** The traces of `traces` that `query` lists, in the same order. */
|
|
115
|
+
declare function filterTraces<T extends TraceSummary>(traces: T[], query: TraceQuery | undefined): T[];
|
|
116
|
+
/**
|
|
117
|
+
* The `[min, max)` of the hour, day, week or month `ms` falls in, on
|
|
118
|
+
* `timeZone`'s clock.
|
|
119
|
+
*/
|
|
120
|
+
declare function traceDateBucket(ms: number, resolution?: TraceDateResolution, timeZone?: string): {
|
|
121
|
+
min: number;
|
|
122
|
+
max: number;
|
|
123
|
+
};
|
|
124
|
+
/**
|
|
125
|
+
* The bucket of `column` that `value` falls in, as the filter listing it:
|
|
126
|
+
* see {@link TRACE_BUCKET_BOUNDS}.
|
|
127
|
+
*/
|
|
128
|
+
declare function traceBucketFilter(column: Exclude<TraceNumberColumn, "startTime">, value: number | undefined): TraceFilter;
|
|
129
|
+
/**
|
|
130
|
+
* The `index`th bucket of `column`, counting from the one below every bound
|
|
131
|
+
* of {@link TRACE_BUCKET_BOUNDS}, as the filter listing it.
|
|
132
|
+
*/
|
|
133
|
+
declare function traceBucketAt(column: Exclude<TraceNumberColumn, "startTime">, index: number): TraceFilter;
|
|
134
|
+
/** The filter listing the group of `groupBy` that `trace` belongs to. */
|
|
135
|
+
declare function traceGroupFilter(trace: TraceSummary, groupBy: TraceGroupBy, timeZone?: string): TraceFilter;
|
|
136
|
+
/** {@link TraceGroup.key} for the group `filter` lists. */
|
|
137
|
+
declare function traceGroupKey(filter: TraceFilter): string;
|
|
138
|
+
/**
|
|
139
|
+
* What `traces` (non-empty) add up to. Takes summaries of groups as well as
|
|
140
|
+
* traces, so the groups of several listings can be added together.
|
|
141
|
+
*/
|
|
142
|
+
declare function summarizeTraces(traces: readonly TraceGroupSummary[]): TraceGroupSummary;
|
|
143
|
+
/**
|
|
144
|
+
* `traces` as the groups of `groupBy`, in the order their first traces come.
|
|
145
|
+
* Filter them first: this groups every trace it's given.
|
|
146
|
+
*/
|
|
147
|
+
declare function groupTraceSummaries(traces: TraceSummary[], groupBy: TraceGroupBy, timeZone?: string): TraceGroup[];
|
|
148
|
+
/**
|
|
149
|
+
* The groups of several listings as one: those with the same key added
|
|
150
|
+
* together, in the order their keys first come.
|
|
151
|
+
*/
|
|
152
|
+
declare function mergeTraceGroups(groups: TraceGroup[]): TraceGroup[];
|
|
153
|
+
/** The query parameters a {@link TraceQuery} or {@link TraceGroupQuery} travels as. */
|
|
154
|
+
declare function traceQueryParams(query: TraceQuery & Partial<TraceGroupQuery>): Record<string, string>;
|
|
155
|
+
/**
|
|
156
|
+
* The {@link TraceQuery} that {@link traceQueryParams} wrote into `params`.
|
|
157
|
+
* Throws a `TypeError` naming what's wrong with one that can't be read.
|
|
158
|
+
*/
|
|
159
|
+
declare function parseTraceQuery(params: Record<string, string | undefined>): TraceQuery;
|
|
160
|
+
/** {@link parseTraceQuery} for the parameters of a {@link TraceGroupQuery}. */
|
|
161
|
+
declare function parseTraceGroupQuery(params: Record<string, string | undefined>): TraceGroupQuery;
|
|
162
|
+
//#endregion
|
|
9
163
|
//#region src/client.d.ts
|
|
10
164
|
/** The Evalution Cloud API, unless `EVALUTION_API_URL` says otherwise. */
|
|
11
165
|
declare const DEFAULT_API_URL = "https://api.evalut.io";
|
|
@@ -57,13 +211,21 @@ interface ListTracesQuery {
|
|
|
57
211
|
to?: number;
|
|
58
212
|
/** At most this many, most recent first. */
|
|
59
213
|
limit?: number;
|
|
214
|
+
/** Only traces whose name or id contains this text. */
|
|
215
|
+
search?: string;
|
|
216
|
+
/** Only traces that meet every one of these. */
|
|
217
|
+
filters?: TraceFilter[];
|
|
60
218
|
}
|
|
219
|
+
/** Which traces {@link Evalution.listTraceGroups} groups, and how. */
|
|
220
|
+
interface ListTraceGroupsQuery extends Omit<ListTracesQuery, "limit">, Pick<TraceGroupQuery, "groupBy" | "timeZone"> {}
|
|
61
221
|
/** A project, as the cloud describes it. */
|
|
62
222
|
interface CloudProject {
|
|
63
223
|
/** The `prj_…` id committed in a repo's config. */
|
|
64
224
|
id: string;
|
|
65
225
|
slug: string;
|
|
66
226
|
name: string;
|
|
227
|
+
/** The slug of the organization it belongs to. */
|
|
228
|
+
orgSlug: string;
|
|
67
229
|
}
|
|
68
230
|
/** What {@link Evalution.whoami} resolves a key to. */
|
|
69
231
|
type CloudIdentity = {
|
|
@@ -90,21 +252,55 @@ interface InstanceRegistration {
|
|
|
90
252
|
/** The cloud's answer to {@link Evalution.registerInstance}. */
|
|
91
253
|
interface RegisteredInstance {
|
|
92
254
|
id: string;
|
|
93
|
-
/** The project's shareable
|
|
255
|
+
/** The project's shareable URL, `https://l.evalut.io/<org>/<project>`. */
|
|
94
256
|
projectUrl: string;
|
|
257
|
+
/**
|
|
258
|
+
* Where to open the browser on this machine: it remembers this instance
|
|
259
|
+
* there, so the project URL goes straight to it, then opens it.
|
|
260
|
+
*/
|
|
261
|
+
openUrl?: string;
|
|
95
262
|
}
|
|
96
263
|
/** Names one synced database of a project. */
|
|
97
264
|
interface SyncDatabaseKey {
|
|
98
265
|
kind: "traces" | "evals" | "dataset";
|
|
266
|
+
/**
|
|
267
|
+
* The id of the local provider whose database it is. Each provider of a
|
|
268
|
+
* kind syncs its own databases.
|
|
269
|
+
*/
|
|
270
|
+
provider: string;
|
|
99
271
|
/** The dataset's id, for `kind: "dataset"`. */
|
|
100
272
|
key?: string;
|
|
101
273
|
}
|
|
274
|
+
/** What {@link Evalution.getSyncDatabase} asks for. */
|
|
275
|
+
interface SyncDatabaseRequest extends SyncDatabaseKey {
|
|
276
|
+
/**
|
|
277
|
+
* How many migrations this client's schema for the kind has. The cloud
|
|
278
|
+
* refuses a client whose schema is older than one that synced before, with
|
|
279
|
+
* a 409.
|
|
280
|
+
*/
|
|
281
|
+
schemaVersion?: number;
|
|
282
|
+
/**
|
|
283
|
+
* Set when creating a dataset. Without it, the cloud answers 410 for a
|
|
284
|
+
* dataset that was deleted, so a machine that still has a copy can't
|
|
285
|
+
* bring it back. The 410's {@link EvalutionApiError.body} has the deleted
|
|
286
|
+
* database's `syncId`: a copy that synced with it should go too.
|
|
287
|
+
*/
|
|
288
|
+
create?: boolean;
|
|
289
|
+
}
|
|
102
290
|
/** Where a synced database lives, and a short-lived token for it. */
|
|
103
291
|
interface SyncDatabaseCredentials {
|
|
104
292
|
url: string;
|
|
105
293
|
token: string;
|
|
106
294
|
/** When {@link token} expires (ms). */
|
|
107
295
|
expiresAt: number;
|
|
296
|
+
/**
|
|
297
|
+
* Names this one cloud database: a new one, made after a dataset was
|
|
298
|
+
* deleted, say, has a new id. A local file records the ids it has synced
|
|
299
|
+
* with, so it can tell whether it holds this database's rows.
|
|
300
|
+
*/
|
|
301
|
+
syncId: string;
|
|
302
|
+
/** The linked project's id. */
|
|
303
|
+
projectId: string;
|
|
108
304
|
}
|
|
109
305
|
/**
|
|
110
306
|
* The client for Evalution's API. As a {@link TraceSink} it sends an app's
|
|
@@ -113,7 +309,7 @@ interface SyncDatabaseCredentials {
|
|
|
113
309
|
* Where traces go is worked out once, on the first one sent:
|
|
114
310
|
*
|
|
115
311
|
* 1. the {@link EvalutionOptions.endpoint} option;
|
|
116
|
-
* 2.
|
|
312
|
+
* 2. Evalution Cloud, when the API key is an ingest key;
|
|
117
313
|
* 3. the local instance `EVALUTION_ENDPOINT` names;
|
|
118
314
|
* 4. on Node, the instance portkit knows for this directory;
|
|
119
315
|
* 5. otherwise nowhere, and sending does nothing.
|
|
@@ -130,6 +326,8 @@ declare class Evalution implements TraceSink {
|
|
|
130
326
|
private readonly project?;
|
|
131
327
|
private readonly fetch;
|
|
132
328
|
private readonly sink;
|
|
329
|
+
/** The start of the API key, safe to show in messages, or `<none>`. */
|
|
330
|
+
get printableApiKey(): string;
|
|
133
331
|
constructor(options?: EvalutionOptions);
|
|
134
332
|
private resolveTraceTarget;
|
|
135
333
|
recordSpanStart(span: Span): Promise<Span>;
|
|
@@ -141,6 +339,11 @@ declare class Evalution implements TraceSink {
|
|
|
141
339
|
whoami(): Promise<CloudIdentity>;
|
|
142
340
|
/** The most recent production traces of the project. */
|
|
143
341
|
listTraces(query?: ListTracesQuery): Promise<TraceSummary[]>;
|
|
342
|
+
/**
|
|
343
|
+
* The project's production traces as groups, one per value or bucket of
|
|
344
|
+
* a column, counting every trace in the window rather than the most recent.
|
|
345
|
+
*/
|
|
346
|
+
listTraceGroups(query: ListTraceGroupsQuery): Promise<TraceGroup[]>;
|
|
144
347
|
/** One production trace with its spans, or `undefined` if there's none. */
|
|
145
348
|
getTrace(id: string): Promise<TraceWithSpans | undefined>;
|
|
146
349
|
/** The projects the key's user is a member of. */
|
|
@@ -157,11 +360,28 @@ declare class Evalution implements TraceSink {
|
|
|
157
360
|
/** Tells the cloud a registered instance has stopped. */
|
|
158
361
|
endInstanceSession(id: string): Promise<void>;
|
|
159
362
|
/** Ensures a synced database exists, and returns how to reach it. */
|
|
160
|
-
getSyncDatabase(
|
|
363
|
+
getSyncDatabase(request: SyncDatabaseRequest): Promise<SyncDatabaseCredentials>;
|
|
364
|
+
/** The project's synced databases. A dataset's has its id as the `key`; the others' `key` is empty. */
|
|
365
|
+
listSyncDatabases(): Promise<Required<SyncDatabaseKey>[]>;
|
|
366
|
+
/** Deletes a dataset's synced database, for every member's machine. */
|
|
367
|
+
deleteSyncDatabase(key: SyncDatabaseKey & {
|
|
368
|
+
kind: "dataset";
|
|
369
|
+
key: string;
|
|
370
|
+
}): Promise<void>;
|
|
161
371
|
private instanceRequest;
|
|
162
372
|
private json;
|
|
163
373
|
private request;
|
|
164
374
|
}
|
|
375
|
+
/** How {@link Evalution} rejects when the cloud refuses a call. */
|
|
376
|
+
declare class EvalutionApiError extends Error {
|
|
377
|
+
/** The HTTP status the cloud answered with. */
|
|
378
|
+
readonly status: number;
|
|
379
|
+
/** The cloud's explanation, if it gave one. */
|
|
380
|
+
readonly detail?: string;
|
|
381
|
+
/** The cloud's whole answer, when it was JSON: a 410's `syncId`, say. */
|
|
382
|
+
readonly body?: Record<string, unknown>;
|
|
383
|
+
constructor(status: number, detail?: string, body?: Record<string, unknown>);
|
|
384
|
+
}
|
|
165
385
|
//#endregion
|
|
166
386
|
//#region src/evaluation-answers.d.ts
|
|
167
387
|
/**
|
|
@@ -264,4 +484,4 @@ declare class HttpTraceSink implements TraceSink {
|
|
|
264
484
|
private send;
|
|
265
485
|
}
|
|
266
486
|
//#endregion
|
|
267
|
-
export { Annotation, AnnotationChanges, AnnotationEvent, AnnotationEventOp, AnnotationKind, AnnotationSource, ApiKeyKind, BaseTraceIngestor, CHECK_ATTRIBUTE, CloudIdentity, CloudProject, DEFAULT_API_URL, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, EvalCounts, Evalution, EvalutionOptions, HttpTraceSink, HttpTraceSinkOptions, HttpTraceTarget, InstanceRegistration, LLMSpanDetails, ListTracesQuery, PLAYGROUND_ENVIRONMENT, PLAYGROUND_RESOURCE, PROMPT_ID_ATTRIBUTE, PROMPT_INPUTS_ATTRIBUTE, PROMPT_NAME_ATTRIBUTE, PROMPT_PROVIDER_ID_ATTRIBUTE, PROMPT_VARIATION_ATTRIBUTE, PROMPT_VERSION_ATTRIBUTE, PromptID, PromptSpanInfo, PromptsFactory, PromptsHelper, PromptsHelperOptions, RegisteredInstance, SERVICE_NAME_ATTRIBUTE, SPAN_KIND_ATTRIBUTE, Span, SpanContentPart, SpanImagePart, SpanKind, SpanMessage, SpanMessageRole, SpanReasoningPart, SpanTextPart, SpanToolCallPart, SpanToolResultPart, SyncDatabaseCredentials, SyncDatabaseKey, TRACE_EVENTS_PATH, ToolSpanDetails, Trace, TraceChangeEvent, TraceChangeType, TraceEvalRun, TraceEventsRequest, TraceIngestor, TraceLiveEvent, TraceProviderInfo, TraceSink, TraceSinkEvent, TraceStreamEvent, TraceSummary, TraceWithSpans, answersWithConfidence, apiKeyKind, createTracerForPrompt, evaluationConfidence, getPromptSpanAttributes, isCheckSpan, isTraceSink, makeBrand, mergePromptIdentity, outputMessages, spanMessages, toolCallingOutput, unwrapToolResultOutput };
|
|
487
|
+
export { Annotation, AnnotationChanges, AnnotationEvent, AnnotationEventOp, AnnotationKind, AnnotationSource, ApiKeyKind, BaseTraceIngestor, CHECK_ATTRIBUTE, CloudIdentity, CloudProject, DEFAULT_API_URL, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, EvalCounts, Evalution, EvalutionApiError, EvalutionOptions, HttpTraceSink, HttpTraceSinkOptions, HttpTraceTarget, InstanceRegistration, LLMSpanDetails, ListTraceGroupsQuery, ListTracesQuery, PLAYGROUND_ENVIRONMENT, PLAYGROUND_RESOURCE, PROMPT_ID_ATTRIBUTE, PROMPT_INPUTS_ATTRIBUTE, PROMPT_NAME_ATTRIBUTE, PROMPT_PROVIDER_ID_ATTRIBUTE, PROMPT_VARIATION_ATTRIBUTE, PROMPT_VERSION_ATTRIBUTE, PromptID, PromptSpanInfo, PromptsFactory, PromptsHelper, PromptsHelperOptions, RegisteredInstance, SERVICE_NAME_ATTRIBUTE, SPAN_KIND_ATTRIBUTE, Span, SpanContentPart, SpanImagePart, SpanKind, SpanMessage, SpanMessageRole, SpanReasoningPart, SpanTextPart, SpanToolCallPart, SpanToolResultPart, SyncDatabaseCredentials, SyncDatabaseKey, SyncDatabaseRequest, TRACE_BUCKET_BOUNDS, TRACE_EVENTS_PATH, ToolSpanDetails, Trace, TraceChangeEvent, TraceChangeType, TraceDateResolution, TraceEvalRun, TraceEventsRequest, TraceFilter, TraceGroup, TraceGroupBy, TraceGroupQuery, TraceGroupSummary, TraceIngestor, TraceLiveEvent, TraceNumberColumn, TraceProviderInfo, TraceQuery, TraceQueryColumn, TraceSink, TraceSinkEvent, TraceStreamEvent, TraceSummary, TraceTextColumn, TraceWithSpans, answersWithConfidence, apiKeyKind, createTracerForPrompt, evaluationConfidence, filterTraces, getPromptSpanAttributes, groupTraceSummaries, isCheckSpan, isEmptyTraceQuery, isTraceNumberColumn, isTraceSink, isTraceTextColumn, makeBrand, matchesTraceFilter, matchesTraceQuery, mergePromptIdentity, mergeTraceGroups, outputMessages, parseTraceGroupQuery, parseTraceQuery, spanMessages, summarizeTraces, toolCallingOutput, traceBucketAt, traceBucketFilter, traceDateBucket, traceGroupFilter, traceGroupKey, traceNumberValue, traceQueryParams, traceTextValue, unwrapToolResultOutput };
|
package/dist/index.js
CHANGED
|
@@ -115,9 +115,437 @@ var HttpTraceSink = class {
|
|
|
115
115
|
}
|
|
116
116
|
};
|
|
117
117
|
//#endregion
|
|
118
|
+
//#region src/trace-query.ts
|
|
119
|
+
const TEXT_COLUMNS = [
|
|
120
|
+
"name",
|
|
121
|
+
"status",
|
|
122
|
+
"model",
|
|
123
|
+
"environment",
|
|
124
|
+
"annotations",
|
|
125
|
+
"checks",
|
|
126
|
+
"version",
|
|
127
|
+
"variation"
|
|
128
|
+
];
|
|
129
|
+
const NUMBER_COLUMNS = [
|
|
130
|
+
"startTime",
|
|
131
|
+
"spanCount",
|
|
132
|
+
"duration",
|
|
133
|
+
"totalTokens",
|
|
134
|
+
"cost"
|
|
135
|
+
];
|
|
136
|
+
/** Whether `column` names a {@link TraceTextColumn}. */
|
|
137
|
+
function isTraceTextColumn(column) {
|
|
138
|
+
return TEXT_COLUMNS.includes(column);
|
|
139
|
+
}
|
|
140
|
+
/** Whether `column` names a {@link TraceNumberColumn}. */
|
|
141
|
+
function isTraceNumberColumn(column) {
|
|
142
|
+
return NUMBER_COLUMNS.includes(column);
|
|
143
|
+
}
|
|
144
|
+
const DATE_RESOLUTIONS = [
|
|
145
|
+
"hour",
|
|
146
|
+
"day",
|
|
147
|
+
"week",
|
|
148
|
+
"month"
|
|
149
|
+
];
|
|
150
|
+
/**
|
|
151
|
+
* The lower bounds of every bucket but the first, for the number columns
|
|
152
|
+
* grouped into buckets: `[1000, 5000]` makes the buckets `< 1000`,
|
|
153
|
+
* `1000 – 5000` and `≥ 5000`. Durations are in ms and costs in dollars.
|
|
154
|
+
*/
|
|
155
|
+
const TRACE_BUCKET_BOUNDS = {
|
|
156
|
+
spanCount: [
|
|
157
|
+
2,
|
|
158
|
+
6,
|
|
159
|
+
21,
|
|
160
|
+
101
|
|
161
|
+
],
|
|
162
|
+
duration: [
|
|
163
|
+
1e3,
|
|
164
|
+
5e3,
|
|
165
|
+
15e3,
|
|
166
|
+
6e4,
|
|
167
|
+
3e5
|
|
168
|
+
],
|
|
169
|
+
totalTokens: [
|
|
170
|
+
1e3,
|
|
171
|
+
1e4,
|
|
172
|
+
1e5,
|
|
173
|
+
1e6
|
|
174
|
+
],
|
|
175
|
+
cost: [
|
|
176
|
+
.001,
|
|
177
|
+
.01,
|
|
178
|
+
.1,
|
|
179
|
+
1,
|
|
180
|
+
10
|
|
181
|
+
]
|
|
182
|
+
};
|
|
183
|
+
/** A trace's value in a text column, or `null` if it has none. */
|
|
184
|
+
function traceTextValue(trace, column) {
|
|
185
|
+
switch (column) {
|
|
186
|
+
case "name": return trace.name;
|
|
187
|
+
case "status": return trace.status;
|
|
188
|
+
case "model": return trace.model ?? null;
|
|
189
|
+
case "environment": return trace.environment ?? null;
|
|
190
|
+
case "version": return trace.promptVersion ?? null;
|
|
191
|
+
case "variation": return trace.promptVariation ?? null;
|
|
192
|
+
case "annotations": {
|
|
193
|
+
const counts = trace.annotationCounts;
|
|
194
|
+
return counts.issue > 0 ? "issue" : counts.good > 0 ? "good" : counts.note > 0 ? "note" : null;
|
|
195
|
+
}
|
|
196
|
+
case "checks": {
|
|
197
|
+
const counts = trace.checkCounts;
|
|
198
|
+
if (!counts) return null;
|
|
199
|
+
return [
|
|
200
|
+
"fail",
|
|
201
|
+
"error",
|
|
202
|
+
"pass",
|
|
203
|
+
"scored",
|
|
204
|
+
"skipped"
|
|
205
|
+
].find((outcome) => counts[outcome] > 0) ?? null;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
/** A trace's value in a number column, or `undefined` if it has none. */
|
|
210
|
+
function traceNumberValue(trace, column) {
|
|
211
|
+
switch (column) {
|
|
212
|
+
case "startTime": return trace.startTime;
|
|
213
|
+
case "spanCount": return trace.spanCount;
|
|
214
|
+
case "duration": return trace.endTime !== void 0 ? trace.endTime - trace.startTime : void 0;
|
|
215
|
+
case "totalTokens": return trace.totalTokens;
|
|
216
|
+
case "cost": return trace.cost;
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
/** Whether `trace` meets `filter`. */
|
|
220
|
+
function matchesTraceFilter(trace, filter) {
|
|
221
|
+
switch (filter.op) {
|
|
222
|
+
case "in": return filter.values.includes(traceTextValue(trace, filter.column));
|
|
223
|
+
case "notIn": return !filter.values.includes(traceTextValue(trace, filter.column));
|
|
224
|
+
case "unset": return traceNumberValue(trace, filter.column) === void 0;
|
|
225
|
+
case "range": {
|
|
226
|
+
const value = traceNumberValue(trace, filter.column);
|
|
227
|
+
return value !== void 0 && (filter.min === void 0 || value >= filter.min) && (filter.max === void 0 || value < filter.max);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
/** Whether `trace` is one `query` lists. */
|
|
232
|
+
function matchesTraceQuery(trace, query) {
|
|
233
|
+
const search = query.search?.trim().toLowerCase();
|
|
234
|
+
if (search && !trace.name.toLowerCase().includes(search) && !trace.id.toLowerCase().includes(search)) return false;
|
|
235
|
+
return (query.filters ?? []).every((filter) => matchesTraceFilter(trace, filter));
|
|
236
|
+
}
|
|
237
|
+
/** Whether `query` lists every trace. */
|
|
238
|
+
function isEmptyTraceQuery(query) {
|
|
239
|
+
return !query?.search?.trim() && !query?.filters?.length;
|
|
240
|
+
}
|
|
241
|
+
/** The traces of `traces` that `query` lists, in the same order. */
|
|
242
|
+
function filterTraces(traces, query) {
|
|
243
|
+
if (!query || isEmptyTraceQuery(query)) return traces;
|
|
244
|
+
return traces.filter((trace) => matchesTraceQuery(trace, query));
|
|
245
|
+
}
|
|
246
|
+
const HOUR_MS = 36e5;
|
|
247
|
+
/** Each time zone's wall-clock formatter: slow to make, so made once. */
|
|
248
|
+
const wallClocks = /* @__PURE__ */ new Map();
|
|
249
|
+
function wallClock(timeZone) {
|
|
250
|
+
let format = wallClocks.get(timeZone);
|
|
251
|
+
if (!format) {
|
|
252
|
+
format = new Intl.DateTimeFormat("en-US", {
|
|
253
|
+
timeZone,
|
|
254
|
+
hourCycle: "h23",
|
|
255
|
+
year: "numeric",
|
|
256
|
+
month: "numeric",
|
|
257
|
+
day: "numeric",
|
|
258
|
+
hour: "numeric",
|
|
259
|
+
minute: "numeric",
|
|
260
|
+
second: "numeric"
|
|
261
|
+
});
|
|
262
|
+
wallClocks.set(timeZone, format);
|
|
263
|
+
}
|
|
264
|
+
return format;
|
|
265
|
+
}
|
|
266
|
+
/** How far `timeZone`'s wall clock is ahead of UTC at `ms`, in ms. */
|
|
267
|
+
function zoneOffset(ms, timeZone) {
|
|
268
|
+
if (timeZone === "UTC") return 0;
|
|
269
|
+
const parts = Object.fromEntries(wallClock(timeZone).formatToParts(ms).map((part) => [part.type, Number(part.value)]));
|
|
270
|
+
return Date.UTC(parts.year, parts.month - 1, parts.day, parts.hour, parts.minute, parts.second) - Math.floor(ms / 1e3) * 1e3;
|
|
271
|
+
}
|
|
272
|
+
/** The instant `timeZone`'s wall clock reads `wall` (a time written as if UTC). */
|
|
273
|
+
function fromWallTime(wall, timeZone) {
|
|
274
|
+
return wall - zoneOffset(wall - zoneOffset(wall, timeZone), timeZone);
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* The `[min, max)` of the hour, day, week or month `ms` falls in, on
|
|
278
|
+
* `timeZone`'s clock.
|
|
279
|
+
*/
|
|
280
|
+
function traceDateBucket(ms, resolution = "day", timeZone = "UTC") {
|
|
281
|
+
const offset = zoneOffset(ms, timeZone);
|
|
282
|
+
if (resolution === "hour") {
|
|
283
|
+
const min = Math.floor((ms + offset) / HOUR_MS) * HOUR_MS - offset;
|
|
284
|
+
return {
|
|
285
|
+
min,
|
|
286
|
+
max: min + HOUR_MS
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
const wall = new Date(ms + offset);
|
|
290
|
+
const year = wall.getUTCFullYear();
|
|
291
|
+
const month = wall.getUTCMonth();
|
|
292
|
+
const day = wall.getUTCDate();
|
|
293
|
+
let from;
|
|
294
|
+
let to;
|
|
295
|
+
if (resolution === "month") {
|
|
296
|
+
from = Date.UTC(year, month, 1);
|
|
297
|
+
to = Date.UTC(year, month + 1, 1);
|
|
298
|
+
} else if (resolution === "week") {
|
|
299
|
+
const sinceMonday = (wall.getUTCDay() + 6) % 7;
|
|
300
|
+
from = Date.UTC(year, month, day - sinceMonday);
|
|
301
|
+
to = Date.UTC(year, month, day - sinceMonday + 7);
|
|
302
|
+
} else {
|
|
303
|
+
from = Date.UTC(year, month, day);
|
|
304
|
+
to = Date.UTC(year, month, day + 1);
|
|
305
|
+
}
|
|
306
|
+
return {
|
|
307
|
+
min: fromWallTime(from, timeZone),
|
|
308
|
+
max: fromWallTime(to, timeZone)
|
|
309
|
+
};
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* The bucket of `column` that `value` falls in, as the filter listing it:
|
|
313
|
+
* see {@link TRACE_BUCKET_BOUNDS}.
|
|
314
|
+
*/
|
|
315
|
+
function traceBucketFilter(column, value) {
|
|
316
|
+
if (value === void 0) return {
|
|
317
|
+
column,
|
|
318
|
+
op: "unset"
|
|
319
|
+
};
|
|
320
|
+
const bounds = TRACE_BUCKET_BOUNDS[column];
|
|
321
|
+
return traceBucketAt(column, bounds.filter((bound) => value >= bound).length);
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* The `index`th bucket of `column`, counting from the one below every bound
|
|
325
|
+
* of {@link TRACE_BUCKET_BOUNDS}, as the filter listing it.
|
|
326
|
+
*/
|
|
327
|
+
function traceBucketAt(column, index) {
|
|
328
|
+
const bounds = TRACE_BUCKET_BOUNDS[column];
|
|
329
|
+
const min = index > 0 ? bounds[index - 1] : void 0;
|
|
330
|
+
const max = index < bounds.length ? bounds[index] : void 0;
|
|
331
|
+
return {
|
|
332
|
+
column,
|
|
333
|
+
op: "range",
|
|
334
|
+
...min !== void 0 && { min },
|
|
335
|
+
...max !== void 0 && { max }
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
/** The filter listing the group of `groupBy` that `trace` belongs to. */
|
|
339
|
+
function traceGroupFilter(trace, groupBy, timeZone) {
|
|
340
|
+
const { column } = groupBy;
|
|
341
|
+
if (isTraceTextColumn(column)) return {
|
|
342
|
+
column,
|
|
343
|
+
op: "in",
|
|
344
|
+
values: [traceTextValue(trace, column)]
|
|
345
|
+
};
|
|
346
|
+
if (column === "startTime") return {
|
|
347
|
+
column,
|
|
348
|
+
op: "range",
|
|
349
|
+
...traceDateBucket(trace.startTime, groupBy.resolution, timeZone)
|
|
350
|
+
};
|
|
351
|
+
return traceBucketFilter(column, traceNumberValue(trace, column));
|
|
352
|
+
}
|
|
353
|
+
/** {@link TraceGroup.key} for the group `filter` lists. */
|
|
354
|
+
function traceGroupKey(filter) {
|
|
355
|
+
switch (filter.op) {
|
|
356
|
+
case "in":
|
|
357
|
+
case "notIn": return `${filter.column}:${filter.op}:${JSON.stringify(filter.values)}`;
|
|
358
|
+
case "unset": return `${filter.column}:unset`;
|
|
359
|
+
case "range": return `${filter.column}:range:${filter.min ?? ""}:${filter.max ?? ""}`;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
/** The one value every item agrees on, else `undefined`. */
|
|
363
|
+
function shared(values) {
|
|
364
|
+
const first = values[0];
|
|
365
|
+
return values.every((v) => v === first) ? first : void 0;
|
|
366
|
+
}
|
|
367
|
+
/** The sum of the values that are set; `undefined` when none is. */
|
|
368
|
+
function sum(values) {
|
|
369
|
+
const set = values.filter((v) => v !== void 0);
|
|
370
|
+
return set.length > 0 ? set.reduce((a, b) => a + b, 0) : void 0;
|
|
371
|
+
}
|
|
372
|
+
/** `counts` added up; `undefined` when none is set. */
|
|
373
|
+
function sumCounts(counts) {
|
|
374
|
+
const set = counts.filter((c) => c !== void 0);
|
|
375
|
+
if (set.length === 0) return void 0;
|
|
376
|
+
const total = {
|
|
377
|
+
pass: 0,
|
|
378
|
+
fail: 0,
|
|
379
|
+
error: 0,
|
|
380
|
+
skipped: 0,
|
|
381
|
+
scored: 0
|
|
382
|
+
};
|
|
383
|
+
for (const c of set) for (const outcome of Object.keys(total)) total[outcome] += c[outcome];
|
|
384
|
+
return total;
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* What `traces` (non-empty) add up to. Takes summaries of groups as well as
|
|
388
|
+
* traces, so the groups of several listings can be added together.
|
|
389
|
+
*/
|
|
390
|
+
function summarizeTraces(traces) {
|
|
391
|
+
const running = traces.some((t) => t.status === "running");
|
|
392
|
+
const totalTokens = sum(traces.map((t) => t.totalTokens));
|
|
393
|
+
const cost = sum(traces.map((t) => t.cost));
|
|
394
|
+
const checkCounts = sumCounts(traces.map((t) => t.checkCounts));
|
|
395
|
+
const model = shared(traces.map((t) => t.model));
|
|
396
|
+
const environment = shared(traces.map((t) => t.environment));
|
|
397
|
+
const promptVersion = shared(traces.map((t) => t.promptVersion));
|
|
398
|
+
const promptVariation = shared(traces.map((t) => t.promptVariation));
|
|
399
|
+
return {
|
|
400
|
+
startTime: Math.min(...traces.map((t) => t.startTime)),
|
|
401
|
+
...!running && { endTime: Math.max(...traces.map((t) => t.endTime ?? t.startTime)) },
|
|
402
|
+
status: running ? "running" : traces.some((t) => t.status === "error") ? "error" : "ok",
|
|
403
|
+
spanCount: traces.reduce((n, t) => n + t.spanCount, 0),
|
|
404
|
+
...totalTokens !== void 0 && { totalTokens },
|
|
405
|
+
...cost !== void 0 && { cost },
|
|
406
|
+
...checkCounts && { checkCounts },
|
|
407
|
+
...model !== void 0 && { model },
|
|
408
|
+
...environment !== void 0 && { environment },
|
|
409
|
+
...promptVersion !== void 0 && { promptVersion },
|
|
410
|
+
...promptVariation !== void 0 && { promptVariation },
|
|
411
|
+
annotationCounts: {
|
|
412
|
+
issue: traces.reduce((n, t) => n + t.annotationCounts.issue, 0),
|
|
413
|
+
good: traces.reduce((n, t) => n + t.annotationCounts.good, 0),
|
|
414
|
+
note: traces.reduce((n, t) => n + t.annotationCounts.note, 0)
|
|
415
|
+
}
|
|
416
|
+
};
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* `traces` as the groups of `groupBy`, in the order their first traces come.
|
|
420
|
+
* Filter them first: this groups every trace it's given.
|
|
421
|
+
*/
|
|
422
|
+
function groupTraceSummaries(traces, groupBy, timeZone) {
|
|
423
|
+
const members = /* @__PURE__ */ new Map();
|
|
424
|
+
for (const trace of traces) {
|
|
425
|
+
const filter = traceGroupFilter(trace, groupBy, timeZone);
|
|
426
|
+
const key = traceGroupKey(filter);
|
|
427
|
+
const group = members.get(key);
|
|
428
|
+
if (group) group.traces.push(trace);
|
|
429
|
+
else members.set(key, {
|
|
430
|
+
filter,
|
|
431
|
+
traces: [trace]
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
return Array.from(members, ([key, group]) => ({
|
|
435
|
+
key,
|
|
436
|
+
filter: group.filter,
|
|
437
|
+
count: group.traces.length,
|
|
438
|
+
summary: summarizeTraces(group.traces)
|
|
439
|
+
}));
|
|
440
|
+
}
|
|
441
|
+
/**
|
|
442
|
+
* The groups of several listings as one: those with the same key added
|
|
443
|
+
* together, in the order their keys first come.
|
|
444
|
+
*/
|
|
445
|
+
function mergeTraceGroups(groups) {
|
|
446
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
447
|
+
for (const group of groups) {
|
|
448
|
+
const same = byKey.get(group.key);
|
|
449
|
+
if (same) same.push(group);
|
|
450
|
+
else byKey.set(group.key, [group]);
|
|
451
|
+
}
|
|
452
|
+
return Array.from(byKey.values(), (same) => same.length === 1 ? same[0] : {
|
|
453
|
+
key: same[0].key,
|
|
454
|
+
filter: same[0].filter,
|
|
455
|
+
count: same.reduce((n, group) => n + group.count, 0),
|
|
456
|
+
summary: summarizeTraces(same.map((group) => group.summary))
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
const optionalNumber = (value) => value === void 0 || typeof value === "number" && Number.isFinite(value);
|
|
460
|
+
/** `value` as a {@link TraceFilter}. Throws a `TypeError` if it isn't one. */
|
|
461
|
+
function parseTraceFilter(value) {
|
|
462
|
+
const filter = value;
|
|
463
|
+
if (filter && typeof filter === "object") {
|
|
464
|
+
const { column, op } = filter;
|
|
465
|
+
if (op === "in" || op === "notIn") {
|
|
466
|
+
const { values } = filter;
|
|
467
|
+
if (isTraceTextColumn(column) && Array.isArray(values) && values.every((v) => v === null || typeof v === "string")) return {
|
|
468
|
+
column,
|
|
469
|
+
op,
|
|
470
|
+
values
|
|
471
|
+
};
|
|
472
|
+
} else if (op === "unset") {
|
|
473
|
+
if (isTraceNumberColumn(column)) return {
|
|
474
|
+
column,
|
|
475
|
+
op
|
|
476
|
+
};
|
|
477
|
+
} else if (op === "range") {
|
|
478
|
+
const { min, max } = filter;
|
|
479
|
+
if (isTraceNumberColumn(column) && optionalNumber(min) && optionalNumber(max)) return {
|
|
480
|
+
column,
|
|
481
|
+
op,
|
|
482
|
+
...min !== void 0 && { min },
|
|
483
|
+
...max !== void 0 && { max }
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
throw new TypeError(`Not a trace filter: ${JSON.stringify(value)}`);
|
|
488
|
+
}
|
|
489
|
+
/** The query parameters a {@link TraceQuery} or {@link TraceGroupQuery} travels as. */
|
|
490
|
+
function traceQueryParams(query) {
|
|
491
|
+
const search = query.search?.trim();
|
|
492
|
+
return {
|
|
493
|
+
...search && { q: search },
|
|
494
|
+
...query.filters?.length && { filters: JSON.stringify(query.filters) },
|
|
495
|
+
...query.groupBy && {
|
|
496
|
+
groupBy: query.groupBy.column,
|
|
497
|
+
...query.groupBy.resolution && { resolution: query.groupBy.resolution }
|
|
498
|
+
},
|
|
499
|
+
...query.timeZone && { tz: query.timeZone }
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* The {@link TraceQuery} that {@link traceQueryParams} wrote into `params`.
|
|
504
|
+
* Throws a `TypeError` naming what's wrong with one that can't be read.
|
|
505
|
+
*/
|
|
506
|
+
function parseTraceQuery(params) {
|
|
507
|
+
const search = params.q?.trim();
|
|
508
|
+
let filters;
|
|
509
|
+
if (params.filters) {
|
|
510
|
+
let parsed;
|
|
511
|
+
try {
|
|
512
|
+
parsed = JSON.parse(params.filters);
|
|
513
|
+
} catch {
|
|
514
|
+
throw new TypeError("`filters` isn't JSON.");
|
|
515
|
+
}
|
|
516
|
+
if (!Array.isArray(parsed)) throw new TypeError("`filters` isn't an array.");
|
|
517
|
+
filters = parsed.map(parseTraceFilter);
|
|
518
|
+
}
|
|
519
|
+
return {
|
|
520
|
+
...search && { search },
|
|
521
|
+
...filters?.length && { filters }
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
/** {@link parseTraceQuery} for the parameters of a {@link TraceGroupQuery}. */
|
|
525
|
+
function parseTraceGroupQuery(params) {
|
|
526
|
+
const { groupBy: column, resolution, tz } = params;
|
|
527
|
+
if (!isTraceTextColumn(column) && !isTraceNumberColumn(column)) throw new TypeError("`groupBy` isn't a trace column.");
|
|
528
|
+
if (resolution !== void 0 && !DATE_RESOLUTIONS.includes(resolution)) throw new TypeError("`resolution` isn't hour, day, week or month.");
|
|
529
|
+
if (tz !== void 0) try {
|
|
530
|
+
new Intl.DateTimeFormat("en-US", { timeZone: tz });
|
|
531
|
+
} catch {
|
|
532
|
+
throw new TypeError("`tz` isn't a time zone.");
|
|
533
|
+
}
|
|
534
|
+
return {
|
|
535
|
+
...parseTraceQuery(params),
|
|
536
|
+
groupBy: {
|
|
537
|
+
column,
|
|
538
|
+
...resolution && { resolution }
|
|
539
|
+
},
|
|
540
|
+
...tz && { timeZone: tz }
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
//#endregion
|
|
118
544
|
//#region src/client.ts
|
|
119
545
|
/** The Evalution Cloud API, unless `EVALUTION_API_URL` says otherwise. */
|
|
120
546
|
const DEFAULT_API_URL = "https://api.evalut.io";
|
|
547
|
+
/** How much of an API key is safe to print: its kind and a few characters. */
|
|
548
|
+
const PRINTABLE_API_KEY_PREFIX = 15;
|
|
121
549
|
/**
|
|
122
550
|
* The kind of an API key, read from its prefix, or `undefined` for a string
|
|
123
551
|
* that isn't one.
|
|
@@ -126,6 +554,18 @@ function apiKeyKind(key) {
|
|
|
126
554
|
if (key.startsWith("evl_ingest_")) return "ingest";
|
|
127
555
|
if (key.startsWith("evl_user_")) return "personal";
|
|
128
556
|
}
|
|
557
|
+
/** A listing's query as a URL's search, `?` included; empty if it has none. */
|
|
558
|
+
function listParams(query) {
|
|
559
|
+
const { search, filters, groupBy, timeZone, ...window } = query;
|
|
560
|
+
const params = new URLSearchParams(traceQueryParams({
|
|
561
|
+
search,
|
|
562
|
+
filters,
|
|
563
|
+
groupBy,
|
|
564
|
+
timeZone
|
|
565
|
+
}));
|
|
566
|
+
for (const [name, value] of Object.entries(window)) if (value !== void 0) params.set(name, String(value));
|
|
567
|
+
return params.size > 0 ? `?${params}` : "";
|
|
568
|
+
}
|
|
129
569
|
/**
|
|
130
570
|
* The client for Evalution's API. As a {@link TraceSink} it sends an app's
|
|
131
571
|
* traces, and it makes the read and control calls the local server needs.
|
|
@@ -133,7 +573,7 @@ function apiKeyKind(key) {
|
|
|
133
573
|
* Where traces go is worked out once, on the first one sent:
|
|
134
574
|
*
|
|
135
575
|
* 1. the {@link EvalutionOptions.endpoint} option;
|
|
136
|
-
* 2.
|
|
576
|
+
* 2. Evalution Cloud, when the API key is an ingest key;
|
|
137
577
|
* 3. the local instance `EVALUTION_ENDPOINT` names;
|
|
138
578
|
* 4. on Node, the instance portkit knows for this directory;
|
|
139
579
|
* 5. otherwise nowhere, and sending does nothing.
|
|
@@ -150,6 +590,10 @@ var Evalution = class {
|
|
|
150
590
|
project;
|
|
151
591
|
fetch;
|
|
152
592
|
sink;
|
|
593
|
+
/** The start of the API key, safe to show in messages, or `<none>`. */
|
|
594
|
+
get printableApiKey() {
|
|
595
|
+
return this.apiKey ? `${this.apiKey.slice(0, PRINTABLE_API_KEY_PREFIX)}…` : "<none>";
|
|
596
|
+
}
|
|
153
597
|
constructor(options = {}) {
|
|
154
598
|
const env = globalThis.process?.env ?? {};
|
|
155
599
|
this.endpoint = options.endpoint;
|
|
@@ -196,10 +640,14 @@ var Evalution = class {
|
|
|
196
640
|
}
|
|
197
641
|
/** The most recent production traces of the project. */
|
|
198
642
|
listTraces(query = {}) {
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
643
|
+
return this.json("GET", `/api/traces${listParams(query)}`, { project: true });
|
|
644
|
+
}
|
|
645
|
+
/**
|
|
646
|
+
* The project's production traces as groups, one per value or bucket of
|
|
647
|
+
* a column, counting every trace in the window rather than the most recent.
|
|
648
|
+
*/
|
|
649
|
+
listTraceGroups(query) {
|
|
650
|
+
return this.json("GET", `/api/traces/groups${listParams(query)}`, { project: true });
|
|
203
651
|
}
|
|
204
652
|
/** One production trace with its spans, or `undefined` if there's none. */
|
|
205
653
|
async getTrace(id) {
|
|
@@ -231,12 +679,21 @@ var Evalution = class {
|
|
|
231
679
|
await readOk(await this.instanceRequest("DELETE", id, "session"));
|
|
232
680
|
}
|
|
233
681
|
/** Ensures a synced database exists, and returns how to reach it. */
|
|
234
|
-
getSyncDatabase(
|
|
682
|
+
getSyncDatabase(request) {
|
|
235
683
|
return this.json("POST", "/cloud/v1/sync/databases", {
|
|
236
684
|
project: true,
|
|
237
|
-
body:
|
|
685
|
+
body: request
|
|
238
686
|
});
|
|
239
687
|
}
|
|
688
|
+
/** The project's synced databases. A dataset's has its id as the `key`; the others' `key` is empty. */
|
|
689
|
+
listSyncDatabases() {
|
|
690
|
+
return this.json("GET", "/cloud/v1/sync/databases", { project: true });
|
|
691
|
+
}
|
|
692
|
+
/** Deletes a dataset's synced database, for every member's machine. */
|
|
693
|
+
async deleteSyncDatabase(key) {
|
|
694
|
+
const path = `/cloud/v1/sync/databases/dataset/${encodeURIComponent(key.provider)}/${encodeURIComponent(key.key)}`;
|
|
695
|
+
await readOk(await this.request("DELETE", path, { project: true }));
|
|
696
|
+
}
|
|
240
697
|
instanceRequest(method, id, action) {
|
|
241
698
|
const path = `/cloud/v1/instances/${encodeURIComponent(id)}/${action}`;
|
|
242
699
|
return this.request(method, path, { project: true });
|
|
@@ -259,11 +716,36 @@ var Evalution = class {
|
|
|
259
716
|
});
|
|
260
717
|
}
|
|
261
718
|
};
|
|
719
|
+
/** How {@link Evalution} rejects when the cloud refuses a call. */
|
|
720
|
+
var EvalutionApiError = class extends Error {
|
|
721
|
+
/** The HTTP status the cloud answered with. */
|
|
722
|
+
status;
|
|
723
|
+
/** The cloud's explanation, if it gave one. */
|
|
724
|
+
detail;
|
|
725
|
+
/** The cloud's whole answer, when it was JSON: a 410's `syncId`, say. */
|
|
726
|
+
body;
|
|
727
|
+
constructor(status, detail, body) {
|
|
728
|
+
super(`Evalution API: ${status} ${detail ?? ""}`.trim());
|
|
729
|
+
this.name = "EvalutionApiError";
|
|
730
|
+
this.status = status;
|
|
731
|
+
this.detail = detail;
|
|
732
|
+
this.body = body;
|
|
733
|
+
}
|
|
734
|
+
};
|
|
262
735
|
/** `res`, or a rejection saying why the cloud refused. */
|
|
263
736
|
async function readOk(res) {
|
|
264
737
|
if (res.ok) return res;
|
|
265
|
-
const
|
|
266
|
-
|
|
738
|
+
const text = await res.text().catch(() => "");
|
|
739
|
+
let detail = text || void 0;
|
|
740
|
+
let body;
|
|
741
|
+
try {
|
|
742
|
+
const parsed = JSON.parse(text);
|
|
743
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
744
|
+
body = parsed;
|
|
745
|
+
if (typeof body.error === "string") detail = body.error;
|
|
746
|
+
}
|
|
747
|
+
} catch {}
|
|
748
|
+
throw new EvalutionApiError(res.status, detail, body);
|
|
267
749
|
}
|
|
268
750
|
//#endregion
|
|
269
751
|
//#region src/evaluation-answers.ts
|
|
@@ -585,4 +1067,4 @@ function isTraceSink(o) {
|
|
|
585
1067
|
return typeof o === "object" && o !== null && typeof o.recordSpanStart === "function" && typeof o.recordSpanEnd === "function" && typeof o.failTrace === "function";
|
|
586
1068
|
}
|
|
587
1069
|
//#endregion
|
|
588
|
-
export { BaseTraceIngestor, CHECK_ATTRIBUTE, DEFAULT_API_URL, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, Evalution, HttpTraceSink, PLAYGROUND_ENVIRONMENT, PLAYGROUND_RESOURCE, PROMPT_ID_ATTRIBUTE, PROMPT_INPUTS_ATTRIBUTE, PROMPT_NAME_ATTRIBUTE, PROMPT_PROVIDER_ID_ATTRIBUTE, PROMPT_VARIATION_ATTRIBUTE, PROMPT_VERSION_ATTRIBUTE, SERVICE_NAME_ATTRIBUTE, SPAN_KIND_ATTRIBUTE, TRACE_EVENTS_PATH, answersWithConfidence, apiKeyKind, createTracerForPrompt, evaluationConfidence, getPromptSpanAttributes, isCheckSpan, isTraceSink, makeBrand, mergePromptIdentity, outputMessages, spanMessages, toolCallingOutput, unwrapToolResultOutput };
|
|
1070
|
+
export { BaseTraceIngestor, CHECK_ATTRIBUTE, DEFAULT_API_URL, DEPLOYMENT_ENVIRONMENT_ATTRIBUTE, Evalution, EvalutionApiError, HttpTraceSink, PLAYGROUND_ENVIRONMENT, PLAYGROUND_RESOURCE, PROMPT_ID_ATTRIBUTE, PROMPT_INPUTS_ATTRIBUTE, PROMPT_NAME_ATTRIBUTE, PROMPT_PROVIDER_ID_ATTRIBUTE, PROMPT_VARIATION_ATTRIBUTE, PROMPT_VERSION_ATTRIBUTE, SERVICE_NAME_ATTRIBUTE, SPAN_KIND_ATTRIBUTE, TRACE_BUCKET_BOUNDS, TRACE_EVENTS_PATH, answersWithConfidence, apiKeyKind, createTracerForPrompt, evaluationConfidence, filterTraces, getPromptSpanAttributes, groupTraceSummaries, isCheckSpan, isEmptyTraceQuery, isTraceNumberColumn, isTraceSink, isTraceTextColumn, makeBrand, matchesTraceFilter, matchesTraceQuery, mergePromptIdentity, mergeTraceGroups, outputMessages, parseTraceGroupQuery, parseTraceQuery, spanMessages, summarizeTraces, toolCallingOutput, traceBucketAt, traceBucketFilter, traceDateBucket, traceGroupFilter, traceGroupKey, traceNumberValue, traceQueryParams, traceTextValue, unwrapToolResultOutput };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@evalution/sdk",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.1",
|
|
4
4
|
"description": "The base of Evalution's SDK packages: the trace model and API client",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"files": [
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
],
|
|
9
9
|
"scripts": {
|
|
10
10
|
"build": "tsdown",
|
|
11
|
-
"prepare": "
|
|
11
|
+
"prepare": "pnpm run build",
|
|
12
12
|
"typecheck": "tsc --noEmit"
|
|
13
13
|
},
|
|
14
14
|
"exports": {
|
|
@@ -43,5 +43,17 @@
|
|
|
43
43
|
"ai",
|
|
44
44
|
"prompt"
|
|
45
45
|
],
|
|
46
|
-
"license": "MIT"
|
|
46
|
+
"license": "MIT",
|
|
47
|
+
"publishConfig": {
|
|
48
|
+
"exports": {
|
|
49
|
+
".": {
|
|
50
|
+
"types": "./dist/index.d.ts",
|
|
51
|
+
"import": "./dist/index.js"
|
|
52
|
+
},
|
|
53
|
+
"./adapter": {
|
|
54
|
+
"types": "./dist/adapter.d.ts",
|
|
55
|
+
"import": "./dist/adapter.js"
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
47
59
|
}
|