@evalution/sdk 0.0.0-stage → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/adapter.d.ts +764 -0
- package/dist/adapter.js +53 -0
- package/dist/index.d.ts +487 -0
- package/dist/index.js +1070 -0
- package/dist/portkit-DV-XcagM.js +84 -0
- package/dist/trace-ingestor-Cu8xQW6B.d.ts +674 -0
- package/package.json +57 -4
- package/README.md +0 -3
|
@@ -0,0 +1,764 @@
|
|
|
1
|
+
import { X as PromptSpanInfo, r as TraceIngestor } from "./trace-ingestor-Cu8xQW6B.js";
|
|
2
|
+
import { ExtractedProps, PropDefinition, PropValue, ValueFactory } from "ts-proppy";
|
|
3
|
+
|
|
4
|
+
//#region src/prompt-types.d.ts
|
|
5
|
+
/**
|
|
6
|
+
* Low-level prompt representation produced by a
|
|
7
|
+
* `PromptFileType`. It exposes the raw `extractedProps` from the parser
|
|
8
|
+
* and uses SDK-specific property names (e.g. `model`, `system`, `messages`).
|
|
9
|
+
*
|
|
10
|
+
* Not intended for public consumption — `PromptProvider` implementations
|
|
11
|
+
* convert this into a {@link NormalizedPrompt} via an {@link SDKAdapter} before
|
|
12
|
+
* returning it to callers.
|
|
13
|
+
*/
|
|
14
|
+
interface ParsedPrompt {
|
|
15
|
+
id: string;
|
|
16
|
+
providerId?: string;
|
|
17
|
+
/**
|
|
18
|
+
* Author-supplied stable alias for this prompt, globally unique across
|
|
19
|
+
* providers. Survives file moves/renames and is registered in the prompt
|
|
20
|
+
* registry so runtime traces can resolve back to this prompt.
|
|
21
|
+
*/
|
|
22
|
+
globalId?: string;
|
|
23
|
+
name: string;
|
|
24
|
+
functionParameters: PropDefinition[];
|
|
25
|
+
extractedProps: ExtractedProps;
|
|
26
|
+
metadata?: unknown;
|
|
27
|
+
treePath?: string[];
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* A single message in a conversation, in a form that is independent of any
|
|
31
|
+
* particular SDK's message shape.
|
|
32
|
+
*/
|
|
33
|
+
interface NormalizedMessage {
|
|
34
|
+
/** Role identifier (`'system'`, `'user'`, `'assistant'`, `'tool'`, …). */
|
|
35
|
+
role: string;
|
|
36
|
+
/** Message content. Typically a primitive or template string, but any
|
|
37
|
+
* {@link PropValue} is allowed so non-string content (e.g. structured
|
|
38
|
+
* content arrays) can round-trip through the editor. */
|
|
39
|
+
content: PropValue;
|
|
40
|
+
/** Optional tool calls attached to an assistant message. */
|
|
41
|
+
toolCalls?: NormalizedToolCall[];
|
|
42
|
+
}
|
|
43
|
+
/** A tool invocation emitted by an assistant message. */
|
|
44
|
+
interface NormalizedToolCall {
|
|
45
|
+
/** The name of the tool / function to invoke. */
|
|
46
|
+
toolName: string;
|
|
47
|
+
/** Arguments to the tool, as a JSON string. */
|
|
48
|
+
args: string;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* A model parameter (`temperature`, `maxTokens`, …) attached to a prompt.
|
|
52
|
+
* Mirrors the `PropDefinition` + current value pair that the playground UI
|
|
53
|
+
* needs for rendering a parameter editor.
|
|
54
|
+
*/
|
|
55
|
+
interface NormalizedParameter {
|
|
56
|
+
/** Describes the parameter's name, type, and documentation. */
|
|
57
|
+
def: PropDefinition;
|
|
58
|
+
/** The parameter's current value, or `undefined` if not set on the prompt. */
|
|
59
|
+
value?: PropValue;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* What every {@link NormalizedPrompt} has, whatever its {@link PromptStyle}.
|
|
63
|
+
*/
|
|
64
|
+
interface NormalizedPromptBase {
|
|
65
|
+
id: string;
|
|
66
|
+
providerId?: string;
|
|
67
|
+
/**
|
|
68
|
+
* Author-supplied stable alias for this prompt, globally unique across
|
|
69
|
+
* providers. Survives file moves/renames and is registered in the prompt
|
|
70
|
+
* registry so runtime traces can resolve back to this prompt.
|
|
71
|
+
*/
|
|
72
|
+
globalId?: string;
|
|
73
|
+
name: string;
|
|
74
|
+
functionParameters: PropDefinition[];
|
|
75
|
+
metadata?: unknown;
|
|
76
|
+
/**
|
|
77
|
+
* Controls where this prompt appears in the sidebar tree.
|
|
78
|
+
*
|
|
79
|
+
* Each element is a path segment. The last segment is treated as the leaf
|
|
80
|
+
* group label (analogous to a file name) and all preceding segments are
|
|
81
|
+
* rendered as collapsible directory nodes.
|
|
82
|
+
*
|
|
83
|
+
* Prompts that share the same `treePath` are grouped under the same leaf
|
|
84
|
+
* node. When omitted, the prompt is placed at the root level.
|
|
85
|
+
*
|
|
86
|
+
* @example ['src', 'prompts', 'greeting.prompt.ts']
|
|
87
|
+
*/
|
|
88
|
+
treePath?: string[];
|
|
89
|
+
/** The model reference (e.g. a provider call or gateway string). */
|
|
90
|
+
model?: PropValue;
|
|
91
|
+
/**
|
|
92
|
+
* Whether this SDK lets the model be chosen from the playground.
|
|
93
|
+
*
|
|
94
|
+
* A capability of the SDK, not a verdict on the current value: an editor
|
|
95
|
+
* also checks that the value itself has an editable shape (see
|
|
96
|
+
* `isEditable`), and renders read-only when either answer is no.
|
|
97
|
+
*/
|
|
98
|
+
modelEditable: boolean;
|
|
99
|
+
/**
|
|
100
|
+
* Model parameters currently set on the prompt (everything that isn't
|
|
101
|
+
* part of the style's own authored content).
|
|
102
|
+
*/
|
|
103
|
+
modelParameters: NormalizedParameter[];
|
|
104
|
+
/**
|
|
105
|
+
* False when the model parameters can't be changed here — a saved
|
|
106
|
+
* variation shown read-only, say. Absent means they can.
|
|
107
|
+
*/
|
|
108
|
+
modelParametersEditable?: boolean;
|
|
109
|
+
/**
|
|
110
|
+
* Named values the SDK needs to **execute** the config this prompt renders,
|
|
111
|
+
* supplied at run time rather than authored in the file.
|
|
112
|
+
*
|
|
113
|
+
* Distinct from {@link functionParameters}, which answers *what does this
|
|
114
|
+
* prompt take?* — this answers *what does running it require?*, which the
|
|
115
|
+
* function signature cannot express. The Vercel AI SDK's `toolsContext` is
|
|
116
|
+
* the motivating case: tools that declare a `contextSchema` need a context
|
|
117
|
+
* object the prompt function never sees.
|
|
118
|
+
*
|
|
119
|
+
* Absent when the prompt needs nothing beyond its arguments. Present with an
|
|
120
|
+
* unresolved type when the requirement is known but its shape is not — a
|
|
121
|
+
* usable state, and better than the silent failure at the first tool call
|
|
122
|
+
* that omitting it would produce.
|
|
123
|
+
*/
|
|
124
|
+
executeParameters?: PropDefinition[];
|
|
125
|
+
/**
|
|
126
|
+
* Non-authored sources that can fill this prompt's input slots — currently
|
|
127
|
+
* code-defined resources. Absent when the provider offers none.
|
|
128
|
+
*/
|
|
129
|
+
inputSources?: PromptInputSources;
|
|
130
|
+
/**
|
|
131
|
+
* Slot path → how to lay out a fan-out slot's fields in the execute panel.
|
|
132
|
+
* Any path not named here uses `"expanded"`.
|
|
133
|
+
*
|
|
134
|
+
* Set by an {@link SDKAdapter} whose own API shape is the reason a slot
|
|
135
|
+
* fans out — e.g. the Vercel AI SDK's `toolsContext`, which is keyed per
|
|
136
|
+
* tool because tools are resolved individually, not because prompt authors
|
|
137
|
+
* think in terms of separate contexts. Mirrors the `functionSlots` /
|
|
138
|
+
* `executeSlots` split on {@link PromptInputSources}, which exists for the
|
|
139
|
+
* same reason: a function parameter and an execute parameter may share a
|
|
140
|
+
* name.
|
|
141
|
+
*/
|
|
142
|
+
inputLayout?: {
|
|
143
|
+
functionSlots?: Record<string, InputLayout>;
|
|
144
|
+
executeSlots?: Record<string, InputLayout>;
|
|
145
|
+
};
|
|
146
|
+
/**
|
|
147
|
+
* Where this prompt was read from. Absent when the provider has no notion
|
|
148
|
+
* of versions, in which case the prompt is always head.
|
|
149
|
+
*/
|
|
150
|
+
ref?: PromptRef;
|
|
151
|
+
/** The base version, when {@link ref} is a version or a variation. */
|
|
152
|
+
version?: VersionInfo;
|
|
153
|
+
/** The variation, when {@link ref} names one. */
|
|
154
|
+
variation?: VariationInfo;
|
|
155
|
+
/**
|
|
156
|
+
* True when a WIP variation exists for this prompt at head — the dirty
|
|
157
|
+
* indicator. Reported on the head prompt, whatever ref is open.
|
|
158
|
+
*/
|
|
159
|
+
dirty?: boolean;
|
|
160
|
+
/** The id of the head WIP variation, when {@link dirty}. */
|
|
161
|
+
wipId?: VariationId;
|
|
162
|
+
/**
|
|
163
|
+
* True when this prompt's base is the working tree, so it can run. False
|
|
164
|
+
* for an old version, or a variation made against one.
|
|
165
|
+
*/
|
|
166
|
+
atHead?: boolean;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* An opaque, immutable name for a state of the world — for a git project, a
|
|
170
|
+
* commit.
|
|
171
|
+
*/
|
|
172
|
+
type VersionId = string;
|
|
173
|
+
/** An opaque id for a {@link VariationInfo | variation}. */
|
|
174
|
+
type VariationId = string;
|
|
175
|
+
/**
|
|
176
|
+
* Which prompt, in which state.
|
|
177
|
+
*
|
|
178
|
+
* A variation already knows its base, so `{ variation }` is complete. There is
|
|
179
|
+
* deliberately no `{ version, variation }` pair: that is a rebase, which
|
|
180
|
+
* produces a new variation with its own id.
|
|
181
|
+
*/
|
|
182
|
+
type PromptRef = /** Head: the prompt as it is on disk now. */{
|
|
183
|
+
promptId: string;
|
|
184
|
+
version?: undefined;
|
|
185
|
+
variation?: undefined;
|
|
186
|
+
} /** The prompt as it was at a saved version. */ | {
|
|
187
|
+
promptId: string;
|
|
188
|
+
version: VersionId;
|
|
189
|
+
variation?: undefined;
|
|
190
|
+
} /** A variation, applied to its base commit (or the working tree, without one). */ | {
|
|
191
|
+
promptId: string;
|
|
192
|
+
variation: VariationId;
|
|
193
|
+
version?: undefined;
|
|
194
|
+
};
|
|
195
|
+
/** A saved state of the world: a commit. See {@link VersionId}. */
|
|
196
|
+
interface VersionInfo {
|
|
197
|
+
id: VersionId;
|
|
198
|
+
/** Commit subject. */
|
|
199
|
+
message?: string;
|
|
200
|
+
author?: string;
|
|
201
|
+
/** When the version was made (ms). */
|
|
202
|
+
time: number;
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* A prompt's field values, keyed like a variation's fields (`system`,
|
|
206
|
+
* `modelParameters.temperature`, …) — `null` for a field the prompt doesn't
|
|
207
|
+
* have.
|
|
208
|
+
*/
|
|
209
|
+
type FieldValues = Record<string, unknown>;
|
|
210
|
+
/**
|
|
211
|
+
* One field that could not be merged: both sides changed it, differently.
|
|
212
|
+
* `field` is `"system"`, `"modelParameters.temperature"`, or the pseudo-field
|
|
213
|
+
* `"prompt"` when the prompt itself no longer exists at the target.
|
|
214
|
+
*/
|
|
215
|
+
interface VariationConflict {
|
|
216
|
+
field: string;
|
|
217
|
+
/** The value both sides started from. */
|
|
218
|
+
base: unknown;
|
|
219
|
+
/** The value on the side being merged into (head, or the unsaved edits). */
|
|
220
|
+
target: unknown;
|
|
221
|
+
/** The value the variation wants. */
|
|
222
|
+
variation: unknown;
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Conflicts a WIP variation is waiting on the user to resolve. While present,
|
|
226
|
+
* the WIP can neither run nor be saved.
|
|
227
|
+
*/
|
|
228
|
+
interface PendingConflicts {
|
|
229
|
+
/**
|
|
230
|
+
* The working tree's values for every field involved, as they were when
|
|
231
|
+
* the conflict arose: what the resolved WIP is based on.
|
|
232
|
+
*/
|
|
233
|
+
targetValues: FieldValues;
|
|
234
|
+
/** Every field that did merge, already applied. */
|
|
235
|
+
updates: NormalizedPromptUpdates;
|
|
236
|
+
conflicts: VariationConflict[];
|
|
237
|
+
/** How to label each side in the UI. */
|
|
238
|
+
labels: {
|
|
239
|
+
target: string;
|
|
240
|
+
variation: string;
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
/**
|
|
244
|
+
* An immutable set of {@link NormalizedPromptUpdates} for one prompt — or,
|
|
245
|
+
* when {@link wip}, the one mutable set of unsaved edits per (prompt, base).
|
|
246
|
+
* It applies to the working tree; its {@link base} commit, when it has one,
|
|
247
|
+
* is where it can also be shown (and one day run) as it was made.
|
|
248
|
+
*/
|
|
249
|
+
interface VariationInfo {
|
|
250
|
+
id: VariationId;
|
|
251
|
+
promptId: string;
|
|
252
|
+
/**
|
|
253
|
+
* The commit whose copy of the prompt's file the variation was made
|
|
254
|
+
* against. Absent when that file had uncommitted changes (or there's no
|
|
255
|
+
* git), and for a WIP of unsaved edits to the working tree.
|
|
256
|
+
*/
|
|
257
|
+
base?: VersionId;
|
|
258
|
+
updates: NormalizedPromptUpdates;
|
|
259
|
+
/** Whether this is a work-in-progress (unsaved, mutable) variation. */
|
|
260
|
+
wip: boolean;
|
|
261
|
+
/** For a WIP: whether it holds the unsaved edits to head. */
|
|
262
|
+
onHead?: boolean;
|
|
263
|
+
/** Names pointing at this variation. */
|
|
264
|
+
names: string[];
|
|
265
|
+
/** For a WIP opened from a named variation: that name. */
|
|
266
|
+
originName?: string;
|
|
267
|
+
/** For a WIP: conflicts awaiting resolution. */
|
|
268
|
+
pending?: PendingConflicts;
|
|
269
|
+
createdAt: number;
|
|
270
|
+
updatedAt: number;
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Which kind of editor a prompt is authored in. Chosen by the
|
|
274
|
+
* {@link SDKAdapter}, since it is the SDK's request shape that decides it.
|
|
275
|
+
*
|
|
276
|
+
* A closed set: the client is a prebuilt bundle, so an adapter picks among
|
|
277
|
+
* the editors it has rather than shipping its own.
|
|
278
|
+
*/
|
|
279
|
+
type PromptStyle = NormalizedPrompt["style"];
|
|
280
|
+
/** A model driven by a system message and a conversation. */
|
|
281
|
+
interface NormalizedChatPrompt extends NormalizedPromptBase {
|
|
282
|
+
style: "chat";
|
|
283
|
+
/** Top-level system prompt, if the SDK supports one. */
|
|
284
|
+
system?: PropValue;
|
|
285
|
+
/** Whether this SDK supports an arbitrary system message. */
|
|
286
|
+
systemEditable: boolean;
|
|
287
|
+
/** Conversation messages. */
|
|
288
|
+
messages: NormalizedMessage[];
|
|
289
|
+
/** Whether this SDK supports an arbitrary message list. */
|
|
290
|
+
messagesEditable: boolean;
|
|
291
|
+
}
|
|
292
|
+
/** A model that answers a set of named questions about a state. */
|
|
293
|
+
interface NormalizedQuestionsPrompt extends NormalizedPromptBase {
|
|
294
|
+
style: "questions";
|
|
295
|
+
/** What the questions are asked about. `def` is the SDK's state type. */
|
|
296
|
+
state: NormalizedParameter;
|
|
297
|
+
/** Whether this SDK supports an arbitrary state. */
|
|
298
|
+
stateEditable: boolean;
|
|
299
|
+
/**
|
|
300
|
+
* The questions, as one value: an object whose property names are question
|
|
301
|
+
* ids. `def` is the SDK's type for the whole collection — a `record` whose
|
|
302
|
+
* value definition carries a catalog of question factories.
|
|
303
|
+
*/
|
|
304
|
+
questions: NormalizedParameter;
|
|
305
|
+
/** Whether this SDK supports adding, removing and editing questions. */
|
|
306
|
+
questionsEditable: boolean;
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Prompt representation used by `PromptProvider` public methods and by
|
|
310
|
+
* the playground UI. It hides SDK-specific property names behind a stable
|
|
311
|
+
* shape, discriminated by {@link PromptStyle}.
|
|
312
|
+
*
|
|
313
|
+
* {@link SDKAdapter.normalizePrompt} converts a {@link ParsedPrompt} produced
|
|
314
|
+
* by a `PromptFileType` into this form.
|
|
315
|
+
*/
|
|
316
|
+
type NormalizedPrompt = NormalizedChatPrompt | NormalizedQuestionsPrompt;
|
|
317
|
+
/** How a fan-out slot's fields are laid out in the execute panel. */
|
|
318
|
+
type InputLayout = "combined" | "expanded";
|
|
319
|
+
/** How long a resource's value lives, and how often `create()` runs. */
|
|
320
|
+
type ResourceScope = "run" | "server";
|
|
321
|
+
/**
|
|
322
|
+
* A code-defined resource offered as an input source, as the UI sees it.
|
|
323
|
+
*
|
|
324
|
+
* The resource's *value* never crosses the wire — the whole point is that only
|
|
325
|
+
* running code can produce it — so this is just enough to render a chip and
|
|
326
|
+
* send the choice back as an `ExecutionInput`.
|
|
327
|
+
*/
|
|
328
|
+
interface ResourceInfo {
|
|
329
|
+
/**
|
|
330
|
+
* Provider-interpreted reference to this resource, relative to the
|
|
331
|
+
* provider's root so it stays valid on another machine (e.g.
|
|
332
|
+
* `.evalution/playground/db.ts#db`).
|
|
333
|
+
*/
|
|
334
|
+
uri: string;
|
|
335
|
+
/** Human-readable label shown on the chip. */
|
|
336
|
+
label: string;
|
|
337
|
+
/** How long the created value lives. See {@link ResourceScope}. */
|
|
338
|
+
scope: ResourceScope;
|
|
339
|
+
/**
|
|
340
|
+
* Why this resource is unavailable, if it is — typically its module threw
|
|
341
|
+
* at import time. Surfaced in the panel rather than hidden, so a broken
|
|
342
|
+
* playground module is visible instead of silently absent.
|
|
343
|
+
*/
|
|
344
|
+
error?: string;
|
|
345
|
+
/**
|
|
346
|
+
* A snapshot of the resource's current value, so the panel can preview it
|
|
347
|
+
* instead of just naming it. Present only when the server already has an
|
|
348
|
+
* instance to peek at — a static `value` resource, or a `scope: 'server'`
|
|
349
|
+
* one it has already created — and that instance's value is plain JSON
|
|
350
|
+
* data. A run-scoped resource, or one whose value isn't plain data (a live
|
|
351
|
+
* handle), has nothing here; the chip is all there is to show.
|
|
352
|
+
*/
|
|
353
|
+
value?: unknown;
|
|
354
|
+
/**
|
|
355
|
+
* Group path this source is displayed under, outermost first. Absent means
|
|
356
|
+
* top level. Purely display metadata — never part of {@link uri} — so
|
|
357
|
+
* renaming a group invalidates no saved selection. See
|
|
358
|
+
* `specs/resource-hierarchy.md` §B.
|
|
359
|
+
*/
|
|
360
|
+
group?: string[];
|
|
361
|
+
/**
|
|
362
|
+
* For a value source (one read off another resource): the `uri` of the
|
|
363
|
+
* resource it is read from. Absent for the resource itself.
|
|
364
|
+
*/
|
|
365
|
+
parent?: string;
|
|
366
|
+
/**
|
|
367
|
+
* For a value source: how many values that resource exposes in total,
|
|
368
|
+
* before slot matching narrows them down for a particular slot. Only the
|
|
369
|
+
* provider can supply this — after filtering, the panel cannot tell "this
|
|
370
|
+
* resource exposes one value" from "it exposes five and four don't fit
|
|
371
|
+
* here", and those read differently once a submenu collapses to its one
|
|
372
|
+
* remaining entry.
|
|
373
|
+
*/
|
|
374
|
+
siblings?: number;
|
|
375
|
+
/**
|
|
376
|
+
* The resource's declared arguments, as slots for the panel to render —
|
|
377
|
+
* one per schema-valued entry of its `inputs`, in declaration order.
|
|
378
|
+
* Absent when the resource takes none. Present with an unresolved (generic
|
|
379
|
+
* string) editor when the requirement is known but its shape is not —
|
|
380
|
+
* the checker-less degradation, same idiom as
|
|
381
|
+
* {@link NormalizedPrompt.executeParameters}.
|
|
382
|
+
*
|
|
383
|
+
* Only the resource itself carries this — a value source (`parent` set)
|
|
384
|
+
* shares its root resource's arguments rather than repeating them. See
|
|
385
|
+
* `specs/resource-arguments.md` §G.
|
|
386
|
+
*/
|
|
387
|
+
parameters?: PropDefinition[];
|
|
388
|
+
/**
|
|
389
|
+
* The resource's code-wired dependencies — its resource-valued `inputs`
|
|
390
|
+
* entries — as input name → the dependency's {@link uri}. Only the
|
|
391
|
+
* resource itself carries this, and only for dependencies discovery
|
|
392
|
+
* registered. The panel lists them in its Resources section. See
|
|
393
|
+
* `specs/resource-instances.md` §D.
|
|
394
|
+
*/
|
|
395
|
+
dependencies?: Record<string, string>;
|
|
396
|
+
}
|
|
397
|
+
/**
|
|
398
|
+
* Which non-authored sources can fill which of a prompt's input slots.
|
|
399
|
+
*
|
|
400
|
+
* Matching runs provider-side, where the checker lives (see
|
|
401
|
+
* `matchResourcesToSlots`), and ships as slot paths so the panel can offer a
|
|
402
|
+
* source without re-deriving the rules.
|
|
403
|
+
*/
|
|
404
|
+
interface PromptInputSources {
|
|
405
|
+
/** Every resource in scope for this prompt, identified by `uri`. */
|
|
406
|
+
resources: ResourceInfo[];
|
|
407
|
+
/**
|
|
408
|
+
* Dotted slot path within {@link NormalizedPrompt.functionParameters}
|
|
409
|
+
* (`taskId`, `ctx.db`) → URIs of the resources that can fill it.
|
|
410
|
+
*/
|
|
411
|
+
functionSlots: Record<string, string[]>;
|
|
412
|
+
/**
|
|
413
|
+
* Dotted slot path within {@link NormalizedPrompt.executeParameters}
|
|
414
|
+
* (`toolsContext.list_tasks.db`) → URIs of the resources that can fill it.
|
|
415
|
+
*/
|
|
416
|
+
executeSlots: Record<string, string[]>;
|
|
417
|
+
/**
|
|
418
|
+
* Resource URI → (argument slot path → URIs of sources that can fill it).
|
|
419
|
+
*
|
|
420
|
+
* Separate from {@link functionSlots} / {@link executeSlots} and keyed by
|
|
421
|
+
* URI rather than folded into the slot path, because an argument slot is a
|
|
422
|
+
* property of the *resource*, not of the prompt: the same resource offers
|
|
423
|
+
* the same arguments in every prompt that can see it. Absent when no
|
|
424
|
+
* resource in scope declares arguments. See
|
|
425
|
+
* `specs/resource-arguments.md` §G.
|
|
426
|
+
*/
|
|
427
|
+
resourceSlots?: Record<string, Record<string, string[]>>;
|
|
428
|
+
}
|
|
429
|
+
/** Updates to a {@link NormalizedChatPrompt}. `null` removes a field. */
|
|
430
|
+
interface ChatPromptUpdates {
|
|
431
|
+
style: "chat";
|
|
432
|
+
model?: PropValue | null;
|
|
433
|
+
system?: PropValue | null;
|
|
434
|
+
messages?: NormalizedMessage[] | null;
|
|
435
|
+
/** Per-parameter updates, keyed by parameter name. `null` removes. */
|
|
436
|
+
modelParameters?: Record<string, PropValue | null>;
|
|
437
|
+
}
|
|
438
|
+
/** Updates to a {@link NormalizedQuestionsPrompt}. `null` removes a field. */
|
|
439
|
+
interface QuestionsPromptUpdates {
|
|
440
|
+
style: "questions";
|
|
441
|
+
model?: PropValue | null;
|
|
442
|
+
state?: PropValue | null;
|
|
443
|
+
questions?: PropValue | null;
|
|
444
|
+
/** Per-parameter updates, keyed by parameter name. `null` removes. */
|
|
445
|
+
modelParameters?: Record<string, PropValue | null>;
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Updates that can be applied to a {@link NormalizedPrompt} via
|
|
449
|
+
* `PromptProvider.updatePromptProperties`, in the vocabulary of the
|
|
450
|
+
* prompt's {@link PromptStyle}. A value of `null` removes that field from the
|
|
451
|
+
* underlying source.
|
|
452
|
+
*/
|
|
453
|
+
type NormalizedPromptUpdates = ChatPromptUpdates | QuestionsPromptUpdates;
|
|
454
|
+
/**
|
|
455
|
+
* A type question for a `PromptFileType` to answer on an
|
|
456
|
+
* {@link SDKAdapter}'s behalf.
|
|
457
|
+
*
|
|
458
|
+
* Neither side can answer alone. Knowing that a Vercel AI SDK config's tools
|
|
459
|
+
* imply a `toolsContext` is SDK semantics; *evaluating* that mapping is a
|
|
460
|
+
* TypeScript capability. So the adapter says what to look for and the file
|
|
461
|
+
* type contributes the checker.
|
|
462
|
+
*/
|
|
463
|
+
type TypeProbe = TypeExpressionProbe | FactoriesProbe;
|
|
464
|
+
/** A probe that resolves a type expression to a {@link PropDefinition}. */
|
|
465
|
+
interface TypeExpressionProbe {
|
|
466
|
+
kind: "type";
|
|
467
|
+
/**
|
|
468
|
+
* The name the result is reported under — for an execute parameter, its
|
|
469
|
+
* name, and the name of the resolved {@link PropDefinition}.
|
|
470
|
+
*/
|
|
471
|
+
name: string;
|
|
472
|
+
/**
|
|
473
|
+
* Source text of a type expression, in the file type's
|
|
474
|
+
* `PromptFileType.language`, that resolves to the type.
|
|
475
|
+
*
|
|
476
|
+
* In a prompt probe, the token `$config` stands for the type of whatever the
|
|
477
|
+
* prompt returns, and the file type substitutes a way to name it — which
|
|
478
|
+
* shape a prompt file uses is the file type's knowledge, not the adapter's.
|
|
479
|
+
* A project probe has no prompt, so `$config` resolves to `never` there.
|
|
480
|
+
*
|
|
481
|
+
* Write it defensively: a probe that cannot find what it is looking for
|
|
482
|
+
* should resolve to `never` (which means "there is no such thing"), never
|
|
483
|
+
* fail to compile. That is what lets an adapter emit a probe without first
|
|
484
|
+
* inspecting the parse result.
|
|
485
|
+
*
|
|
486
|
+
* For instance, the Vercel AI SDK derives its `toolsContext` from whichever
|
|
487
|
+
* of a config's tools declare a `contextSchema`, with the `never` branch
|
|
488
|
+
* covering a config that has no tools at all:
|
|
489
|
+
*
|
|
490
|
+
* ```ts
|
|
491
|
+
* "$config extends { tools: infer T extends ToolSet } ? InferToolSetContext<T> : never"
|
|
492
|
+
* ```
|
|
493
|
+
*/
|
|
494
|
+
expression: string;
|
|
495
|
+
/**
|
|
496
|
+
* How to label the resolved type in the UI, when the checker's own spelling
|
|
497
|
+
* would be unreadable — a deeply-instantiated mapped type is accurate and
|
|
498
|
+
* useless as a label. The resolved *shape* is unaffected.
|
|
499
|
+
*/
|
|
500
|
+
syntax?: string;
|
|
501
|
+
/** Documentation shown beside the resolved definition in the UI. */
|
|
502
|
+
description?: string;
|
|
503
|
+
}
|
|
504
|
+
/**
|
|
505
|
+
* A probe that resolves to the {@link ValueFactory}s among some modules'
|
|
506
|
+
* exports: every export that can be called to produce a value assignable to
|
|
507
|
+
* {@link produces}.
|
|
508
|
+
*
|
|
509
|
+
* The adapter says *what* to look for; the file type knows *how* to ask in its
|
|
510
|
+
* language. A module that isn't installed contributes nothing rather than
|
|
511
|
+
* failing the probe.
|
|
512
|
+
*/
|
|
513
|
+
interface FactoriesProbe {
|
|
514
|
+
kind: "factories";
|
|
515
|
+
/** The name the result is reported under. */
|
|
516
|
+
name: string;
|
|
517
|
+
/** Module specifiers whose exports to consider, e.g. `"@ai-sdk/openai"`. */
|
|
518
|
+
modules: string[];
|
|
519
|
+
/**
|
|
520
|
+
* A type expression, in the file type's language, naming what a factory's
|
|
521
|
+
* call must produce, e.g. `import("ai").LanguageModel`.
|
|
522
|
+
*/
|
|
523
|
+
produces: string;
|
|
524
|
+
/**
|
|
525
|
+
* Consider a method of each export rather than the export itself: with
|
|
526
|
+
* `"evaluationModel"`, an exported provider object `typeSafeAi` whose
|
|
527
|
+
* `evaluationModel(…)` produces the type is found as the factory
|
|
528
|
+
* `typeSafeAi.evaluationModel`, still bound to the import of `typeSafeAi`.
|
|
529
|
+
*/
|
|
530
|
+
member?: string;
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* What one probe resolved to: a {@link PropDefinition} for a `type` probe or
|
|
534
|
+
* the {@link ValueFactory}s found by a `factories` probe; `null` for a definite
|
|
535
|
+
* "no such thing" (the expression evaluated to `never`); or `undefined` for
|
|
536
|
+
* "could not be evaluated".
|
|
537
|
+
*/
|
|
538
|
+
type ProbeResult = PropDefinition | ValueFactory[] | null | undefined;
|
|
539
|
+
/**
|
|
540
|
+
* Project probe results, keyed by probe name. See
|
|
541
|
+
* {@link SDKAdapter.getProjectProbes}.
|
|
542
|
+
*/
|
|
543
|
+
type ProbeResults = Record<string, ProbeResult>;
|
|
544
|
+
//#endregion
|
|
545
|
+
//#region src/sdk-adapter.d.ts
|
|
546
|
+
/** Options for {@link SDKAdapter.executeConfig}. */
|
|
547
|
+
interface ExecuteConfigOptions {
|
|
548
|
+
/** The ID to use for the trace created by this execution, if any. */
|
|
549
|
+
traceId?: string;
|
|
550
|
+
/**
|
|
551
|
+
* The ID the route gave this execution's root span, if any. An adapter that
|
|
552
|
+
* records its own spans names the root span with this, so the span the
|
|
553
|
+
* client selects when a run starts is the one that actually arrives.
|
|
554
|
+
*/
|
|
555
|
+
rootSpanId?: string;
|
|
556
|
+
/**
|
|
557
|
+
* The prompt's identity (id, name, inputs). Used to name and link the
|
|
558
|
+
* trace when the config wasn't built by the `prompts()` helper (which would
|
|
559
|
+
* otherwise carry that identity itself).
|
|
560
|
+
*/
|
|
561
|
+
identity?: PromptSpanInfo;
|
|
562
|
+
/**
|
|
563
|
+
* Named values the SDK needs to execute this config, resolved from the
|
|
564
|
+
* prompt's `executeParameters` — the Vercel AI SDK's `toolsContext`, say.
|
|
565
|
+
*
|
|
566
|
+
* How they reach the underlying call is the adapter's business: merging
|
|
567
|
+
* them generically would assume an execute parameter's name is always a
|
|
568
|
+
* config key, which is true for `toolsContext` but is the adapter's fact to
|
|
569
|
+
* know rather than the caller's.
|
|
570
|
+
*/
|
|
571
|
+
executeValues?: Record<string, any>;
|
|
572
|
+
}
|
|
573
|
+
/** What Evalution hands an adapter when setting up its tracing. */
|
|
574
|
+
interface TraceIngestionContext {
|
|
575
|
+
/**
|
|
576
|
+
* Stands up the process-global OpenTelemetry pipeline, if not already done,
|
|
577
|
+
* and returns the ingestor feeding it. Safe to call from several adapters:
|
|
578
|
+
* the setup runs at most once per process.
|
|
579
|
+
*/
|
|
580
|
+
setupGlobalOTelPipeline(): Promise<TraceIngestor>;
|
|
581
|
+
}
|
|
582
|
+
/**
|
|
583
|
+
* Handle on an execution that has been dispatched.
|
|
584
|
+
*
|
|
585
|
+
* `executeConfig` deliberately resolves as soon as the call is in flight — the
|
|
586
|
+
* route answers with a trace id and the generation continues in the background
|
|
587
|
+
* — so this is how a caller learns when the run is actually over, which is
|
|
588
|
+
* what run-scoped resource teardown hangs off.
|
|
589
|
+
*/
|
|
590
|
+
interface ExecutionHandle {
|
|
591
|
+
/**
|
|
592
|
+
* Settles when the run is over, successfully or not. Never rejects: a
|
|
593
|
+
* failure is already reported through the trace, and a caller awaiting this
|
|
594
|
+
* only wants to know that it is safe to tear down.
|
|
595
|
+
*/
|
|
596
|
+
done: Promise<void>;
|
|
597
|
+
}
|
|
598
|
+
/**
|
|
599
|
+
* Adapter that provides values and execution for a particular AI SDK.
|
|
600
|
+
*
|
|
601
|
+
* Pass an instance of this to `FilePromptProvider` via the
|
|
602
|
+
* `sdk` option.
|
|
603
|
+
*
|
|
604
|
+
* Each `SDKAdapter` implementation should be paired with a companion package
|
|
605
|
+
* (named by {@link SDKAdapter.promptsHelperImport}) that exports a `prompts`
|
|
606
|
+
* function satisfying the `PromptsHelper` type. That function is the user-facing entry
|
|
607
|
+
* point for defining prompts that work with Evalution. It accepts a `PromptsHelperOptions`
|
|
608
|
+
* and a factory that can optionally receive SDK-specific parameters. The factory should return a
|
|
609
|
+
* record of prompt functions. Prompt functions should return a configuration that enables OpenTelemetry
|
|
610
|
+
* reporting, if possible, with the attributes returned by `getPromptSpanAttributes`.
|
|
611
|
+
*/
|
|
612
|
+
interface SDKAdapter {
|
|
613
|
+
/**
|
|
614
|
+
* The package that exports the `prompts()` helper used in new prompt files
|
|
615
|
+
* (e.g. `'@evalution/vercel-ai-sdk'`). Used by `PromptFileType.newPromptSkeleton`.
|
|
616
|
+
*/
|
|
617
|
+
promptsHelperImport: string;
|
|
618
|
+
/**
|
|
619
|
+
* Returns the definition of this SDK's model slot: its type, and the
|
|
620
|
+
* catalogs (providers, presets, free-form entries) the playground's model
|
|
621
|
+
* picker offers.
|
|
622
|
+
*
|
|
623
|
+
* A per-SDK call rather than something copied onto every prompt, since the
|
|
624
|
+
* suggestions a checker derives can run to hundreds of model IDs. It is
|
|
625
|
+
* asked per {@link PromptStyle}, since one SDK may drive different kinds of
|
|
626
|
+
* model — the Vercel AI SDK's chat prompts take a language model, and its
|
|
627
|
+
* evaluation prompts an evaluation model. An adapter that only produces one
|
|
628
|
+
* style may ignore it.
|
|
629
|
+
*
|
|
630
|
+
* @param project - What {@link getProjectProbes} resolved to, by probe name.
|
|
631
|
+
* Substitute a fallback for any `undefined` result.
|
|
632
|
+
* @param style - The style of the prompts the slot belongs to.
|
|
633
|
+
*/
|
|
634
|
+
getModelDefinition(project: ProbeResults, style: PromptStyle): Promise<PropDefinition>;
|
|
635
|
+
/**
|
|
636
|
+
* Returns the list of model parameters that can be edited in the playground
|
|
637
|
+
* UI for projects rooted at `rootDir`. Typically extracted from the SDK's
|
|
638
|
+
* published TypeScript type definitions.
|
|
639
|
+
*
|
|
640
|
+
* @param rootDir - Absolute path to the project root.
|
|
641
|
+
*/
|
|
642
|
+
getModelParameters(rootDir: string): PropDefinition[];
|
|
643
|
+
/**
|
|
644
|
+
* Executes a prompt config object.
|
|
645
|
+
*
|
|
646
|
+
* Resolves once the call has been dispatched, not once it has finished. Use
|
|
647
|
+
* the returned {@link ExecutionHandle} to await completion.
|
|
648
|
+
*
|
|
649
|
+
* @param config - The config object returned by the prompt function.
|
|
650
|
+
* @param options - Optional execution options (trace id, prompt identity,
|
|
651
|
+
* resolved execute values).
|
|
652
|
+
*/
|
|
653
|
+
executeConfig(config: any, options?: ExecuteConfigOptions): Promise<ExecutionHandle | undefined>;
|
|
654
|
+
/**
|
|
655
|
+
* Called once, during server startup, before any prompt config is built,
|
|
656
|
+
* to perform whatever setup this SDK's tracing mechanism needs
|
|
657
|
+
* (e.g. registering a native telemetry integration, or standing up an
|
|
658
|
+
* OpenTelemetry pipeline).
|
|
659
|
+
*
|
|
660
|
+
* If this SDK's tracing is built on OpenTelemetry, its implementation of
|
|
661
|
+
* this method should call {@link TraceIngestionContext.setupGlobalOTelPipeline}, which
|
|
662
|
+
* ensures the global pipeline is only set up once, even if other SDK adapters
|
|
663
|
+
* also call it.
|
|
664
|
+
*
|
|
665
|
+
* Optional — adapters with no tracing support may omit it.
|
|
666
|
+
*
|
|
667
|
+
* @returns An ingestor to which {@link TraceSink}s should be added, or
|
|
668
|
+
* `undefined` if tracing cannot be set up.
|
|
669
|
+
*/
|
|
670
|
+
setupTraceIngestion?(context: TraceIngestionContext): Promise<TraceIngestor | undefined>;
|
|
671
|
+
/**
|
|
672
|
+
* Convert a low-level {@link ParsedPrompt} produced by a
|
|
673
|
+
* `PromptFileType` into a {@link NormalizedPrompt} that the UI can
|
|
674
|
+
* consume without knowing the SDK's specific property names or message shape.
|
|
675
|
+
*
|
|
676
|
+
* Stays synchronous: the expensive, asynchronous work of resolving probes
|
|
677
|
+
* happens in the provider, which passes the results back in.
|
|
678
|
+
*
|
|
679
|
+
* The adapter also chooses the prompt's {@link PromptStyle},
|
|
680
|
+
* which decides the editor the playground renders it in.
|
|
681
|
+
*
|
|
682
|
+
* @param prompt - The raw parsed prompt.
|
|
683
|
+
* @param promptProbes - What this prompt's {@link getPromptProbes} resolved
|
|
684
|
+
* to, in the order the probes were returned — see {@link ProbeResult}.
|
|
685
|
+
* Absent when the file type cannot evaluate probes at all, which an
|
|
686
|
+
* adapter should treat the same as `undefined`: declaring a parameter with
|
|
687
|
+
* an unresolved type rather than omitting it.
|
|
688
|
+
* @param project - What {@link getProjectProbes} resolved to, by probe name.
|
|
689
|
+
* An adapter substitutes its own fallback for any `undefined` result.
|
|
690
|
+
*/
|
|
691
|
+
normalizePrompt(prompt: ParsedPrompt, promptProbes?: readonly ProbeResult[], project?: ProbeResults): NormalizedPrompt;
|
|
692
|
+
/**
|
|
693
|
+
* Returns the type expressions to evaluate for one prompt — chiefly to
|
|
694
|
+
* discover its **execute parameters**, the named values this SDK needs at
|
|
695
|
+
* run time that the prompt function's signature cannot express. Resolved
|
|
696
|
+
* per prompt, so an expression may reference `$config`.
|
|
697
|
+
*
|
|
698
|
+
* The file type declares the language and this method is asked *in* it, so
|
|
699
|
+
* the decision of whether the expression can be written at all belongs here,
|
|
700
|
+
* where the SDK knowledge is. Return `[]` for a language this adapter does
|
|
701
|
+
* not speak, rather than emitting every variant and hoping.
|
|
702
|
+
*
|
|
703
|
+
* Optional — an SDK whose configs are self-contained omits it.
|
|
704
|
+
*
|
|
705
|
+
* @param prompt - The parsed prompt to probe.
|
|
706
|
+
* @param language - The file type's `PromptFileType.language`.
|
|
707
|
+
*/
|
|
708
|
+
getPromptProbes?(prompt: ParsedPrompt, language: string): TypeProbe[];
|
|
709
|
+
/**
|
|
710
|
+
* Returns the probes to evaluate about the project as a whole — the SDK's
|
|
711
|
+
* own types, as installed — rather than about any one prompt. They are
|
|
712
|
+
* resolved once per program build, in a virtual module at the project root
|
|
713
|
+
* (so they see the project's module resolution), and cached; `$config` has
|
|
714
|
+
* no meaning here.
|
|
715
|
+
*
|
|
716
|
+
* Optional, and like {@link getPromptProbes}, return `[]` for a language
|
|
717
|
+
* this adapter does not speak.
|
|
718
|
+
*
|
|
719
|
+
* @param language - The file type's `PromptFileType.language`.
|
|
720
|
+
*/
|
|
721
|
+
getProjectProbes?(language: string): TypeProbe[];
|
|
722
|
+
/**
|
|
723
|
+
* Convert {@link NormalizedPromptUpdates} (what the UI sends back) into the
|
|
724
|
+
* raw property-name-keyed updates that `PromptFileType.updateProperty`
|
|
725
|
+
* and friends operate on.
|
|
726
|
+
*
|
|
727
|
+
* Throws on updates of a {@link PromptStyle} this adapter never
|
|
728
|
+
* produces: they were written against a different editor, and guessing at
|
|
729
|
+
* property names for them would corrupt the file.
|
|
730
|
+
*
|
|
731
|
+
* @param updates - Updates expressed in the normalized vocabulary.
|
|
732
|
+
* @returns A `Record` keyed by the SDK's actual property names. Values may
|
|
733
|
+
* be `null` (to remove) or a `PropValue`.
|
|
734
|
+
*/
|
|
735
|
+
denormalizeUpdates(updates: NormalizedPromptUpdates, currentValues?: Record<string, PropValue>): Record<string, PropValue | null>;
|
|
736
|
+
}
|
|
737
|
+
/**
|
|
738
|
+
* Walk up the directory tree from both `rootDir` and `process.cwd()` looking
|
|
739
|
+
* for `node_modules/<packageName>/<dtsRelPath>`.
|
|
740
|
+
*/
|
|
741
|
+
declare function findPackageDts(packageName: string, dtsRelPath: string, rootDir: string): string | null;
|
|
742
|
+
/**
|
|
743
|
+
* Whether `err` is Node's "package not installed" error for `packageName`
|
|
744
|
+
* itself, as opposed to a resolution failure *inside* that package (a broken
|
|
745
|
+
* transitive dependency), which is a real error worth surfacing.
|
|
746
|
+
*
|
|
747
|
+
* Used to tell an uninstalled optional peer dependency apart from a genuine
|
|
748
|
+
* failure when lazily importing one.
|
|
749
|
+
*/
|
|
750
|
+
declare function isMissingPackage(err: unknown, packageName: string): boolean;
|
|
751
|
+
/**
|
|
752
|
+
* Throws unless `updates` are in the style the adapter produces. For use at
|
|
753
|
+
* the top of {@link SDKAdapter.denormalizeUpdates}; narrows `updates` to that
|
|
754
|
+
* style's update type.
|
|
755
|
+
*/
|
|
756
|
+
declare function assertUpdateStyle<S extends NormalizedPromptUpdates["style"]>(updates: NormalizedPromptUpdates, style: S): asserts updates is Extract<NormalizedPromptUpdates, {
|
|
757
|
+
style: S;
|
|
758
|
+
}>;
|
|
759
|
+
/**
|
|
760
|
+
* Message shown when an optional peer dependency an adapter needs is missing.
|
|
761
|
+
*/
|
|
762
|
+
declare function missingPackageMessage(packageName: string): string;
|
|
763
|
+
//#endregion
|
|
764
|
+
export { ChatPromptUpdates, ExecuteConfigOptions, ExecutionHandle, FactoriesProbe, FieldValues, InputLayout, NormalizedChatPrompt, NormalizedMessage, NormalizedParameter, NormalizedPrompt, NormalizedPromptBase, NormalizedPromptUpdates, NormalizedQuestionsPrompt, NormalizedToolCall, ParsedPrompt, PendingConflicts, ProbeResult, ProbeResults, PromptInputSources, PromptRef, PromptStyle, QuestionsPromptUpdates, ResourceInfo, ResourceScope, SDKAdapter, TraceIngestionContext, TypeExpressionProbe, TypeProbe, VariationConflict, VariationId, VariationInfo, VersionId, VersionInfo, assertUpdateStyle, findPackageDts, isMissingPackage, missingPackageMessage };
|