@hue-run/sdk 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ENVIRONMENTS.md +182 -0
- package/EVALUATIONS.md +12 -0
- package/README.md +194 -18
- package/dist/ai-sdk.d.ts +9 -1
- package/dist/ai-sdk.js +34 -8
- package/dist/client.d.ts +121 -6
- package/dist/client.js +329 -56
- package/dist/config.d.ts +11 -2
- package/dist/config.js +36 -7
- package/dist/environment/client.d.ts +73 -0
- package/dist/environment/client.js +209 -0
- package/dist/environment/tools.d.ts +30 -0
- package/dist/environment/tools.js +24 -0
- package/dist/environment/types.d.ts +429 -0
- package/dist/environment/types.js +1 -0
- package/dist/environment.d.ts +5 -0
- package/dist/environment.js +2 -0
- package/dist/evals/attempt.d.ts +454 -0
- package/dist/evals/attempt.js +687 -0
- package/dist/evals/client.d.ts +99 -5
- package/dist/evals/client.js +136 -7
- package/dist/evals/environment-evidence.d.ts +6 -0
- package/dist/evals/environment-evidence.js +123 -0
- package/dist/evals/environment-json.d.ts +3 -0
- package/dist/evals/environment-json.js +76 -0
- package/dist/evals/json.d.ts +9 -1
- package/dist/evals/json.js +14 -6
- package/dist/evals/runner.d.ts +61 -2
- package/dist/evals/runner.js +71 -9
- package/dist/evals/scorer-publication.d.ts +2 -0
- package/dist/evals/scorer-publication.js +84 -0
- package/dist/evals/scorers.d.ts +11 -0
- package/dist/evals/scorers.js +56 -5
- package/dist/evals/simulation.d.ts +184 -0
- package/dist/evals/simulation.js +603 -0
- package/dist/evals/types.d.ts +304 -0
- package/dist/evals.d.ts +5 -1
- package/dist/evals.js +3 -1
- package/dist/experimental-telemetry.d.ts +8 -0
- package/dist/experimental-telemetry.js +13 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +2 -0
- package/dist/managed.d.ts +51 -1
- package/dist/managed.js +11 -1
- package/dist/privacy.d.ts +2 -0
- package/dist/privacy.js +16 -1
- package/dist/receipt.d.ts +12 -1
- package/dist/receipt.js +10 -1
- package/dist/safety.d.ts +1 -2
- package/dist/snapshot.js +4 -0
- package/dist/transport.d.ts +41 -9
- package/dist/transport.js +80 -22
- package/dist/types.d.ts +144 -8
- package/dist/version.d.ts +2 -0
- package/dist/version.js +3 -0
- package/package.json +51 -15
package/dist/evals/types.d.ts
CHANGED
|
@@ -1,301 +1,605 @@
|
|
|
1
1
|
import type { JsonValue } from "../types.js";
|
|
2
|
+
import type { EnvironmentCoverage } from "../environment/types.js";
|
|
2
3
|
export type { JsonValue } from "../types.js";
|
|
4
|
+
/** One page of a paginated list. */
|
|
3
5
|
export interface Page<T> {
|
|
6
|
+
/** Items on this page. */
|
|
4
7
|
items: T[];
|
|
8
|
+
/** Cursor for the next page, or `null` on the last page. */
|
|
5
9
|
nextCursor: string | null;
|
|
6
10
|
}
|
|
11
|
+
/** Cursor pagination for list methods. */
|
|
7
12
|
export interface PageOptions {
|
|
13
|
+
/** Cursor returned as `nextCursor` by the previous page. */
|
|
8
14
|
after?: string;
|
|
15
|
+
/** Page size, 1–100. */
|
|
9
16
|
limit?: number;
|
|
10
17
|
}
|
|
18
|
+
/** Cursor pagination for registry methods that can include archived identities. */
|
|
19
|
+
export interface RegistryPageOptions extends PageOptions {
|
|
20
|
+
/** Include archived identities so callers can diagnose slug conflicts explicitly. */
|
|
21
|
+
includeArchived?: boolean;
|
|
22
|
+
}
|
|
23
|
+
/** Human-readable identity of a dataset or scorer. */
|
|
11
24
|
export interface Identity {
|
|
25
|
+
/** Display name. */
|
|
12
26
|
name: string;
|
|
27
|
+
/** URL-safe slug, unique within the project. */
|
|
13
28
|
slug: string;
|
|
29
|
+
/** Optional description. */
|
|
14
30
|
description?: string;
|
|
15
31
|
}
|
|
32
|
+
/** A dataset and its versions. */
|
|
16
33
|
export interface Dataset extends Identity {
|
|
34
|
+
/** Dataset ID. */
|
|
17
35
|
id: string;
|
|
36
|
+
/** Archive timestamp on current servers; absent on older compatible responses. */
|
|
37
|
+
archivedAt?: string | null;
|
|
38
|
+
/** Versions of this dataset. */
|
|
18
39
|
versions: DatasetVersion[];
|
|
19
40
|
}
|
|
41
|
+
/** One version of a dataset; frozen versions are immutable and can back experiments. */
|
|
20
42
|
export interface DatasetVersion {
|
|
43
|
+
/** Version ID. */
|
|
21
44
|
id: string;
|
|
45
|
+
/** Owning dataset ID. */
|
|
22
46
|
datasetId: string;
|
|
47
|
+
/** Sequential version number within the dataset. */
|
|
23
48
|
version: number;
|
|
49
|
+
/** Optimistic-concurrency revision; pass it as `expectedRevision` when writing. */
|
|
24
50
|
revision: number;
|
|
51
|
+
/** When the version was frozen, or `null` while it is still a draft. */
|
|
25
52
|
frozenAt: string | null;
|
|
53
|
+
/** Digest of the frozen content, or `null` for a draft. */
|
|
26
54
|
contentDigest: string | null;
|
|
27
55
|
}
|
|
56
|
+
/** A stored case: inputs, an optional reference output and metadata. */
|
|
28
57
|
export interface DatasetCase {
|
|
58
|
+
/** Case ID. */
|
|
29
59
|
id: string;
|
|
60
|
+
/** Caller-chosen key, unique within the version. */
|
|
30
61
|
externalKey: string;
|
|
62
|
+
/** Inputs handed to the target. */
|
|
31
63
|
inputs: JsonValue;
|
|
64
|
+
/** Reference output for scorers, when known. */
|
|
32
65
|
expected?: JsonValue;
|
|
66
|
+
/** Caller-owned metadata. */
|
|
33
67
|
metadata: Record<string, JsonValue>;
|
|
68
|
+
/** Dataset version the case belongs to. */
|
|
34
69
|
datasetVersionId: string;
|
|
70
|
+
/** Exact immutable simulated world selected for this case, when present. */
|
|
71
|
+
environmentVersionId?: string | null;
|
|
72
|
+
/** Immutable trace provenance retained when the case was promoted from a trace. */
|
|
73
|
+
sourceTraceId?: string | null;
|
|
74
|
+
/** Immutable source trace revision paired with `sourceTraceId`. */
|
|
75
|
+
sourceTraceRevision?: number | null;
|
|
76
|
+
/** Immutable input-file manifest identity, when files are attached. */
|
|
77
|
+
artifactManifestId?: string | null;
|
|
35
78
|
}
|
|
79
|
+
/** A frozen case as an experiment sees it. */
|
|
36
80
|
export interface ExperimentCase extends DatasetCase {
|
|
81
|
+
/** Whether a reference output is stored; JSON `null` counts as present. */
|
|
37
82
|
hasExpected: boolean;
|
|
38
83
|
}
|
|
84
|
+
/** Input for {@link EvaluationClient.addCase}. */
|
|
39
85
|
export interface CaseWrite {
|
|
86
|
+
/** Current `revision` of the draft version; the write fails when it has moved. */
|
|
40
87
|
expectedRevision: number;
|
|
88
|
+
/** Caller-chosen key, unique within the version. */
|
|
41
89
|
externalKey: string;
|
|
90
|
+
/** Inputs handed to the target. */
|
|
42
91
|
inputs: JsonValue;
|
|
92
|
+
/** Reference output for scorers. */
|
|
43
93
|
expected?: JsonValue;
|
|
94
|
+
/** Caller-owned metadata. */
|
|
44
95
|
metadata?: Record<string, JsonValue>;
|
|
96
|
+
/** Immutable simulated-world version selected for this case. */
|
|
97
|
+
environmentVersionId?: string | null;
|
|
45
98
|
}
|
|
99
|
+
/** A typed metric a scorer declares and must report exactly once per scored result. */
|
|
46
100
|
export type MetricDefinition = {
|
|
101
|
+
/** Metric name. */
|
|
47
102
|
name: string;
|
|
103
|
+
/** Boolean or free-text metric. */
|
|
48
104
|
type: "boolean" | "text";
|
|
49
105
|
} | {
|
|
106
|
+
/** Metric name. */
|
|
50
107
|
name: string;
|
|
108
|
+
/** Numeric metric. */
|
|
51
109
|
type: "number";
|
|
110
|
+
/** Inclusive lower bound. */
|
|
52
111
|
min?: number;
|
|
112
|
+
/** Inclusive upper bound. */
|
|
53
113
|
max?: number;
|
|
54
114
|
} | {
|
|
115
|
+
/** Metric name. */
|
|
55
116
|
name: string;
|
|
117
|
+
/** Categorical metric. */
|
|
56
118
|
type: "category";
|
|
119
|
+
/** Allowed values. */
|
|
57
120
|
categories: string[];
|
|
58
121
|
};
|
|
122
|
+
/** A pinned scorer definition: a Hue built-in, trusted local code, a manual rubric or a hosted judge. */
|
|
59
123
|
export type ScorerDefinition = {
|
|
124
|
+
/** Hue built-in scorer. */
|
|
60
125
|
kind: "builtin";
|
|
126
|
+
/** Exact typed JSON equality with the reference output. */
|
|
61
127
|
entry: "hue.exact_match.v1";
|
|
128
|
+
/** No configuration. */
|
|
62
129
|
config: Record<string, never>;
|
|
63
130
|
} | {
|
|
131
|
+
/** Hue built-in scorer. */
|
|
64
132
|
kind: "builtin";
|
|
133
|
+
/** String inclusion of the reference output in the output. */
|
|
65
134
|
entry: "hue.includes.v1";
|
|
135
|
+
/** Pinned comparison options. */
|
|
66
136
|
config: {
|
|
137
|
+
/** Compare case-sensitively. */
|
|
67
138
|
caseSensitive: boolean;
|
|
68
139
|
};
|
|
69
140
|
} | {
|
|
141
|
+
/** Hue built-in scorer. */
|
|
70
142
|
kind: "builtin";
|
|
143
|
+
/** JSON Schema draft 2020-12 validation of the output. */
|
|
71
144
|
entry: "hue.json_schema.v1";
|
|
145
|
+
/** Pinned schema. */
|
|
72
146
|
config: {
|
|
147
|
+
/** JSON Schema the output must satisfy. */
|
|
73
148
|
schema: JsonValue;
|
|
74
149
|
};
|
|
75
150
|
} | {
|
|
151
|
+
/** Trusted local callback bound by digest; Hue never downloads or runs the source. */
|
|
76
152
|
kind: "local_code";
|
|
153
|
+
/** Implementation language. */
|
|
77
154
|
language: "typescript" | "python";
|
|
155
|
+
/** Exported function name in the source. */
|
|
78
156
|
entrypoint: string;
|
|
157
|
+
/** SHA-256 of the declared source. */
|
|
79
158
|
sourceDigest: string;
|
|
159
|
+
/** Metrics the callback reports. */
|
|
80
160
|
metrics: MetricDefinition[];
|
|
81
161
|
} | {
|
|
162
|
+
/** Scored by a person in Hue; the local runner defers it. */
|
|
82
163
|
kind: "manual";
|
|
164
|
+
/** Metrics the reviewer records. */
|
|
83
165
|
metrics: MetricDefinition[];
|
|
84
166
|
} | {
|
|
167
|
+
/** Hosted model judge dispatched through `createJudgeJobs`; the local runner defers it. */
|
|
85
168
|
kind: "llm_judge";
|
|
169
|
+
/** Judge model and rubric. */
|
|
86
170
|
config: JudgeConfig;
|
|
171
|
+
/** Metrics the judge reports. */
|
|
87
172
|
metrics: MetricDefinition[];
|
|
88
173
|
};
|
|
174
|
+
/** Configuration of a hosted judge scorer. */
|
|
89
175
|
export interface JudgeConfig {
|
|
176
|
+
/** Judge model identifier. */
|
|
90
177
|
model: string;
|
|
178
|
+
/** Judge model provider. */
|
|
91
179
|
provider: string;
|
|
180
|
+
/** Rubric prompt the judge follows. */
|
|
92
181
|
rubric: string;
|
|
182
|
+
/** Subject fields bound into the rubric. */
|
|
93
183
|
bindings: {
|
|
184
|
+
/** Placeholder name in the rubric. */
|
|
94
185
|
name: string;
|
|
186
|
+
/** JSON path into the subject. */
|
|
95
187
|
path: string;
|
|
188
|
+
/** Whether the bound value must be present. */
|
|
96
189
|
required: boolean;
|
|
97
190
|
}[];
|
|
191
|
+
/** Output token cap for the judge call. */
|
|
98
192
|
maxOutputTokens: number;
|
|
193
|
+
/** Judge call timeout in milliseconds. */
|
|
99
194
|
timeoutMs: number;
|
|
195
|
+
/** Sampling temperature, when set. */
|
|
100
196
|
temperature?: number;
|
|
101
197
|
}
|
|
198
|
+
/** A result as listed by {@link EvaluationClient.listResults}. */
|
|
102
199
|
export interface ResultSummary {
|
|
200
|
+
/** Result ID. */
|
|
103
201
|
id: string;
|
|
202
|
+
/** Evaluation item the result scores. */
|
|
104
203
|
itemId: string;
|
|
204
|
+
/** Scorer version that produced it. */
|
|
105
205
|
scorerVersionId: string;
|
|
206
|
+
/** Outcome state. */
|
|
106
207
|
state: "scored" | "error" | "skipped";
|
|
107
208
|
}
|
|
209
|
+
/** A full stored result from {@link EvaluationClient.getResult}. */
|
|
108
210
|
export interface StoredResult extends ResultSummary {
|
|
211
|
+
/** Evaluation run the result belongs to. */
|
|
109
212
|
runId: string;
|
|
213
|
+
/** Reported metric values. */
|
|
110
214
|
metrics: Metric[];
|
|
215
|
+
/** Scorer explanation, or `null` when absent or not persisted. */
|
|
111
216
|
explanation: string | null;
|
|
217
|
+
/** Scorer evidence. */
|
|
112
218
|
evidence: JsonValue;
|
|
219
|
+
/** Failure for `state: "error"`, otherwise `null`. */
|
|
113
220
|
error: TypedError | null;
|
|
221
|
+
/** Source digest of the local scorer that produced it, when applicable. */
|
|
114
222
|
sourceDigest: string | null;
|
|
115
223
|
}
|
|
224
|
+
/** A hosted judge job and its charge accounting. */
|
|
116
225
|
export interface JudgeJob {
|
|
226
|
+
/** Job ID. */
|
|
117
227
|
id: string;
|
|
228
|
+
/** Evaluation run. */
|
|
118
229
|
runId: string;
|
|
230
|
+
/** Evaluation item being judged. */
|
|
119
231
|
itemId: string;
|
|
232
|
+
/** Judge scorer version. */
|
|
120
233
|
scorerVersionId: string;
|
|
234
|
+
/** Execution state. */
|
|
121
235
|
state: "queued" | "running" | "completed" | "cancelled" | "uncertain";
|
|
236
|
+
/** Charge state after any separately verified reconciliation. */
|
|
122
237
|
chargeState: "unreserved" | "reserved" | "settled" | "uncertain";
|
|
123
238
|
/** Preserved charge state before any separately verified reconciliation. */
|
|
124
239
|
originalChargeState: JudgeJob["chargeState"];
|
|
240
|
+
/** Budget reserved for the job, in micro-USD. */
|
|
125
241
|
reservationMicroUsd: number;
|
|
242
|
+
/** Verified actual charge in micro-USD, or `null` until settled. */
|
|
126
243
|
actualMicroUsd: number | null;
|
|
244
|
+
/** Separately verified settlement, or `null` when there is none. */
|
|
127
245
|
reconciliation: {
|
|
246
|
+
/** Reconciled job ID. */
|
|
128
247
|
jobId: string;
|
|
248
|
+
/** Verified charge in micro-USD. */
|
|
129
249
|
actualMicroUsd: number;
|
|
250
|
+
/** Reference to the settlement evidence. */
|
|
130
251
|
evidenceReference: string;
|
|
252
|
+
/** Why the charge was reconciled. */
|
|
131
253
|
reason: string;
|
|
254
|
+
/** When the reconciliation was recorded. */
|
|
132
255
|
createdAt: string;
|
|
133
256
|
} | null;
|
|
257
|
+
/** Provider price quote behind the reservation. */
|
|
134
258
|
priceQuote: JsonValue;
|
|
259
|
+
/** Original provider receipt. */
|
|
135
260
|
receipt: JsonValue;
|
|
261
|
+
/** Platform workflow ID once dispatched, otherwise `null`. */
|
|
136
262
|
workflowId: string | null;
|
|
263
|
+
/** When cancellation was requested, or `null`. */
|
|
137
264
|
cancelRequestedAt: string | null;
|
|
265
|
+
/** Caller-supplied cancellation reason, or `null`. */
|
|
138
266
|
cancellationReason: string | null;
|
|
267
|
+
/** Creation time. */
|
|
139
268
|
createdAt: string;
|
|
269
|
+
/** Start time, or `null` while queued. */
|
|
140
270
|
startedAt: string | null;
|
|
271
|
+
/** Finish time, or `null` until the job is terminal. */
|
|
141
272
|
finishedAt: string | null;
|
|
142
273
|
}
|
|
274
|
+
/** The project's hosted judge budget and admission controls. */
|
|
143
275
|
export interface JudgeBudget {
|
|
276
|
+
/** Project ID. */
|
|
144
277
|
projectId: string;
|
|
278
|
+
/** A judge provider credential is configured. */
|
|
145
279
|
configured: boolean;
|
|
146
280
|
/** Credential resolution does not establish provider acceptance or available funds. */
|
|
147
281
|
authentication?: {
|
|
282
|
+
/** Whether a credential resolved. */
|
|
148
283
|
status: "available" | "unavailable";
|
|
284
|
+
/** How the credential resolved, or `null`. */
|
|
149
285
|
method: "api-key" | "oidc" | null;
|
|
286
|
+
/** What was verified: credential resolution only. */
|
|
150
287
|
verification: "credential_resolution";
|
|
151
288
|
};
|
|
289
|
+
/** Hosted judging is enabled for the project. */
|
|
152
290
|
enabled: boolean;
|
|
291
|
+
/** Total allowance in micro-USD. */
|
|
153
292
|
allowanceMicroUsd: number;
|
|
293
|
+
/** Currently reserved micro-USD. */
|
|
154
294
|
reservedMicroUsd: number;
|
|
295
|
+
/** Settled spend in micro-USD. */
|
|
155
296
|
spentMicroUsd: number;
|
|
297
|
+
/** Maximum concurrently running jobs. */
|
|
156
298
|
maxInFlight: number;
|
|
299
|
+
/** Dispatch is currently blocked. */
|
|
157
300
|
blocked: boolean;
|
|
158
301
|
}
|
|
302
|
+
/** A scorer and its published versions. */
|
|
159
303
|
export interface Scorer extends Identity {
|
|
304
|
+
/** Scorer ID. */
|
|
160
305
|
id: string;
|
|
306
|
+
/** Archive timestamp on current servers; absent on older compatible responses. */
|
|
307
|
+
archivedAt?: string | null;
|
|
308
|
+
/** Published versions, when included in the response. */
|
|
161
309
|
versions?: ScorerVersion[];
|
|
162
310
|
}
|
|
311
|
+
/** An immutable published scorer definition. */
|
|
163
312
|
export interface ScorerVersion {
|
|
313
|
+
/** Version ID; pin it in experiments and runs. */
|
|
164
314
|
id: string;
|
|
315
|
+
/** Digest of the definition. */
|
|
165
316
|
contentDigest: string;
|
|
317
|
+
/** The pinned definition. */
|
|
166
318
|
definition: ScorerDefinition;
|
|
167
319
|
}
|
|
320
|
+
/** Final state of a target execution. */
|
|
168
321
|
export type TerminalState = "succeeded" | "error" | "cancelled";
|
|
322
|
+
/** One attempt to run the target for a case. */
|
|
169
323
|
export interface Execution {
|
|
324
|
+
/** Execution ID. */
|
|
170
325
|
id: string;
|
|
326
|
+
/** Attempt number for the case, starting at 1. */
|
|
171
327
|
attempt: number;
|
|
328
|
+
/** Current state; `uncertain` means no outcome was saved. */
|
|
172
329
|
state: TerminalState | "started" | "uncertain";
|
|
330
|
+
/** OpenTelemetry trace ID declared for the attempt, or `null`. */
|
|
173
331
|
traceExternalId: string | null;
|
|
332
|
+
/** Subject created on completion, when known. */
|
|
174
333
|
subjectId?: string | null;
|
|
175
334
|
}
|
|
335
|
+
/** A case within an experiment and its latest execution. */
|
|
176
336
|
export interface ExperimentItem {
|
|
337
|
+
/** Item ID, used with the experiment to address the case. */
|
|
177
338
|
id: string;
|
|
339
|
+
/** The case's caller-chosen key. */
|
|
178
340
|
externalKey: string;
|
|
341
|
+
/** Whether a reference output is stored. */
|
|
179
342
|
hasExpected: boolean;
|
|
343
|
+
/** Latest execution, or `null` before the first start. */
|
|
180
344
|
execution: Execution | null;
|
|
181
345
|
}
|
|
346
|
+
/** Scoring progress for an experiment or a historical rescore. */
|
|
182
347
|
export interface EvaluationRun {
|
|
348
|
+
/** Run ID. */
|
|
183
349
|
id: string;
|
|
350
|
+
/** Display name. */
|
|
184
351
|
name: string;
|
|
352
|
+
/** Scorer versions pinned to the run. */
|
|
185
353
|
scorerVersions: ScorerVersion[];
|
|
354
|
+
/** Subjects in the run. */
|
|
186
355
|
itemCount: number;
|
|
356
|
+
/** Result counts by state. */
|
|
187
357
|
scores: {
|
|
358
|
+
/** Results with metrics. */
|
|
188
359
|
scored: number;
|
|
360
|
+
/** Scorer errors. */
|
|
189
361
|
error: number;
|
|
362
|
+
/** Skipped results. */
|
|
190
363
|
skipped: number;
|
|
364
|
+
/** Results not yet recorded. */
|
|
191
365
|
pending: number;
|
|
192
366
|
};
|
|
193
367
|
}
|
|
368
|
+
/** An experiment: a frozen dataset version, a configuration and pinned scorers. */
|
|
194
369
|
export interface Experiment {
|
|
370
|
+
/** Experiment ID. */
|
|
195
371
|
id: string;
|
|
372
|
+
/** Display name. */
|
|
196
373
|
name: string;
|
|
374
|
+
/** Frozen dataset version under test. */
|
|
197
375
|
datasetVersionId: string;
|
|
376
|
+
/** Configuration handed to the target. */
|
|
198
377
|
config: JsonValue;
|
|
378
|
+
/** Digest of `config`. */
|
|
199
379
|
configDigest: string;
|
|
380
|
+
/** The experiment's evaluation run. */
|
|
200
381
|
evaluation: EvaluationRun;
|
|
382
|
+
/** Cases in the frozen version. */
|
|
201
383
|
caseCount: number;
|
|
384
|
+
/** When the experiment was finished, or `null`. */
|
|
202
385
|
finishedAt: string | null;
|
|
386
|
+
/** Case counts by execution state. */
|
|
203
387
|
execution: {
|
|
388
|
+
/** Cases never started. */
|
|
204
389
|
unstarted: number;
|
|
390
|
+
/** Cases with a started attempt. */
|
|
205
391
|
started: number;
|
|
392
|
+
/** Cases whose attempt has no saved outcome. */
|
|
206
393
|
uncertain: number;
|
|
394
|
+
/** Cases that succeeded. */
|
|
207
395
|
succeeded: number;
|
|
396
|
+
/** Cases that failed. */
|
|
208
397
|
error: number;
|
|
398
|
+
/** Cases that were cancelled. */
|
|
209
399
|
cancelled: number;
|
|
210
400
|
};
|
|
211
401
|
}
|
|
402
|
+
/** A sanitized error type with an optional bounded message. */
|
|
212
403
|
export interface TypedError {
|
|
404
|
+
/** Stable error type. */
|
|
213
405
|
type: string;
|
|
406
|
+
/** Optional message; stored only when result content is persisted. */
|
|
214
407
|
message?: string;
|
|
215
408
|
}
|
|
409
|
+
/** Input for {@link EvaluationClient.startExecution}. */
|
|
216
410
|
export interface StartExecution {
|
|
411
|
+
/** Stable key; replaying it returns the same execution. */
|
|
217
412
|
idempotencyKey: string;
|
|
413
|
+
/** OpenTelemetry trace ID the attempt will emit under. */
|
|
218
414
|
traceExternalId?: string;
|
|
415
|
+
/** Execution being replaced; required for a new attempt. */
|
|
219
416
|
previousExecutionId?: string;
|
|
417
|
+
/** Explicitly allow replacing a still-started (uncertain) attempt. */
|
|
220
418
|
allowUncertainRetry?: boolean;
|
|
221
419
|
}
|
|
420
|
+
/** Input for {@link EvaluationClient.completeExecution}. */
|
|
222
421
|
export interface CompleteExecution {
|
|
422
|
+
/** Stable key; replaying it returns the same completion. */
|
|
223
423
|
idempotencyKey: string;
|
|
424
|
+
/** Final state of the attempt. */
|
|
224
425
|
state: TerminalState;
|
|
426
|
+
/** Target output; omit when unavailable. */
|
|
225
427
|
output?: JsonValue;
|
|
428
|
+
/** Sanitized failure for `state: "error"`. */
|
|
226
429
|
error?: TypedError;
|
|
430
|
+
/** Trace revision the stored snapshot must have reached. */
|
|
227
431
|
expectedTraceRevision?: number;
|
|
432
|
+
/** Whether stored trace evidence is required or explicitly omitted. */
|
|
228
433
|
traceEvidence?: "required" | "omit";
|
|
434
|
+
/** Why trace evidence was omitted. */
|
|
229
435
|
omissionReason?: string;
|
|
230
436
|
}
|
|
437
|
+
/** Result of {@link EvaluationClient.completeExecution}. */
|
|
231
438
|
export interface Completion {
|
|
439
|
+
/** Completed execution ID. */
|
|
232
440
|
executionId: string;
|
|
441
|
+
/** Immutable subject created from the outcome. */
|
|
233
442
|
subjectId: string;
|
|
443
|
+
/** Stored trace snapshot, or `null` when omitted. */
|
|
234
444
|
traceSnapshotId: string | null;
|
|
445
|
+
/** Evaluation item to score. */
|
|
235
446
|
evaluationItemId: string;
|
|
236
447
|
}
|
|
448
|
+
/** A subject within an evaluation run. */
|
|
237
449
|
export interface EvaluationItem {
|
|
450
|
+
/** Item ID. */
|
|
238
451
|
id: string;
|
|
452
|
+
/** Subject being scored. */
|
|
239
453
|
subjectId: string;
|
|
454
|
+
/** Whether the subject has an output. */
|
|
240
455
|
hasOutput: boolean;
|
|
456
|
+
/** Stored trace snapshot, or `null`. */
|
|
241
457
|
traceSnapshotId: string | null;
|
|
242
458
|
}
|
|
459
|
+
/** An immutable saved outcome: inputs, output, reference and evidence for one case attempt. */
|
|
243
460
|
export interface Subject {
|
|
461
|
+
/** Subject ID. */
|
|
244
462
|
id: string;
|
|
463
|
+
/** Execution that produced this subject. */
|
|
464
|
+
executionId: string;
|
|
465
|
+
/** Case inputs. */
|
|
245
466
|
inputs: JsonValue;
|
|
467
|
+
/** Whether an output is stored; JSON `null` counts as present. */
|
|
246
468
|
hasOutput: boolean;
|
|
469
|
+
/** Target output, when available. */
|
|
247
470
|
output?: JsonValue;
|
|
471
|
+
/** Whether a reference output is stored. */
|
|
248
472
|
hasExpected: boolean;
|
|
473
|
+
/** Reference output, when stored. */
|
|
249
474
|
expected?: JsonValue;
|
|
475
|
+
/** Case metadata. */
|
|
250
476
|
metadata: Record<string, JsonValue>;
|
|
477
|
+
/** Digest of the subject content. */
|
|
251
478
|
contentDigest: string;
|
|
479
|
+
/** Whether output evidence can be read. */
|
|
252
480
|
outputEvidence: "available" | "unavailable";
|
|
481
|
+
/** Final state of the execution. */
|
|
253
482
|
executionState: TerminalState;
|
|
483
|
+
/** Stored trace snapshot, or `null`. */
|
|
254
484
|
traceSnapshotId: string | null;
|
|
485
|
+
/** Source case ID. */
|
|
255
486
|
caseId: string;
|
|
487
|
+
/** Source dataset version ID. */
|
|
256
488
|
datasetVersionId: string;
|
|
489
|
+
/** Source case key. */
|
|
257
490
|
caseExternalKey: string;
|
|
491
|
+
/** Source experiment ID. */
|
|
258
492
|
experimentId: string;
|
|
493
|
+
/** Attempt number of the execution. */
|
|
259
494
|
attempt: number;
|
|
495
|
+
/** Whether trace evidence was captured, omitted with a reason or not requested. */
|
|
260
496
|
traceEvidence: "captured" | "omitted" | "not_requested";
|
|
497
|
+
/** Declared OpenTelemetry trace ID, or `null`. */
|
|
261
498
|
traceExternalId: string | null;
|
|
499
|
+
/** Reason trace evidence was omitted, or `null`. */
|
|
262
500
|
omissionReason: string | null;
|
|
263
501
|
}
|
|
502
|
+
/** A reported metric value. */
|
|
264
503
|
export interface Metric {
|
|
504
|
+
/** Declared metric name. */
|
|
265
505
|
name: string;
|
|
506
|
+
/** Value matching the declared type. */
|
|
266
507
|
value: boolean | number | string;
|
|
508
|
+
/** Quality verdict; a failed metric stays `state: "scored"`. */
|
|
267
509
|
passed?: boolean;
|
|
268
510
|
}
|
|
511
|
+
/** Outcome of one scorer for one subject. */
|
|
269
512
|
export type Score = {
|
|
513
|
+
/** Metrics were produced. */
|
|
270
514
|
state: "scored";
|
|
515
|
+
/** Every declared metric exactly once. */
|
|
271
516
|
metrics: Metric[];
|
|
517
|
+
/** Human-readable reasoning; required unless `evidence` is given. */
|
|
272
518
|
explanation?: string;
|
|
519
|
+
/** Supporting data; required unless `explanation` is given. */
|
|
273
520
|
evidence?: JsonValue;
|
|
274
521
|
} | {
|
|
522
|
+
/** The scorer failed. */
|
|
275
523
|
state: "error";
|
|
524
|
+
/** Sanitized failure. */
|
|
276
525
|
error: TypedError;
|
|
277
526
|
} | {
|
|
527
|
+
/** The scorer did not apply. */
|
|
278
528
|
state: "skipped";
|
|
529
|
+
/** Why it was skipped. */
|
|
279
530
|
explanation: string;
|
|
280
531
|
};
|
|
532
|
+
/** A score addressed to an evaluation item, as uploaded by {@link EvaluationClient.submitResults}. */
|
|
281
533
|
export type Result = Score & {
|
|
534
|
+
/** Evaluation item the score belongs to. */
|
|
282
535
|
evaluationItemId: string;
|
|
536
|
+
/** Scorer version that produced the score. */
|
|
283
537
|
scorerVersionId: string;
|
|
538
|
+
/** Source digest of the local scorer, for `local_code` pins. */
|
|
284
539
|
sourceDigest?: string;
|
|
285
540
|
};
|
|
541
|
+
/** What a local scorer callback receives. */
|
|
286
542
|
export interface ScoreContext {
|
|
543
|
+
/** Case inputs. */
|
|
287
544
|
inputs: JsonValue;
|
|
545
|
+
/** Target output; `undefined` means unavailable. */
|
|
288
546
|
output?: JsonValue;
|
|
547
|
+
/** Reference output, when stored. */
|
|
289
548
|
expected?: JsonValue;
|
|
549
|
+
/** Whether `output` is present; JSON `null` counts. */
|
|
290
550
|
hasOutput: boolean;
|
|
551
|
+
/** Whether `expected` is present. */
|
|
291
552
|
hasExpected: boolean;
|
|
553
|
+
/** Case metadata. */
|
|
292
554
|
metadata: Record<string, JsonValue>;
|
|
555
|
+
/** Final state of the target execution. */
|
|
293
556
|
executionState: TerminalState;
|
|
557
|
+
/** Authoritative sealed world and complete journal, when required by the runner. */
|
|
558
|
+
environment?: EnvironmentEvidence;
|
|
294
559
|
}
|
|
560
|
+
/** Sealed environment evidence resolved through one target execution. */
|
|
561
|
+
export interface EnvironmentEvidenceSnapshot extends EnvironmentCoverage {
|
|
562
|
+
/** Environment-run identity. */
|
|
563
|
+
runId: string;
|
|
564
|
+
/** Linked target execution identity. */
|
|
565
|
+
executionId: string;
|
|
566
|
+
/** Immutable environment version used by the world. */
|
|
567
|
+
environmentVersionId: string;
|
|
568
|
+
/** Digest of the stored environment definition. */
|
|
569
|
+
definitionDigest: string;
|
|
570
|
+
/** Deterministic world seed. */
|
|
571
|
+
seed: string;
|
|
572
|
+
/** Terminal world status. */
|
|
573
|
+
status: "completed" | "abandoned" | "expired";
|
|
574
|
+
/** Number of recorded journal steps. */
|
|
575
|
+
stepCount: number;
|
|
576
|
+
/** Digest of final state. */
|
|
577
|
+
stateDigest: string;
|
|
578
|
+
/** State before the first action. */
|
|
579
|
+
initialState: JsonValue;
|
|
580
|
+
/** State at sealing. */
|
|
581
|
+
finalState: JsonValue;
|
|
582
|
+
}
|
|
583
|
+
/** Sealed environment evidence with its full ordered journal. */
|
|
584
|
+
export interface EnvironmentEvidence extends EnvironmentEvidenceSnapshot {
|
|
585
|
+
/** Complete steps ordered by ordinal. */
|
|
586
|
+
steps: import("../environment/types.js").Step[];
|
|
587
|
+
}
|
|
588
|
+
/** A local scorer: its pinned definition and the callback bound to it. */
|
|
295
589
|
export interface LocalScorer {
|
|
590
|
+
/** Definition to publish and pin; the runner matches it by digest. */
|
|
296
591
|
definition: Extract<ScorerDefinition, {
|
|
297
592
|
kind: "local_code";
|
|
298
593
|
}>;
|
|
299
594
|
/** Trusted local code. There is no callback timeout or side-effect cancellation. */
|
|
300
595
|
score(context: ScoreContext): Score | Promise<Score>;
|
|
301
596
|
}
|
|
597
|
+
/** Short-lived execution-scoped MCP connection for one simulated world. */
|
|
598
|
+
export interface SimulationMcpCapability {
|
|
599
|
+
/** HTTPS MCP endpoint. */
|
|
600
|
+
url: string;
|
|
601
|
+
/** Attempt-scoped bearer; never persist or expose it. */
|
|
602
|
+
token: string;
|
|
603
|
+
/** Credential expiry timestamp. */
|
|
604
|
+
expiresAt: string;
|
|
605
|
+
}
|
package/dist/evals.d.ts
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
export { createEvaluationClient, EvaluationClient, HueApiError } from "./evals/client.js";
|
|
2
2
|
export type { EvaluationClientOptions } from "./evals/client.js";
|
|
3
|
-
export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, } from "./evals/runner.js";
|
|
3
|
+
export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, } from "./evals/runner.js";
|
|
4
4
|
export type { RunExperimentOptions, RescoreOptions, RunnerReport } from "./evals/runner.js";
|
|
5
|
+
export { runSimulation } from "./evals/simulation.js";
|
|
6
|
+
export type { RepositorySimulationCase, RepositorySimulationScorer, RunSimulationOptions, SimulationProgress, SimulationReport, SimulationScenario, SimulationTargetContext, } from "./evals/simulation.js";
|
|
7
|
+
export { actualAgentManifestV2, agentManifestDigestV2, attemptBaselineV2, attemptBindingRead, attemptConnectionBundleV2, attemptIdentityV2, dependencyManifestV2, dependencyProviderV2, expectedAgentManifestV2, executionManifestDigestV2, parityEvidenceV2, preflightFindingV2, preflightReportV2, prepareAttemptInputV2, projectMcpConnectionV2, secretFreeBindingV2, surfaceBindingV2, } from "./evals/attempt.js";
|
|
8
|
+
export type { ActualAgentManifestInputV2, ActualAgentManifestV2, AttemptBaselineV2, AttemptBindingRead, AttemptConnectionBundleV2, AttemptIdentityV2, DependencyManifestV2, DependencyProviderV2, ExpectedAgentManifestV2, ParityEvidenceV2, PreflightFindingV2, PreflightReportV2, PrepareAttemptIncompleteV2, PrepareAttemptInputV2, PrepareAttemptReadyV2, PrepareAttemptRequestV2, PrepareAttemptResultV2, RefreshAttemptResultV2, RequestedAttemptProviderV2, RevokeAttemptResult, SurfaceBindingV2, } from "./evals/attempt.js";
|
|
5
9
|
export { builtins, defineLocalScorer, scoreLocally } from "./evals/scorers.js";
|
|
6
10
|
export { sourceDigest } from "./evals/json.js";
|
|
7
11
|
export type * from "./evals/types.js";
|
package/dist/evals.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export { createEvaluationClient, EvaluationClient, HueApiError } from "./evals/client.js";
|
|
2
|
-
export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, } from "./evals/runner.js";
|
|
2
|
+
export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, } from "./evals/runner.js";
|
|
3
|
+
export { runSimulation } from "./evals/simulation.js";
|
|
4
|
+
export { actualAgentManifestV2, agentManifestDigestV2, attemptBaselineV2, attemptBindingRead, attemptConnectionBundleV2, attemptIdentityV2, dependencyManifestV2, dependencyProviderV2, expectedAgentManifestV2, executionManifestDigestV2, parityEvidenceV2, preflightFindingV2, preflightReportV2, prepareAttemptInputV2, projectMcpConnectionV2, secretFreeBindingV2, surfaceBindingV2, } from "./evals/attempt.js";
|
|
3
5
|
export { builtins, defineLocalScorer, scoreLocally } from "./evals/scorers.js";
|
|
4
6
|
export { sourceDigest } from "./evals/json.js";
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { HueClient } from "./client.js";
|
|
2
|
+
import type { ExperimentalTelemetrySettings } from "./types.js";
|
|
3
|
+
/**
|
|
4
|
+
* Per-call telemetry for AI SDK 6: pass as `experimental_telemetry`. Spans are created with Hue's
|
|
5
|
+
* tracer, so they parent under `withSpan` and inherit session/user identifiers, and prompt/response
|
|
6
|
+
* recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry` from `@hue-run/sdk/ai-sdk`.
|
|
7
|
+
*/
|
|
8
|
+
export declare function hueExperimentalTelemetry(hue: HueClient): ExperimentalTelemetrySettings;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-call telemetry for AI SDK 6: pass as `experimental_telemetry`. Spans are created with Hue's
|
|
3
|
+
* tracer, so they parent under `withSpan` and inherit session/user identifiers, and prompt/response
|
|
4
|
+
* recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry` from `@hue-run/sdk/ai-sdk`.
|
|
5
|
+
*/
|
|
6
|
+
export function hueExperimentalTelemetry(hue) {
|
|
7
|
+
return {
|
|
8
|
+
isEnabled: hue.enabled,
|
|
9
|
+
recordInputs: hue.captureContent,
|
|
10
|
+
recordOutputs: hue.captureContent,
|
|
11
|
+
tracer: hue.tracer,
|
|
12
|
+
};
|
|
13
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
export { createHue, createHueSafe, HueClient, HueConnectionError, type ExistingHueProviders, } from "./client.js";
|
|
2
2
|
export { createHueTransport, HueTransport, HueExportError } from "./transport.js";
|
|
3
|
+
export type { RecordValue } from "./transport.js";
|
|
4
|
+
export { hueExperimentalTelemetry } from "./experimental-telemetry.js";
|
|
5
|
+
export { contentPrefixes } from "./privacy.js";
|
|
3
6
|
export { HueTraceVerificationError } from "./receipt.js";
|
|
4
7
|
export type * from "./types.js";
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
export { createHue, createHueSafe, HueClient, HueConnectionError, } from "./client.js";
|
|
2
2
|
export { createHueTransport, HueTransport, HueExportError } from "./transport.js";
|
|
3
|
+
export { hueExperimentalTelemetry } from "./experimental-telemetry.js";
|
|
4
|
+
export { contentPrefixes } from "./privacy.js";
|
|
3
5
|
export { HueTraceVerificationError } from "./receipt.js";
|