@hue-run/sdk 0.1.5 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/ENVIRONMENTS.md +182 -0
  2. package/EVALUATIONS.md +12 -0
  3. package/README.md +194 -18
  4. package/dist/ai-sdk.d.ts +9 -1
  5. package/dist/ai-sdk.js +34 -8
  6. package/dist/client.d.ts +121 -6
  7. package/dist/client.js +329 -56
  8. package/dist/config.d.ts +11 -2
  9. package/dist/config.js +36 -7
  10. package/dist/environment/client.d.ts +73 -0
  11. package/dist/environment/client.js +209 -0
  12. package/dist/environment/tools.d.ts +30 -0
  13. package/dist/environment/tools.js +24 -0
  14. package/dist/environment/types.d.ts +429 -0
  15. package/dist/environment/types.js +1 -0
  16. package/dist/environment.d.ts +5 -0
  17. package/dist/environment.js +2 -0
  18. package/dist/evals/attempt.d.ts +454 -0
  19. package/dist/evals/attempt.js +687 -0
  20. package/dist/evals/client.d.ts +99 -5
  21. package/dist/evals/client.js +136 -7
  22. package/dist/evals/environment-evidence.d.ts +6 -0
  23. package/dist/evals/environment-evidence.js +123 -0
  24. package/dist/evals/environment-json.d.ts +3 -0
  25. package/dist/evals/environment-json.js +76 -0
  26. package/dist/evals/json.d.ts +9 -1
  27. package/dist/evals/json.js +14 -6
  28. package/dist/evals/runner.d.ts +61 -2
  29. package/dist/evals/runner.js +71 -9
  30. package/dist/evals/scorer-publication.d.ts +2 -0
  31. package/dist/evals/scorer-publication.js +84 -0
  32. package/dist/evals/scorers.d.ts +11 -0
  33. package/dist/evals/scorers.js +56 -5
  34. package/dist/evals/simulation.d.ts +184 -0
  35. package/dist/evals/simulation.js +603 -0
  36. package/dist/evals/types.d.ts +304 -0
  37. package/dist/evals.d.ts +5 -1
  38. package/dist/evals.js +3 -1
  39. package/dist/experimental-telemetry.d.ts +8 -0
  40. package/dist/experimental-telemetry.js +13 -0
  41. package/dist/index.d.ts +3 -0
  42. package/dist/index.js +2 -0
  43. package/dist/managed.d.ts +51 -1
  44. package/dist/managed.js +11 -1
  45. package/dist/privacy.d.ts +2 -0
  46. package/dist/privacy.js +16 -1
  47. package/dist/receipt.d.ts +12 -1
  48. package/dist/receipt.js +10 -1
  49. package/dist/safety.d.ts +1 -2
  50. package/dist/snapshot.js +4 -0
  51. package/dist/transport.d.ts +41 -9
  52. package/dist/transport.js +80 -22
  53. package/dist/types.d.ts +144 -8
  54. package/dist/version.d.ts +2 -0
  55. package/dist/version.js +3 -0
  56. package/package.json +51 -15
@@ -1,301 +1,605 @@
1
1
  import type { JsonValue } from "../types.js";
2
+ import type { EnvironmentCoverage } from "../environment/types.js";
2
3
  export type { JsonValue } from "../types.js";
4
+ /** One page of a paginated list. */
3
5
  export interface Page<T> {
6
+ /** Items on this page. */
4
7
  items: T[];
8
+ /** Cursor for the next page, or `null` on the last page. */
5
9
  nextCursor: string | null;
6
10
  }
11
+ /** Cursor pagination for list methods. */
7
12
  export interface PageOptions {
13
+ /** Cursor returned as `nextCursor` by the previous page. */
8
14
  after?: string;
15
+ /** Page size, 1–100. */
9
16
  limit?: number;
10
17
  }
18
+ /** Cursor pagination for registry methods that can include archived identities. */
19
+ export interface RegistryPageOptions extends PageOptions {
20
+ /** Include archived identities so callers can diagnose slug conflicts explicitly. */
21
+ includeArchived?: boolean;
22
+ }
23
+ /** Human-readable identity of a dataset or scorer. */
11
24
  export interface Identity {
25
+ /** Display name. */
12
26
  name: string;
27
+ /** URL-safe slug, unique within the project. */
13
28
  slug: string;
29
+ /** Optional description. */
14
30
  description?: string;
15
31
  }
32
+ /** A dataset and its versions. */
16
33
  export interface Dataset extends Identity {
34
+ /** Dataset ID. */
17
35
  id: string;
36
+ /** Archive timestamp on current servers; absent on older compatible responses. */
37
+ archivedAt?: string | null;
38
+ /** Versions of this dataset. */
18
39
  versions: DatasetVersion[];
19
40
  }
41
+ /** One version of a dataset; frozen versions are immutable and can back experiments. */
20
42
  export interface DatasetVersion {
43
+ /** Version ID. */
21
44
  id: string;
45
+ /** Owning dataset ID. */
22
46
  datasetId: string;
47
+ /** Sequential version number within the dataset. */
23
48
  version: number;
49
+ /** Optimistic-concurrency revision; pass it as `expectedRevision` when writing. */
24
50
  revision: number;
51
+ /** When the version was frozen, or `null` while it is still a draft. */
25
52
  frozenAt: string | null;
53
+ /** Digest of the frozen content, or `null` for a draft. */
26
54
  contentDigest: string | null;
27
55
  }
56
+ /** A stored case: inputs, an optional reference output and metadata. */
28
57
  export interface DatasetCase {
58
+ /** Case ID. */
29
59
  id: string;
60
+ /** Caller-chosen key, unique within the version. */
30
61
  externalKey: string;
62
+ /** Inputs handed to the target. */
31
63
  inputs: JsonValue;
64
+ /** Reference output for scorers, when known. */
32
65
  expected?: JsonValue;
66
+ /** Caller-owned metadata. */
33
67
  metadata: Record<string, JsonValue>;
68
+ /** Dataset version the case belongs to. */
34
69
  datasetVersionId: string;
70
+ /** Exact immutable simulated world selected for this case, when present. */
71
+ environmentVersionId?: string | null;
72
+ /** Immutable trace provenance retained when the case was promoted from a trace. */
73
+ sourceTraceId?: string | null;
74
+ /** Immutable source trace revision paired with `sourceTraceId`. */
75
+ sourceTraceRevision?: number | null;
76
+ /** Immutable input-file manifest identity, when files are attached. */
77
+ artifactManifestId?: string | null;
35
78
  }
79
+ /** A frozen case as an experiment sees it. */
36
80
  export interface ExperimentCase extends DatasetCase {
81
+ /** Whether a reference output is stored; JSON `null` counts as present. */
37
82
  hasExpected: boolean;
38
83
  }
84
+ /** Input for {@link EvaluationClient.addCase}. */
39
85
  export interface CaseWrite {
86
+ /** Current `revision` of the draft version; the write fails when it has moved. */
40
87
  expectedRevision: number;
88
+ /** Caller-chosen key, unique within the version. */
41
89
  externalKey: string;
90
+ /** Inputs handed to the target. */
42
91
  inputs: JsonValue;
92
+ /** Reference output for scorers. */
43
93
  expected?: JsonValue;
94
+ /** Caller-owned metadata. */
44
95
  metadata?: Record<string, JsonValue>;
96
+ /** Immutable simulated-world version selected for this case. */
97
+ environmentVersionId?: string | null;
45
98
  }
99
+ /** A typed metric a scorer declares and must report exactly once per scored result. */
46
100
  export type MetricDefinition = {
101
+ /** Metric name. */
47
102
  name: string;
103
+ /** Boolean or free-text metric. */
48
104
  type: "boolean" | "text";
49
105
  } | {
106
+ /** Metric name. */
50
107
  name: string;
108
+ /** Numeric metric. */
51
109
  type: "number";
110
+ /** Inclusive lower bound. */
52
111
  min?: number;
112
+ /** Inclusive upper bound. */
53
113
  max?: number;
54
114
  } | {
115
+ /** Metric name. */
55
116
  name: string;
117
+ /** Categorical metric. */
56
118
  type: "category";
119
+ /** Allowed values. */
57
120
  categories: string[];
58
121
  };
122
+ /** A pinned scorer definition: a Hue built-in, trusted local code, a manual rubric or a hosted judge. */
59
123
  export type ScorerDefinition = {
124
+ /** Hue built-in scorer. */
60
125
  kind: "builtin";
126
+ /** Exact typed JSON equality with the reference output. */
61
127
  entry: "hue.exact_match.v1";
128
+ /** No configuration. */
62
129
  config: Record<string, never>;
63
130
  } | {
131
+ /** Hue built-in scorer. */
64
132
  kind: "builtin";
133
+ /** String inclusion of the reference output in the output. */
65
134
  entry: "hue.includes.v1";
135
+ /** Pinned comparison options. */
66
136
  config: {
137
+ /** Compare case-sensitively. */
67
138
  caseSensitive: boolean;
68
139
  };
69
140
  } | {
141
+ /** Hue built-in scorer. */
70
142
  kind: "builtin";
143
+ /** JSON Schema draft 2020-12 validation of the output. */
71
144
  entry: "hue.json_schema.v1";
145
+ /** Pinned schema. */
72
146
  config: {
147
+ /** JSON Schema the output must satisfy. */
73
148
  schema: JsonValue;
74
149
  };
75
150
  } | {
151
+ /** Trusted local callback bound by digest; Hue never downloads or runs the source. */
76
152
  kind: "local_code";
153
+ /** Implementation language. */
77
154
  language: "typescript" | "python";
155
+ /** Exported function name in the source. */
78
156
  entrypoint: string;
157
+ /** SHA-256 of the declared source. */
79
158
  sourceDigest: string;
159
+ /** Metrics the callback reports. */
80
160
  metrics: MetricDefinition[];
81
161
  } | {
162
+ /** Scored by a person in Hue; the local runner defers it. */
82
163
  kind: "manual";
164
+ /** Metrics the reviewer records. */
83
165
  metrics: MetricDefinition[];
84
166
  } | {
167
+ /** Hosted model judge dispatched through `createJudgeJobs`; the local runner defers it. */
85
168
  kind: "llm_judge";
169
+ /** Judge model and rubric. */
86
170
  config: JudgeConfig;
171
+ /** Metrics the judge reports. */
87
172
  metrics: MetricDefinition[];
88
173
  };
174
+ /** Configuration of a hosted judge scorer. */
89
175
  export interface JudgeConfig {
176
+ /** Judge model identifier. */
90
177
  model: string;
178
+ /** Judge model provider. */
91
179
  provider: string;
180
+ /** Rubric prompt the judge follows. */
92
181
  rubric: string;
182
+ /** Subject fields bound into the rubric. */
93
183
  bindings: {
184
+ /** Placeholder name in the rubric. */
94
185
  name: string;
186
+ /** JSON path into the subject. */
95
187
  path: string;
188
+ /** Whether the bound value must be present. */
96
189
  required: boolean;
97
190
  }[];
191
+ /** Output token cap for the judge call. */
98
192
  maxOutputTokens: number;
193
+ /** Judge call timeout in milliseconds. */
99
194
  timeoutMs: number;
195
+ /** Sampling temperature, when set. */
100
196
  temperature?: number;
101
197
  }
198
+ /** A result as listed by {@link EvaluationClient.listResults}. */
102
199
  export interface ResultSummary {
200
+ /** Result ID. */
103
201
  id: string;
202
+ /** Evaluation item the result scores. */
104
203
  itemId: string;
204
+ /** Scorer version that produced it. */
105
205
  scorerVersionId: string;
206
+ /** Outcome state. */
106
207
  state: "scored" | "error" | "skipped";
107
208
  }
209
+ /** A full stored result from {@link EvaluationClient.getResult}. */
108
210
  export interface StoredResult extends ResultSummary {
211
+ /** Evaluation run the result belongs to. */
109
212
  runId: string;
213
+ /** Reported metric values. */
110
214
  metrics: Metric[];
215
+ /** Scorer explanation, or `null` when absent or not persisted. */
111
216
  explanation: string | null;
217
+ /** Scorer evidence. */
112
218
  evidence: JsonValue;
219
+ /** Failure for `state: "error"`, otherwise `null`. */
113
220
  error: TypedError | null;
221
+ /** Source digest of the local scorer that produced it, when applicable. */
114
222
  sourceDigest: string | null;
115
223
  }
224
+ /** A hosted judge job and its charge accounting. */
116
225
  export interface JudgeJob {
226
+ /** Job ID. */
117
227
  id: string;
228
+ /** Evaluation run. */
118
229
  runId: string;
230
+ /** Evaluation item being judged. */
119
231
  itemId: string;
232
+ /** Judge scorer version. */
120
233
  scorerVersionId: string;
234
+ /** Execution state. */
121
235
  state: "queued" | "running" | "completed" | "cancelled" | "uncertain";
236
+ /** Charge state after any separately verified reconciliation. */
122
237
  chargeState: "unreserved" | "reserved" | "settled" | "uncertain";
123
238
  /** Preserved charge state before any separately verified reconciliation. */
124
239
  originalChargeState: JudgeJob["chargeState"];
240
+ /** Budget reserved for the job, in micro-USD. */
125
241
  reservationMicroUsd: number;
242
+ /** Verified actual charge in micro-USD, or `null` until settled. */
126
243
  actualMicroUsd: number | null;
244
+ /** Separately verified settlement, or `null` when there is none. */
127
245
  reconciliation: {
246
+ /** Reconciled job ID. */
128
247
  jobId: string;
248
+ /** Verified charge in micro-USD. */
129
249
  actualMicroUsd: number;
250
+ /** Reference to the settlement evidence. */
130
251
  evidenceReference: string;
252
+ /** Why the charge was reconciled. */
131
253
  reason: string;
254
+ /** When the reconciliation was recorded. */
132
255
  createdAt: string;
133
256
  } | null;
257
+ /** Provider price quote behind the reservation. */
134
258
  priceQuote: JsonValue;
259
+ /** Original provider receipt. */
135
260
  receipt: JsonValue;
261
+ /** Platform workflow ID once dispatched, otherwise `null`. */
136
262
  workflowId: string | null;
263
+ /** When cancellation was requested, or `null`. */
137
264
  cancelRequestedAt: string | null;
265
+ /** Caller-supplied cancellation reason, or `null`. */
138
266
  cancellationReason: string | null;
267
+ /** Creation time. */
139
268
  createdAt: string;
269
+ /** Start time, or `null` while queued. */
140
270
  startedAt: string | null;
271
+ /** Finish time, or `null` until the job is terminal. */
141
272
  finishedAt: string | null;
142
273
  }
274
+ /** The project's hosted judge budget and admission controls. */
143
275
  export interface JudgeBudget {
276
+ /** Project ID. */
144
277
  projectId: string;
278
+ /** A judge provider credential is configured. */
145
279
  configured: boolean;
146
280
  /** Credential resolution does not establish provider acceptance or available funds. */
147
281
  authentication?: {
282
+ /** Whether a credential resolved. */
148
283
  status: "available" | "unavailable";
284
+ /** How the credential resolved, or `null`. */
149
285
  method: "api-key" | "oidc" | null;
286
+ /** What was verified: credential resolution only. */
150
287
  verification: "credential_resolution";
151
288
  };
289
+ /** Hosted judging is enabled for the project. */
152
290
  enabled: boolean;
291
+ /** Total allowance in micro-USD. */
153
292
  allowanceMicroUsd: number;
293
+ /** Currently reserved micro-USD. */
154
294
  reservedMicroUsd: number;
295
+ /** Settled spend in micro-USD. */
155
296
  spentMicroUsd: number;
297
+ /** Maximum concurrently running jobs. */
156
298
  maxInFlight: number;
299
+ /** Dispatch is currently blocked. */
157
300
  blocked: boolean;
158
301
  }
302
+ /** A scorer and its published versions. */
159
303
  export interface Scorer extends Identity {
304
+ /** Scorer ID. */
160
305
  id: string;
306
+ /** Archive timestamp on current servers; absent on older compatible responses. */
307
+ archivedAt?: string | null;
308
+ /** Published versions, when included in the response. */
161
309
  versions?: ScorerVersion[];
162
310
  }
311
+ /** An immutable published scorer definition. */
163
312
  export interface ScorerVersion {
313
+ /** Version ID; pin it in experiments and runs. */
164
314
  id: string;
315
+ /** Digest of the definition. */
165
316
  contentDigest: string;
317
+ /** The pinned definition. */
166
318
  definition: ScorerDefinition;
167
319
  }
320
+ /** Final state of a target execution. */
168
321
  export type TerminalState = "succeeded" | "error" | "cancelled";
322
+ /** One attempt to run the target for a case. */
169
323
  export interface Execution {
324
+ /** Execution ID. */
170
325
  id: string;
326
+ /** Attempt number for the case, starting at 1. */
171
327
  attempt: number;
328
+ /** Current state; `uncertain` means no outcome was saved. */
172
329
  state: TerminalState | "started" | "uncertain";
330
+ /** OpenTelemetry trace ID declared for the attempt, or `null`. */
173
331
  traceExternalId: string | null;
332
+ /** Subject created on completion, when known. */
174
333
  subjectId?: string | null;
175
334
  }
335
+ /** A case within an experiment and its latest execution. */
176
336
  export interface ExperimentItem {
337
+ /** Item ID, used with the experiment to address the case. */
177
338
  id: string;
339
+ /** The case's caller-chosen key. */
178
340
  externalKey: string;
341
+ /** Whether a reference output is stored. */
179
342
  hasExpected: boolean;
343
+ /** Latest execution, or `null` before the first start. */
180
344
  execution: Execution | null;
181
345
  }
346
+ /** Scoring progress for an experiment or a historical rescore. */
182
347
  export interface EvaluationRun {
348
+ /** Run ID. */
183
349
  id: string;
350
+ /** Display name. */
184
351
  name: string;
352
+ /** Scorer versions pinned to the run. */
185
353
  scorerVersions: ScorerVersion[];
354
+ /** Subjects in the run. */
186
355
  itemCount: number;
356
+ /** Result counts by state. */
187
357
  scores: {
358
+ /** Results with metrics. */
188
359
  scored: number;
360
+ /** Scorer errors. */
189
361
  error: number;
362
+ /** Skipped results. */
190
363
  skipped: number;
364
+ /** Results not yet recorded. */
191
365
  pending: number;
192
366
  };
193
367
  }
368
+ /** An experiment: a frozen dataset version, a configuration and pinned scorers. */
194
369
  export interface Experiment {
370
+ /** Experiment ID. */
195
371
  id: string;
372
+ /** Display name. */
196
373
  name: string;
374
+ /** Frozen dataset version under test. */
197
375
  datasetVersionId: string;
376
+ /** Configuration handed to the target. */
198
377
  config: JsonValue;
378
+ /** Digest of `config`. */
199
379
  configDigest: string;
380
+ /** The experiment's evaluation run. */
200
381
  evaluation: EvaluationRun;
382
+ /** Cases in the frozen version. */
201
383
  caseCount: number;
384
+ /** When the experiment was finished, or `null`. */
202
385
  finishedAt: string | null;
386
+ /** Case counts by execution state. */
203
387
  execution: {
388
+ /** Cases never started. */
204
389
  unstarted: number;
390
+ /** Cases with a started attempt. */
205
391
  started: number;
392
+ /** Cases whose attempt has no saved outcome. */
206
393
  uncertain: number;
394
+ /** Cases that succeeded. */
207
395
  succeeded: number;
396
+ /** Cases that failed. */
208
397
  error: number;
398
+ /** Cases that were cancelled. */
209
399
  cancelled: number;
210
400
  };
211
401
  }
402
+ /** A sanitized error type with an optional bounded message. */
212
403
  export interface TypedError {
404
+ /** Stable error type. */
213
405
  type: string;
406
+ /** Optional message; stored only when result content is persisted. */
214
407
  message?: string;
215
408
  }
409
+ /** Input for {@link EvaluationClient.startExecution}. */
216
410
  export interface StartExecution {
411
+ /** Stable key; replaying it returns the same execution. */
217
412
  idempotencyKey: string;
413
+ /** OpenTelemetry trace ID the attempt will emit under. */
218
414
  traceExternalId?: string;
415
+ /** Execution being replaced; required for a new attempt. */
219
416
  previousExecutionId?: string;
417
+ /** Explicitly allow replacing a still-started (uncertain) attempt. */
220
418
  allowUncertainRetry?: boolean;
221
419
  }
420
+ /** Input for {@link EvaluationClient.completeExecution}. */
222
421
  export interface CompleteExecution {
422
+ /** Stable key; replaying it returns the same completion. */
223
423
  idempotencyKey: string;
424
+ /** Final state of the attempt. */
224
425
  state: TerminalState;
426
+ /** Target output; omit when unavailable. */
225
427
  output?: JsonValue;
428
+ /** Sanitized failure for `state: "error"`. */
226
429
  error?: TypedError;
430
+ /** Trace revision the stored snapshot must have reached. */
227
431
  expectedTraceRevision?: number;
432
+ /** Whether stored trace evidence is required or explicitly omitted. */
228
433
  traceEvidence?: "required" | "omit";
434
+ /** Why trace evidence was omitted. */
229
435
  omissionReason?: string;
230
436
  }
437
+ /** Result of {@link EvaluationClient.completeExecution}. */
231
438
  export interface Completion {
439
+ /** Completed execution ID. */
232
440
  executionId: string;
441
+ /** Immutable subject created from the outcome. */
233
442
  subjectId: string;
443
+ /** Stored trace snapshot, or `null` when omitted. */
234
444
  traceSnapshotId: string | null;
445
+ /** Evaluation item to score. */
235
446
  evaluationItemId: string;
236
447
  }
448
+ /** A subject within an evaluation run. */
237
449
  export interface EvaluationItem {
450
+ /** Item ID. */
238
451
  id: string;
452
+ /** Subject being scored. */
239
453
  subjectId: string;
454
+ /** Whether the subject has an output. */
240
455
  hasOutput: boolean;
456
+ /** Stored trace snapshot, or `null`. */
241
457
  traceSnapshotId: string | null;
242
458
  }
459
+ /** An immutable saved outcome: inputs, output, reference and evidence for one case attempt. */
243
460
  export interface Subject {
461
+ /** Subject ID. */
244
462
  id: string;
463
+ /** Execution that produced this subject. */
464
+ executionId: string;
465
+ /** Case inputs. */
245
466
  inputs: JsonValue;
467
+ /** Whether an output is stored; JSON `null` counts as present. */
246
468
  hasOutput: boolean;
469
+ /** Target output, when available. */
247
470
  output?: JsonValue;
471
+ /** Whether a reference output is stored. */
248
472
  hasExpected: boolean;
473
+ /** Reference output, when stored. */
249
474
  expected?: JsonValue;
475
+ /** Case metadata. */
250
476
  metadata: Record<string, JsonValue>;
477
+ /** Digest of the subject content. */
251
478
  contentDigest: string;
479
+ /** Whether output evidence can be read. */
252
480
  outputEvidence: "available" | "unavailable";
481
+ /** Final state of the execution. */
253
482
  executionState: TerminalState;
483
+ /** Stored trace snapshot, or `null`. */
254
484
  traceSnapshotId: string | null;
485
+ /** Source case ID. */
255
486
  caseId: string;
487
+ /** Source dataset version ID. */
256
488
  datasetVersionId: string;
489
+ /** Source case key. */
257
490
  caseExternalKey: string;
491
+ /** Source experiment ID. */
258
492
  experimentId: string;
493
+ /** Attempt number of the execution. */
259
494
  attempt: number;
495
+ /** Whether trace evidence was captured, omitted with a reason or not requested. */
260
496
  traceEvidence: "captured" | "omitted" | "not_requested";
497
+ /** Declared OpenTelemetry trace ID, or `null`. */
261
498
  traceExternalId: string | null;
499
+ /** Reason trace evidence was omitted, or `null`. */
262
500
  omissionReason: string | null;
263
501
  }
502
+ /** A reported metric value. */
264
503
  export interface Metric {
504
+ /** Declared metric name. */
265
505
  name: string;
506
+ /** Value matching the declared type. */
266
507
  value: boolean | number | string;
508
+ /** Quality verdict; a failed metric stays `state: "scored"`. */
267
509
  passed?: boolean;
268
510
  }
511
+ /** Outcome of one scorer for one subject. */
269
512
  export type Score = {
513
+ /** Metrics were produced. */
270
514
  state: "scored";
515
+ /** Every declared metric exactly once. */
271
516
  metrics: Metric[];
517
+ /** Human-readable reasoning; required unless `evidence` is given. */
272
518
  explanation?: string;
519
+ /** Supporting data; required unless `explanation` is given. */
273
520
  evidence?: JsonValue;
274
521
  } | {
522
+ /** The scorer failed. */
275
523
  state: "error";
524
+ /** Sanitized failure. */
276
525
  error: TypedError;
277
526
  } | {
527
+ /** The scorer did not apply. */
278
528
  state: "skipped";
529
+ /** Why it was skipped. */
279
530
  explanation: string;
280
531
  };
532
+ /** A score addressed to an evaluation item, as uploaded by {@link EvaluationClient.submitResults}. */
281
533
  export type Result = Score & {
534
+ /** Evaluation item the score belongs to. */
282
535
  evaluationItemId: string;
536
+ /** Scorer version that produced the score. */
283
537
  scorerVersionId: string;
538
+ /** Source digest of the local scorer, for `local_code` pins. */
284
539
  sourceDigest?: string;
285
540
  };
541
+ /** What a local scorer callback receives. */
286
542
  export interface ScoreContext {
543
+ /** Case inputs. */
287
544
  inputs: JsonValue;
545
+ /** Target output; `undefined` means unavailable. */
288
546
  output?: JsonValue;
547
+ /** Reference output, when stored. */
289
548
  expected?: JsonValue;
549
+ /** Whether `output` is present; JSON `null` counts. */
290
550
  hasOutput: boolean;
551
+ /** Whether `expected` is present. */
291
552
  hasExpected: boolean;
553
+ /** Case metadata. */
292
554
  metadata: Record<string, JsonValue>;
555
+ /** Final state of the target execution. */
293
556
  executionState: TerminalState;
557
+ /** Authoritative sealed world and complete journal, when required by the runner. */
558
+ environment?: EnvironmentEvidence;
294
559
  }
560
+ /** Sealed environment evidence resolved through one target execution. */
561
+ export interface EnvironmentEvidenceSnapshot extends EnvironmentCoverage {
562
+ /** Environment-run identity. */
563
+ runId: string;
564
+ /** Linked target execution identity. */
565
+ executionId: string;
566
+ /** Immutable environment version used by the world. */
567
+ environmentVersionId: string;
568
+ /** Digest of the stored environment definition. */
569
+ definitionDigest: string;
570
+ /** Deterministic world seed. */
571
+ seed: string;
572
+ /** Terminal world status. */
573
+ status: "completed" | "abandoned" | "expired";
574
+ /** Number of recorded journal steps. */
575
+ stepCount: number;
576
+ /** Digest of final state. */
577
+ stateDigest: string;
578
+ /** State before the first action. */
579
+ initialState: JsonValue;
580
+ /** State at sealing. */
581
+ finalState: JsonValue;
582
+ }
583
+ /** Sealed environment evidence with its full ordered journal. */
584
+ export interface EnvironmentEvidence extends EnvironmentEvidenceSnapshot {
585
+ /** Complete steps ordered by ordinal. */
586
+ steps: import("../environment/types.js").Step[];
587
+ }
588
+ /** A local scorer: its pinned definition and the callback bound to it. */
295
589
  export interface LocalScorer {
590
+ /** Definition to publish and pin; the runner matches it by digest. */
296
591
  definition: Extract<ScorerDefinition, {
297
592
  kind: "local_code";
298
593
  }>;
299
594
  /** Trusted local code. There is no callback timeout or side-effect cancellation. */
300
595
  score(context: ScoreContext): Score | Promise<Score>;
301
596
  }
597
+ /** Short-lived execution-scoped MCP connection for one simulated world. */
598
+ export interface SimulationMcpCapability {
599
+ /** HTTPS MCP endpoint. */
600
+ url: string;
601
+ /** Attempt-scoped bearer; never persist or expose it. */
602
+ token: string;
603
+ /** Credential expiry timestamp. */
604
+ expiresAt: string;
605
+ }
package/dist/evals.d.ts CHANGED
@@ -1,7 +1,11 @@
1
1
  export { createEvaluationClient, EvaluationClient, HueApiError } from "./evals/client.js";
2
2
  export type { EvaluationClientOptions } from "./evals/client.js";
3
- export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, } from "./evals/runner.js";
3
+ export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, } from "./evals/runner.js";
4
4
  export type { RunExperimentOptions, RescoreOptions, RunnerReport } from "./evals/runner.js";
5
+ export { runSimulation } from "./evals/simulation.js";
6
+ export type { RepositorySimulationCase, RepositorySimulationScorer, RunSimulationOptions, SimulationProgress, SimulationReport, SimulationScenario, SimulationTargetContext, } from "./evals/simulation.js";
7
+ export { actualAgentManifestV2, agentManifestDigestV2, attemptBaselineV2, attemptBindingRead, attemptConnectionBundleV2, attemptIdentityV2, dependencyManifestV2, dependencyProviderV2, expectedAgentManifestV2, executionManifestDigestV2, parityEvidenceV2, preflightFindingV2, preflightReportV2, prepareAttemptInputV2, projectMcpConnectionV2, secretFreeBindingV2, surfaceBindingV2, } from "./evals/attempt.js";
8
+ export type { ActualAgentManifestInputV2, ActualAgentManifestV2, AttemptBaselineV2, AttemptBindingRead, AttemptConnectionBundleV2, AttemptIdentityV2, DependencyManifestV2, DependencyProviderV2, ExpectedAgentManifestV2, ParityEvidenceV2, PreflightFindingV2, PreflightReportV2, PrepareAttemptIncompleteV2, PrepareAttemptInputV2, PrepareAttemptReadyV2, PrepareAttemptRequestV2, PrepareAttemptResultV2, RefreshAttemptResultV2, RequestedAttemptProviderV2, RevokeAttemptResult, SurfaceBindingV2, } from "./evals/attempt.js";
5
9
  export { builtins, defineLocalScorer, scoreLocally } from "./evals/scorers.js";
6
10
  export { sourceDigest } from "./evals/json.js";
7
11
  export type * from "./evals/types.js";
package/dist/evals.js CHANGED
@@ -1,4 +1,6 @@
1
1
  export { createEvaluationClient, EvaluationClient, HueApiError } from "./evals/client.js";
2
- export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, } from "./evals/runner.js";
2
+ export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, } from "./evals/runner.js";
3
+ export { runSimulation } from "./evals/simulation.js";
4
+ export { actualAgentManifestV2, agentManifestDigestV2, attemptBaselineV2, attemptBindingRead, attemptConnectionBundleV2, attemptIdentityV2, dependencyManifestV2, dependencyProviderV2, expectedAgentManifestV2, executionManifestDigestV2, parityEvidenceV2, preflightFindingV2, preflightReportV2, prepareAttemptInputV2, projectMcpConnectionV2, secretFreeBindingV2, surfaceBindingV2, } from "./evals/attempt.js";
3
5
  export { builtins, defineLocalScorer, scoreLocally } from "./evals/scorers.js";
4
6
  export { sourceDigest } from "./evals/json.js";
@@ -0,0 +1,8 @@
1
+ import type { HueClient } from "./client.js";
2
+ import type { ExperimentalTelemetrySettings } from "./types.js";
3
+ /**
4
+ * Per-call telemetry for AI SDK 6: pass as `experimental_telemetry`. Spans are created with Hue's
5
+ * tracer, so they parent under `withSpan` and inherit session/user identifiers, and prompt/response
6
+ * recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry` from `@hue-run/sdk/ai-sdk`.
7
+ */
8
+ export declare function hueExperimentalTelemetry(hue: HueClient): ExperimentalTelemetrySettings;
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Per-call telemetry for AI SDK 6: pass as `experimental_telemetry`. Spans are created with Hue's
3
+ * tracer, so they parent under `withSpan` and inherit session/user identifiers, and prompt/response
4
+ * recording follows `captureContent`. AI SDK 7 applications use `hueTelemetry` from `@hue-run/sdk/ai-sdk`.
5
+ */
6
+ export function hueExperimentalTelemetry(hue) {
7
+ return {
8
+ isEnabled: hue.enabled,
9
+ recordInputs: hue.captureContent,
10
+ recordOutputs: hue.captureContent,
11
+ tracer: hue.tracer,
12
+ };
13
+ }
package/dist/index.d.ts CHANGED
@@ -1,4 +1,7 @@
1
1
  export { createHue, createHueSafe, HueClient, HueConnectionError, type ExistingHueProviders, } from "./client.js";
2
2
  export { createHueTransport, HueTransport, HueExportError } from "./transport.js";
3
+ export type { RecordValue } from "./transport.js";
4
+ export { hueExperimentalTelemetry } from "./experimental-telemetry.js";
5
+ export { contentPrefixes } from "./privacy.js";
3
6
  export { HueTraceVerificationError } from "./receipt.js";
4
7
  export type * from "./types.js";
package/dist/index.js CHANGED
@@ -1,3 +1,5 @@
1
1
  export { createHue, createHueSafe, HueClient, HueConnectionError, } from "./client.js";
2
2
  export { createHueTransport, HueTransport, HueExportError } from "./transport.js";
3
+ export { hueExperimentalTelemetry } from "./experimental-telemetry.js";
4
+ export { contentPrefixes } from "./privacy.js";
3
5
  export { HueTraceVerificationError } from "./receipt.js";