@hue-run/sdk 0.8.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CLI.md +100 -29
  2. package/ENVIRONMENTS.md +73 -8
  3. package/EVALUATIONS.md +39 -19
  4. package/README.md +51 -22
  5. package/dist/cli/eval-direct.d.ts +12 -1
  6. package/dist/cli/eval-direct.js +145 -33
  7. package/dist/cli/eval.d.ts +28 -1
  8. package/dist/cli/eval.js +469 -72
  9. package/dist/client.js +10 -20
  10. package/dist/environment/client.d.ts +10 -2
  11. package/dist/environment/client.js +43 -12
  12. package/dist/environment/types.d.ts +30 -10
  13. package/dist/environment.d.ts +1 -1
  14. package/dist/environment.js +1 -1
  15. package/dist/evals/checkpoint.d.ts +15 -1
  16. package/dist/evals/checkpoint.js +34 -3
  17. package/dist/evals/client.d.ts +19 -3
  18. package/dist/evals/client.js +82 -34
  19. package/dist/evals/environment-target.d.ts +23 -3
  20. package/dist/evals/environment-target.js +73 -10
  21. package/dist/evals/exit-cleanup.d.ts +4 -0
  22. package/dist/evals/exit-cleanup.js +26 -0
  23. package/dist/evals/files.d.ts +59 -1
  24. package/dist/evals/files.js +187 -27
  25. package/dist/evals/local-worker.d.ts +21 -3
  26. package/dist/evals/local-worker.js +14 -0
  27. package/dist/evals/runner.d.ts +45 -0
  28. package/dist/evals/runner.js +164 -13
  29. package/dist/evals/scenarios.d.ts +8 -6
  30. package/dist/evals/scenarios.js +75 -15
  31. package/dist/evals/simulation.d.ts +17 -4
  32. package/dist/evals/simulation.js +8 -0
  33. package/dist/evals/types.d.ts +10 -1
  34. package/dist/evals/verdicts.d.ts +15 -2
  35. package/dist/evals/verdicts.js +43 -9
  36. package/dist/evals.d.ts +4 -3
  37. package/dist/evals.js +2 -2
  38. package/dist/inline-files.d.ts +4 -0
  39. package/dist/inline-files.js +40 -17
  40. package/dist/provider-tools.js +30 -3
  41. package/dist/snapshot.d.ts +2 -2
  42. package/dist/snapshot.js +47 -12
  43. package/dist/transport.js +2 -2
  44. package/dist/version.d.ts +1 -1
  45. package/dist/version.js +1 -1
  46. package/package.json +1 -1
package/dist/client.js CHANGED
@@ -40,7 +40,7 @@ function identifier(value) {
40
40
  throw new TypeError("Session/user/workspace identifiers must contain 1–4096 valid characters");
41
41
  return value;
42
42
  }
43
- /** A usable metadata label: a non-blank string of at most 256 characters. */
43
+ /** A usable metadata label: non-blank, at most 256 UTF-16 code units, with no NUL or unpaired surrogate. */
44
44
  function isLabel(value) {
45
45
  return (typeof value === "string" &&
46
46
  value.trim() !== "" &&
@@ -48,18 +48,6 @@ function isLabel(value) {
48
48
  !value.includes("\u0000") &&
49
49
  value.isWellFormed());
50
50
  }
51
- /** A source label uses the stricter wire-safe validation without changing existing labels. */
52
- function isSourceLabel(value) {
53
- return (typeof value === "string" &&
54
- value.trim() !== "" &&
55
- value.length <= 256 &&
56
- !value.includes("\u0000") &&
57
- value.isWellFormed());
58
- }
59
- /** A label that is also free of NUL and unpaired surrogates, which export would reject. */
60
- function isTextLabel(value) {
61
- return isLabel(value) && !value.includes("\u0000") && value.isWellFormed();
62
- }
63
51
  /**
64
52
  * Runs `work` exactly once with `active` as OpenTelemetry's current context, so instrumentations
65
53
  * that use the global API parent under the Hue span. Without a registered context manager
@@ -330,11 +318,11 @@ export class HueClient {
330
318
  "gen_ai.operation.name": "execute_tool",
331
319
  "gen_ai.tool.name": name,
332
320
  };
333
- const stamp = (key, value, valid = isLabel) => {
321
+ const stamp = (key, value) => {
334
322
  if (value === undefined)
335
323
  return;
336
324
  // A blank or non-string label is omitted and counted; the tool call itself still runs.
337
- if (valid(value))
325
+ if (isLabel(value))
338
326
  attributes[key] = value;
339
327
  else if (this.enabled && !this.closed)
340
328
  this.transport.instrumentationFailure();
@@ -342,8 +330,8 @@ export class HueClient {
342
330
  stamp("gen_ai.tool.call.id", options.callId);
343
331
  stamp("mcp.server.name", options.mcp?.name);
344
332
  stamp("mcp.server.version", options.mcp?.version);
345
- stamp("hue.mcp.provider", options.mcp?.provider, isSourceLabel);
346
- stamp("hue.mcp.surface", options.mcp?.surface, isSourceLabel);
333
+ stamp("hue.mcp.provider", options.mcp?.provider);
334
+ stamp("hue.mcp.surface", options.mcp?.surface);
347
335
  return this.withSpan(`execute_tool ${name}`, async ({ span }) => {
348
336
  this.setContent(span, "gen_ai.tool.call.arguments", input);
349
337
  const result = await execute();
@@ -555,7 +543,9 @@ export class HueClient {
555
543
  const attributes = {};
556
544
  if (label === undefined)
557
545
  return attributes;
558
- const info = options.servers?.[label];
546
+ const info = options.servers && Object.hasOwn(options.servers, label)
547
+ ? options.servers[label]
548
+ : undefined;
559
549
  for (const [key, value] of [
560
550
  ["mcp.server.name", info?.name ?? label],
561
551
  ["mcp.server.version", info?.version],
@@ -630,7 +620,7 @@ export class HueClient {
630
620
  const { role, mediaType, data, name } = file;
631
621
  if (role !== "input" && role !== "attachment" && role !== "output")
632
622
  throw new TypeError("Invalid file role");
633
- if (!isTextLabel(mediaType))
623
+ if (!isLabel(mediaType))
634
624
  throw new TypeError("Invalid media type");
635
625
  let sha256 = typeof file.sha256 === "string" ? file.sha256.toLowerCase() : file.sha256;
636
626
  let byteSize = file.byteSize;
@@ -673,7 +663,7 @@ export class HueClient {
673
663
  if (byteSize !== undefined)
674
664
  attributes["hue.file.size"] = byteSize;
675
665
  if (this.captureContent && name !== undefined) {
676
- if (isTextLabel(name))
666
+ if (isLabel(name))
677
667
  attributes["hue.file.name"] = name;
678
668
  else
679
669
  this.transport.instrumentationFailure();
@@ -16,12 +16,18 @@ export declare class HueEnvironmentError extends Error {
16
16
  readonly status?: number | undefined;
17
17
  /** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
18
18
  readonly retryAfterMs?: number | undefined;
19
+ /** The server's `X-Hue-Diagnostic` code, when it is a short snake-case value. */
20
+ readonly diagnostic?: string | undefined;
19
21
  constructor(
20
22
  /** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
21
23
  status?: number | undefined,
22
24
  /** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
23
- retryAfterMs?: number | undefined);
25
+ retryAfterMs?: number | undefined,
26
+ /** The server's `X-Hue-Diagnostic` code, when it is a short snake-case value. */
27
+ diagnostic?: string | undefined);
24
28
  }
29
+ /** A connection failure or a status the client retries; seal polling continues through these. */
30
+ export declare function isTransientEnvironmentError(error: unknown): boolean;
25
31
  /** Typed client for authored environments, isolated runs and immutable journals. */
26
32
  export declare class EnvironmentClient {
27
33
  /** Validated Hue origin. */
@@ -47,7 +53,9 @@ export declare class EnvironmentClient {
47
53
  /** Creates or recovers one fresh isolated world using a stable idempotency key. */
48
54
  createRun(input: CreateRunInput): Promise<EnvironmentRun>;
49
55
  /** Reads authoritative current or sealed world state. */
50
- getRun(runId: string): Promise<{
56
+ getRun(runId: string, options?: {
57
+ signal?: AbortSignal;
58
+ }): Promise<{
51
59
  id: string;
52
60
  environmentVersionId: string;
53
61
  executionId: string | null;
@@ -4,20 +4,35 @@ import { aggregateBounds, json, uuid, valueBounds } from "../evals/json.js";
4
4
  export class HueEnvironmentError extends Error {
5
5
  status;
6
6
  retryAfterMs;
7
+ diagnostic;
7
8
  constructor(
8
9
  /** HTTP status when Hue answered; absent for transport, timeout or parse failure. */
9
10
  status,
10
11
  /** Hue's `Retry-After` in milliseconds, bounded, when a 429 or 503 carried one. */
11
- retryAfterMs) {
12
+ retryAfterMs,
13
+ /** The server's `X-Hue-Diagnostic` code, when it is a short snake-case value. */
14
+ diagnostic) {
12
15
  super(status
13
- ? `Hue environment request failed (HTTP ${status})`
16
+ ? `Hue environment request failed (HTTP ${status}${diagnostic ? `, ${diagnostic}` : ""})`
14
17
  : "Hue environment connection or response failed");
15
18
  this.status = status;
16
19
  this.retryAfterMs = retryAfterMs;
20
+ this.diagnostic = diagnostic;
17
21
  this.name = "HueEnvironmentError";
18
22
  }
19
23
  }
24
+ const DIAGNOSTIC = /^[a-z_]{1,64}$/;
25
+ /** The response's diagnostic code, or undefined when absent or invalid. */
26
+ function diagnosticOf(response) {
27
+ const value = response.headers.get("x-hue-diagnostic");
28
+ return value !== null && DIAGNOSTIC.test(value) ? value : undefined;
29
+ }
20
30
  const RETRYABLE = new Set([408, 429, 500, 502, 503, 504]);
31
+ /** A connection failure or a status the client retries; seal polling continues through these. */
32
+ export function isTransientEnvironmentError(error) {
33
+ return (error instanceof HueEnvironmentError &&
34
+ (error.status === undefined || RETRYABLE.has(error.status)));
35
+ }
21
36
  /** A `Retry-After` longer than this waits this long: Hue asks for a second, never minutes. */
22
37
  const MAX_RETRY_AFTER_MS = 10_000;
23
38
  const TRACEPARENT = /^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$/;
@@ -64,9 +79,10 @@ export class EnvironmentClient {
64
79
  throw new RangeError("maxAttempts must be 1–10");
65
80
  this.maxAttempts = attempts;
66
81
  }
67
- async send(method, path, payload) {
82
+ async send(method, path, payload, options = {}) {
68
83
  let response;
69
84
  try {
85
+ const timeout = AbortSignal.timeout(this.timeoutMillis);
70
86
  response = await fetch(`${this.baseUrl}/api/v1${path}`, {
71
87
  method,
72
88
  headers: {
@@ -75,7 +91,7 @@ export class EnvironmentClient {
75
91
  },
76
92
  body: payload,
77
93
  redirect: "error",
78
- signal: AbortSignal.timeout(this.timeoutMillis),
94
+ signal: options.signal ? AbortSignal.any([options.signal, timeout]) : timeout,
79
95
  });
80
96
  }
81
97
  catch {
@@ -83,7 +99,7 @@ export class EnvironmentClient {
83
99
  }
84
100
  if (!response.ok) {
85
101
  await response.body?.cancel();
86
- throw new HueEnvironmentError(response.status, response.status === 429 || response.status === 503 ? retryAfterMillis(response) : undefined);
102
+ throw new HueEnvironmentError(response.status, response.status === 429 || response.status === 503 ? retryAfterMillis(response) : undefined, diagnosticOf(response));
87
103
  }
88
104
  try {
89
105
  const reader = response.body?.getReader();
@@ -111,25 +127,40 @@ export class EnvironmentClient {
111
127
  throw new HueEnvironmentError();
112
128
  }
113
129
  }
114
- async request(method, path, body) {
130
+ async request(method, path, body, options = {}) {
115
131
  // Serialize once: a body this client cannot encode is a caller error that no retry fixes.
116
132
  const payload = body === undefined
117
133
  ? undefined
118
134
  : JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), REQUEST_BOUNDS));
119
135
  for (let attempt = 1;; attempt++) {
120
136
  try {
121
- return await this.send(method, path, payload);
137
+ return await this.send(method, path, payload, options);
122
138
  }
123
139
  catch (error) {
124
140
  if (!(error instanceof HueEnvironmentError))
125
141
  throw error;
126
- const recoverable = error.status === undefined || RETRYABLE.has(error.status);
127
- if (!recoverable || attempt >= this.maxAttempts)
142
+ if (options.signal?.aborted)
143
+ throw error;
144
+ if (!isTransientEnvironmentError(error) || attempt >= this.maxAttempts)
128
145
  throw error;
129
146
  // Hue's admission refusals say how long to wait; anything else backs off.
130
147
  const backoff = Math.min(100 * 2 ** (attempt - 1), 2000);
131
148
  const wait = error.retryAfterMs ?? backoff + Math.random() * backoff;
132
- await new Promise((resolve) => setTimeout(resolve, wait));
149
+ await new Promise((resolve, reject) => {
150
+ const timer = setTimeout(() => {
151
+ options.signal?.removeEventListener("abort", abort);
152
+ resolve();
153
+ }, wait);
154
+ const abort = () => {
155
+ clearTimeout(timer);
156
+ options.signal?.removeEventListener("abort", abort);
157
+ reject(new HueEnvironmentError());
158
+ };
159
+ if (options.signal?.aborted)
160
+ abort();
161
+ else
162
+ options.signal?.addEventListener("abort", abort, { once: true });
163
+ });
133
164
  }
134
165
  }
135
166
  }
@@ -194,8 +225,8 @@ export class EnvironmentClient {
194
225
  });
195
226
  }
196
227
  /** Reads authoritative current or sealed world state. */
197
- async getRun(runId) {
198
- const run = await this.request("GET", `/environment-runs/${uuid(runId)}`);
228
+ async getRun(runId, options = {}) {
229
+ const run = await this.request("GET", `/environment-runs/${uuid(runId)}`, undefined, options);
199
230
  return { validity: "not_assessed", coverageGap: null, ...run };
200
231
  }
201
232
  /** Record a known coverage gap with durable identity; retries reuse the exact request. */
@@ -173,6 +173,35 @@ export interface EnvironmentDefinition {
173
173
  /** The extendable legacy name remains V1. Publication and runs select their
174
174
  * explicit version; provider context is validated by the authoritative server. */
175
175
  export type EnvironmentDefinitionV1 = EnvironmentDefinition;
176
+ /** Gmail mailbox carrier holding messages and drafts in the simplified shape Hue's
177
+ * `hue.gmail.*` actions read. */
178
+ export interface GmailMailboxConfigurationV1 {
179
+ /** Gmail mailbox configuration discriminator. */
180
+ kind: "gmail_mailbox/v1";
181
+ /** Collection containing synthetic messages. */
182
+ messagesCollection: string;
183
+ /** Collection containing synthetic drafts. */
184
+ draftsCollection: string;
185
+ /** Synthetic mailbox address. */
186
+ mailboxAddress: string;
187
+ }
188
+ /** Gmail mailbox carrier served by Hue's simulation gateway: messages, drafts and labels in the
189
+ * entity shapes the Gmail mirrors serve, with threads derived from each message's `threadId`. A
190
+ * definition whose provider instances all use it may publish with no actions. */
191
+ export interface GmailMailboxConfigurationV2 {
192
+ /** Gmail mailbox configuration discriminator. */
193
+ kind: "gmail_mailbox/v2";
194
+ /** Collection containing synthetic messages. */
195
+ messagesCollection: string;
196
+ /** Collection containing synthetic drafts, each naming a `DRAFT`-labelled message. */
197
+ draftsCollection: string;
198
+ /** Collection containing synthetic labels. */
199
+ labelsCollection: string;
200
+ /** Synthetic mailbox address; Hue accepts only `owner@example.test`, in any letter case. */
201
+ mailboxAddress: string;
202
+ }
203
+ /** Either Gmail mailbox carrier, discriminated by `kind`; Hue validates both at publication. */
204
+ export type GmailMailboxConfiguration = GmailMailboxConfigurationV1 | GmailMailboxConfigurationV2;
176
205
  /** One synthetic Gmail principal and its world-state collection bindings. */
177
206
  export interface GmailProviderInstance {
178
207
  /** Stable instance key referenced by attempt provider selection. */
@@ -182,16 +211,7 @@ export interface GmailProviderInstance {
182
211
  /** Synthetic principal UUID, canonicalized to lowercase by Hue. */
183
212
  syntheticPrincipalId: string;
184
213
  /** Versioned mapping from Gmail concepts to authored-world collections. */
185
- configuration: {
186
- /** Gmail mailbox configuration discriminator. */
187
- kind: "gmail_mailbox/v1";
188
- /** Collection containing synthetic messages. */
189
- messagesCollection: string;
190
- /** Collection containing synthetic drafts. */
191
- draftsCollection: string;
192
- /** Synthetic mailbox address. */
193
- mailboxAddress: string;
194
- };
214
+ configuration: GmailMailboxConfiguration;
195
215
  }
196
216
  /** V2 authored world with immutable provider-instance bindings. */
197
217
  export interface EnvironmentDefinitionV2 extends Omit<EnvironmentDefinition, "schemaVersion"> {
@@ -1,4 +1,4 @@
1
- export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, } from "./environment/client.js";
1
+ export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, isTransientEnvironmentError, } from "./environment/client.js";
2
2
  export type { EnvironmentClientOptions } from "./environment/client.js";
3
3
  export { bindEnvironmentTools } from "./environment/tools.js";
4
4
  export type { BindEnvironmentToolsOptions, EnvironmentTool } from "./environment/tools.js";
@@ -1,3 +1,3 @@
1
- export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, } from "./environment/client.js";
1
+ export { createEnvironmentClient, EnvironmentClient, HueEnvironmentError, isTransientEnvironmentError, } from "./environment/client.js";
2
2
  export { bindEnvironmentTools } from "./environment/tools.js";
3
3
  export { agentEnvironment, HUE_CONTROL_PLANE_VARIABLES, isHueControlPlaneCredential, legacyMcpCapability, stripHueControlPlaneCredentials, worldHandoff, writeMcpConfig, } from "./environment/world.js";
@@ -1,6 +1,20 @@
1
- /** One owner per directory. A crash leaves .lock for explicit operator recovery. */
1
+ /** The checkpoint belongs to a run started with a different identity. When only its content
2
+ * policy differs, `startedWith` holds the policy the unfinished run was started with. */
3
+ export declare class CheckpointIdentityError extends Error {
4
+ readonly startedWith?: {
5
+ persistResultContent?: unknown;
6
+ captureContent?: unknown;
7
+ } | undefined;
8
+ constructor(startedWith?: {
9
+ persistResultContent?: unknown;
10
+ captureContent?: unknown;
11
+ } | undefined);
12
+ }
13
+ /** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
14
+ * `hue eval`, which stops its agents first, releases it. */
2
15
  export declare class CheckpointStore {
3
16
  readonly directory: string;
17
+ private untrack;
4
18
  private constructor();
5
19
  static acquire(directory: string, identity: unknown): Promise<CheckpointStore>;
6
20
  read<T>(key: string): Promise<T | undefined>;
@@ -1,11 +1,40 @@
1
- import { constants } from "node:fs";
1
+ import { constants, rmSync } from "node:fs";
2
2
  import { lstat, mkdir, open, rename, rm } from "node:fs/promises";
3
3
  import { join, resolve } from "node:path";
4
4
  import { randomUUID } from "node:crypto";
5
+ import { onForcedExit } from "./exit-cleanup.js";
5
6
  import { digest } from "./json.js";
6
- /** One owner per directory. A crash leaves .lock for explicit operator recovery. */
7
+ const CONTENT_POLICY = ["persistResultContent", "captureContent"];
8
+ /** The checkpoint belongs to a run started with a different identity. When only its content
9
+ * policy differs, `startedWith` holds the policy the unfinished run was started with. */
10
+ export class CheckpointIdentityError extends Error {
11
+ startedWith;
12
+ constructor(startedWith) {
13
+ super(startedWith
14
+ ? "Checkpoint content policy differs from the one this unfinished run was started with"
15
+ : "Checkpoint identity differs from this project, run, pins or content policy");
16
+ this.startedWith = startedWith;
17
+ this.name = "CheckpointIdentityError";
18
+ }
19
+ }
20
+ /** The prior policy when two identities differ only in their content policy. */
21
+ function contentPolicyOnly(prior, identity) {
22
+ if (!prior || !identity || typeof prior !== "object" || typeof identity !== "object")
23
+ return undefined;
24
+ const before = prior;
25
+ const after = identity;
26
+ const keys = new Set([...Object.keys(before), ...Object.keys(after)]);
27
+ for (const key of keys)
28
+ if (!CONTENT_POLICY.includes(key) &&
29
+ JSON.stringify(before[key]) !== JSON.stringify(after[key]))
30
+ return undefined;
31
+ return Object.fromEntries(CONTENT_POLICY.map((key) => [key, before[key]]));
32
+ }
33
+ /** One owner per directory. A crash leaves .lock for explicit operator recovery; a forced exit of
34
+ * `hue eval`, which stops its agents first, releases it. */
7
35
  export class CheckpointStore {
8
36
  directory;
37
+ untrack = () => { };
9
38
  constructor(directory) {
10
39
  this.directory = directory;
11
40
  }
@@ -18,6 +47,7 @@ export class CheckpointStore {
18
47
  const store = new CheckpointStore(root);
19
48
  try {
20
49
  await mkdir(join(root, ".lock"), { mode: 0o700 });
50
+ store.untrack = onForcedExit(() => rmSync(join(root, ".lock"), { recursive: true, force: true }));
21
51
  }
22
52
  catch {
23
53
  throw new Error("Checkpoint directory is locked; confirm its owner stopped before explicitly removing .lock");
@@ -27,7 +57,7 @@ export class CheckpointStore {
27
57
  const expected = { format: 1, identity, digest: digest(identity) };
28
58
  const prior = await store.read("manifest");
29
59
  if (prior && (prior.format !== 1 || prior.digest !== expected.digest))
30
- throw new Error("Checkpoint identity differs from this project, run, pins or content policy");
60
+ throw new CheckpointIdentityError(prior.format === 1 ? contentPolicyOnly(prior.identity, identity) : undefined);
31
61
  if (!prior)
32
62
  await store.write("manifest", expected);
33
63
  return store;
@@ -85,5 +115,6 @@ export class CheckpointStore {
85
115
  }
86
116
  async release() {
87
117
  await rm(join(this.directory, ".lock"), { recursive: true });
118
+ this.untrack();
88
119
  }
89
120
  }
@@ -18,10 +18,20 @@ export declare class HueApiError extends Error {
18
18
  /** HTTP status when Hue answered; absent for network, timeout and parsing failures. */
19
19
  status?: number | undefined);
20
20
  }
21
+ /** An artifact download that ran past the size its caller expected; the rest is not read. */
22
+ export declare class ArtifactSizeError extends Error {
23
+ /** The size the caller expected, in bytes. */
24
+ readonly maxBytes: number;
25
+ constructor(
26
+ /** The size the caller expected, in bytes. */
27
+ maxBytes: number);
28
+ }
21
29
  /**
22
30
  * Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
23
31
  * results and hosted judge jobs. No implicit mutation retry: callers retain stable idempotency keys
24
- * for experiments and results. Responses are bounded to 4 MiB.
32
+ * for experiments and results. The one exception is a request Hue refused before acting on it with
33
+ * a short `Retry-After` (HTTP 429 or 503, at most 5 seconds): it is sent again after that wait, up
34
+ * to four times. Responses are bounded to 4 MiB.
25
35
  */
26
36
  export declare class EvaluationClient {
27
37
  /** Validated Hue origin. */
@@ -29,9 +39,15 @@ export declare class EvaluationClient {
29
39
  private readonly apiKey;
30
40
  private readonly timeoutMillis;
31
41
  constructor(options: EvaluationClientOptions);
42
+ /** Sends one request to Hue, again after the wait while Hue refuses it with a short
43
+ * `Retry-After`. `init` runs for every attempt, so each has its own timeout. */
44
+ private send;
32
45
  private request;
33
- /** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size. */
34
- downloadArtifact(id: string): Promise<Uint8Array>;
46
+ /** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
47
+ * With `maxBytes`, the download stops one byte past it and throws `ArtifactSizeError`. */
48
+ downloadArtifact(id: string, options?: {
49
+ maxBytes?: number;
50
+ }): Promise<Uint8Array>;
35
51
  /** Stage bytes at the storage capability Hue issued. The Hue key is never sent to storage. */
36
52
  uploadArtifactBytes(upload: ArtifactUpload, bytes: Uint8Array, contentType: string): Promise<void>;
37
53
  private page;
@@ -12,6 +12,17 @@ export class HueApiError extends Error {
12
12
  this.name = "HueApiError";
13
13
  }
14
14
  }
15
+ /** An artifact download that ran past the size its caller expected; the rest is not read. */
16
+ export class ArtifactSizeError extends Error {
17
+ maxBytes;
18
+ constructor(
19
+ /** The size the caller expected, in bytes. */
20
+ maxBytes) {
21
+ super(`Artifact download exceeded the expected ${maxBytes} bytes`);
22
+ this.maxBytes = maxBytes;
23
+ this.name = "ArtifactSizeError";
24
+ }
25
+ }
15
26
  const registryFieldAliases = [
16
27
  ["datasetId", "evalSetId"],
17
28
  ["datasetVersionId", "evalSetVersionId"],
@@ -91,10 +102,30 @@ function productRunFields(value, kind) {
91
102
  result.scoring = result.evaluation;
92
103
  return result;
93
104
  }
105
+ /** Times a refused request is sent again before its refusal is the caller's error. */
106
+ const REFUSAL_RETRIES = 4;
107
+ /** A refusal asking for a longer wait fails at once rather than holding the caller. */
108
+ const MAX_REFUSAL_RETRY_AFTER_SECONDS = 5;
109
+ /**
110
+ * The whole-second `Retry-After` of a 429 or 503 asking for at most 5 seconds. Hue sends one only
111
+ * when it refused the request before acting on it, such as a busy key check, so sending any method
112
+ * again is safe. A date, a longer wait or any other failure, a timeout included, is not retried.
113
+ */
114
+ function refusalRetryAfter(response) {
115
+ if (response.status !== 429 && response.status !== 503)
116
+ return undefined;
117
+ const header = response.headers.get("retry-after")?.trim() ?? "";
118
+ if (!/^\d{1,6}$/.test(header))
119
+ return undefined;
120
+ const seconds = Number(header);
121
+ return seconds <= MAX_REFUSAL_RETRY_AFTER_SECONDS ? seconds : undefined;
122
+ }
94
123
  /**
95
124
  * Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
96
125
  * results and hosted judge jobs. No implicit mutation retry: callers retain stable idempotency keys
97
- * for experiments and results. Responses are bounded to 4 MiB.
126
+ * for experiments and results. The one exception is a request Hue refused before acting on it with
127
+ * a short `Retry-After` (HTTP 429 or 503, at most 5 seconds): it is sent again after that wait, up
128
+ * to four times. Responses are bounded to 4 MiB.
98
129
  */
99
130
  export class EvaluationClient {
100
131
  /** Validated Hue origin. */
@@ -111,27 +142,40 @@ export class EvaluationClient {
111
142
  this.apiKey = validated.apiKey;
112
143
  this.timeoutMillis = validated.timeoutMillis;
113
144
  }
145
+ /** Sends one request to Hue, again after the wait while Hue refuses it with a short
146
+ * `Retry-After`. `init` runs for every attempt, so each has its own timeout. */
147
+ async send(url, init) {
148
+ for (let attempt = 0;; attempt++) {
149
+ let response;
150
+ try {
151
+ response = await fetch(url, init());
152
+ }
153
+ catch {
154
+ throw new HueApiError();
155
+ }
156
+ const seconds = attempt < REFUSAL_RETRIES ? refusalRetryAfter(response) : undefined;
157
+ if (seconds === undefined)
158
+ return response;
159
+ await response.body?.cancel().catch(() => undefined);
160
+ // Never sooner than asked; the jitter spreads out parallel requests refused together.
161
+ await new Promise((resolve) => setTimeout(resolve, seconds * 1000 * (1 + Math.random() * 0.5)));
162
+ }
163
+ }
114
164
  async request(method, path, body, bounds = { ...valueBounds, bytes: 1024 * 1024 }) {
115
165
  // Optional top-level fields are omitted intentionally; nested undefined remains invalid.
116
166
  const payload = body === undefined
117
167
  ? undefined
118
168
  : JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), bounds));
119
- let response;
120
- try {
121
- response = await fetch(`${this.baseUrl}/api/v1${path}`, {
122
- method,
123
- headers: {
124
- Authorization: `Bearer ${this.apiKey}`,
125
- ...(payload ? { "Content-Type": "application/json" } : {}),
126
- },
127
- body: payload,
128
- redirect: "error",
129
- signal: AbortSignal.timeout(this.timeoutMillis),
130
- });
131
- }
132
- catch {
133
- throw new HueApiError();
134
- }
169
+ const response = await this.send(`${this.baseUrl}/api/v1${path}`, () => ({
170
+ method,
171
+ headers: {
172
+ Authorization: `Bearer ${this.apiKey}`,
173
+ ...(payload ? { "Content-Type": "application/json" } : {}),
174
+ },
175
+ body: payload,
176
+ redirect: "error",
177
+ signal: AbortSignal.timeout(this.timeoutMillis),
178
+ }));
135
179
  if (!response.ok) {
136
180
  await response.body?.cancel();
137
181
  throw new HueApiError(response.status);
@@ -162,20 +206,18 @@ export class EvaluationClient {
162
206
  throw new HueApiError();
163
207
  }
164
208
  }
165
- /** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size. */
166
- async downloadArtifact(id) {
167
- let response;
168
- try {
169
- response = await fetch(`${this.baseUrl}/api/v1/artifacts/${uuid(id)}/download`, {
170
- method: "GET",
171
- headers: { Authorization: `Bearer ${this.apiKey}` },
172
- redirect: "error",
173
- signal: AbortSignal.timeout(Math.max(this.timeoutMillis, 120_000)),
174
- });
175
- }
176
- catch {
177
- throw new HueApiError();
178
- }
209
+ /** Verified bytes of one ready artifact in this project, bounded to the 25 MiB pilot file size.
210
+ * With `maxBytes`, the download stops one byte past it and throws `ArtifactSizeError`. */
211
+ async downloadArtifact(id, options = {}) {
212
+ const expected = options.maxBytes;
213
+ if (expected !== undefined && (!Number.isSafeInteger(expected) || expected < 0))
214
+ throw new RangeError("maxBytes must be a non-negative integer");
215
+ const response = await this.send(`${this.baseUrl}/api/v1/artifacts/${uuid(id)}/download`, () => ({
216
+ method: "GET",
217
+ headers: { Authorization: `Bearer ${this.apiKey}` },
218
+ redirect: "error",
219
+ signal: AbortSignal.timeout(Math.max(this.timeoutMillis, 120_000)),
220
+ }));
179
221
  if (!response.ok) {
180
222
  await response.body?.cancel();
181
223
  throw new HueApiError(response.status);
@@ -192,6 +234,8 @@ export class EvaluationClient {
192
234
  if (done)
193
235
  break;
194
236
  size += value.byteLength;
237
+ if (expected !== undefined && size > expected)
238
+ throw new ArtifactSizeError(expected);
195
239
  if (size > 25 * 1024 * 1024)
196
240
  throw new Error("Oversized artifact");
197
241
  chunks.push(value);
@@ -202,7 +246,9 @@ export class EvaluationClient {
202
246
  }
203
247
  return new Uint8Array(Buffer.concat(chunks));
204
248
  }
205
- catch {
249
+ catch (error) {
250
+ if (error instanceof ArtifactSizeError)
251
+ throw error;
206
252
  throw new HueApiError();
207
253
  }
208
254
  }
@@ -592,8 +638,10 @@ export class EvaluationClient {
592
638
  return this.request("GET", "/judge-budget");
593
639
  }
594
640
  /** Register or refresh the fixed local agent key and revision. */
595
- registerLocalAgent(input) {
596
- return this.request("POST", "/local-agent-worker/register", input);
641
+ async registerLocalAgent(input) {
642
+ const agent = await this.request("POST", "/local-agent-worker/register", input);
643
+ // Hue's response names the key `agentKey`; read either name.
644
+ return { ...agent, key: agent.key ?? agent.agentKey ?? input.key };
597
645
  }
598
646
  /** Claim a queued run for this agent and durable worker identity. */
599
647
  claimLocalAgentRun(input) {