@hue-run/sdk 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +8 -8
- package/EVALUATIONS.md +53 -2
- package/README.md +262 -37
- package/dist/attachment.d.ts +82 -0
- package/dist/attachment.js +225 -0
- package/dist/blobs.d.ts +41 -3
- package/dist/blobs.js +126 -41
- package/dist/client.d.ts +18 -0
- package/dist/client.js +148 -13
- package/dist/evals/client.d.ts +14 -1
- package/dist/evals/client.js +73 -9
- package/dist/evals/files.d.ts +11 -1
- package/dist/evals/files.js +160 -20
- package/dist/evals/media.d.ts +217 -0
- package/dist/evals/media.js +907 -0
- package/dist/evals/runner.js +130 -106
- package/dist/evals/types.d.ts +48 -0
- package/dist/evals.d.ts +2 -0
- package/dist/evals.js +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/inline-files.d.ts +131 -8
- package/dist/inline-files.js +565 -47
- package/dist/json-edits.d.ts +24 -0
- package/dist/json-edits.js +187 -0
- package/dist/privacy.d.ts +13 -0
- package/dist/privacy.js +84 -17
- package/dist/safety.d.ts +6 -2
- package/dist/safety.js +15 -3
- package/dist/snapshot.d.ts +1 -1
- package/dist/snapshot.js +123 -29
- package/dist/transport.d.ts +7 -0
- package/dist/transport.js +285 -55
- package/dist/types.d.ts +25 -6
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/client.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
|
-
import { isMediaName } from "./inline-files.js";
|
|
3
|
+
import { digestFormBytes, INLINE_FILE_TEXT_PER_RECORD, isMediaName } from "./inline-files.js";
|
|
4
4
|
import { context, isSpanContextValid, ROOT_CONTEXT, SpanKind, SpanStatusCode, trace, } from "@opentelemetry/api";
|
|
5
5
|
import { defaultTextMapGetter, defaultTextMapSetter } from "@opentelemetry/api";
|
|
6
6
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
7
|
-
import { W3CTraceContextPropagator } from "@opentelemetry/core";
|
|
7
|
+
import { isAttributeValue, W3CTraceContextPropagator } from "@opentelemetry/core";
|
|
8
8
|
import { LoggerProvider } from "@opentelemetry/sdk-logs";
|
|
9
9
|
import { TracerProvider } from "@opentelemetry/sdk-trace";
|
|
10
10
|
import { defaultResource, resourceFromAttributes } from "@opentelemetry/resources";
|
|
@@ -13,7 +13,8 @@ import { hostedServerAddresses, hostedToolActivity, hostedToolProvider, provider
|
|
|
13
13
|
import { toolCatalogSummary } from "./tool-definitions.js";
|
|
14
14
|
import { contentLimitsOf, encodeBoundedContent, encodeContent, noopSpan, safeSpan, spanSource, } from "./safety.js";
|
|
15
15
|
import { TRUNCATED_KEY, withTruncatedKeys } from "./privacy.js";
|
|
16
|
-
import { MAX_BLOB_BYTES } from "./blobs.js";
|
|
16
|
+
import { countFallback, MAX_BLOB_BYTES, newTally } from "./blobs.js";
|
|
17
|
+
import { attachmentPart, attachmentState, holdsAttachment, MAX_ATTACHMENT_DEPTH, } from "./attachment.js";
|
|
17
18
|
import { createHueTransport, exportBudgetMillis, HueExportError, } from "./transport.js";
|
|
18
19
|
import { verifyTrace } from "./receipt.js";
|
|
19
20
|
import { sdkVersion } from "./version.js";
|
|
@@ -170,6 +171,19 @@ const OWNED_SPAN_LIMITS = {
|
|
|
170
171
|
/** A provider tool listing's bounds before its catalog summary: one export request, 64 levels and
|
|
171
172
|
* 65,536 values, as the Python SDK's content snapshot. */
|
|
172
173
|
const catalogLimits = { bytes: MAX_BODY_BYTES, nodes: 65_536, depth: 64 };
|
|
174
|
+
/** Whether a span attribute passed to a helper is recorded as JSON text: an attachment, a value
|
|
175
|
+
* holding one, or JSON data OpenTelemetry cannot hold as an attribute (a plain object, or an array
|
|
176
|
+
* that is not of one primitive type). */
|
|
177
|
+
function recordedAsJson(value) {
|
|
178
|
+
if (holdsAttachment(value))
|
|
179
|
+
return true;
|
|
180
|
+
if (value === null || typeof value !== "object" || isAttributeValue(value))
|
|
181
|
+
return false;
|
|
182
|
+
if (Array.isArray(value))
|
|
183
|
+
return true;
|
|
184
|
+
const prototype = Object.getPrototypeOf(value);
|
|
185
|
+
return prototype === Object.prototype || prototype === null;
|
|
186
|
+
}
|
|
173
187
|
/**
|
|
174
188
|
* Hue tracing client. Helpers create spans through a private tracer and local async context, never
|
|
175
189
|
* through global OpenTelemetry registration, and they fail open: capture problems are counted in
|
|
@@ -286,7 +300,10 @@ export class HueClient {
|
|
|
286
300
|
workspaceId: identifier(options.workspaceId ?? inherited?.workspaceId),
|
|
287
301
|
model: inherited?.model,
|
|
288
302
|
};
|
|
289
|
-
|
|
303
|
+
const { attributes, placed } = this.attributeValues(options.attributes);
|
|
304
|
+
span = this.storage.run(active, () => this.tracer.startSpan(name, { kind: options.kind ?? SpanKind.INTERNAL, attributes }, active.context));
|
|
305
|
+
for (const [key, found] of placed)
|
|
306
|
+
this.stageAttachments(span, key, found, this.captureContent);
|
|
290
307
|
}
|
|
291
308
|
catch {
|
|
292
309
|
this.instrumentationFailed();
|
|
@@ -535,18 +552,127 @@ export class HueClient {
|
|
|
535
552
|
stamp("gen_ai.provider.name", messages.provider, enclosing?.model?.provider);
|
|
536
553
|
stamp("gen_ai.request.model", messages.model, enclosing?.model?.requestModel);
|
|
537
554
|
stamp("gen_ai.conversation.id", undefined, enclosing?.sessionId);
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
555
|
+
// While media may be uploaded, the body may hold files past the value cap: it is emitted
|
|
556
|
+
// when it fits the cap with each file as its digest, and export uploads the files (under
|
|
557
|
+
// `body#<pointer>` keys) or cuts the body as before. Otherwise a body over the cap is
|
|
558
|
+
// refused, as before. Each attachment is written as its digest part.
|
|
559
|
+
const limits = this.contentLimits();
|
|
560
|
+
const placed = [];
|
|
561
|
+
const place = this.placer(placed, true);
|
|
562
|
+
let text;
|
|
563
|
+
if (this.transport.holdsValues()) {
|
|
564
|
+
text = encodeContent(body, { ...limits, bytes: limits.bytes + INLINE_FILE_TEXT_PER_RECORD }, place);
|
|
565
|
+
if (Buffer.byteLength(text) > limits.bytes &&
|
|
566
|
+
digestFormBytes(JSON.parse(text)) > limits.bytes)
|
|
567
|
+
throw new RangeError("Content limit exceeded");
|
|
568
|
+
}
|
|
569
|
+
else
|
|
570
|
+
text = encodeContent(body, limits, place);
|
|
571
|
+
const recorded = JSON.parse(text);
|
|
572
|
+
// Attachments in the body are staged with the body until this transport admits the record;
|
|
573
|
+
// export uploads them with the media the body holds.
|
|
574
|
+
const traceId = trace.getSpanContext(active).traceId;
|
|
575
|
+
const files = this.attachmentFiles(placed, true, "logs", traceId, false);
|
|
576
|
+
if (files.length)
|
|
577
|
+
this.transport.stageLogParts(recorded, traceId, files);
|
|
578
|
+
try {
|
|
579
|
+
this.logger.emit({
|
|
580
|
+
context: active,
|
|
581
|
+
severityNumber: SeverityNumber.INFO,
|
|
582
|
+
eventName: "gen_ai.client.inference.operation.details",
|
|
583
|
+
attributes,
|
|
584
|
+
body: recorded,
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
finally {
|
|
588
|
+
this.transport.unstageLogParts(recorded);
|
|
589
|
+
}
|
|
545
590
|
}
|
|
546
591
|
catch {
|
|
547
592
|
this.instrumentationFailed("logs");
|
|
548
593
|
}
|
|
549
594
|
}
|
|
595
|
+
/** Writes each attachment as its digest part, `null` for an invalid one, collecting them. */
|
|
596
|
+
placer(placed, captureContent) {
|
|
597
|
+
return (attachment, pointer, depth) => {
|
|
598
|
+
const state = attachmentState(attachment);
|
|
599
|
+
const part = attachmentPart(state, captureContent);
|
|
600
|
+
placed.push({ state, ...(part ? { part } : {}), pointer, depth });
|
|
601
|
+
return part ?? null;
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
/**
|
|
605
|
+
* The files of the attachments a value placed that are uploaded: an invalid attachment, and
|
|
606
|
+
* an invalid name with content capture, count as instrumentation failures; nothing is uploaded
|
|
607
|
+
* without content capture or with setup credentials; a value that was `cut`, or a part deeper
|
|
608
|
+
* than Hue reads, keeps its digest part and counts as not uploaded.
|
|
609
|
+
*/
|
|
610
|
+
attachmentFiles(placed, captureContent, signal, traceId, cut) {
|
|
611
|
+
const files = [];
|
|
612
|
+
const tally = newTally();
|
|
613
|
+
for (const { state, part, depth } of placed) {
|
|
614
|
+
if (!part || !state.bytes || !state.sha256) {
|
|
615
|
+
this.instrumentationFailed(signal);
|
|
616
|
+
continue;
|
|
617
|
+
}
|
|
618
|
+
if (state.invalidName && captureContent)
|
|
619
|
+
this.instrumentationFailed(signal);
|
|
620
|
+
if (!this.transport.offloadsValues() || !state.bytes.byteLength)
|
|
621
|
+
continue;
|
|
622
|
+
if (cut || depth > MAX_ATTACHMENT_DEPTH) {
|
|
623
|
+
countFallback(tally, "budget", traceId);
|
|
624
|
+
continue;
|
|
625
|
+
}
|
|
626
|
+
files.push({
|
|
627
|
+
bytes: state.bytes,
|
|
628
|
+
digest: { sha256: state.sha256, size: state.bytes.byteLength },
|
|
629
|
+
contentType: state.contentType,
|
|
630
|
+
...(state.name === undefined ? {} : { name: state.name }),
|
|
631
|
+
});
|
|
632
|
+
}
|
|
633
|
+
if (tally.fallbacks.size)
|
|
634
|
+
this.transport.reportOffload(tally, signal);
|
|
635
|
+
return files;
|
|
636
|
+
}
|
|
637
|
+
/** Stages the attachments a span's `attribute` now holds, replacing its earlier ones. */
|
|
638
|
+
stageAttachments(span, attribute, placed, captureContent, cut = false) {
|
|
639
|
+
const traceId = span.spanContext().traceId;
|
|
640
|
+
const files = this.attachmentFiles(placed, captureContent, "traces", traceId, cut);
|
|
641
|
+
if (span.isRecording())
|
|
642
|
+
this.transport.stageParts(span, attribute, files);
|
|
643
|
+
}
|
|
644
|
+
/**
|
|
645
|
+
* Span attributes as recorded: a value that is an {@link Attachment}, an object or array holding
|
|
646
|
+
* one, or JSON data OpenTelemetry cannot hold as an attribute (an object, or an array that is not
|
|
647
|
+
* of one primitive type), becomes its JSON text with each attachment as its digest part (an
|
|
648
|
+
* invalid attachment alone is omitted); every other value is left to OpenTelemetry as given.
|
|
649
|
+
*/
|
|
650
|
+
attributeValues(input) {
|
|
651
|
+
const placed = [];
|
|
652
|
+
if (!input || typeof input !== "object")
|
|
653
|
+
return { attributes: input, placed };
|
|
654
|
+
let attributes;
|
|
655
|
+
for (const [key, value] of Object.entries(input)) {
|
|
656
|
+
if (!recordedAsJson(value))
|
|
657
|
+
continue;
|
|
658
|
+
attributes ??= { ...input };
|
|
659
|
+
const found = [];
|
|
660
|
+
try {
|
|
661
|
+
const text = encodeContent(value, this.contentLimits(), this.placer(found, this.captureContent));
|
|
662
|
+
if (text === "null")
|
|
663
|
+
delete attributes[key];
|
|
664
|
+
else
|
|
665
|
+
attributes[key] = text;
|
|
666
|
+
}
|
|
667
|
+
catch {
|
|
668
|
+
delete attributes[key];
|
|
669
|
+
this.instrumentationFailed();
|
|
670
|
+
continue;
|
|
671
|
+
}
|
|
672
|
+
placed.push([key, found]);
|
|
673
|
+
}
|
|
674
|
+
return { attributes: attributes ?? input, placed };
|
|
675
|
+
}
|
|
550
676
|
/**
|
|
551
677
|
* Records the tools a model provider executed itself while producing `response`, which no
|
|
552
678
|
* `hue.tool()` call saw: OpenAI Responses `mcp_call`, `web_search_call`, `file_search_call` and
|
|
@@ -799,10 +925,17 @@ export class HueClient {
|
|
|
799
925
|
if (!this.enabled || this.closed || !this.captureContent)
|
|
800
926
|
return;
|
|
801
927
|
try {
|
|
928
|
+
const encode = (limits) => {
|
|
929
|
+
const placed = [];
|
|
930
|
+
return {
|
|
931
|
+
...encodeBoundedContent(value, limits, true, this.placer(placed, true)),
|
|
932
|
+
placed,
|
|
933
|
+
};
|
|
934
|
+
};
|
|
802
935
|
let encoded;
|
|
803
936
|
if (this.transport.holdsValues())
|
|
804
937
|
try {
|
|
805
|
-
encoded =
|
|
938
|
+
encoded = encode({
|
|
806
939
|
...this.contentLimits(),
|
|
807
940
|
bytes: MAX_BLOB_BYTES,
|
|
808
941
|
nodes: MAX_RECORD_NODES,
|
|
@@ -811,8 +944,10 @@ export class HueClient {
|
|
|
811
944
|
catch {
|
|
812
945
|
// Past the runtime's longest string, or its memory: cut to the cap instead.
|
|
813
946
|
}
|
|
814
|
-
const { text, truncated } = encoded ??
|
|
947
|
+
const { text, truncated, placed } = encoded ?? encode(this.contentLimits());
|
|
815
948
|
span.setAttribute(key, text);
|
|
949
|
+
// The attachments the value holds are staged for upload in place of any it held before.
|
|
950
|
+
this.stageAttachments(span, key, placed, true, truncated);
|
|
816
951
|
// The span's list as it stands: the application's own entries (read from the SDK span,
|
|
817
952
|
// which carries its attributes; the API alone shows none) and this helper's earlier ones.
|
|
818
953
|
const listed = withTruncatedKeys(span.attributes?.[TRUNCATED_KEY], this.truncatedKeys.get(span) ?? []);
|
package/dist/evals/client.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ProjectConnection } from "../types.js";
|
|
2
|
+
import { MediaClient } from "./media.js";
|
|
2
3
|
import { type AttemptConnectionBundleV2, type PrepareAttemptRequestV2 } from "./attempt.js";
|
|
3
4
|
import type { ArtifactReservation, ArtifactUpload, CaseConversion, CaseConversionSummary, CaseWrite, CompleteExecution, Completion, Dataset, DatasetCase, DatasetVersion, EvalSet, EvalSetCase, EvalSetVersion, EvaluationItem, EvaluationRun, EvaluationRunSummary, EnvironmentEvidenceSnapshot, Execution, Experiment, ExperimentCase, ExperimentItem, Evaluator, EvaluatorVersion, JsonValue, LocalAgentClaim, LocalAgentRegistration, RegisteredLocalAgent, JudgeBudget, JudgeJob, Page, PageOptions, RegistryPageOptions, Result, ResultSummary, Run, RunCase, Scorer, ScorerDefinition, ScorerVersion, Scoring, ScoringResultInput, ScoringResultSummary, ScoringSubject, ScoringSummary, SimulationMcpCapability, StartExecution, Subject, StoredResult, StoredScoringResult, IdentityUpdate, Tag, TaggedIdentity, UpdatedRun } from "./types.js";
|
|
4
5
|
/** Connection options for {@link createEvaluationClient}. */
|
|
@@ -59,6 +60,9 @@ export declare class EvaluationClient {
|
|
|
59
60
|
#private;
|
|
60
61
|
/** Validated Hue origin. */
|
|
61
62
|
readonly baseUrl: string;
|
|
63
|
+
/** Recorded media read back by digest: one file (`fetch`) or every digest part of a recorded
|
|
64
|
+
* value (`resolve`). Needs a key that may read recorded content (Read, or Read and write). */
|
|
65
|
+
readonly media: MediaClient;
|
|
62
66
|
private readonly timeoutMillis;
|
|
63
67
|
private readonly maxAttempts;
|
|
64
68
|
constructor(options: EvaluationClientOptions);
|
|
@@ -97,7 +101,16 @@ export declare class EvaluationClient {
|
|
|
97
101
|
getDatasetVersion(id: string): Promise<DatasetVersion>;
|
|
98
102
|
/** Lists the full cases of a dataset version; use small page limits for large values. */
|
|
99
103
|
listCases(id: string, page?: PageOptions): Promise<Page<DatasetCase>>;
|
|
100
|
-
/**
|
|
104
|
+
/**
|
|
105
|
+
* Adds a case to a draft version using optimistic concurrency on `expectedRevision`. Each of
|
|
106
|
+
* `files` given as bytes or a path is checked against the artifact policy and uploaded as a
|
|
107
|
+
* verified artifact first, under a key derived from the version, the case's `externalKey`, the
|
|
108
|
+
* role and the bytes, so sending the same case again reuses the artifact. Every file and the
|
|
109
|
+
* case itself are checked before the first upload.
|
|
110
|
+
*
|
|
111
|
+
* @throws TypeError or RangeError for `files` outside the artifact policy, or a case Hue would
|
|
112
|
+
* refuse for its fields or size, before any request.
|
|
113
|
+
*/
|
|
101
114
|
addCase(id: string, input: CaseWrite): Promise<{
|
|
102
115
|
/** The stored case. */
|
|
103
116
|
item: DatasetCase;
|
package/dist/evals/client.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isLoopbackHost, validateOptions } from "../config.js";
|
|
2
2
|
import { json, uuid, valueBounds } from "./json.js";
|
|
3
|
+
import { MediaClient } from "./media.js";
|
|
3
4
|
import { attemptBindingRead, parsePrepareAttemptResultV2, parseRefreshedAttemptResultV2, parseRevocationResult, prepareAttemptInputV2, validateAttemptConnectionBundleV2, } from "./attempt.js";
|
|
4
5
|
/** Thrown for a failed evaluation API request; the message is fixed and never includes response text. */
|
|
5
6
|
export class HueApiError extends Error {
|
|
@@ -149,6 +150,42 @@ function productRunFields(value, kind) {
|
|
|
149
150
|
result.scoring = result.evaluation;
|
|
150
151
|
return result;
|
|
151
152
|
}
|
|
153
|
+
/** The bounds of a request body as a whole: 1 MiB, with each value's node and depth bounds. */
|
|
154
|
+
const REQUEST_BOUNDS = { ...valueBounds, bytes: 1024 * 1024 };
|
|
155
|
+
/** A request body as it is sent: checked as JSON within `bounds`. Optional top-level fields are
|
|
156
|
+
* omitted intentionally; nested undefined remains invalid. */
|
|
157
|
+
function payloadOf(body, bounds) {
|
|
158
|
+
return JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), bounds));
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Checks what a case write's fields can be checked for here, as Hue checks them: a positive
|
|
162
|
+
* integer `expectedRevision`, an `externalKey` of 1–200 UTF-16 code units of well-formed text
|
|
163
|
+
* without NUL, `inputs`, `expected` and an object `metadata` within one value's JSON bounds, and an
|
|
164
|
+
* `environmentVersionId` that is a UUID or null.
|
|
165
|
+
*/
|
|
166
|
+
function checkCaseWrite(input) {
|
|
167
|
+
if (!input || typeof input !== "object")
|
|
168
|
+
throw new TypeError("A case write must be an object");
|
|
169
|
+
if (!Number.isSafeInteger(input.expectedRevision) || input.expectedRevision < 1)
|
|
170
|
+
throw new TypeError("expectedRevision must be a positive integer");
|
|
171
|
+
const key = input.externalKey;
|
|
172
|
+
if (typeof key !== "string" ||
|
|
173
|
+
!key.length ||
|
|
174
|
+
key.length > 200 ||
|
|
175
|
+
!key.isWellFormed() ||
|
|
176
|
+
key.includes("\u0000"))
|
|
177
|
+
throw new TypeError("externalKey must be 1–200 UTF-16 code units of well-formed text");
|
|
178
|
+
json(input.inputs, valueBounds);
|
|
179
|
+
if (input.expected !== undefined)
|
|
180
|
+
json(input.expected, valueBounds);
|
|
181
|
+
if (input.metadata !== undefined) {
|
|
182
|
+
if (!input.metadata || typeof input.metadata !== "object" || Array.isArray(input.metadata))
|
|
183
|
+
throw new TypeError("metadata must be an object");
|
|
184
|
+
json(input.metadata, valueBounds);
|
|
185
|
+
}
|
|
186
|
+
if (input.environmentVersionId !== undefined && input.environmentVersionId !== null)
|
|
187
|
+
uuid(input.environmentVersionId);
|
|
188
|
+
}
|
|
152
189
|
/** Times a refused request is sent again before its refusal is the caller's error. */
|
|
153
190
|
const REFUSAL_RETRIES = 4;
|
|
154
191
|
/** A refusal asking for a longer wait fails at once rather than holding the caller. */
|
|
@@ -183,6 +220,9 @@ function refusalRetryAfter(response) {
|
|
|
183
220
|
export class EvaluationClient {
|
|
184
221
|
/** Validated Hue origin. */
|
|
185
222
|
baseUrl;
|
|
223
|
+
/** Recorded media read back by digest: one file (`fetch`) or every digest part of a recorded
|
|
224
|
+
* value (`resolve`). Needs a key that may read recorded content (Read, or Read and write). */
|
|
225
|
+
media;
|
|
186
226
|
#apiKey;
|
|
187
227
|
timeoutMillis;
|
|
188
228
|
maxAttempts;
|
|
@@ -199,6 +239,11 @@ export class EvaluationClient {
|
|
|
199
239
|
if (!Number.isInteger(attempts) || attempts < 1 || attempts > 10)
|
|
200
240
|
throw new RangeError("maxAttempts must be 1–10");
|
|
201
241
|
this.maxAttempts = attempts;
|
|
242
|
+
this.media = new MediaClient({
|
|
243
|
+
baseUrl: this.baseUrl,
|
|
244
|
+
apiKey: validated.apiKey,
|
|
245
|
+
timeoutMillis: this.timeoutMillis,
|
|
246
|
+
});
|
|
202
247
|
}
|
|
203
248
|
/** Sends one request to Hue, again after the wait while Hue refuses it with a short
|
|
204
249
|
* `Retry-After`. `init` runs for every attempt, so each has its own timeout; `signal` also ends
|
|
@@ -235,15 +280,12 @@ export class EvaluationClient {
|
|
|
235
280
|
throw new HueApiError(response.status, askedRetryAfter(response));
|
|
236
281
|
}
|
|
237
282
|
}
|
|
238
|
-
async request(method, path, body, bounds =
|
|
283
|
+
async request(method, path, body, bounds = REQUEST_BOUNDS, signal,
|
|
239
284
|
/** Overrides the retry decision `retryable` takes from the method and body. */
|
|
240
285
|
idempotent) {
|
|
241
286
|
// Serialized once, outside the attempts: a body this client cannot encode is a caller error
|
|
242
|
-
// no attempt fixes.
|
|
243
|
-
|
|
244
|
-
const payload = body === undefined
|
|
245
|
-
? undefined
|
|
246
|
-
: JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), bounds));
|
|
287
|
+
// no attempt fixes.
|
|
288
|
+
const payload = body === undefined ? undefined : payloadOf(body, bounds);
|
|
247
289
|
const attempts = retryable(method, body, idempotent) ? this.maxAttempts : 1;
|
|
248
290
|
for (let attempt = 1;; attempt++) {
|
|
249
291
|
try {
|
|
@@ -433,9 +475,31 @@ export class EvaluationClient {
|
|
|
433
475
|
listCases(id, page) {
|
|
434
476
|
return this.request("GET", `/dataset-versions/${uuid(id)}/cases${this.page(page)}`);
|
|
435
477
|
}
|
|
436
|
-
/**
|
|
437
|
-
|
|
438
|
-
|
|
478
|
+
/**
|
|
479
|
+
* Adds a case to a draft version using optimistic concurrency on `expectedRevision`. Each of
|
|
480
|
+
* `files` given as bytes or a path is checked against the artifact policy and uploaded as a
|
|
481
|
+
* verified artifact first, under a key derived from the version, the case's `externalKey`, the
|
|
482
|
+
* role and the bytes, so sending the same case again reuses the artifact. Every file and the
|
|
483
|
+
* case itself are checked before the first upload.
|
|
484
|
+
*
|
|
485
|
+
* @throws TypeError or RangeError for `files` outside the artifact policy, or a case Hue would
|
|
486
|
+
* refuse for its fields or size, before any request.
|
|
487
|
+
*/
|
|
488
|
+
async addCase(id, input) {
|
|
489
|
+
// The canonical spelling: the reservation keys of uploaded files name the version.
|
|
490
|
+
const versionId = uuid(id).toLowerCase();
|
|
491
|
+
checkCaseWrite(input);
|
|
492
|
+
let body = input;
|
|
493
|
+
if (input.files !== undefined) {
|
|
494
|
+
// Loaded on use: the artifact helpers import this client.
|
|
495
|
+
const { caseFileReferences } = await import("./files.js");
|
|
496
|
+
body = {
|
|
497
|
+
...input,
|
|
498
|
+
// The case is checked as it will be sent, with a stand-in for each upload's artifact.
|
|
499
|
+
files: await caseFileReferences(this, versionId, input, (files) => payloadOf({ ...input, files }, REQUEST_BOUNDS)),
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
return this.request("POST", `/dataset-versions/${versionId}/cases`, body);
|
|
439
503
|
}
|
|
440
504
|
/** Freezes a draft version at the given revision; frozen versions are immutable. */
|
|
441
505
|
freezeDatasetVersion(id, expectedRevision) {
|
package/dist/evals/files.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type EvaluationClient } from "./client.js";
|
|
2
|
-
import type { CaseFile, LocalFile, OutputFile, SubjectFile } from "./types.js";
|
|
2
|
+
import type { CaseFile, CaseFileReference, CaseWrite, LocalFile, OutputFile, SubjectFile } from "./types.js";
|
|
3
3
|
/** Roles the target receives. Organization templates stay with grading, as in the managed protocol. */
|
|
4
4
|
export declare const targetFileRoles: readonly CaseFile["role"][];
|
|
5
5
|
/** Hue's artifact policy: the pilot file size and the accepted document types. */
|
|
@@ -112,5 +112,15 @@ export declare const artifactSettling: {
|
|
|
112
112
|
* finished verifying after an earlier attempt stopped waiting.
|
|
113
113
|
*/
|
|
114
114
|
export declare function uploadOutputFiles(client: EvaluationClient, executionId: string, files: StagedOutputFile[], save: () => Promise<void>, timing?: typeof artifactSettling): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* The `files` of a case write as the case API takes them: references kept, and each file given
|
|
117
|
+
* as bytes or a path checked against the artifact policy (role, name, one of
|
|
118
|
+
* `outputContentTypes`, 1 byte to 25 MiB) and uploaded as a verified artifact. Every file is
|
|
119
|
+
* checked, and then `beforeUpload` is given the references with a stand-in ID for each upload,
|
|
120
|
+
* before the first upload. The reservation key names the version, the case's `externalKey`, the
|
|
121
|
+
* role and the bytes, so the same write sent again reuses its artifacts. Each artifact is pinned
|
|
122
|
+
* once with a role, even when a reference names the artifact a file was uploaded as.
|
|
123
|
+
*/
|
|
124
|
+
export declare function caseFileReferences(client: EvaluationClient, versionId: string, input: CaseWrite, beforeUpload?: (references: CaseFileReference[]) => void, timing?: typeof artifactSettling): Promise<CaseFileReference[]>;
|
|
115
125
|
/** Project uploaded staged files as the verified local files handed to scorers. */
|
|
116
126
|
export declare function localOutputFiles(files: StagedOutputFile[]): LocalFile[];
|
package/dist/evals/files.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { constants } from "node:fs";
|
|
3
|
-
import { lstat, mkdir, open, readFile, rename, rm } from "node:fs/promises";
|
|
3
|
+
import { lstat, mkdir, open, readFile, rename, rm, stat } from "node:fs/promises";
|
|
4
4
|
import { extname, join, resolve } from "node:path";
|
|
5
5
|
import { randomUUID } from "node:crypto";
|
|
6
6
|
import { ArtifactSizeError, HueApiError } from "./client.js";
|
|
7
|
+
import { uuid } from "./json.js";
|
|
7
8
|
/** Roles the target receives. Organization templates stay with grading, as in the managed protocol. */
|
|
8
9
|
export const targetFileRoles = [
|
|
9
10
|
"source",
|
|
@@ -413,33 +414,172 @@ export async function uploadOutputFiles(client, executionId, files, save, timing
|
|
|
413
414
|
if (file.artifactId)
|
|
414
415
|
continue;
|
|
415
416
|
const bytes = await verifiedBytes(file.path, file);
|
|
416
|
-
|
|
417
|
+
file.artifactId = await publishArtifact(client, {
|
|
417
418
|
idempotencyKey: `hue-sdk:${executionId}:${file.sha256}:${sha256(Buffer.from(file.filename)).slice(0, 16)}`,
|
|
418
419
|
filename: file.filename,
|
|
419
420
|
contentType: file.contentType,
|
|
420
421
|
byteSize: file.byteSize,
|
|
421
422
|
sha256: file.sha256,
|
|
422
|
-
});
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
423
|
+
}, bytes, `Generated file ${file.filename}`, timing);
|
|
424
|
+
await save();
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
/** Reserve, stage and verify one artifact; its ID once Hue verified it. Replaying the key settles
|
|
428
|
+
* on the same artifact, including one Hue finished verifying after an earlier attempt stopped. */
|
|
429
|
+
async function publishArtifact(client, declared, bytes, label, timing) {
|
|
430
|
+
const reserved = await client.reserveArtifact(declared);
|
|
431
|
+
let state = reserved.state;
|
|
432
|
+
if (state !== "ready") {
|
|
433
|
+
// A verification already running holds the uploaded bytes; Hue refuses another upload then.
|
|
434
|
+
if (reserved.copyState !== "acknowledged" && state !== "verifying") {
|
|
435
|
+
const upload = await client.requestArtifactUpload(reserved.id);
|
|
436
|
+
try {
|
|
437
|
+
await client.uploadArtifactBytes(upload, bytes, declared.contentType);
|
|
438
|
+
}
|
|
439
|
+
catch {
|
|
440
|
+
// A lost staging acknowledgement or an immutable object already present can only be
|
|
441
|
+
// settled by verified completion; never rewrite a final object here.
|
|
435
442
|
}
|
|
436
|
-
state = await settleArtifact(client, reserved.id, timing);
|
|
437
443
|
}
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
444
|
+
state = await settleArtifact(client, reserved.id, timing);
|
|
445
|
+
}
|
|
446
|
+
if (state !== "ready")
|
|
447
|
+
throw new Error(`${label} was not verified by Hue (${state})`);
|
|
448
|
+
return reserved.id;
|
|
449
|
+
}
|
|
450
|
+
/** Roles a pinned case input can carry. */
|
|
451
|
+
const caseFileRoles = [
|
|
452
|
+
...targetFileRoles,
|
|
453
|
+
"org_template",
|
|
454
|
+
"evaluator_reference",
|
|
455
|
+
];
|
|
456
|
+
/** The most input files one case pins. */
|
|
457
|
+
const MAX_CASE_FILES = 32;
|
|
458
|
+
/** A case file name as the artifact API takes it: well-formed Unicode of 1–255 UTF-16 code units,
|
|
459
|
+
* as Hue counts them, without a control character or a path separator. */
|
|
460
|
+
function caseFileName(name) {
|
|
461
|
+
if (typeof name !== "string" ||
|
|
462
|
+
!name.length ||
|
|
463
|
+
name.length > 255 ||
|
|
464
|
+
!name.isWellFormed() ||
|
|
465
|
+
[...name].some((c) => c.charCodeAt(0) < 32 || c.charCodeAt(0) === 127 || c === "/" || c === "\\"))
|
|
466
|
+
throw new TypeError("A case file needs a well-formed filename of 1–255 UTF-16 code units without path separators or control characters");
|
|
467
|
+
return name;
|
|
468
|
+
}
|
|
469
|
+
/** The bytes of a case file given by path: a regular file within the size limit, read once. */
|
|
470
|
+
async function caseFileBytes(path, label) {
|
|
471
|
+
if (typeof path !== "string" || !path)
|
|
472
|
+
throw new TypeError(`${label} needs bytes or a path`);
|
|
473
|
+
// A path with no file behind it, or one that cannot be read, is refused as any invalid file.
|
|
474
|
+
const unreadable = (error) => new TypeError(`${label} could not be read`, { cause: error });
|
|
475
|
+
const info = await stat(path).catch((error) => {
|
|
476
|
+
throw unreadable(error);
|
|
477
|
+
});
|
|
478
|
+
if (!info.isFile())
|
|
479
|
+
throw new TypeError(`${label} is not a regular file`);
|
|
480
|
+
if (info.size > outputFileLimits.bytes)
|
|
481
|
+
throw new RangeError(`${label} exceeds 25 MiB`);
|
|
482
|
+
return readFile(path).catch((error) => {
|
|
483
|
+
throw unreadable(error);
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
/** A stand-in for an artifact a case file is uploaded as, the length of a real ID. */
|
|
487
|
+
const STAND_IN_ID = "00000000-0000-4000-8000-000000000000";
|
|
488
|
+
/**
|
|
489
|
+
* The `files` of a case write as the case API takes them: references kept, and each file given
|
|
490
|
+
* as bytes or a path checked against the artifact policy (role, name, one of
|
|
491
|
+
* `outputContentTypes`, 1 byte to 25 MiB) and uploaded as a verified artifact. Every file is
|
|
492
|
+
* checked, and then `beforeUpload` is given the references with a stand-in ID for each upload,
|
|
493
|
+
* before the first upload. The reservation key names the version, the case's `externalKey`, the
|
|
494
|
+
* role and the bytes, so the same write sent again reuses its artifacts. Each artifact is pinned
|
|
495
|
+
* once with a role, even when a reference names the artifact a file was uploaded as.
|
|
496
|
+
*/
|
|
497
|
+
export async function caseFileReferences(client, versionId, input, beforeUpload = () => undefined, timing = artifactSettling) {
|
|
498
|
+
const files = input.files;
|
|
499
|
+
if (!Array.isArray(files))
|
|
500
|
+
throw new TypeError("files must be a list of case files");
|
|
501
|
+
if (files.length > MAX_CASE_FILES)
|
|
502
|
+
throw new RangeError(`A case pins at most ${MAX_CASE_FILES} files`);
|
|
503
|
+
const prepared = [];
|
|
504
|
+
const seen = new Set();
|
|
505
|
+
for (const [index, file] of files.entries()) {
|
|
506
|
+
const label = `Case file ${index + 1}`;
|
|
507
|
+
if (!file || typeof file !== "object")
|
|
508
|
+
throw new TypeError(`${label} is not a file`);
|
|
509
|
+
if (!caseFileRoles.includes(file.role))
|
|
510
|
+
throw new TypeError(`${label} has an unsupported role ${String(file.role)}`);
|
|
511
|
+
let identity;
|
|
512
|
+
if (file.artifactId !== undefined) {
|
|
513
|
+
// A reference keeps the artifact's own name and type; a file field beside it is refused
|
|
514
|
+
// rather than ignored.
|
|
515
|
+
const fields = file;
|
|
516
|
+
if (fields.filename !== undefined ||
|
|
517
|
+
fields.contentType !== undefined ||
|
|
518
|
+
fields.bytes !== undefined ||
|
|
519
|
+
fields.path !== undefined)
|
|
520
|
+
throw new TypeError(`${label} names an artifactId and a file; give one`);
|
|
521
|
+
const reference = { artifactId: uuid(file.artifactId).toLowerCase(), role: file.role };
|
|
522
|
+
prepared.push(reference);
|
|
523
|
+
identity = `artifact:${reference.artifactId}`;
|
|
524
|
+
}
|
|
525
|
+
else {
|
|
526
|
+
const filename = caseFileName(file.filename);
|
|
527
|
+
if (!outputContentTypes.includes(file.contentType))
|
|
528
|
+
throw new TypeError(`${label} has an unsupported content type ${String(file.contentType)}`);
|
|
529
|
+
if (file.bytes !== undefined && file.path !== undefined)
|
|
530
|
+
throw new TypeError(`${label} names bytes and a path; give one`);
|
|
531
|
+
if (file.bytes !== undefined && !(file.bytes instanceof Uint8Array))
|
|
532
|
+
throw new TypeError(`${label} bytes must be a Uint8Array`);
|
|
533
|
+
const bytes = file.bytes ?? (await caseFileBytes(file.path, label));
|
|
534
|
+
if (!bytes.byteLength)
|
|
535
|
+
throw new RangeError(`${label} is empty`);
|
|
536
|
+
if (bytes.byteLength > outputFileLimits.bytes)
|
|
537
|
+
throw new RangeError(`${label} exceeds 25 MiB`);
|
|
538
|
+
const digest = sha256(bytes);
|
|
539
|
+
prepared.push({
|
|
540
|
+
role: file.role,
|
|
541
|
+
filename,
|
|
542
|
+
contentType: file.contentType,
|
|
543
|
+
bytes,
|
|
544
|
+
sha256: digest,
|
|
545
|
+
});
|
|
546
|
+
identity = `file:${digest}:${file.contentType}:${filename}`;
|
|
547
|
+
}
|
|
548
|
+
const pinned = `${identity}:${file.role}`;
|
|
549
|
+
if (seen.has(pinned))
|
|
550
|
+
throw new TypeError(`${label} is pinned twice with the role ${file.role}`);
|
|
551
|
+
seen.add(pinned);
|
|
552
|
+
}
|
|
553
|
+
beforeUpload(prepared.map((item) => "artifactId" in item ? item : { artifactId: STAND_IN_ID, role: item.role }));
|
|
554
|
+
const caseKey = sha256(Buffer.from(String(input.externalKey))).slice(0, 16);
|
|
555
|
+
const references = [];
|
|
556
|
+
// A reference and an upload that settle on the same artifact are pinned once: Hue refuses an
|
|
557
|
+
// artifact pinned twice with one role.
|
|
558
|
+
const pins = new Set();
|
|
559
|
+
const pin = (reference) => {
|
|
560
|
+
const identity = `${reference.artifactId.toLowerCase()}:${reference.role}`;
|
|
561
|
+
if (pins.has(identity))
|
|
562
|
+
return;
|
|
563
|
+
pins.add(identity);
|
|
564
|
+
references.push(reference);
|
|
565
|
+
};
|
|
566
|
+
for (const item of prepared) {
|
|
567
|
+
if ("artifactId" in item) {
|
|
568
|
+
pin(item);
|
|
569
|
+
continue;
|
|
570
|
+
}
|
|
571
|
+
const artifactId = await publishArtifact(client, {
|
|
572
|
+
// The key names everything the reservation declares, so a corrected name or type reserves
|
|
573
|
+
// a new artifact instead of colliding with the earlier one.
|
|
574
|
+
idempotencyKey: `hue-sdk:case:${versionId}:${caseKey}:${item.role}:${item.sha256}:${sha256(Buffer.from(item.filename)).slice(0, 16)}:${sha256(Buffer.from(item.contentType)).slice(0, 16)}`,
|
|
575
|
+
filename: item.filename,
|
|
576
|
+
contentType: item.contentType,
|
|
577
|
+
byteSize: item.bytes.byteLength,
|
|
578
|
+
sha256: item.sha256,
|
|
579
|
+
}, item.bytes, `Case file ${item.filename}`, timing);
|
|
580
|
+
pin({ artifactId, role: item.role });
|
|
442
581
|
}
|
|
582
|
+
return references;
|
|
443
583
|
}
|
|
444
584
|
/** Project uploaded staged files as the verified local files handed to scorers. */
|
|
445
585
|
export function localOutputFiles(files) {
|