@hue-run/sdk 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +130 -8
- package/EVALUATIONS.md +56 -2
- package/README.md +299 -37
- package/dist/attachment.d.ts +82 -0
- package/dist/attachment.js +225 -0
- package/dist/blobs.d.ts +41 -3
- package/dist/blobs.js +126 -41
- package/dist/cli/import.d.ts +35 -0
- package/dist/cli/import.js +692 -0
- package/dist/cli/langfuse.d.ts +309 -0
- package/dist/cli/langfuse.js +1075 -0
- package/dist/client.d.ts +18 -0
- package/dist/client.js +148 -13
- package/dist/evals/client.d.ts +19 -1
- package/dist/evals/client.js +91 -11
- package/dist/evals/files.d.ts +11 -1
- package/dist/evals/files.js +248 -20
- package/dist/evals/media.d.ts +221 -0
- package/dist/evals/media.js +919 -0
- package/dist/evals/runner.js +130 -106
- package/dist/evals/types.d.ts +49 -0
- package/dist/evals.d.ts +2 -0
- package/dist/evals.js +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/inline-files.d.ts +140 -8
- package/dist/inline-files.js +572 -51
- package/dist/json-edits.d.ts +24 -0
- package/dist/json-edits.js +187 -0
- package/dist/privacy.d.ts +13 -0
- package/dist/privacy.js +84 -17
- package/dist/safety.d.ts +6 -2
- package/dist/safety.js +15 -3
- package/dist/setup/cli.js +7 -3
- package/dist/snapshot.d.ts +1 -1
- package/dist/snapshot.js +123 -29
- package/dist/transport.d.ts +10 -0
- package/dist/transport.js +296 -56
- package/dist/types.d.ts +25 -6
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/client.d.ts
CHANGED
|
@@ -127,6 +127,24 @@ export declare class HueClient {
|
|
|
127
127
|
/** `gen_ai.request.model` for the record; defaults to the enclosing `model()` span's value. */
|
|
128
128
|
model?: string;
|
|
129
129
|
}, explicitContext?: Context): void;
|
|
130
|
+
/** Writes each attachment as its digest part, `null` for an invalid one, collecting them. */
|
|
131
|
+
private placer;
|
|
132
|
+
/**
|
|
133
|
+
* The files of the attachments a value placed that are uploaded: an invalid attachment, and
|
|
134
|
+
* an invalid name with content capture, count as instrumentation failures; nothing is uploaded
|
|
135
|
+
* without content capture or with setup credentials; a value that was `cut`, or a part deeper
|
|
136
|
+
* than Hue reads, keeps its digest part and counts as not uploaded.
|
|
137
|
+
*/
|
|
138
|
+
private attachmentFiles;
|
|
139
|
+
/** Stages the attachments a span's `attribute` now holds, replacing its earlier ones. */
|
|
140
|
+
private stageAttachments;
|
|
141
|
+
/**
|
|
142
|
+
* Span attributes as recorded: a value that is an {@link Attachment}, an object or array holding
|
|
143
|
+
* one, or JSON data OpenTelemetry cannot hold as an attribute (an object, or an array that is not
|
|
144
|
+
* of one primitive type), becomes its JSON text with each attachment as its digest part (an
|
|
145
|
+
* invalid attachment alone is omitted); every other value is left to OpenTelemetry as given.
|
|
146
|
+
*/
|
|
147
|
+
private attributeValues;
|
|
130
148
|
/**
|
|
131
149
|
* Records the tools a model provider executed itself while producing `response`, which no
|
|
132
150
|
* `hue.tool()` call saw: OpenAI Responses `mcp_call`, `web_search_call`, `file_search_call` and
|
package/dist/client.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
|
-
import { isMediaName } from "./inline-files.js";
|
|
3
|
+
import { digestFormBytes, INLINE_FILE_TEXT_PER_RECORD, isMediaName } from "./inline-files.js";
|
|
4
4
|
import { context, isSpanContextValid, ROOT_CONTEXT, SpanKind, SpanStatusCode, trace, } from "@opentelemetry/api";
|
|
5
5
|
import { defaultTextMapGetter, defaultTextMapSetter } from "@opentelemetry/api";
|
|
6
6
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
7
|
-
import { W3CTraceContextPropagator } from "@opentelemetry/core";
|
|
7
|
+
import { isAttributeValue, W3CTraceContextPropagator } from "@opentelemetry/core";
|
|
8
8
|
import { LoggerProvider } from "@opentelemetry/sdk-logs";
|
|
9
9
|
import { TracerProvider } from "@opentelemetry/sdk-trace";
|
|
10
10
|
import { defaultResource, resourceFromAttributes } from "@opentelemetry/resources";
|
|
@@ -13,7 +13,8 @@ import { hostedServerAddresses, hostedToolActivity, hostedToolProvider, provider
|
|
|
13
13
|
import { toolCatalogSummary } from "./tool-definitions.js";
|
|
14
14
|
import { contentLimitsOf, encodeBoundedContent, encodeContent, noopSpan, safeSpan, spanSource, } from "./safety.js";
|
|
15
15
|
import { TRUNCATED_KEY, withTruncatedKeys } from "./privacy.js";
|
|
16
|
-
import { MAX_BLOB_BYTES } from "./blobs.js";
|
|
16
|
+
import { countFallback, MAX_BLOB_BYTES, newTally } from "./blobs.js";
|
|
17
|
+
import { attachmentPart, attachmentState, holdsAttachment, MAX_ATTACHMENT_DEPTH, } from "./attachment.js";
|
|
17
18
|
import { createHueTransport, exportBudgetMillis, HueExportError, } from "./transport.js";
|
|
18
19
|
import { verifyTrace } from "./receipt.js";
|
|
19
20
|
import { sdkVersion } from "./version.js";
|
|
@@ -170,6 +171,19 @@ const OWNED_SPAN_LIMITS = {
|
|
|
170
171
|
/** A provider tool listing's bounds before its catalog summary: one export request, 64 levels and
|
|
171
172
|
* 65,536 values, as the Python SDK's content snapshot. */
|
|
172
173
|
const catalogLimits = { bytes: MAX_BODY_BYTES, nodes: 65_536, depth: 64 };
|
|
174
|
+
/** Whether a span attribute passed to a helper is recorded as JSON text: an attachment, a value
|
|
175
|
+
* holding one, or JSON data OpenTelemetry cannot hold as an attribute (a plain object, or an array
|
|
176
|
+
* that is not of one primitive type). */
|
|
177
|
+
function recordedAsJson(value) {
|
|
178
|
+
if (holdsAttachment(value))
|
|
179
|
+
return true;
|
|
180
|
+
if (value === null || typeof value !== "object" || isAttributeValue(value))
|
|
181
|
+
return false;
|
|
182
|
+
if (Array.isArray(value))
|
|
183
|
+
return true;
|
|
184
|
+
const prototype = Object.getPrototypeOf(value);
|
|
185
|
+
return prototype === Object.prototype || prototype === null;
|
|
186
|
+
}
|
|
173
187
|
/**
|
|
174
188
|
* Hue tracing client. Helpers create spans through a private tracer and local async context, never
|
|
175
189
|
* through global OpenTelemetry registration, and they fail open: capture problems are counted in
|
|
@@ -286,7 +300,10 @@ export class HueClient {
|
|
|
286
300
|
workspaceId: identifier(options.workspaceId ?? inherited?.workspaceId),
|
|
287
301
|
model: inherited?.model,
|
|
288
302
|
};
|
|
289
|
-
|
|
303
|
+
const { attributes, placed } = this.attributeValues(options.attributes);
|
|
304
|
+
span = this.storage.run(active, () => this.tracer.startSpan(name, { kind: options.kind ?? SpanKind.INTERNAL, attributes }, active.context));
|
|
305
|
+
for (const [key, found] of placed)
|
|
306
|
+
this.stageAttachments(span, key, found, this.captureContent);
|
|
290
307
|
}
|
|
291
308
|
catch {
|
|
292
309
|
this.instrumentationFailed();
|
|
@@ -535,18 +552,127 @@ export class HueClient {
|
|
|
535
552
|
stamp("gen_ai.provider.name", messages.provider, enclosing?.model?.provider);
|
|
536
553
|
stamp("gen_ai.request.model", messages.model, enclosing?.model?.requestModel);
|
|
537
554
|
stamp("gen_ai.conversation.id", undefined, enclosing?.sessionId);
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
555
|
+
// While media may be uploaded, the body may hold files past the value cap: it is emitted
|
|
556
|
+
// when it fits the cap with each file as its digest, and export uploads the files (under
|
|
557
|
+
// `body#<pointer>` keys) or cuts the body as before. Otherwise a body over the cap is
|
|
558
|
+
// refused, as before. Each attachment is written as its digest part.
|
|
559
|
+
const limits = this.contentLimits();
|
|
560
|
+
const placed = [];
|
|
561
|
+
const place = this.placer(placed, true);
|
|
562
|
+
let text;
|
|
563
|
+
if (this.transport.holdsValues()) {
|
|
564
|
+
text = encodeContent(body, { ...limits, bytes: limits.bytes + INLINE_FILE_TEXT_PER_RECORD }, place);
|
|
565
|
+
if (Buffer.byteLength(text) > limits.bytes &&
|
|
566
|
+
digestFormBytes(JSON.parse(text)) > limits.bytes)
|
|
567
|
+
throw new RangeError("Content limit exceeded");
|
|
568
|
+
}
|
|
569
|
+
else
|
|
570
|
+
text = encodeContent(body, limits, place);
|
|
571
|
+
const recorded = JSON.parse(text);
|
|
572
|
+
// Attachments in the body are staged with the body until this transport admits the record;
|
|
573
|
+
// export uploads them with the media the body holds.
|
|
574
|
+
const traceId = trace.getSpanContext(active).traceId;
|
|
575
|
+
const files = this.attachmentFiles(placed, true, "logs", traceId, false);
|
|
576
|
+
if (files.length)
|
|
577
|
+
this.transport.stageLogParts(recorded, traceId, files);
|
|
578
|
+
try {
|
|
579
|
+
this.logger.emit({
|
|
580
|
+
context: active,
|
|
581
|
+
severityNumber: SeverityNumber.INFO,
|
|
582
|
+
eventName: "gen_ai.client.inference.operation.details",
|
|
583
|
+
attributes,
|
|
584
|
+
body: recorded,
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
finally {
|
|
588
|
+
this.transport.unstageLogParts(recorded);
|
|
589
|
+
}
|
|
545
590
|
}
|
|
546
591
|
catch {
|
|
547
592
|
this.instrumentationFailed("logs");
|
|
548
593
|
}
|
|
549
594
|
}
|
|
595
|
+
/** Writes each attachment as its digest part, `null` for an invalid one, collecting them. */
|
|
596
|
+
placer(placed, captureContent) {
|
|
597
|
+
return (attachment, pointer, depth) => {
|
|
598
|
+
const state = attachmentState(attachment);
|
|
599
|
+
const part = attachmentPart(state, captureContent);
|
|
600
|
+
placed.push({ state, ...(part ? { part } : {}), pointer, depth });
|
|
601
|
+
return part ?? null;
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
/**
|
|
605
|
+
* The files of the attachments a value placed that are uploaded: an invalid attachment, and
|
|
606
|
+
* an invalid name with content capture, count as instrumentation failures; nothing is uploaded
|
|
607
|
+
* without content capture or with setup credentials; a value that was `cut`, or a part deeper
|
|
608
|
+
* than Hue reads, keeps its digest part and counts as not uploaded.
|
|
609
|
+
*/
|
|
610
|
+
attachmentFiles(placed, captureContent, signal, traceId, cut) {
|
|
611
|
+
const files = [];
|
|
612
|
+
const tally = newTally();
|
|
613
|
+
for (const { state, part, depth } of placed) {
|
|
614
|
+
if (!part || !state.bytes || !state.sha256) {
|
|
615
|
+
this.instrumentationFailed(signal);
|
|
616
|
+
continue;
|
|
617
|
+
}
|
|
618
|
+
if (state.invalidName && captureContent)
|
|
619
|
+
this.instrumentationFailed(signal);
|
|
620
|
+
if (!this.transport.offloadsValues() || !state.bytes.byteLength)
|
|
621
|
+
continue;
|
|
622
|
+
if (cut || depth > MAX_ATTACHMENT_DEPTH) {
|
|
623
|
+
countFallback(tally, "budget", traceId);
|
|
624
|
+
continue;
|
|
625
|
+
}
|
|
626
|
+
files.push({
|
|
627
|
+
bytes: state.bytes,
|
|
628
|
+
digest: { sha256: state.sha256, size: state.bytes.byteLength },
|
|
629
|
+
contentType: state.contentType,
|
|
630
|
+
...(state.name === undefined ? {} : { name: state.name }),
|
|
631
|
+
});
|
|
632
|
+
}
|
|
633
|
+
if (tally.fallbacks.size)
|
|
634
|
+
this.transport.reportOffload(tally, signal);
|
|
635
|
+
return files;
|
|
636
|
+
}
|
|
637
|
+
/** Stages the attachments a span's `attribute` now holds, replacing its earlier ones. */
|
|
638
|
+
stageAttachments(span, attribute, placed, captureContent, cut = false) {
|
|
639
|
+
const traceId = span.spanContext().traceId;
|
|
640
|
+
const files = this.attachmentFiles(placed, captureContent, "traces", traceId, cut);
|
|
641
|
+
if (span.isRecording())
|
|
642
|
+
this.transport.stageParts(span, attribute, files);
|
|
643
|
+
}
|
|
644
|
+
/**
|
|
645
|
+
* Span attributes as recorded: a value that is an {@link Attachment}, an object or array holding
|
|
646
|
+
* one, or JSON data OpenTelemetry cannot hold as an attribute (an object, or an array that is not
|
|
647
|
+
* of one primitive type), becomes its JSON text with each attachment as its digest part (an
|
|
648
|
+
* invalid attachment alone is omitted); every other value is left to OpenTelemetry as given.
|
|
649
|
+
*/
|
|
650
|
+
attributeValues(input) {
|
|
651
|
+
const placed = [];
|
|
652
|
+
if (!input || typeof input !== "object")
|
|
653
|
+
return { attributes: input, placed };
|
|
654
|
+
let attributes;
|
|
655
|
+
for (const [key, value] of Object.entries(input)) {
|
|
656
|
+
if (!recordedAsJson(value))
|
|
657
|
+
continue;
|
|
658
|
+
attributes ??= { ...input };
|
|
659
|
+
const found = [];
|
|
660
|
+
try {
|
|
661
|
+
const text = encodeContent(value, this.contentLimits(), this.placer(found, this.captureContent));
|
|
662
|
+
if (text === "null")
|
|
663
|
+
delete attributes[key];
|
|
664
|
+
else
|
|
665
|
+
attributes[key] = text;
|
|
666
|
+
}
|
|
667
|
+
catch {
|
|
668
|
+
delete attributes[key];
|
|
669
|
+
this.instrumentationFailed();
|
|
670
|
+
continue;
|
|
671
|
+
}
|
|
672
|
+
placed.push([key, found]);
|
|
673
|
+
}
|
|
674
|
+
return { attributes: attributes ?? input, placed };
|
|
675
|
+
}
|
|
550
676
|
/**
|
|
551
677
|
* Records the tools a model provider executed itself while producing `response`, which no
|
|
552
678
|
* `hue.tool()` call saw: OpenAI Responses `mcp_call`, `web_search_call`, `file_search_call` and
|
|
@@ -799,10 +925,17 @@ export class HueClient {
|
|
|
799
925
|
if (!this.enabled || this.closed || !this.captureContent)
|
|
800
926
|
return;
|
|
801
927
|
try {
|
|
928
|
+
const encode = (limits) => {
|
|
929
|
+
const placed = [];
|
|
930
|
+
return {
|
|
931
|
+
...encodeBoundedContent(value, limits, true, this.placer(placed, true)),
|
|
932
|
+
placed,
|
|
933
|
+
};
|
|
934
|
+
};
|
|
802
935
|
let encoded;
|
|
803
936
|
if (this.transport.holdsValues())
|
|
804
937
|
try {
|
|
805
|
-
encoded =
|
|
938
|
+
encoded = encode({
|
|
806
939
|
...this.contentLimits(),
|
|
807
940
|
bytes: MAX_BLOB_BYTES,
|
|
808
941
|
nodes: MAX_RECORD_NODES,
|
|
@@ -811,8 +944,10 @@ export class HueClient {
|
|
|
811
944
|
catch {
|
|
812
945
|
// Past the runtime's longest string, or its memory: cut to the cap instead.
|
|
813
946
|
}
|
|
814
|
-
const { text, truncated } = encoded ??
|
|
947
|
+
const { text, truncated, placed } = encoded ?? encode(this.contentLimits());
|
|
815
948
|
span.setAttribute(key, text);
|
|
949
|
+
// The attachments the value holds are staged for upload in place of any it held before.
|
|
950
|
+
this.stageAttachments(span, key, placed, true, truncated);
|
|
816
951
|
// The span's list as it stands: the application's own entries (read from the SDK span,
|
|
817
952
|
// which carries its attributes; the API alone shows none) and this helper's earlier ones.
|
|
818
953
|
const listed = withTruncatedKeys(span.attributes?.[TRUNCATED_KEY], this.truncatedKeys.get(span) ?? []);
|
package/dist/evals/client.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ProjectConnection } from "../types.js";
|
|
2
|
+
import { MediaClient } from "./media.js";
|
|
2
3
|
import { type AttemptConnectionBundleV2, type PrepareAttemptRequestV2 } from "./attempt.js";
|
|
3
4
|
import type { ArtifactReservation, ArtifactUpload, CaseConversion, CaseConversionSummary, CaseWrite, CompleteExecution, Completion, Dataset, DatasetCase, DatasetVersion, EvalSet, EvalSetCase, EvalSetVersion, EvaluationItem, EvaluationRun, EvaluationRunSummary, EnvironmentEvidenceSnapshot, Execution, Experiment, ExperimentCase, ExperimentItem, Evaluator, EvaluatorVersion, JsonValue, LocalAgentClaim, LocalAgentRegistration, RegisteredLocalAgent, JudgeBudget, JudgeJob, Page, PageOptions, RegistryPageOptions, Result, ResultSummary, Run, RunCase, Scorer, ScorerDefinition, ScorerVersion, Scoring, ScoringResultInput, ScoringResultSummary, ScoringSubject, ScoringSummary, SimulationMcpCapability, StartExecution, Subject, StoredResult, StoredScoringResult, IdentityUpdate, Tag, TaggedIdentity, UpdatedRun } from "./types.js";
|
|
4
5
|
/** Connection options for {@link createEvaluationClient}. */
|
|
@@ -59,6 +60,9 @@ export declare class EvaluationClient {
|
|
|
59
60
|
#private;
|
|
60
61
|
/** Validated Hue origin. */
|
|
61
62
|
readonly baseUrl: string;
|
|
63
|
+
/** Recorded media read back by digest: one file (`fetch`) or every digest part of a recorded
|
|
64
|
+
* value (`resolve`). Needs a key that may read recorded content (Read, or Read and write). */
|
|
65
|
+
readonly media: MediaClient;
|
|
62
66
|
private readonly timeoutMillis;
|
|
63
67
|
private readonly maxAttempts;
|
|
64
68
|
constructor(options: EvaluationClientOptions);
|
|
@@ -75,6 +79,7 @@ export declare class EvaluationClient {
|
|
|
75
79
|
}): Promise<Uint8Array>;
|
|
76
80
|
/** Stage bytes at the storage capability Hue issued. The Hue key is never sent to storage. */
|
|
77
81
|
uploadArtifactBytes(upload: ArtifactUpload, bytes: Uint8Array, contentType: string): Promise<void>;
|
|
82
|
+
private putArtifact;
|
|
78
83
|
private page;
|
|
79
84
|
private registryPage;
|
|
80
85
|
/** Reads the current project to confirm the key and origin. */
|
|
@@ -97,7 +102,20 @@ export declare class EvaluationClient {
|
|
|
97
102
|
getDatasetVersion(id: string): Promise<DatasetVersion>;
|
|
98
103
|
/** Lists the full cases of a dataset version; use small page limits for large values. */
|
|
99
104
|
listCases(id: string, page?: PageOptions): Promise<Page<DatasetCase>>;
|
|
100
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* Adds a case to a draft version using optimistic concurrency on `expectedRevision`. Each of
|
|
107
|
+
* `files` given as bytes or a path is checked against the artifact policy and uploaded as a
|
|
108
|
+
* verified artifact first, under a key derived from the version, the case's `externalKey`, the
|
|
109
|
+
* role and the bytes, so sending the same case again reuses the artifact. Every file and the
|
|
110
|
+
* case itself are checked before the first upload. A file given by `path` is hashed a slice at
|
|
111
|
+
* a time when it is checked and read again when it is uploaded, one file at a time, so no more
|
|
112
|
+
* than one file is held in memory; only bytes that match the checked SHA-256 are sent.
|
|
113
|
+
*
|
|
114
|
+
* @throws TypeError or RangeError for `files` outside the artifact policy, or a case Hue would
|
|
115
|
+
* refuse for its fields or size, before any request. TypeError when a file given by `path`
|
|
116
|
+
* changed after it was checked, or can no longer be read, before any of its bytes are sent: it
|
|
117
|
+
* is not staged and the case is not written.
|
|
118
|
+
*/
|
|
101
119
|
addCase(id: string, input: CaseWrite): Promise<{
|
|
102
120
|
/** The stored case. */
|
|
103
121
|
item: DatasetCase;
|
package/dist/evals/client.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isLoopbackHost, validateOptions } from "../config.js";
|
|
2
2
|
import { json, uuid, valueBounds } from "./json.js";
|
|
3
|
+
import { MediaClient } from "./media.js";
|
|
3
4
|
import { attemptBindingRead, parsePrepareAttemptResultV2, parseRefreshedAttemptResultV2, parseRevocationResult, prepareAttemptInputV2, validateAttemptConnectionBundleV2, } from "./attempt.js";
|
|
4
5
|
/** Thrown for a failed evaluation API request; the message is fixed and never includes response text. */
|
|
5
6
|
export class HueApiError extends Error {
|
|
@@ -149,6 +150,42 @@ function productRunFields(value, kind) {
|
|
|
149
150
|
result.scoring = result.evaluation;
|
|
150
151
|
return result;
|
|
151
152
|
}
|
|
153
|
+
/** The bounds of a request body as a whole: 1 MiB, with each value's node and depth bounds. */
|
|
154
|
+
const REQUEST_BOUNDS = { ...valueBounds, bytes: 1024 * 1024 };
|
|
155
|
+
/** A request body as it is sent: checked as JSON within `bounds`. Optional top-level fields are
|
|
156
|
+
* omitted intentionally; nested undefined remains invalid. */
|
|
157
|
+
function payloadOf(body, bounds) {
|
|
158
|
+
return JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), bounds));
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Checks what a case write's fields can be checked for here, as Hue checks them: a positive
|
|
162
|
+
* integer `expectedRevision`, an `externalKey` of 1–200 UTF-16 code units of well-formed text
|
|
163
|
+
* without NUL, `inputs`, `expected` and an object `metadata` within one value's JSON bounds, and an
|
|
164
|
+
* `environmentVersionId` that is a UUID or null.
|
|
165
|
+
*/
|
|
166
|
+
function checkCaseWrite(input) {
|
|
167
|
+
if (!input || typeof input !== "object")
|
|
168
|
+
throw new TypeError("A case write must be an object");
|
|
169
|
+
if (!Number.isSafeInteger(input.expectedRevision) || input.expectedRevision < 1)
|
|
170
|
+
throw new TypeError("expectedRevision must be a positive integer");
|
|
171
|
+
const key = input.externalKey;
|
|
172
|
+
if (typeof key !== "string" ||
|
|
173
|
+
!key.length ||
|
|
174
|
+
key.length > 200 ||
|
|
175
|
+
!key.isWellFormed() ||
|
|
176
|
+
key.includes("\u0000"))
|
|
177
|
+
throw new TypeError("externalKey must be 1–200 UTF-16 code units of well-formed text");
|
|
178
|
+
json(input.inputs, valueBounds);
|
|
179
|
+
if (input.expected !== undefined)
|
|
180
|
+
json(input.expected, valueBounds);
|
|
181
|
+
if (input.metadata !== undefined) {
|
|
182
|
+
if (!input.metadata || typeof input.metadata !== "object" || Array.isArray(input.metadata))
|
|
183
|
+
throw new TypeError("metadata must be an object");
|
|
184
|
+
json(input.metadata, valueBounds);
|
|
185
|
+
}
|
|
186
|
+
if (input.environmentVersionId !== undefined && input.environmentVersionId !== null)
|
|
187
|
+
uuid(input.environmentVersionId);
|
|
188
|
+
}
|
|
152
189
|
/** Times a refused request is sent again before its refusal is the caller's error. */
|
|
153
190
|
const REFUSAL_RETRIES = 4;
|
|
154
191
|
/** A refusal asking for a longer wait fails at once rather than holding the caller. */
|
|
@@ -183,6 +220,9 @@ function refusalRetryAfter(response) {
|
|
|
183
220
|
export class EvaluationClient {
|
|
184
221
|
/** Validated Hue origin. */
|
|
185
222
|
baseUrl;
|
|
223
|
+
/** Recorded media read back by digest: one file (`fetch`) or every digest part of a recorded
|
|
224
|
+
* value (`resolve`). Needs a key that may read recorded content (Read, or Read and write). */
|
|
225
|
+
media;
|
|
186
226
|
#apiKey;
|
|
187
227
|
timeoutMillis;
|
|
188
228
|
maxAttempts;
|
|
@@ -199,6 +239,11 @@ export class EvaluationClient {
|
|
|
199
239
|
if (!Number.isInteger(attempts) || attempts < 1 || attempts > 10)
|
|
200
240
|
throw new RangeError("maxAttempts must be 1–10");
|
|
201
241
|
this.maxAttempts = attempts;
|
|
242
|
+
this.media = new MediaClient({
|
|
243
|
+
baseUrl: this.baseUrl,
|
|
244
|
+
apiKey: validated.apiKey,
|
|
245
|
+
timeoutMillis: this.timeoutMillis,
|
|
246
|
+
});
|
|
202
247
|
}
|
|
203
248
|
/** Sends one request to Hue, again after the wait while Hue refuses it with a short
|
|
204
249
|
* `Retry-After`. `init` runs for every attempt, so each has its own timeout; `signal` also ends
|
|
@@ -235,15 +280,12 @@ export class EvaluationClient {
|
|
|
235
280
|
throw new HueApiError(response.status, askedRetryAfter(response));
|
|
236
281
|
}
|
|
237
282
|
}
|
|
238
|
-
async request(method, path, body, bounds =
|
|
283
|
+
async request(method, path, body, bounds = REQUEST_BOUNDS, signal,
|
|
239
284
|
/** Overrides the retry decision `retryable` takes from the method and body. */
|
|
240
285
|
idempotent) {
|
|
241
286
|
// Serialized once, outside the attempts: a body this client cannot encode is a caller error
|
|
242
|
-
// no attempt fixes.
|
|
243
|
-
|
|
244
|
-
const payload = body === undefined
|
|
245
|
-
? undefined
|
|
246
|
-
: JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), bounds));
|
|
287
|
+
// no attempt fixes.
|
|
288
|
+
const payload = body === undefined ? undefined : payloadOf(body, bounds);
|
|
247
289
|
const attempts = retryable(method, body, idempotent) ? this.maxAttempts : 1;
|
|
248
290
|
for (let attempt = 1;; attempt++) {
|
|
249
291
|
try {
|
|
@@ -360,17 +402,29 @@ export class EvaluationClient {
|
|
|
360
402
|
}
|
|
361
403
|
}
|
|
362
404
|
/** Stage bytes at the storage capability Hue issued. The Hue key is never sent to storage. */
|
|
363
|
-
|
|
405
|
+
uploadArtifactBytes(upload, bytes, contentType) {
|
|
406
|
+
return this.putArtifact(upload, contentType, bytes);
|
|
407
|
+
}
|
|
408
|
+
/** @internal Stage `byteSize` bytes read from `body` a slice at a time, without the copy a
|
|
409
|
+
* byte body is given; the request declares their length, as it does for bytes. */
|
|
410
|
+
uploadArtifactStream(upload, body, byteSize, contentType) {
|
|
411
|
+
return this.putArtifact(upload, contentType, body, byteSize);
|
|
412
|
+
}
|
|
413
|
+
async putArtifact(upload, contentType, body, byteSize) {
|
|
364
414
|
const uploadUrl = signedUploadUrl(upload.uploadUrl);
|
|
365
415
|
if (upload.method !== "PUT")
|
|
366
416
|
throw new HueApiError();
|
|
367
417
|
const headers = signedUploadHeaders(upload.headers, contentType);
|
|
418
|
+
// Storage refuses a body of unknown length, which a stream would otherwise be sent as.
|
|
419
|
+
if (byteSize !== undefined)
|
|
420
|
+
headers["content-length"] = String(byteSize);
|
|
368
421
|
let response;
|
|
369
422
|
try {
|
|
370
423
|
response = await fetch(uploadUrl, {
|
|
371
424
|
method: "PUT",
|
|
372
425
|
headers,
|
|
373
|
-
body
|
|
426
|
+
body,
|
|
427
|
+
...(byteSize !== undefined ? { duplex: "half" } : {}),
|
|
374
428
|
redirect: "error",
|
|
375
429
|
signal: AbortSignal.timeout(Math.max(this.timeoutMillis, 120_000)),
|
|
376
430
|
});
|
|
@@ -433,9 +487,35 @@ export class EvaluationClient {
|
|
|
433
487
|
listCases(id, page) {
|
|
434
488
|
return this.request("GET", `/dataset-versions/${uuid(id)}/cases${this.page(page)}`);
|
|
435
489
|
}
|
|
436
|
-
/**
|
|
437
|
-
|
|
438
|
-
|
|
490
|
+
/**
|
|
491
|
+
* Adds a case to a draft version using optimistic concurrency on `expectedRevision`. Each of
|
|
492
|
+
* `files` given as bytes or a path is checked against the artifact policy and uploaded as a
|
|
493
|
+
* verified artifact first, under a key derived from the version, the case's `externalKey`, the
|
|
494
|
+
* role and the bytes, so sending the same case again reuses the artifact. Every file and the
|
|
495
|
+
* case itself are checked before the first upload. A file given by `path` is hashed a slice at
|
|
496
|
+
* a time when it is checked and read again when it is uploaded, one file at a time, so no more
|
|
497
|
+
* than one file is held in memory; only bytes that match the checked SHA-256 are sent.
|
|
498
|
+
*
|
|
499
|
+
* @throws TypeError or RangeError for `files` outside the artifact policy, or a case Hue would
|
|
500
|
+
* refuse for its fields or size, before any request. TypeError when a file given by `path`
|
|
501
|
+
* changed after it was checked, or can no longer be read, before any of its bytes are sent: it
|
|
502
|
+
* is not staged and the case is not written.
|
|
503
|
+
*/
|
|
504
|
+
async addCase(id, input) {
|
|
505
|
+
// The canonical spelling: the reservation keys of uploaded files name the version.
|
|
506
|
+
const versionId = uuid(id).toLowerCase();
|
|
507
|
+
checkCaseWrite(input);
|
|
508
|
+
let body = input;
|
|
509
|
+
if (input.files !== undefined) {
|
|
510
|
+
// Loaded on use: the artifact helpers import this client.
|
|
511
|
+
const { caseFileReferences } = await import("./files.js");
|
|
512
|
+
body = {
|
|
513
|
+
...input,
|
|
514
|
+
// The case is checked as it will be sent, with a stand-in for each upload's artifact.
|
|
515
|
+
files: await caseFileReferences(this, versionId, input, (files) => payloadOf({ ...input, files }, REQUEST_BOUNDS)),
|
|
516
|
+
};
|
|
517
|
+
}
|
|
518
|
+
return this.request("POST", `/dataset-versions/${versionId}/cases`, body);
|
|
439
519
|
}
|
|
440
520
|
/** Freezes a draft version at the given revision; frozen versions are immutable. */
|
|
441
521
|
freezeDatasetVersion(id, expectedRevision) {
|
package/dist/evals/files.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type EvaluationClient } from "./client.js";
|
|
2
|
-
import type { CaseFile, LocalFile, OutputFile, SubjectFile } from "./types.js";
|
|
2
|
+
import type { CaseFile, CaseFileReference, CaseWrite, LocalFile, OutputFile, SubjectFile } from "./types.js";
|
|
3
3
|
/** Roles the target receives. Organization templates stay with grading, as in the managed protocol. */
|
|
4
4
|
export declare const targetFileRoles: readonly CaseFile["role"][];
|
|
5
5
|
/** Hue's artifact policy: the pilot file size and the accepted document types. */
|
|
@@ -112,5 +112,15 @@ export declare const artifactSettling: {
|
|
|
112
112
|
* finished verifying after an earlier attempt stopped waiting.
|
|
113
113
|
*/
|
|
114
114
|
export declare function uploadOutputFiles(client: EvaluationClient, executionId: string, files: StagedOutputFile[], save: () => Promise<void>, timing?: typeof artifactSettling): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* The `files` of a case write as the case API takes them: references kept, and each file given
|
|
117
|
+
* as bytes or a path checked against the artifact policy (role, name, one of
|
|
118
|
+
* `outputContentTypes`, 1 byte to 25 MiB) and uploaded as a verified artifact. Every file is
|
|
119
|
+
* checked, and then `beforeUpload` is given the references with a stand-in ID for each upload,
|
|
120
|
+
* before the first upload. The reservation key names the version, the case's `externalKey`, the
|
|
121
|
+
* role and the bytes, so the same write sent again reuses its artifacts. Each artifact is pinned
|
|
122
|
+
* once with a role, even when a reference names the artifact a file was uploaded as.
|
|
123
|
+
*/
|
|
124
|
+
export declare function caseFileReferences(client: EvaluationClient, versionId: string, input: CaseWrite, beforeUpload?: (references: CaseFileReference[]) => void, timing?: typeof artifactSettling): Promise<CaseFileReference[]>;
|
|
115
125
|
/** Project uploaded staged files as the verified local files handed to scorers. */
|
|
116
126
|
export declare function localOutputFiles(files: StagedOutputFile[]): LocalFile[];
|