@hue-run/sdk 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +18 -11
- package/EVALUATIONS.md +53 -2
- package/README.md +295 -45
- package/dist/attachment.d.ts +82 -0
- package/dist/attachment.js +225 -0
- package/dist/blobs.d.ts +73 -17
- package/dist/blobs.js +247 -96
- package/dist/cli/eval-direct.d.ts +3 -2
- package/dist/cli/eval-direct.js +49 -6
- package/dist/client.d.ts +21 -1
- package/dist/client.js +217 -32
- package/dist/evals/client.d.ts +14 -1
- package/dist/evals/client.js +73 -9
- package/dist/evals/files.d.ts +11 -1
- package/dist/evals/files.js +172 -20
- package/dist/evals/media.d.ts +217 -0
- package/dist/evals/media.js +907 -0
- package/dist/evals/runner.js +130 -106
- package/dist/evals/types.d.ts +48 -0
- package/dist/evals.d.ts +2 -0
- package/dist/evals.js +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/inline-files.d.ts +160 -28
- package/dist/inline-files.js +809 -86
- package/dist/json-edits.d.ts +24 -0
- package/dist/json-edits.js +187 -0
- package/dist/privacy.d.ts +14 -1
- package/dist/privacy.js +86 -25
- package/dist/safety.d.ts +7 -2
- package/dist/safety.js +22 -4
- package/dist/snapshot.d.ts +7 -4
- package/dist/snapshot.js +177 -38
- package/dist/transport.d.ts +18 -0
- package/dist/transport.js +544 -38
- package/dist/types.d.ts +28 -7
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +2 -2
package/dist/client.d.ts
CHANGED
|
@@ -127,6 +127,24 @@ export declare class HueClient {
|
|
|
127
127
|
/** `gen_ai.request.model` for the record; defaults to the enclosing `model()` span's value. */
|
|
128
128
|
model?: string;
|
|
129
129
|
}, explicitContext?: Context): void;
|
|
130
|
+
/** Writes each attachment as its digest part, `null` for an invalid one, collecting them. */
|
|
131
|
+
private placer;
|
|
132
|
+
/**
|
|
133
|
+
* The files of the attachments a value placed that are uploaded: an invalid attachment, and
|
|
134
|
+
* an invalid name with content capture, count as instrumentation failures; nothing is uploaded
|
|
135
|
+
* without content capture or with setup credentials; a value that was `cut`, or a part deeper
|
|
136
|
+
* than Hue reads, keeps its digest part and counts as not uploaded.
|
|
137
|
+
*/
|
|
138
|
+
private attachmentFiles;
|
|
139
|
+
/** Stages the attachments a span's `attribute` now holds, replacing its earlier ones. */
|
|
140
|
+
private stageAttachments;
|
|
141
|
+
/**
|
|
142
|
+
* Span attributes as recorded: a value that is an {@link Attachment}, an object or array holding
|
|
143
|
+
* one, or JSON data OpenTelemetry cannot hold as an attribute (an object, or an array that is not
|
|
144
|
+
* of one primitive type), becomes its JSON text with each attachment as its digest part (an
|
|
145
|
+
* invalid attachment alone is omitted); every other value is left to OpenTelemetry as given.
|
|
146
|
+
*/
|
|
147
|
+
private attributeValues;
|
|
130
148
|
/**
|
|
131
149
|
* Records the tools a model provider executed itself while producing `response`, which no
|
|
132
150
|
* `hue.tool()` call saw: OpenAI Responses `mcp_call`, `web_search_call`, `file_search_call` and
|
|
@@ -149,13 +167,15 @@ export declare class HueClient {
|
|
|
149
167
|
* Adds a `hue.file` event to the active (or given) span for a file the work read, received or
|
|
150
168
|
* produced: `hue.file.sha256`, `hue.file.role`, `hue.file.media_type`, `hue.file.size` when known
|
|
151
169
|
* and, when `captureContent` is true, `hue.file.name`. `data` is hashed and measured locally and
|
|
152
|
-
*
|
|
170
|
+
* uploaded only with `upload: true`. The event is metadata, so it is recorded in both capture modes. An invalid
|
|
153
171
|
* record, or one without an active span, is omitted and counted, never thrown; an invalid name
|
|
154
172
|
* alone is omitted and counted while the rest is recorded.
|
|
155
173
|
*/
|
|
156
174
|
recordFile(file: FileRecord, explicitContext?: Context): void;
|
|
157
175
|
/** The keys of each span's content values that were cut to Hue's value cap, for `hue.truncated`. */
|
|
158
176
|
private truncatedKeys;
|
|
177
|
+
/** Distinguishes requests whose helper events otherwise have identical metadata. */
|
|
178
|
+
private fileEventTimes;
|
|
159
179
|
/** A content value's bounds: Hue's value cap as the receiver last advertised it. */
|
|
160
180
|
private contentLimits;
|
|
161
181
|
/**
|
package/dist/client.js
CHANGED
|
@@ -1,18 +1,20 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
|
+
import { digestFormBytes, INLINE_FILE_TEXT_PER_RECORD, isMediaName } from "./inline-files.js";
|
|
3
4
|
import { context, isSpanContextValid, ROOT_CONTEXT, SpanKind, SpanStatusCode, trace, } from "@opentelemetry/api";
|
|
4
5
|
import { defaultTextMapGetter, defaultTextMapSetter } from "@opentelemetry/api";
|
|
5
6
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
6
|
-
import { W3CTraceContextPropagator } from "@opentelemetry/core";
|
|
7
|
+
import { isAttributeValue, W3CTraceContextPropagator } from "@opentelemetry/core";
|
|
7
8
|
import { LoggerProvider } from "@opentelemetry/sdk-logs";
|
|
8
9
|
import { TracerProvider } from "@opentelemetry/sdk-trace";
|
|
9
10
|
import { defaultResource, resourceFromAttributes } from "@opentelemetry/resources";
|
|
10
11
|
import { HUE_SCOPE } from "./config.js";
|
|
11
12
|
import { hostedServerAddresses, hostedToolActivity, hostedToolProvider, providerErrorDescription, } from "./provider-tools.js";
|
|
12
13
|
import { toolCatalogSummary } from "./tool-definitions.js";
|
|
13
|
-
import { contentLimitsOf, encodeBoundedContent, encodeContent, noopSpan, safeSpan, } from "./safety.js";
|
|
14
|
+
import { contentLimitsOf, encodeBoundedContent, encodeContent, noopSpan, safeSpan, spanSource, } from "./safety.js";
|
|
14
15
|
import { TRUNCATED_KEY, withTruncatedKeys } from "./privacy.js";
|
|
15
|
-
import { MAX_BLOB_BYTES } from "./blobs.js";
|
|
16
|
+
import { countFallback, MAX_BLOB_BYTES, newTally } from "./blobs.js";
|
|
17
|
+
import { attachmentPart, attachmentState, holdsAttachment, MAX_ATTACHMENT_DEPTH, } from "./attachment.js";
|
|
16
18
|
import { createHueTransport, exportBudgetMillis, HueExportError, } from "./transport.js";
|
|
17
19
|
import { verifyTrace } from "./receipt.js";
|
|
18
20
|
import { sdkVersion } from "./version.js";
|
|
@@ -169,6 +171,19 @@ const OWNED_SPAN_LIMITS = {
|
|
|
169
171
|
/** A provider tool listing's bounds before its catalog summary: one export request, 64 levels and
|
|
170
172
|
* 65,536 values, as the Python SDK's content snapshot. */
|
|
171
173
|
const catalogLimits = { bytes: MAX_BODY_BYTES, nodes: 65_536, depth: 64 };
|
|
174
|
+
/** Whether a span attribute passed to a helper is recorded as JSON text: an attachment, a value
|
|
175
|
+
* holding one, or JSON data OpenTelemetry cannot hold as an attribute (a plain object, or an array
|
|
176
|
+
* that is not of one primitive type). */
|
|
177
|
+
function recordedAsJson(value) {
|
|
178
|
+
if (holdsAttachment(value))
|
|
179
|
+
return true;
|
|
180
|
+
if (value === null || typeof value !== "object" || isAttributeValue(value))
|
|
181
|
+
return false;
|
|
182
|
+
if (Array.isArray(value))
|
|
183
|
+
return true;
|
|
184
|
+
const prototype = Object.getPrototypeOf(value);
|
|
185
|
+
return prototype === Object.prototype || prototype === null;
|
|
186
|
+
}
|
|
172
187
|
/**
|
|
173
188
|
* Hue tracing client. Helpers create spans through a private tracer and local async context, never
|
|
174
189
|
* through global OpenTelemetry registration, and they fail open: capture problems are counted in
|
|
@@ -285,7 +300,10 @@ export class HueClient {
|
|
|
285
300
|
workspaceId: identifier(options.workspaceId ?? inherited?.workspaceId),
|
|
286
301
|
model: inherited?.model,
|
|
287
302
|
};
|
|
288
|
-
|
|
303
|
+
const { attributes, placed } = this.attributeValues(options.attributes);
|
|
304
|
+
span = this.storage.run(active, () => this.tracer.startSpan(name, { kind: options.kind ?? SpanKind.INTERNAL, attributes }, active.context));
|
|
305
|
+
for (const [key, found] of placed)
|
|
306
|
+
this.stageAttachments(span, key, found, this.captureContent);
|
|
289
307
|
}
|
|
290
308
|
catch {
|
|
291
309
|
this.instrumentationFailed();
|
|
@@ -534,18 +552,127 @@ export class HueClient {
|
|
|
534
552
|
stamp("gen_ai.provider.name", messages.provider, enclosing?.model?.provider);
|
|
535
553
|
stamp("gen_ai.request.model", messages.model, enclosing?.model?.requestModel);
|
|
536
554
|
stamp("gen_ai.conversation.id", undefined, enclosing?.sessionId);
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
555
|
+
// While media may be uploaded, the body may hold files past the value cap: it is emitted
|
|
556
|
+
// when it fits the cap with each file as its digest, and export uploads the files (under
|
|
557
|
+
// `body#<pointer>` keys) or cuts the body as before. Otherwise a body over the cap is
|
|
558
|
+
// refused, as before. Each attachment is written as its digest part.
|
|
559
|
+
const limits = this.contentLimits();
|
|
560
|
+
const placed = [];
|
|
561
|
+
const place = this.placer(placed, true);
|
|
562
|
+
let text;
|
|
563
|
+
if (this.transport.holdsValues()) {
|
|
564
|
+
text = encodeContent(body, { ...limits, bytes: limits.bytes + INLINE_FILE_TEXT_PER_RECORD }, place);
|
|
565
|
+
if (Buffer.byteLength(text) > limits.bytes &&
|
|
566
|
+
digestFormBytes(JSON.parse(text)) > limits.bytes)
|
|
567
|
+
throw new RangeError("Content limit exceeded");
|
|
568
|
+
}
|
|
569
|
+
else
|
|
570
|
+
text = encodeContent(body, limits, place);
|
|
571
|
+
const recorded = JSON.parse(text);
|
|
572
|
+
// Attachments in the body are staged with the body until this transport admits the record;
|
|
573
|
+
// export uploads them with the media the body holds.
|
|
574
|
+
const traceId = trace.getSpanContext(active).traceId;
|
|
575
|
+
const files = this.attachmentFiles(placed, true, "logs", traceId, false);
|
|
576
|
+
if (files.length)
|
|
577
|
+
this.transport.stageLogParts(recorded, traceId, files);
|
|
578
|
+
try {
|
|
579
|
+
this.logger.emit({
|
|
580
|
+
context: active,
|
|
581
|
+
severityNumber: SeverityNumber.INFO,
|
|
582
|
+
eventName: "gen_ai.client.inference.operation.details",
|
|
583
|
+
attributes,
|
|
584
|
+
body: recorded,
|
|
585
|
+
});
|
|
586
|
+
}
|
|
587
|
+
finally {
|
|
588
|
+
this.transport.unstageLogParts(recorded);
|
|
589
|
+
}
|
|
544
590
|
}
|
|
545
591
|
catch {
|
|
546
592
|
this.instrumentationFailed("logs");
|
|
547
593
|
}
|
|
548
594
|
}
|
|
595
|
+
/** Writes each attachment as its digest part, `null` for an invalid one, collecting them. */
|
|
596
|
+
placer(placed, captureContent) {
|
|
597
|
+
return (attachment, pointer, depth) => {
|
|
598
|
+
const state = attachmentState(attachment);
|
|
599
|
+
const part = attachmentPart(state, captureContent);
|
|
600
|
+
placed.push({ state, ...(part ? { part } : {}), pointer, depth });
|
|
601
|
+
return part ?? null;
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
/**
|
|
605
|
+
* The files of the attachments a value placed that are uploaded: an invalid attachment, and
|
|
606
|
+
* an invalid name with content capture, count as instrumentation failures; nothing is uploaded
|
|
607
|
+
* without content capture or with setup credentials; a value that was `cut`, or a part deeper
|
|
608
|
+
* than Hue reads, keeps its digest part and counts as not uploaded.
|
|
609
|
+
*/
|
|
610
|
+
attachmentFiles(placed, captureContent, signal, traceId, cut) {
|
|
611
|
+
const files = [];
|
|
612
|
+
const tally = newTally();
|
|
613
|
+
for (const { state, part, depth } of placed) {
|
|
614
|
+
if (!part || !state.bytes || !state.sha256) {
|
|
615
|
+
this.instrumentationFailed(signal);
|
|
616
|
+
continue;
|
|
617
|
+
}
|
|
618
|
+
if (state.invalidName && captureContent)
|
|
619
|
+
this.instrumentationFailed(signal);
|
|
620
|
+
if (!this.transport.offloadsValues() || !state.bytes.byteLength)
|
|
621
|
+
continue;
|
|
622
|
+
if (cut || depth > MAX_ATTACHMENT_DEPTH) {
|
|
623
|
+
countFallback(tally, "budget", traceId);
|
|
624
|
+
continue;
|
|
625
|
+
}
|
|
626
|
+
files.push({
|
|
627
|
+
bytes: state.bytes,
|
|
628
|
+
digest: { sha256: state.sha256, size: state.bytes.byteLength },
|
|
629
|
+
contentType: state.contentType,
|
|
630
|
+
...(state.name === undefined ? {} : { name: state.name }),
|
|
631
|
+
});
|
|
632
|
+
}
|
|
633
|
+
if (tally.fallbacks.size)
|
|
634
|
+
this.transport.reportOffload(tally, signal);
|
|
635
|
+
return files;
|
|
636
|
+
}
|
|
637
|
+
/** Stages the attachments a span's `attribute` now holds, replacing its earlier ones. */
|
|
638
|
+
stageAttachments(span, attribute, placed, captureContent, cut = false) {
|
|
639
|
+
const traceId = span.spanContext().traceId;
|
|
640
|
+
const files = this.attachmentFiles(placed, captureContent, "traces", traceId, cut);
|
|
641
|
+
if (span.isRecording())
|
|
642
|
+
this.transport.stageParts(span, attribute, files);
|
|
643
|
+
}
|
|
644
|
+
/**
|
|
645
|
+
* Span attributes as recorded: a value that is an {@link Attachment}, an object or array holding
|
|
646
|
+
* one, or JSON data OpenTelemetry cannot hold as an attribute (an object, or an array that is not
|
|
647
|
+
* of one primitive type), becomes its JSON text with each attachment as its digest part (an
|
|
648
|
+
* invalid attachment alone is omitted); every other value is left to OpenTelemetry as given.
|
|
649
|
+
*/
|
|
650
|
+
attributeValues(input) {
|
|
651
|
+
const placed = [];
|
|
652
|
+
if (!input || typeof input !== "object")
|
|
653
|
+
return { attributes: input, placed };
|
|
654
|
+
let attributes;
|
|
655
|
+
for (const [key, value] of Object.entries(input)) {
|
|
656
|
+
if (!recordedAsJson(value))
|
|
657
|
+
continue;
|
|
658
|
+
attributes ??= { ...input };
|
|
659
|
+
const found = [];
|
|
660
|
+
try {
|
|
661
|
+
const text = encodeContent(value, this.contentLimits(), this.placer(found, this.captureContent));
|
|
662
|
+
if (text === "null")
|
|
663
|
+
delete attributes[key];
|
|
664
|
+
else
|
|
665
|
+
attributes[key] = text;
|
|
666
|
+
}
|
|
667
|
+
catch {
|
|
668
|
+
delete attributes[key];
|
|
669
|
+
this.instrumentationFailed();
|
|
670
|
+
continue;
|
|
671
|
+
}
|
|
672
|
+
placed.push([key, found]);
|
|
673
|
+
}
|
|
674
|
+
return { attributes: attributes ?? input, placed };
|
|
675
|
+
}
|
|
549
676
|
/**
|
|
550
677
|
* Records the tools a model provider executed itself while producing `response`, which no
|
|
551
678
|
* `hue.tool()` call saw: OpenAI Responses `mcp_call`, `web_search_call`, `file_search_call` and
|
|
@@ -662,7 +789,7 @@ export class HueClient {
|
|
|
662
789
|
* Adds a `hue.file` event to the active (or given) span for a file the work read, received or
|
|
663
790
|
* produced: `hue.file.sha256`, `hue.file.role`, `hue.file.media_type`, `hue.file.size` when known
|
|
664
791
|
* and, when `captureContent` is true, `hue.file.name`. `data` is hashed and measured locally and
|
|
665
|
-
*
|
|
792
|
+
* uploaded only with `upload: true`. The event is metadata, so it is recorded in both capture modes. An invalid
|
|
666
793
|
* record, or one without an active span, is omitted and counted, never thrown; an invalid name
|
|
667
794
|
* alone is omitted and counted while the rest is recorded.
|
|
668
795
|
*/
|
|
@@ -680,26 +807,30 @@ export class HueClient {
|
|
|
680
807
|
throw new TypeError("Invalid media type");
|
|
681
808
|
let sha256 = typeof file.sha256 === "string" ? file.sha256.toLowerCase() : file.sha256;
|
|
682
809
|
let byteSize = file.byteSize;
|
|
810
|
+
let bytes;
|
|
683
811
|
if (data !== undefined) {
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
return Buffer.from(data, "utf8");
|
|
692
|
-
})()
|
|
693
|
-
: data instanceof Uint8Array
|
|
694
|
-
? data.byteLength <= MAX_FILE_DATA_BYTES
|
|
695
|
-
? data
|
|
696
|
-
: (() => {
|
|
812
|
+
bytes =
|
|
813
|
+
typeof data === "string"
|
|
814
|
+
? (() => {
|
|
815
|
+
// Buffer.byteLength measures UTF-8 without allocating the copy that hashing would
|
|
816
|
+
// otherwise require. Reject before Buffer.from/createHash can retain large input.
|
|
817
|
+
if (data.length > MAX_FILE_DATA_BYTES ||
|
|
818
|
+
Buffer.byteLength(data, "utf8") > MAX_FILE_DATA_BYTES)
|
|
697
819
|
throw new RangeError("File data exceeds Hue's 25 MiB limit");
|
|
698
|
-
|
|
699
|
-
|
|
820
|
+
return Buffer.from(data, "utf8");
|
|
821
|
+
})()
|
|
822
|
+
: data instanceof Uint8Array
|
|
823
|
+
? data.byteLength <= MAX_FILE_DATA_BYTES
|
|
824
|
+
? data
|
|
825
|
+
: (() => {
|
|
826
|
+
throw new RangeError("File data exceeds Hue's 25 MiB limit");
|
|
827
|
+
})()
|
|
828
|
+
: undefined;
|
|
700
829
|
if (!bytes)
|
|
701
830
|
throw new TypeError("File data must be bytes or a string");
|
|
702
|
-
const digest =
|
|
831
|
+
const digest = bytes.byteLength > 0 && file.upload === true && this.transport.holdsValues()
|
|
832
|
+
? this.transport.digestMedia(bytes, span.spanContext().traceId).sha256
|
|
833
|
+
: createHash("sha256").update(bytes).digest("hex");
|
|
703
834
|
// A caller-supplied digest or size must describe the same bytes.
|
|
704
835
|
if ((sha256 !== undefined && sha256 !== digest) ||
|
|
705
836
|
(byteSize !== undefined && byteSize !== bytes.byteLength))
|
|
@@ -719,12 +850,55 @@ export class HueClient {
|
|
|
719
850
|
if (byteSize !== undefined)
|
|
720
851
|
attributes["hue.file.size"] = byteSize;
|
|
721
852
|
if (this.captureContent && name !== undefined) {
|
|
722
|
-
if (
|
|
853
|
+
if (isMediaName(name))
|
|
723
854
|
attributes["hue.file.name"] = name;
|
|
724
855
|
else
|
|
725
856
|
this.instrumentationFailed();
|
|
726
857
|
}
|
|
727
|
-
span
|
|
858
|
+
// Let OpenTelemetry use the span's wall-clock anchor, then keep this event's exact
|
|
859
|
+
// timestamp for upload admission. A dropped event cannot authorize an upload.
|
|
860
|
+
const source = spanSource(span);
|
|
861
|
+
const previous = this.fileEventTimes.get(source);
|
|
862
|
+
const after = (time) => {
|
|
863
|
+
if (previous &&
|
|
864
|
+
(time[0] < previous[0] || (time[0] === previous[0] && time[1] <= previous[1]))) {
|
|
865
|
+
time[0] = previous[0];
|
|
866
|
+
time[1] = previous[1] + 1;
|
|
867
|
+
if (time[1] === 1_000_000_000) {
|
|
868
|
+
time[0]++;
|
|
869
|
+
time[1] = 0;
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
return time;
|
|
873
|
+
};
|
|
874
|
+
let eventTime;
|
|
875
|
+
if (Array.isArray(source.events)) {
|
|
876
|
+
const previousEvent = source.events.at(-1);
|
|
877
|
+
span.addEvent("hue.file", attributes);
|
|
878
|
+
const event = source.events.at(-1);
|
|
879
|
+
eventTime = event && event !== previousEvent ? after(event.time) : undefined;
|
|
880
|
+
}
|
|
881
|
+
else {
|
|
882
|
+
// A span that does not expose its events (an API-only wrapper in attach mode): stamp the
|
|
883
|
+
// event with the wall clock and keep that exact time for upload admission.
|
|
884
|
+
const now = Date.now();
|
|
885
|
+
eventTime = after([Math.floor(now / 1000), (now % 1000) * 1_000_000]);
|
|
886
|
+
span.addEvent("hue.file", attributes, [eventTime[0], eventTime[1]]);
|
|
887
|
+
}
|
|
888
|
+
if (eventTime)
|
|
889
|
+
this.fileEventTimes.set(source, eventTime);
|
|
890
|
+
if (file.upload === true && this.transport.offloadsValues()) {
|
|
891
|
+
if (!this.transport.stageMedia(span, bytes && eventTime
|
|
892
|
+
? {
|
|
893
|
+
bytes,
|
|
894
|
+
digest: { sha256, size: bytes.byteLength },
|
|
895
|
+
contentType: mediaType,
|
|
896
|
+
eventTime,
|
|
897
|
+
...(isMediaName(name) ? { name } : {}),
|
|
898
|
+
}
|
|
899
|
+
: undefined))
|
|
900
|
+
this.instrumentationFailed();
|
|
901
|
+
}
|
|
728
902
|
}
|
|
729
903
|
catch {
|
|
730
904
|
this.instrumentationFailed();
|
|
@@ -732,6 +906,8 @@ export class HueClient {
|
|
|
732
906
|
}
|
|
733
907
|
/** The keys of each span's content values that were cut to Hue's value cap, for `hue.truncated`. */
|
|
734
908
|
truncatedKeys = new WeakMap();
|
|
909
|
+
/** Distinguishes requests whose helper events otherwise have identical metadata. */
|
|
910
|
+
fileEventTimes = new WeakMap();
|
|
735
911
|
/** A content value's bounds: Hue's value cap as the receiver last advertised it. */
|
|
736
912
|
contentLimits() {
|
|
737
913
|
return contentLimitsOf(this.transport.receiverLimits().valueBytes);
|
|
@@ -749,10 +925,17 @@ export class HueClient {
|
|
|
749
925
|
if (!this.enabled || this.closed || !this.captureContent)
|
|
750
926
|
return;
|
|
751
927
|
try {
|
|
928
|
+
const encode = (limits) => {
|
|
929
|
+
const placed = [];
|
|
930
|
+
return {
|
|
931
|
+
...encodeBoundedContent(value, limits, true, this.placer(placed, true)),
|
|
932
|
+
placed,
|
|
933
|
+
};
|
|
934
|
+
};
|
|
752
935
|
let encoded;
|
|
753
936
|
if (this.transport.holdsValues())
|
|
754
937
|
try {
|
|
755
|
-
encoded =
|
|
938
|
+
encoded = encode({
|
|
756
939
|
...this.contentLimits(),
|
|
757
940
|
bytes: MAX_BLOB_BYTES,
|
|
758
941
|
nodes: MAX_RECORD_NODES,
|
|
@@ -761,8 +944,10 @@ export class HueClient {
|
|
|
761
944
|
catch {
|
|
762
945
|
// Past the runtime's longest string, or its memory: cut to the cap instead.
|
|
763
946
|
}
|
|
764
|
-
const { text, truncated } = encoded ??
|
|
947
|
+
const { text, truncated, placed } = encoded ?? encode(this.contentLimits());
|
|
765
948
|
span.setAttribute(key, text);
|
|
949
|
+
// The attachments the value holds are staged for upload in place of any it held before.
|
|
950
|
+
this.stageAttachments(span, key, placed, true, truncated);
|
|
766
951
|
// The span's list as it stands: the application's own entries (read from the SDK span,
|
|
767
952
|
// which carries its attributes; the API alone shows none) and this helper's earlier ones.
|
|
768
953
|
const listed = withTruncatedKeys(span.attributes?.[TRUNCATED_KEY], this.truncatedKeys.get(span) ?? []);
|
package/dist/evals/client.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ProjectConnection } from "../types.js";
|
|
2
|
+
import { MediaClient } from "./media.js";
|
|
2
3
|
import { type AttemptConnectionBundleV2, type PrepareAttemptRequestV2 } from "./attempt.js";
|
|
3
4
|
import type { ArtifactReservation, ArtifactUpload, CaseConversion, CaseConversionSummary, CaseWrite, CompleteExecution, Completion, Dataset, DatasetCase, DatasetVersion, EvalSet, EvalSetCase, EvalSetVersion, EvaluationItem, EvaluationRun, EvaluationRunSummary, EnvironmentEvidenceSnapshot, Execution, Experiment, ExperimentCase, ExperimentItem, Evaluator, EvaluatorVersion, JsonValue, LocalAgentClaim, LocalAgentRegistration, RegisteredLocalAgent, JudgeBudget, JudgeJob, Page, PageOptions, RegistryPageOptions, Result, ResultSummary, Run, RunCase, Scorer, ScorerDefinition, ScorerVersion, Scoring, ScoringResultInput, ScoringResultSummary, ScoringSubject, ScoringSummary, SimulationMcpCapability, StartExecution, Subject, StoredResult, StoredScoringResult, IdentityUpdate, Tag, TaggedIdentity, UpdatedRun } from "./types.js";
|
|
4
5
|
/** Connection options for {@link createEvaluationClient}. */
|
|
@@ -59,6 +60,9 @@ export declare class EvaluationClient {
|
|
|
59
60
|
#private;
|
|
60
61
|
/** Validated Hue origin. */
|
|
61
62
|
readonly baseUrl: string;
|
|
63
|
+
/** Recorded media read back by digest: one file (`fetch`) or every digest part of a recorded
|
|
64
|
+
* value (`resolve`). Needs a key that may read recorded content (Read, or Read and write). */
|
|
65
|
+
readonly media: MediaClient;
|
|
62
66
|
private readonly timeoutMillis;
|
|
63
67
|
private readonly maxAttempts;
|
|
64
68
|
constructor(options: EvaluationClientOptions);
|
|
@@ -97,7 +101,16 @@ export declare class EvaluationClient {
|
|
|
97
101
|
getDatasetVersion(id: string): Promise<DatasetVersion>;
|
|
98
102
|
/** Lists the full cases of a dataset version; use small page limits for large values. */
|
|
99
103
|
listCases(id: string, page?: PageOptions): Promise<Page<DatasetCase>>;
|
|
100
|
-
/**
|
|
104
|
+
/**
|
|
105
|
+
* Adds a case to a draft version using optimistic concurrency on `expectedRevision`. Each of
|
|
106
|
+
* `files` given as bytes or a path is checked against the artifact policy and uploaded as a
|
|
107
|
+
* verified artifact first, under a key derived from the version, the case's `externalKey`, the
|
|
108
|
+
* role and the bytes, so sending the same case again reuses the artifact. Every file and the
|
|
109
|
+
* case itself are checked before the first upload.
|
|
110
|
+
*
|
|
111
|
+
* @throws TypeError or RangeError for `files` outside the artifact policy, or a case Hue would
|
|
112
|
+
* refuse for its fields or size, before any request.
|
|
113
|
+
*/
|
|
101
114
|
addCase(id: string, input: CaseWrite): Promise<{
|
|
102
115
|
/** The stored case. */
|
|
103
116
|
item: DatasetCase;
|
package/dist/evals/client.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isLoopbackHost, validateOptions } from "../config.js";
|
|
2
2
|
import { json, uuid, valueBounds } from "./json.js";
|
|
3
|
+
import { MediaClient } from "./media.js";
|
|
3
4
|
import { attemptBindingRead, parsePrepareAttemptResultV2, parseRefreshedAttemptResultV2, parseRevocationResult, prepareAttemptInputV2, validateAttemptConnectionBundleV2, } from "./attempt.js";
|
|
4
5
|
/** Thrown for a failed evaluation API request; the message is fixed and never includes response text. */
|
|
5
6
|
export class HueApiError extends Error {
|
|
@@ -149,6 +150,42 @@ function productRunFields(value, kind) {
|
|
|
149
150
|
result.scoring = result.evaluation;
|
|
150
151
|
return result;
|
|
151
152
|
}
|
|
153
|
+
/** The bounds of a request body as a whole: 1 MiB, with each value's node and depth bounds. */
|
|
154
|
+
const REQUEST_BOUNDS = { ...valueBounds, bytes: 1024 * 1024 };
|
|
155
|
+
/** A request body as it is sent: checked as JSON within `bounds`. Optional top-level fields are
|
|
156
|
+
* omitted intentionally; nested undefined remains invalid. */
|
|
157
|
+
function payloadOf(body, bounds) {
|
|
158
|
+
return JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), bounds));
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Checks what a case write's fields can be checked for here, as Hue checks them: a positive
|
|
162
|
+
* integer `expectedRevision`, an `externalKey` of 1–200 UTF-16 code units of well-formed text
|
|
163
|
+
* without NUL, `inputs`, `expected` and an object `metadata` within one value's JSON bounds, and an
|
|
164
|
+
* `environmentVersionId` that is a UUID or null.
|
|
165
|
+
*/
|
|
166
|
+
function checkCaseWrite(input) {
|
|
167
|
+
if (!input || typeof input !== "object")
|
|
168
|
+
throw new TypeError("A case write must be an object");
|
|
169
|
+
if (!Number.isSafeInteger(input.expectedRevision) || input.expectedRevision < 1)
|
|
170
|
+
throw new TypeError("expectedRevision must be a positive integer");
|
|
171
|
+
const key = input.externalKey;
|
|
172
|
+
if (typeof key !== "string" ||
|
|
173
|
+
!key.length ||
|
|
174
|
+
key.length > 200 ||
|
|
175
|
+
!key.isWellFormed() ||
|
|
176
|
+
key.includes("\u0000"))
|
|
177
|
+
throw new TypeError("externalKey must be 1–200 UTF-16 code units of well-formed text");
|
|
178
|
+
json(input.inputs, valueBounds);
|
|
179
|
+
if (input.expected !== undefined)
|
|
180
|
+
json(input.expected, valueBounds);
|
|
181
|
+
if (input.metadata !== undefined) {
|
|
182
|
+
if (!input.metadata || typeof input.metadata !== "object" || Array.isArray(input.metadata))
|
|
183
|
+
throw new TypeError("metadata must be an object");
|
|
184
|
+
json(input.metadata, valueBounds);
|
|
185
|
+
}
|
|
186
|
+
if (input.environmentVersionId !== undefined && input.environmentVersionId !== null)
|
|
187
|
+
uuid(input.environmentVersionId);
|
|
188
|
+
}
|
|
152
189
|
/** Times a refused request is sent again before its refusal is the caller's error. */
|
|
153
190
|
const REFUSAL_RETRIES = 4;
|
|
154
191
|
/** A refusal asking for a longer wait fails at once rather than holding the caller. */
|
|
@@ -183,6 +220,9 @@ function refusalRetryAfter(response) {
|
|
|
183
220
|
export class EvaluationClient {
|
|
184
221
|
/** Validated Hue origin. */
|
|
185
222
|
baseUrl;
|
|
223
|
+
/** Recorded media read back by digest: one file (`fetch`) or every digest part of a recorded
|
|
224
|
+
* value (`resolve`). Needs a key that may read recorded content (Read, or Read and write). */
|
|
225
|
+
media;
|
|
186
226
|
#apiKey;
|
|
187
227
|
timeoutMillis;
|
|
188
228
|
maxAttempts;
|
|
@@ -199,6 +239,11 @@ export class EvaluationClient {
|
|
|
199
239
|
if (!Number.isInteger(attempts) || attempts < 1 || attempts > 10)
|
|
200
240
|
throw new RangeError("maxAttempts must be 1–10");
|
|
201
241
|
this.maxAttempts = attempts;
|
|
242
|
+
this.media = new MediaClient({
|
|
243
|
+
baseUrl: this.baseUrl,
|
|
244
|
+
apiKey: validated.apiKey,
|
|
245
|
+
timeoutMillis: this.timeoutMillis,
|
|
246
|
+
});
|
|
202
247
|
}
|
|
203
248
|
/** Sends one request to Hue, again after the wait while Hue refuses it with a short
|
|
204
249
|
* `Retry-After`. `init` runs for every attempt, so each has its own timeout; `signal` also ends
|
|
@@ -235,15 +280,12 @@ export class EvaluationClient {
|
|
|
235
280
|
throw new HueApiError(response.status, askedRetryAfter(response));
|
|
236
281
|
}
|
|
237
282
|
}
|
|
238
|
-
async request(method, path, body, bounds =
|
|
283
|
+
async request(method, path, body, bounds = REQUEST_BOUNDS, signal,
|
|
239
284
|
/** Overrides the retry decision `retryable` takes from the method and body. */
|
|
240
285
|
idempotent) {
|
|
241
286
|
// Serialized once, outside the attempts: a body this client cannot encode is a caller error
|
|
242
|
-
// no attempt fixes.
|
|
243
|
-
|
|
244
|
-
const payload = body === undefined
|
|
245
|
-
? undefined
|
|
246
|
-
: JSON.stringify(json(Object.fromEntries(Object.entries(body).filter(([, value]) => value !== undefined)), bounds));
|
|
287
|
+
// no attempt fixes.
|
|
288
|
+
const payload = body === undefined ? undefined : payloadOf(body, bounds);
|
|
247
289
|
const attempts = retryable(method, body, idempotent) ? this.maxAttempts : 1;
|
|
248
290
|
for (let attempt = 1;; attempt++) {
|
|
249
291
|
try {
|
|
@@ -433,9 +475,31 @@ export class EvaluationClient {
|
|
|
433
475
|
listCases(id, page) {
|
|
434
476
|
return this.request("GET", `/dataset-versions/${uuid(id)}/cases${this.page(page)}`);
|
|
435
477
|
}
|
|
436
|
-
/**
|
|
437
|
-
|
|
438
|
-
|
|
478
|
+
/**
|
|
479
|
+
* Adds a case to a draft version using optimistic concurrency on `expectedRevision`. Each of
|
|
480
|
+
* `files` given as bytes or a path is checked against the artifact policy and uploaded as a
|
|
481
|
+
* verified artifact first, under a key derived from the version, the case's `externalKey`, the
|
|
482
|
+
* role and the bytes, so sending the same case again reuses the artifact. Every file and the
|
|
483
|
+
* case itself are checked before the first upload.
|
|
484
|
+
*
|
|
485
|
+
* @throws TypeError or RangeError for `files` outside the artifact policy, or a case Hue would
|
|
486
|
+
* refuse for its fields or size, before any request.
|
|
487
|
+
*/
|
|
488
|
+
async addCase(id, input) {
|
|
489
|
+
// The canonical spelling: the reservation keys of uploaded files name the version.
|
|
490
|
+
const versionId = uuid(id).toLowerCase();
|
|
491
|
+
checkCaseWrite(input);
|
|
492
|
+
let body = input;
|
|
493
|
+
if (input.files !== undefined) {
|
|
494
|
+
// Loaded on use: the artifact helpers import this client.
|
|
495
|
+
const { caseFileReferences } = await import("./files.js");
|
|
496
|
+
body = {
|
|
497
|
+
...input,
|
|
498
|
+
// The case is checked as it will be sent, with a stand-in for each upload's artifact.
|
|
499
|
+
files: await caseFileReferences(this, versionId, input, (files) => payloadOf({ ...input, files }, REQUEST_BOUNDS)),
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
return this.request("POST", `/dataset-versions/${versionId}/cases`, body);
|
|
439
503
|
}
|
|
440
504
|
/** Freezes a draft version at the given revision; frozen versions are immutable. */
|
|
441
505
|
freezeDatasetVersion(id, expectedRevision) {
|
package/dist/evals/files.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type EvaluationClient } from "./client.js";
|
|
2
|
-
import type { CaseFile, LocalFile, OutputFile, SubjectFile } from "./types.js";
|
|
2
|
+
import type { CaseFile, CaseFileReference, CaseWrite, LocalFile, OutputFile, SubjectFile } from "./types.js";
|
|
3
3
|
/** Roles the target receives. Organization templates stay with grading, as in the managed protocol. */
|
|
4
4
|
export declare const targetFileRoles: readonly CaseFile["role"][];
|
|
5
5
|
/** Hue's artifact policy: the pilot file size and the accepted document types. */
|
|
@@ -112,5 +112,15 @@ export declare const artifactSettling: {
|
|
|
112
112
|
* finished verifying after an earlier attempt stopped waiting.
|
|
113
113
|
*/
|
|
114
114
|
export declare function uploadOutputFiles(client: EvaluationClient, executionId: string, files: StagedOutputFile[], save: () => Promise<void>, timing?: typeof artifactSettling): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* The `files` of a case write as the case API takes them: references kept, and each file given
|
|
117
|
+
* as bytes or a path checked against the artifact policy (role, name, one of
|
|
118
|
+
* `outputContentTypes`, 1 byte to 25 MiB) and uploaded as a verified artifact. Every file is
|
|
119
|
+
* checked, and then `beforeUpload` is given the references with a stand-in ID for each upload,
|
|
120
|
+
* before the first upload. The reservation key names the version, the case's `externalKey`, the
|
|
121
|
+
* role and the bytes, so the same write sent again reuses its artifacts. Each artifact is pinned
|
|
122
|
+
* once with a role, even when a reference names the artifact a file was uploaded as.
|
|
123
|
+
*/
|
|
124
|
+
export declare function caseFileReferences(client: EvaluationClient, versionId: string, input: CaseWrite, beforeUpload?: (references: CaseFileReference[]) => void, timing?: typeof artifactSettling): Promise<CaseFileReference[]>;
|
|
115
125
|
/** Project uploaded staged files as the verified local files handed to scorers. */
|
|
116
126
|
export declare function localOutputFiles(files: StagedOutputFile[]): LocalFile[];
|