@hue-run/sdk 0.13.2 → 0.15.0

package/dist/cli/eval.js

@@ -78,7 +78,8 @@ Output and limits:
   --name <run name>               Run name (default: <agent key> @ <revision>, hashes shortened)
   --baseline <experiment id|url>  Compare verdicts with a previous experiment
   --json                          Print one JSON document on stdout; progress goes to stderr
-  --content                       Capture telemetry content (span inputs, outputs and messages)
+  --content                       Capture telemetry content (the default)
+  --no-content                    Omit telemetry inputs, outputs and messages
   --no-output                     One-shot: do not store case outputs, error messages and
                                   explanations (stored by default; --worker always stores); with
                                   --content the case span still carries the output
@@ -141,7 +142,8 @@ function parse(argv) {
                 name: { type: "string" },
                 baseline: { type: "string" },
                 json: { type: "boolean", default: false },
-                content: { type: "boolean", default: false },
+                content: { type: "boolean" },
+                "no-content": { type: "boolean", default: false },
                 "no-output": { type: "boolean", default: false },
                 "trace-not-accepted": { type: "string" },
                 "save-version": { type: "boolean", default: false },
@@ -817,9 +819,9 @@ export function explain(error) {
     if (error instanceof CheckpointIdentityError && error.startedWith) {
         const flags = [
             error.startedWith.persistResultContent === false ? "--no-output" : "",
-            error.startedWith.captureContent === true ? "--content" : "",
+            error.startedWith.captureContent === false ? "--no-content" : "",
         ].filter(Boolean);
-        return `This unfinished run was started ${flags.length ? `with ${flags.join(" and ")}` : "without --no-output or --content"}; rerun with the same flags to resume it, or remove its checkpoint directory to start over`;
+        return `This unfinished run was started ${flags.length ? `with ${flags.join(" and ")}` : "without --no-output or --no-content"}; rerun with the same flags to resume it, or remove its checkpoint directory to start over`;
     }
     // The error's own message points at an in-memory report the user cannot reach.
     if (error instanceof HueExportError)
@@ -1446,6 +1448,8 @@ export async function runEvalCommand(argv) {
     let json = false;
     try {
         const { values, positionals } = parse(argv);
+        if (values.content && values["no-content"])
+            throw new UsageError("Choose either --content or --no-content");
         // A usage error before anything is prepared or created: no run is left to recover from it.
         traceNotAcceptedPolicy(values["trace-not-accepted"]);
         json = values.json;
@@ -1522,7 +1526,7 @@ export async function runEvalCommand(argv) {
             apiKey,
             baseUrl,
             serviceName: key,
-            captureContent: values.content,
+            captureContent: values.content ?? !values["no-content"],
             timeoutMillis: CASE_TRACE_EXPORT_MILLIS,
         });
         return values.worker

package/dist/client.d.ts

@@ -156,11 +156,16 @@ export declare class HueClient {
     recordFile(file: FileRecord, explicitContext?: Context): void;
     /** The keys of each span's content values that were cut to Hue's value cap, for `hue.truncated`. */
     private truncatedKeys;
+    /** A content value's bounds: Hue's value cap as the receiver last advertised it. */
+    private contentLimits;
     /**
-     * A content value as a JSON attribute within Hue's 256 KiB value cap. A larger value is cut to
-     * a UTF-8 prefix and its key listed under `hue.truncated`, where Hue's receiver lists the
-     * values it cuts itself: the span carries the call and the recorded part of its value instead
-     * of losing the value whole. Only a value that cannot be encoded is omitted and counted.
+     * A content value as a JSON attribute within Hue's value cap (1 MiB unless the receiver
+     * advertises another). A larger value is cut to a UTF-8 prefix and its key listed under
+     * `hue.truncated`, where Hue's receiver lists the values it cuts itself: the span carries the
+     * call and the recorded part of its value instead of losing the value whole. While values over
+     * the cap may be uploaded, the value is encoded whole, up to Hue's 1 GB upload limit and as many
+     * values as a queued record may hold, and export uploads it or cuts it. Only a value that cannot
+     * be encoded is omitted and counted.
      */
     private setContent;
     /**
@@ -173,7 +178,12 @@ export declare class HueClient {
     checkConnection(): Promise<ProjectConnection>;
     /** Production lifecycle path: never rejects; timeouts do not cancel borrowed provider work. */
     flushSafe(options?: SafeLifecycleOptions): Promise<SafeLifecycleResult>;
-    /** Safe for finally blocks; preserves the application's result or original exception. */
+    /**
+     * Safe for finally blocks; preserves the application's result or original exception. When its
+     * budget runs out while the client's own transport holds records for a receiver that limited
+     * the rate, the hold ends: those records are reported lost to the rate limit, and the hold no
+     * longer keeps the process running.
+     */
     shutdownSafe(options?: SafeLifecycleOptions): Promise<SafeLifecycleResult>;
     private safeLifecycle;
     /**

package/dist/client.js

@@ -10,12 +10,13 @@ import { defaultResource, resourceFromAttributes } from "@opentelemetry/resource
 import { HUE_SCOPE } from "./config.js";
 import { hostedServerAddresses, hostedToolActivity, hostedToolProvider, providerErrorDescription, } from "./provider-tools.js";
 import { toolCatalogSummary } from "./tool-definitions.js";
-import { encodeBoundedContent, encodeContent, noopSpan, safeSpan, } from "./safety.js";
+import { contentLimitsOf, encodeBoundedContent, encodeContent, noopSpan, safeSpan, } from "./safety.js";
 import { TRUNCATED_KEY, withTruncatedKeys } from "./privacy.js";
-import { createHueTransport, HueExportError } from "./transport.js";
+import { MAX_BLOB_BYTES } from "./blobs.js";
+import { createHueTransport, exportBudgetMillis, HueExportError, } from "./transport.js";
 import { verifyTrace } from "./receipt.js";
 import { sdkVersion } from "./version.js";
-import { MAX_BODY_BYTES, MAX_FILE_DATA_BYTES } from "./config.js";
+import { MAX_BODY_BYTES, MAX_FILE_DATA_BYTES, MAX_RECORD_NODES } from "./config.js";
 /**
  * Thrown by {@link HueClient.checkConnection} and {@link HueClient.verifyTrace} when Hue cannot be
  * reached, rejects the project key or answers unexpectedly. The message is fixed and safe to log;
@@ -151,6 +152,20 @@ class ContextualTracer {
     }
 }
 const propagator = new W3CTraceContextPropagator();
+/**
+ * Attributes, events and links one span or log record of a client's own providers keeps, and
+ * attributes per event and link. OpenTelemetry's default of 128 silently dropped the rest of a
+ * long conversation's flattened messages; Hue's receiver counts none of them. Providers an
+ * application attaches Hue to keep their own limits.
+ */
+const OWNED_RECORD_LIMIT = 2000;
+const OWNED_SPAN_LIMITS = {
+    attributeCountLimit: OWNED_RECORD_LIMIT,
+    eventCountLimit: OWNED_RECORD_LIMIT,
+    linkCountLimit: OWNED_RECORD_LIMIT,
+    attributePerEventCountLimit: OWNED_RECORD_LIMIT,
+    attributePerLinkCountLimit: OWNED_RECORD_LIMIT,
+};
 /** A provider tool listing's bounds before its catalog summary: one export request, 64 levels and
  * 65,536 values, as the Python SDK's content snapshot. */
 const catalogLimits = { bytes: MAX_BODY_BYTES, nodes: 65_536, depth: 64 };
@@ -214,10 +229,12 @@ export class HueClient {
             const tracer = new TracerProvider({
                 resource,
                 spanProcessors: this.transport.options.enabled === false ? [] : [this.transport.spanProcessor],
+                spanLimits: OWNED_SPAN_LIMITS,
             });
             const logger = new LoggerProvider({
                 resource,
                 processors: this.transport.options.enabled === false ? [] : [this.transport.logRecordProcessor],
+                logRecordLimits: { attributeCountLimit: OWNED_RECORD_LIMIT },
             });
             this.ownedProviders = { tracer, logger };
             this.tracerProvider = tracer;
@@ -522,7 +539,7 @@ export class HueClient {
                 severityNumber: SeverityNumber.INFO,
                 eventName: "gen_ai.client.inference.operation.details",
                 attributes,
-                body: JSON.parse(encodeContent(body)),
+                body: JSON.parse(encodeContent(body, this.contentLimits())),
             });
         }
         catch {
@@ -715,17 +732,36 @@ export class HueClient {
     }
     /** The keys of each span's content values that were cut to Hue's value cap, for `hue.truncated`. */
     truncatedKeys = new WeakMap();
+    /** A content value's bounds: Hue's value cap as the receiver last advertised it. */
+    contentLimits() {
+        return contentLimitsOf(this.transport.receiverLimits().valueBytes);
+    }
     /**
-     * A content value as a JSON attribute within Hue's 256 KiB value cap. A larger value is cut to
-     * a UTF-8 prefix and its key listed under `hue.truncated`, where Hue's receiver lists the
-     * values it cuts itself: the span carries the call and the recorded part of its value instead
-     * of losing the value whole. Only a value that cannot be encoded is omitted and counted.
+     * A content value as a JSON attribute within Hue's value cap (1 MiB unless the receiver
+     * advertises another). A larger value is cut to a UTF-8 prefix and its key listed under
+     * `hue.truncated`, where Hue's receiver lists the values it cuts itself: the span carries the
+     * call and the recorded part of its value instead of losing the value whole. While values over
+     * the cap may be uploaded, the value is encoded whole, up to Hue's 1 GB upload limit and as many
+     * values as a queued record may hold, and export uploads it or cuts it. Only a value that cannot
+     * be encoded is omitted and counted.
      */
     setContent(span, key, value) {
         if (!this.enabled || this.closed || !this.captureContent)
             return;
         try {
-            const { text, truncated } = encodeBoundedContent(value);
+            let encoded;
+            if (this.transport.holdsValues())
+                try {
+                    encoded = encodeBoundedContent(value, {
+                        ...this.contentLimits(),
+                        bytes: MAX_BLOB_BYTES,
+                        nodes: MAX_RECORD_NODES,
+                    });
+                }
+                catch {
+                    // Past the runtime's longest string, or its memory: cut to the cap instead.
+                }
+            const { text, truncated } = encoded ?? encodeBoundedContent(value, this.contentLimits());
             span.setAttribute(key, text);
             // The span's list as it stands: the application's own entries (read from the SDK span,
             // which carries its attributes; the API alone shows none) and this helper's earlier ones.
@@ -826,12 +862,22 @@ export class HueClient {
         void work.then(clear, clear);
         return result;
     }
-    /** Safe for finally blocks; preserves the application's result or original exception. */
+    /**
+     * Safe for finally blocks; preserves the application's result or original exception. When its
+     * budget runs out while the client's own transport holds records for a receiver that limited
+     * the rate, the hold ends: those records are reported lost to the rate limit, and the hold no
+     * longer keeps the process running.
+     */
     shutdownSafe(options = {}) {
         if (this.safeShutdownPromise)
             return this.safeShutdownPromise;
         const work = this.shutdown();
-        const result = this.safeLifecycle(() => work, options);
+        const result = this.safeLifecycle(() => work, options).then((outcome) => {
+            // A transport the application attached keeps running past this client's shutdown.
+            if (outcome.timedOut && this.ownedProviders)
+                this.transport.releaseRateLimitHolds();
+            return outcome;
+        });
         this.safeShutdownPromise = result;
         const clear = () => {
             this.safeShutdownPromise = undefined;
@@ -896,9 +942,13 @@ export class HueClient {
         return next;
     }
     async flushOnce() {
+        // Providers this client owns wait as long as an export may take, a rate-limit hold included,
+        // instead of OpenTelemetry's 30 s; borrowed providers keep their own budget.
+        const owned = this.ownedProviders;
+        const budget = { timeoutMillis: exportBudgetMillis(this.transport.options.timeoutMillis) };
         const results = await Promise.allSettled([
-            Promise.resolve().then(() => this.tracerProvider.forceFlush()),
-            Promise.resolve().then(() => this.loggerProvider.forceFlush()),
+            Promise.resolve().then(() => owned ? owned.tracer.forceFlush(budget) : this.tracerProvider.forceFlush()),
+            Promise.resolve().then(() => owned ? owned.logger.forceFlush(budget) : this.loggerProvider.forceFlush()),
         ]);
         for (const [index, result] of results.entries())
             if (result.status === "rejected")

package/dist/config.d.ts

@@ -1,6 +1,26 @@
 import type { HueOptions, SharedHueOptions } from "./types.js";
+/** One request on the wire (after gzip), as Hue's receiver accepts before it advertises its own
+ * limit with `Hue-Max-Request-Bytes`. */
 export declare const MAX_BODY_BYTES: number;
+/** One request after decompression, before the receiver advertises `Hue-Max-Decoded-Bytes`. */
+export declare const MAX_DECODED_BYTES: number;
+/** One attribute value or log body kept whole, before the receiver advertises
+ * `Hue-Max-Value-Bytes`; a longer value is cut to it and listed under `hue.truncated`. */
 export declare const MAX_CONTENT_BYTES: number;
+/**
+ * Text past the value cap that a record keeps when it is queued, in UTF-16 code units. A string
+ * longer than the cap and this context is cut when it is queued, so a huge value cannot exhaust
+ * the queue's byte budget and drop its record; the context lets `redact` still see a secret that
+ * crosses the cap, and what the redactor saw without its continuation is never exported.
+ */
+export declare const REDACTION_CONTEXT_UNITS: number;
+/**
+ * The values one record may hold when it is queued, redacted and measured for export. A client's
+ * own providers keep 2,000 events and links of 2,000 attributes each, so a record's byte budget
+ * (every value is charged at least 16 bytes against `maxQueueBytes`) bounds it before this does;
+ * this bounds the work on the application's thread when the queue budget is raised.
+ */
+export declare const MAX_RECORD_NODES = 1048576;
 /** Instrumentation scope of the client's own tracer and logger. */
 export declare const HUE_SCOPE = "@hue-run/sdk";
 /** Maximum bytes accepted when recordFile hashes caller-provided data locally. */

package/dist/config.js

@@ -1,5 +1,25 @@
+/** One request on the wire (after gzip), as Hue's receiver accepts before it advertises its own
+ * limit with `Hue-Max-Request-Bytes`. */
 export const MAX_BODY_BYTES = 1024 * 1024;
-export const MAX_CONTENT_BYTES = 256 * 1024;
+/** One request after decompression, before the receiver advertises `Hue-Max-Decoded-Bytes`. */
+export const MAX_DECODED_BYTES = 4 * 1024 * 1024;
+/** One attribute value or log body kept whole, before the receiver advertises
+ * `Hue-Max-Value-Bytes`; a longer value is cut to it and listed under `hue.truncated`. */
+export const MAX_CONTENT_BYTES = 1024 * 1024;
+/**
+ * Text past the value cap that a record keeps when it is queued, in UTF-16 code units. A string
+ * longer than the cap and this context is cut when it is queued, so a huge value cannot exhaust
+ * the queue's byte budget and drop its record; the context lets `redact` still see a secret that
+ * crosses the cap, and what the redactor saw without its continuation is never exported.
+ */
+export const REDACTION_CONTEXT_UNITS = 64 * 1024;
+/**
+ * The values one record may hold when it is queued, redacted and measured for export. A client's
+ * own providers keep 2,000 events and links of 2,000 attributes each, so a record's byte budget
+ * (every value is charged at least 16 bytes against `maxQueueBytes`) bounds it before this does;
+ * this bounds the work on the application's thread when the queue budget is raised.
+ */
+export const MAX_RECORD_NODES = 1_048_576;
 /** Instrumentation scope of the client's own tracer and logger. */
 export const HUE_SCOPE = "@hue-run/sdk";
 /** Maximum bytes accepted when recordFile hashes caller-provided data locally. */
@@ -37,8 +57,8 @@ export function validateOptions(options) {
             liveSpans: false,
         };
     }
-    if (typeof options.captureContent !== "boolean")
-        throw new TypeError("Choose captureContent explicitly: true or false");
+    if (options.captureContent !== undefined && typeof options.captureContent !== "boolean")
+        throw new TypeError("captureContent must be a boolean");
     if (typeof options.apiKey !== "string" ||
         !options.apiKey ||
         options.apiKey.length > 4096 ||
@@ -83,5 +103,12 @@ export function validateOptions(options) {
         throw new TypeError("liveSpans must be a boolean");
     // Setup credentials send installer telemetry only, never in-progress placeholders.
     const liveSpans = options.liveSpans !== false && !options.apiKey.startsWith("hue_setup_");
-    return { ...options, baseUrl: url.origin, timeoutMillis, maxQueueBytes, liveSpans };
+    return {
+        ...options,
+        captureContent: options.captureContent ?? true,
+        baseUrl: url.origin,
+        timeoutMillis,
+        maxQueueBytes,
+        liveSpans,
+    };
 }

package/dist/environment/client.d.ts

@@ -30,9 +30,9 @@ export declare class HueEnvironmentError extends Error {
 export declare function isTransientEnvironmentError(error: unknown): boolean;
 /** Typed client for authored environments, isolated runs and immutable journals. */
 export declare class EnvironmentClient {
+    #private;
     /** Validated Hue origin. */
     readonly baseUrl: string;
-    private readonly apiKey;
     private readonly timeoutMillis;
     private readonly maxAttempts;
     constructor(options: EnvironmentClientOptions);

package/dist/environment/client.js

@@ -62,7 +62,7 @@ const ENVIRONMENT_PUBLICATION_BOUNDS = aggregateBounds(240_000 + 32);
 export class EnvironmentClient {
     /** Validated Hue origin. */
     baseUrl;
-    apiKey;
+    #apiKey;
     timeoutMillis;
     maxAttempts;
     constructor(options) {
@@ -72,7 +72,7 @@ export class EnvironmentClient {
             captureContent: false,
         });
         this.baseUrl = validated.baseUrl;
-        this.apiKey = validated.apiKey;
+        this.#apiKey = validated.apiKey;
         this.timeoutMillis = validated.timeoutMillis;
         const attempts = options.maxAttempts ?? 4;
         if (!Number.isInteger(attempts) || attempts < 1 || attempts > 10)
@@ -86,7 +86,7 @@ export class EnvironmentClient {
             response = await fetch(`${this.baseUrl}/api/v1${path}`, {
                 method,
                 headers: {
-                    Authorization: `Bearer ${this.apiKey}`,
+                    Authorization: `Bearer ${this.#apiKey}`,
                     ...(payload ? { "Content-Type": "application/json" } : {}),
                 },
                 body: payload,

package/dist/evals/client.d.ts

@@ -56,9 +56,9 @@ export declare class ArtifactSizeError extends Error {
  * bounded to 4 MiB.
  */
 export declare class EvaluationClient {
+    #private;
     /** Validated Hue origin. */
     readonly baseUrl: string;
-    private readonly apiKey;
     private readonly timeoutMillis;
     private readonly maxAttempts;
     constructor(options: EvaluationClientOptions);

package/dist/evals/client.js

@@ -183,7 +183,7 @@ function refusalRetryAfter(response) {
 export class EvaluationClient {
     /** Validated Hue origin. */
     baseUrl;
-    apiKey;
+    #apiKey;
     timeoutMillis;
     maxAttempts;
     constructor(options) {
@@ -193,7 +193,7 @@ export class EvaluationClient {
             captureContent: false,
         });
         this.baseUrl = validated.baseUrl;
-        this.apiKey = validated.apiKey;
+        this.#apiKey = validated.apiKey;
         this.timeoutMillis = validated.timeoutMillis;
         const attempts = options.maxAttempts ?? 4;
         if (!Number.isInteger(attempts) || attempts < 1 || attempts > 10)
@@ -276,7 +276,7 @@ export class EvaluationClient {
         const response = await this.send(`${this.baseUrl}/api/v1${path}`, () => ({
             method,
             headers: {
-                Authorization: `Bearer ${this.apiKey}`,
+                Authorization: `Bearer ${this.#apiKey}`,
                 ...(payload ? { "Content-Type": "application/json" } : {}),
             },
             body: payload,
@@ -321,7 +321,7 @@ export class EvaluationClient {
             throw new RangeError("maxBytes must be a non-negative integer");
         const response = await this.send(`${this.baseUrl}/api/v1/artifacts/${uuid(id)}/download`, () => ({
             method: "GET",
-            headers: { Authorization: `Bearer ${this.apiKey}` },
+            headers: { Authorization: `Bearer ${this.#apiKey}` },
             redirect: "error",
             signal: AbortSignal.timeout(Math.max(this.timeoutMillis, 120_000)),
         }));

package/dist/evals/verdicts.d.ts

@@ -1,5 +1,5 @@
 import type { EvaluationClient } from "./client.js";
-import type { ExperimentItem, Metric, Subject, TypedError } from "./types.js";
+import type { ExperimentItem, Metric, ScorerVersion, Subject, TypedError } from "./types.js";
 /** Subset of {@link EvaluationClient} used to wait for results. */
 export type VerdictClient = Pick<EvaluationClient, "listEvaluationItems" | "listResults" | "getResult">;
 /** Options for {@link waitForResults}. */
@@ -44,7 +44,7 @@ export interface VerdictResult {
     notApplicable?: boolean;
     /** What a not-applicable evaluator needs that the case lacks, such as `outcome_criteria`. */
     requires?: string;
-    /** Whether Hue recorded the result as advisory, as it does for every judge today: a result of a
+    /** Whether Hue recorded the result as advisory: a result of a
      * pinned Hue judge whose evidence says `advisory: true`, not an error, and whose metrics carry
      * no `passed`. It is shown with its verdict but never decides a case. Absent means false. */
     advisory?: boolean;
@@ -81,8 +81,8 @@ export interface CaseVerdict {
     /** `passed` and `failed` are scored verdicts; `error`, `skipped` and `pending` carry no verdict.
      * Only the evaluators that apply to the case decide it; one where none applies is an error. */
     state: "passed" | "failed" | "error" | "skipped" | "pending";
-    /** True only for `state: "passed"`: every scored metric of an applicable pin passed and no
-     * applicable pin errored or is missing. */
+    /** True only for `state: "passed"`: every deciding metric passed, required evaluators decided,
+     * and no applicable pin errored or is missing. */
     passed: boolean;
     /** Pinned scorer versions Hue recorded as not applicable to the case; always set by
      * {@link summarizeVerdicts}. */
@@ -114,7 +114,7 @@ export interface VerdictSummary {
         failed: number;
         /** Cases with a scorer error. */
         error: number;
-        /** Cases whose pins were all skipped. */
+        /** Cases whose deciding pins were skipped or whose required evaluator has no decision. */
         skipped: number;
         /** Cases still missing a result. */
         pending: number;
@@ -127,14 +127,17 @@ export interface VerdictSummary {
 export declare function metricPassed(metric: Metric): boolean;
 /**
  * Groups results by experiment case. Cases link to results through their execution's subject;
- * pass `subjects` when the items do not carry `execution.subjectId`. A case passes when every
- * pinned scorer scored it without a failing metric.
+ * pass `subjects` when the items do not carry `execution.subjectId`. Applicable required pins
+ * need a decision before a case can pass; genuine not-applicable and advisory pins are exempt.
  */
 export declare function summarizeVerdicts(results: VerdictResults, options: {
     /** Experiment items, in the order rows should appear. */
     experimentItems: Pick<ExperimentItem, "id" | "externalKey" | "execution">[];
     /** Scorer versions every case needs a result for; defaults to the versions seen in `results`. */
     scorerVersionIds?: string[];
+    /** Immutable pinned definitions, as returned on the experiment. Outcome entries and required
+     * judge versions need a decision; result evidence cannot change their required policy. */
+    scorerVersions?: Pick<ScorerVersion, "id" | "definition">[];
     /** Subject to case links for servers that omit `execution.subjectId`. */
     subjects?: Pick<Subject, "id" | "caseId">[];
 }): VerdictSummary;

package/dist/evals/verdicts.js

@@ -104,13 +104,25 @@ export function metricPassed(metric) {
         return metric.passed;
     return typeof metric.value === "boolean" ? metric.value : true;
 }
+/** Outcome entries decide the case; a separate judge's immutable version opts into that policy.
+ * A named answer reference remains inside its outcome entry, not an advisory judge promotion. */
+function requiresDecision(definition) {
+    if (!definition || typeof definition !== "object" || Array.isArray(definition))
+        return false;
+    const pinned = definition;
+    return (pinned.kind === "world_outcome" ||
+        (pinned.kind === "world_judge" && pinned.config?.required === true));
+}
 /**
  * Groups results by experiment case. Cases link to results through their execution's subject;
- * pass `subjects` when the items do not carry `execution.subjectId`. A case passes when every
- * pinned scorer scored it without a failing metric.
+ * pass `subjects` when the items do not carry `execution.subjectId`. Applicable required pins
+ * need a decision before a case can pass; genuine not-applicable and advisory pins are exempt.
  */
 export function summarizeVerdicts(results, options) {
     const pins = new Set(options.scorerVersionIds ?? results.results.map((result) => result.scorerVersionId));
+    const required = new Set((options.scorerVersions ?? [])
+        .filter((version) => requiresDecision(version.definition))
+        .map((version) => version.id));
     const subjectByCase = new Map();
     for (const subject of options.subjects ?? [])
         subjectByCase.set(subject.caseId, subject.id);
@@ -123,7 +135,8 @@ export function summarizeVerdicts(results, options) {
         const metrics = own.flatMap((result) => result.metrics.map((metric) => ({ ...metric, scorerVersionId: result.scorerVersionId })));
         // An evaluator that does not apply to the case neither passes nor fails it.
         const skippedAsInapplicable = (result) => result.state === "skipped" && result.notApplicable === true;
-        const isAdvisory = (result) => result.advisory === true &&
+        const isAdvisory = (result) => !required.has(result.scorerVersionId) &&
+            result.advisory === true &&
             result.state !== "error" &&
             result.metrics.every((metric) => metric.passed === undefined);
         const inapplicable = own.filter(skippedAsInapplicable);
@@ -134,18 +147,29 @@ export function summarizeVerdicts(results, options) {
         const errors = deciding.filter((result) => result.state === "error");
         const scored = deciding.filter((result) => result.state === "scored");
         const failing = scored.filter((result) => !result.metrics.every(metricPassed));
+        const undecided = deciding.filter((result) => required.has(result.scorerVersionId) &&
+            result.state !== "error" &&
+            (result.state !== "scored" ||
+                !result.metrics.some((metric) => typeof metric.passed === "boolean" || typeof metric.value === "boolean")));
         const missing = [...pins].some((pin) => !own.some((result) => result.scorerVersionId === pin));
         const noneApplies = !missing && own.length > 0 && !applicable.length;
         const noneDecides = !missing && applicable.length > 0 && !deciding.length;
-        const state = errors.length || noneApplies || noneDecides
-            ? "error"
-            : failing.length
-                ? "failed"
-                : missing || !own.length
-                    ? "pending"
-                    : scored.length
-                        ? "passed"
-                        : "skipped";
+        // A real failure still decides the case if only required evaluators are unavailable. Other
+        // scorer errors keep their existing precedence over quality metrics.
+        const failureWins = failing.length > 0 && errors.every((result) => required.has(result.scorerVersionId));
+        const state = failureWins
+            ? "failed"
+            : errors.length || noneApplies || noneDecides
+                ? "error"
+                : failing.length
+                    ? "failed"
+                    : missing || !own.length
+                        ? "pending"
+                        : undecided.length
+                            ? "skipped"
+                            : scored.length
+                                ? "passed"
+                                : "skipped";
         const requires = [
             ...new Set(inapplicable.flatMap((result) => (result.requires ? [result.requires] : []))),
         ];
@@ -159,6 +183,11 @@ export function summarizeVerdicts(results, options) {
             advisory: advisory.map((result) => result.scorerVersionId),
             metrics,
             explanations: [
+                ...(state === "skipped" &&
+                    undecided.length &&
+                    !undecided.some((result) => /^inconclusive:/i.test(result.explanation ?? ""))
+                    ? ["Inconclusive: a required evaluator did not decide this case."]
+                    : []),
                 ...(noneApplies
                     ? [
                         `No pinned evaluator applies to this case${requires.length ? ` (they need ${requires.join(" or ")})` : ""}; pin one that grades it`,
@@ -173,7 +202,7 @@ export function summarizeVerdicts(results, options) {
                     ]
                     : []),
                 ...deciding
-                    .filter((result) => result.state !== "scored" || failing.includes(result))
+                    .filter((result) => result.state !== "scored" || failing.includes(result) || undecided.includes(result))
                     .map((result) => result.explanation)
                     .filter((explanation) => !!explanation),
             ],
@@ -227,9 +256,11 @@ export async function collectExperimentVerdicts(client, options) {
     const experiment = await client.getExperiment(options.experimentId);
     const scorerVersionIds = experiment.evaluation.scorerVersions.map((version) => version.id);
     const { experimentId: _experimentId, ...wait } = options;
-    // Hue's own judges, the only evaluators whose results can be advisory.
+    // Only non-required Hue judges can be advisory. Their immutable policy is already in this
+    // response; a required version's result evidence cannot override it.
     const judgeScorerVersionIds = experiment.evaluation.scorerVersions
-        .filter((version) => version.definition?.kind === "world_judge")
+        .filter((version) => version.definition?.kind === "world_judge" &&
+        !requiresDecision(version.definition))
         .map((version) => version.id);
     const results = await waitForResults(client, {
         ...wait,
@@ -250,6 +281,11 @@ export async function collectExperimentVerdicts(client, options) {
         runId: experiment.evaluation.id,
         scorerVersionIds,
         results,
-        summary: summarizeVerdicts(results, { experimentItems, scorerVersionIds, subjects }),
+        summary: summarizeVerdicts(results, {
+            experimentItems,
+            scorerVersionIds,
+            scorerVersions: experiment.evaluation.scorerVersions,
+            subjects,
+        }),
     };
 }

package/dist/inline-files.d.ts

@@ -1,9 +1,27 @@
 /** Inline file content larger than this many UTF-8 bytes is exported as its digest instead. */
 export declare const INLINE_FILE_LIMIT: number;
+/** The longest message text, in UTF-8 bytes, inspected for inline files. */
+export declare const MAX_INLINE_FILE_TEXT: number;
 /** Message text one record inspects for inline files in all, in UTF-16 code units. */
 export declare const INLINE_FILE_TEXT_PER_RECORD: number;
 /** Whether an attribute is one of the recorded message attributes that can inline files. */
 export declare function isMessageKey(key: string): boolean;
+/** An inline file's bytes, whether they were decoded from the part's text (base64 or
+ * percent-escapes) rather than taken as its UTF-8, and the media type its `data:` URL names. */
+export interface InlineFileBytes {
+    bytes: Buffer;
+    decoded: boolean;
+    urlMediaType?: string;
+}
+/**
+ * The bytes an inline file part carries, the file's own bytes whatever its media type: a `data:`
+ * URL decoded by its own encoding (base64 with `;base64`, percent-escapes otherwise), content in
+ * the base64 alphabet decoded, and anything else, such as a text file's own text, as UTF-8. A
+ * `data:` URL whose data does not decode is taken as UTF-8 too.
+ */
+export declare function inlineFileBytes(content: string): InlineFileBytes;
+/** The key holding a part's inline content: GenAI `blob` parts and AI SDK 6 `file` parts. */
+export declare function contentKey(part: Record<string, unknown>): "content" | "data" | undefined;
 /**
  * Replaces inline files longer than {@link INLINE_FILE_LIMIT} in a recorded message attribute
  * with their SHA-256 and byte size, so a span that inlines a large file exports the file's
@@ -11,4 +29,29 @@ export declare function isMessageKey(key: string): boolean;
  * `mime_type`, `modality`, `mediaType`, …). Other attributes, shorter messages and values that
  * are not JSON are returned unchanged.
  */
-export declare function hashInlineFiles(key: string, value: unknown): unknown;
+export declare function hashInlineFiles(key: string, value: unknown, 
+/** Counts the files replaced by their digest, when given. */
+count?: {
+    files: number;
+}): unknown;
+/** One inline file part found in a parsed message: the part object, the key holding its inline
+ * content, that content, and the part's content as a JSON Pointer (RFC 6901) into the message. */
+export interface InlineFilePart {
+    part: Record<string, unknown>;
+    key: "content" | "data";
+    content: string;
+    pointer: string;
+}
+/**
+ * The inline file parts of a parsed message whose content text is longer than
+ * {@link INLINE_FILE_LIMIT} UTF-8 bytes, in document order: the caller decodes each and keeps the
+ * ones whose file bytes are over the limit. The parts are the parsed value's own objects, so the
+ * caller may replace their content in place.
+ */
+export declare function largeInlineFileParts(value: unknown): InlineFilePart[];
+/** The digest an inline file part is exported as when its bytes are not uploaded: the part's
+ * other fields with the file's SHA-256 and byte size, as {@link hashInlineFiles} writes it. */
+export declare function digestPart(part: Record<string, unknown>, key: "content" | "data", digest: {
+    sha256: string;
+    size: number;
+}): void;