@hue-run/sdk 0.12.1 → 0.13.1

package/dist/evals/client.js

@@ -89,6 +89,15 @@ function productRegistryFields(value) {
     }
     return result;
 }
+/** Tag names travel as given; anything but a list of strings is a caller error. */
+function tagNames(tags) {
+    if (!Array.isArray(tags) || !tags.every((tag) => typeof tag === "string"))
+        throw new TypeError("tags must be a list of tag names");
+    return tags;
+}
+function withTagNames(input) {
+    return input.tags === undefined ? input : { ...input, tags: tagNames(input.tags) };
+}
 const runResponseAliases = [
     ["datasetId", "evalSetId"],
     ["datasetName", "evalSetName"],
@@ -388,15 +397,21 @@ export class EvaluationClient {
         const query = new URLSearchParams(this.page(options).slice(1));
         if (options.includeArchived !== undefined)
             query.set("includeArchived", String(options.includeArchived));
+        for (const tag of tagNames(options.tags ?? []))
+            query.append("tag", tag);
         return query.size ? `?${query}` : "";
     }
     /** Reads the current project to confirm the key and origin. */
     checkConnection() {
         return this.request("GET", "/projects/current");
     }
-    /** Creates a dataset with an initial draft version. */
+    /** Creates a dataset with an initial draft version; `tags` names the tags it starts with. */
     createDataset(input) {
-        return this.request("POST", "/datasets", input);
+        return this.request("POST", "/datasets", withTagNames(input));
+    }
+    /** Renames a dataset or replaces its tags by name; the same values again change nothing. */
+    updateDataset(id, input) {
+        return this.request("PATCH", `/datasets/${uuid(id)}`, withTagNames(input), undefined, undefined, true);
     }
     /** Reads a dataset and its versions. */
     getDataset(id) {
@@ -430,7 +445,15 @@ export class EvaluationClient {
     }
     /** Creates a scorer identity; publish definitions with {@link publishScorerVersion}. */
     createScorer(input) {
-        return this.request("POST", "/scorers", input);
+        return this.request("POST", "/scorers", withTagNames(input));
+    }
+    /** Renames a scorer or replaces its tags by name; the same values again change nothing. */
+    updateScorer(id, input) {
+        return this.request("PATCH", `/scorers/${uuid(id)}`, withTagNames(input), undefined, undefined, true);
+    }
+    /** Lists the project's tags in the order they were created. */
+    listTags() {
+        return this.request("GET", "/tags");
     }
     /** Reads a scorer and its published versions. */
     getScorer(id) {
@@ -452,6 +475,10 @@ export class EvaluationClient {
     async createEvalSet(input) {
         return productRegistryFields(await this.createDataset(input));
     }
+    /** Renames an eval set or replaces its tags by name. */
+    async updateEvalSet(id, input) {
+        return productRegistryFields(await this.updateDataset(id, input));
+    }
     /** Reads an eval set and its versions. */
     async getEvalSet(id) {
         return productRegistryFields(await this.getDataset(id));
@@ -484,6 +511,10 @@ export class EvaluationClient {
     async createEvaluator(input) {
         return productRegistryFields(await this.createScorer(input));
     }
+    /** Renames an evaluator or replaces its tags by name. */
+    async updateEvaluator(id, input) {
+        return productRegistryFields(await this.updateScorer(id, input));
+    }
     /** Reads an evaluator and its published versions. */
     async getEvaluator(id) {
         return productRegistryFields(await this.getScorer(id));
@@ -502,12 +533,16 @@ export class EvaluationClient {
     }
     /** Creates an experiment over a frozen dataset version with pinned scorer versions and a configuration. */
     createExperiment(input) {
-        return this.request("POST", "/experiments", input);
+        return this.request("POST", "/experiments", withTagNames(input));
     }
     /** Reads an experiment with its evaluation run and execution counts. */
     getExperiment(id) {
         return this.request("GET", `/experiments/${uuid(id)}`);
     }
+    /** Renames an experiment or replaces its own tags by name. */
+    updateExperiment(id, input) {
+        return this.request("PATCH", `/experiments/${uuid(id)}`, withTagNames(input), undefined, undefined, true);
+    }
     /** Lists an experiment's cases with their latest executions. */
     listExperimentItems(id, page) {
         return this.request("GET", `/experiments/${uuid(id)}/items${this.page(page)}`);
@@ -656,12 +691,16 @@ export class EvaluationClient {
     }
     /** Creates a run using product request fields on the existing v1 path. */
     async createRun(input) {
-        return productRunFields(await this.request("POST", "/experiments", input), "run");
+        return productRunFields(await this.request("POST", "/experiments", withTagNames(input)), "run");
     }
     /** Reads a run with its scoring progress. */
     async getRun(id) {
         return productRunFields(await this.getExperiment(id), "run");
     }
+    /** Renames a run or replaces its own tags by name; its eval set's tags are not its own. */
+    async updateRun(id, input) {
+        return this.updateExperiment(id, input);
+    }
     /** Lists a run's cases with their latest executions. */
     async listRunItems(id, page) {
         return productRunFields(await this.listExperimentItems(id, page), "run");

package/dist/evals/failure.d.ts

@@ -10,9 +10,26 @@
  * The runner records the type on the execution in place of `TargetError`; Hue classes the first
  * three as infrastructure and the last as configuration, so an error from Hue's own clients is
  * never filed as the agent's. Anything else is the agent's own error.
+ *
+ * A simulated world the agent reaches through its own client (an MCP client, a REST client, a
+ * fetch) answers Hue's gateway refusals with a status and a diagnostic: `503
+ * authorization_unavailable` when Hue could not authorize the world token because its database
+ * could not answer, `503 gateway_failure` when the gateway failed. The MCP SDK's transport throws
+ * these as `StreamableHTTPError` or `SseError` with the status as `code` and no response, so the
+ * status is read from there too; the diagnostic is the error's cause (`serviceFailureCause`),
+ * recorded beside the type so a run's reader sees what refused it.
  */
 export type ServiceFailureType = "ServiceRefused" | "ConnectionFailed" | "TimedOut" | "ConfigurationRejected";
 /** The service failure a target's throw shows, by the errors linked from it, in precedence
  * order: a timeout, a failed connection, a retryable refusal, a status Hue's own client was
  * refused with; null for the agent's own error. */
 export declare function serviceFailureType(error: unknown): ServiceFailureType | null;
+/**
+ * The service's own word for what failed, when the answer a target's throw carries names one:
+ * Hue's gateway diagnostic (`authorization_unavailable`, `gateway_failure`), read from the
+ * `x-hue-diagnostic` header or the JSON body's `diagnostic` of any error linked from the throw,
+ * nearest first. Undefined when no answer names one, or the name is not a diagnostic's token.
+ * Read for a service failure (`serviceFailureType`), where the runner records it as the error's
+ * `cause`.
+ */
+export declare function serviceFailureCause(error: unknown): string | undefined;

package/dist/evals/failure.js

@@ -69,6 +69,17 @@ function names(error) {
     }
     return found;
 }
+/** The MCP TypeScript SDK's transport errors, which carry the HTTP status as `code` and no
+ * response: a POST the server answered with a failing status (`StreamableHTTPError`) or an SSE
+ * stream it refused (`SseError`). Recognized by class name, never imported. */
+const MCP_TRANSPORT_NAMES = new Set(["StreamableHTTPError", "SseError"]);
+/** The status an MCP transport error carries, or undefined for any other error. */
+function transportStatus(error) {
+    const { code } = error;
+    return typeof code === "number" && names(error).some((name) => MCP_TRANSPORT_NAMES.has(name))
+        ? code
+        : undefined;
+}
 const retryable = (status) => typeof status === "number" &&
     (status === 408 || status === 429 || (status >= 500 && status < 600));
 function timedOut(error) {
@@ -99,6 +110,9 @@ function disconnected(error) {
 function refused(error) {
     if (hueClientError(error))
         return retryable(error.status);
+    // An MCP transport error is the exchange itself: the server answered the status it carries.
+    if (retryable(transportStatus(error)))
+        return true;
     const { status, statusCode, headers, response, url, responseHeaders } = error;
     const exchanged = [headers, response, url, responseHeaders].some((item) => item !== undefined && item !== null);
     // A fetch `Response` carries the status itself.
@@ -148,3 +162,91 @@ export function serviceFailureType(error) {
         return "ConfigurationRejected";
     return null;
 }
+/** The header Hue's gateway names its diagnostic in, beside the JSON body's `diagnostic`. */
+const DIAGNOSTIC_HEADER = "x-hue-diagnostic";
+/** A diagnostic's spelling: a short lower-case token (`authorization_unavailable`). */
+const DIAGNOSTIC = /^[a-z][a-z0-9_]{0,63}$/;
+/** The most of a body or message read for a diagnostic. */
+const MAX_BODY = 8192;
+/** A header's value from a `Headers`, a record of names to values (case-insensitive), or a
+ * record of names to arrays, as the HTTP clients spell them; undefined otherwise. */
+function headerValue(headers, name) {
+    if (headers === null || typeof headers !== "object")
+        return undefined;
+    const get = headers.get;
+    if (typeof get === "function") {
+        const value = get.call(headers, name);
+        return typeof value === "string" ? value : undefined;
+    }
+    for (const [key, value] of Object.entries(headers)) {
+        if (key.toLowerCase() !== name)
+            continue;
+        const first = Array.isArray(value) ? value[0] : value;
+        return typeof first === "string" ? first : undefined;
+    }
+    return undefined;
+}
+/** The `diagnostic` a JSON body names: an object's, at the top or in a JSON-RPC error's `data`,
+ * or the object a string carries from its first brace (an MCP transport error's message quotes
+ * the body it was answered). */
+function bodyDiagnostic(body) {
+    if (typeof body === "string") {
+        const start = body.indexOf("{");
+        if (start < 0 || body.length - start > MAX_BODY)
+            return undefined;
+        try {
+            return bodyDiagnostic(JSON.parse(body.slice(start)));
+        }
+        catch {
+            return undefined;
+        }
+    }
+    if (body === null || typeof body !== "object")
+        return undefined;
+    // Hue's REST refusal names it at the top; its MCP refusal, a JSON-RPC error, in `error.data`.
+    const { diagnostic, error } = body;
+    if (typeof diagnostic === "string")
+        return diagnostic;
+    const data = error !== null && typeof error === "object" ? error.data : undefined;
+    const named = data !== null && typeof data === "object"
+        ? data.diagnostic
+        : undefined;
+    return typeof named === "string" ? named : undefined;
+}
+/** The diagnostic one error carries: the one it names itself (Hue's own clients read the header
+ * into `diagnostic`), else the header on its `headers`, `responseHeaders` or `response.headers`,
+ * else the body on its `responseBody`, `body` or, for an MCP transport error, its message. */
+function diagnosticOf(error) {
+    const { diagnostic, headers, responseHeaders, response, responseBody, body, message } = error;
+    const answered = response !== null && typeof response === "object"
+        ? response.headers
+        : undefined;
+    return ((typeof diagnostic === "string" ? diagnostic : undefined) ??
+        headerValue(headers, DIAGNOSTIC_HEADER) ??
+        headerValue(responseHeaders, DIAGNOSTIC_HEADER) ??
+        headerValue(answered, DIAGNOSTIC_HEADER) ??
+        bodyDiagnostic(responseBody) ??
+        bodyDiagnostic(body) ??
+        (transportStatus(error) !== undefined ? bodyDiagnostic(message) : undefined));
+}
+/**
+ * The service's own word for what failed, when the answer a target's throw carries names one:
+ * Hue's gateway diagnostic (`authorization_unavailable`, `gateway_failure`), read from the
+ * `x-hue-diagnostic` header or the JSON body's `diagnostic` of any error linked from the throw,
+ * nearest first. Undefined when no answer names one, or the name is not a diagnostic's token.
+ * Read for a service failure (`serviceFailureType`), where the runner records it as the error's
+ * `cause`.
+ */
+export function serviceFailureCause(error) {
+    for (const item of linked(error)) {
+        try {
+            const diagnostic = diagnosticOf(item);
+            if (diagnostic !== undefined && DIAGNOSTIC.test(diagnostic))
+                return diagnostic;
+        }
+        catch {
+            // A property that throws when read names nothing.
+        }
+    }
+    return undefined;
+}

package/dist/evals/runner.js

@@ -6,7 +6,7 @@ import { ROOT_CONTEXT } from "@opentelemetry/api";
 import { HueExportError } from "../transport.js";
 import { HueApiError } from "./client.js";
 import { loadEnvironmentEvidence } from "./environment-evidence.js";
-import { serviceFailureType } from "./failure.js";
+import { serviceFailureCause, serviceFailureType } from "./failure.js";
 import { CheckpointStore } from "./checkpoint.js";
 import { onForcedExit } from "./exit-cleanup.js";
 import { assertSafeFileNames, downloadCaseFiles, downloadCaseInputs, localOutputFiles, verifyCaseFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
@@ -434,17 +434,26 @@ export async function runExperiment(options) {
         // A target stopped at its world's deadline is a `TargetTimeout`, told apart from an error the
         // target raised itself: the agent did not finish, rather than finishing wrongly. A service the
         // agent called that refused, dropped the connection or timed out is named apart from the
-        // agent's own error (`failure.ts`), so Hue counts it as infrastructure, not as the agent.
-        const errorPayload = (error) => ({
-            type: error instanceof OutputTooLargeError
-                ? "OutputTooLarge"
-                : error instanceof TargetTimeoutError
-                    ? "TargetTimeout"
-                    : (serviceFailureType(error) ?? "TargetError"),
-            ...(options.persistResultContent && error instanceof Error
-                ? { message: sanitize(error.message) }
-                : {}),
-        });
+        // agent's own error (`failure.ts`), so Hue counts it as infrastructure, not as the agent, with
+        // the service's own word for what failed as the error's `cause` when its answer names one
+        // (Hue's gateway diagnostic).
+        const errorPayload = (error) => {
+            const service = error instanceof OutputTooLargeError || error instanceof TargetTimeoutError
+                ? null
+                : serviceFailureType(error);
+            const cause = service ? serviceFailureCause(error) : undefined;
+            return {
+                type: error instanceof OutputTooLargeError
+                    ? "OutputTooLarge"
+                    : error instanceof TargetTimeoutError
+                        ? "TargetTimeout"
+                        : (service ?? "TargetError"),
+                ...(cause === undefined ? {} : { cause }),
+                ...(options.persistResultContent && error instanceof Error
+                    ? { message: sanitize(error.message) }
+                    : {}),
+            };
+        };
         /** Publish staged files, score with every verified file on disk and save the completion. */
         async function prepare(file, saved, frozenCase, caseDirectory, 
         /** The target's output, in memory when result content is not persisted. */

package/dist/evals/scorer-publication.js

@@ -5,18 +5,25 @@ const metricName = z
     .min(1)
     .max(64)
     .regex(/^[a-zA-Z][a-zA-Z0-9_]*$/);
+const metricDescription = z.string().trim().min(1).max(500).optional();
 const metric = z.discriminatedUnion("type", [
-    z.strictObject({ name: metricName, type: z.literal("boolean") }),
-    z.strictObject({ name: metricName, type: z.literal("text") }),
+    z.strictObject({
+        name: metricName,
+        type: z.literal("boolean"),
+        description: metricDescription,
+    }),
+    z.strictObject({ name: metricName, type: z.literal("text"), description: metricDescription }),
     z.strictObject({
         name: metricName,
         type: z.literal("number"),
+        description: metricDescription,
         min: z.number().finite().optional(),
         max: z.number().finite().optional(),
     }),
     z.strictObject({
         name: metricName,
         type: z.literal("category"),
+        description: metricDescription,
         categories: z.array(z.string()).min(1),
     }),
 ]);

package/dist/evals/scorers.js

@@ -32,7 +32,9 @@ export function defineLocalScorer(options) {
             language: "typescript",
             entrypoint: options.entrypoint,
             sourceDigest: sourceDigest(options.source),
-            metrics: options.metrics,
+            metrics: options.metrics.map((metric) => metric.description === undefined
+                ? metric
+                : { ...metric, description: metric.description.trim() }),
         },
         score: options.score,
     };

package/dist/evals/types.d.ts

@@ -19,6 +19,8 @@ export interface PageOptions {
 export interface RegistryPageOptions extends PageOptions {
     /** Include archived identities so callers can diagnose slug conflicts explicitly. */
     includeArchived?: boolean;
+    /** Only identities carrying any of these tag names, matched without case. */
+    tags?: string[];
 }
 /** Human-readable identity of a dataset or scorer. */
 export interface Identity {
@@ -29,12 +31,50 @@ export interface Identity {
     /** Optional description. */
     description?: string;
 }
+/** A dataset or scorer to create, with the names of the tags it starts with. */
+export interface TaggedIdentity extends Identity {
+    /** Tag names; a name the project has no tag for creates one. */
+    tags?: string[];
+}
+/** Identity fields and tags to change on a dataset or scorer; omitted fields stay. */
+export interface IdentityUpdate {
+    /** New display name. */
+    name?: string;
+    /** New description. */
+    description?: string;
+    /** Every tag name the item should carry; replaces its tags. */
+    tags?: string[];
+}
+/** An experiment's (run's) name and tags after an update. */
+export interface UpdatedRun {
+    /** Experiment (run) ID. */
+    id: string;
+    /** Display name. */
+    name: string;
+    /** The experiment's own tags. */
+    tags: Tag[];
+    /** Its dataset's (eval set's) tags. */
+    evalSetTags: Tag[];
+}
+/** The colors a tag can take. */
+export type TagColor = "default" | "gray" | "brown" | "orange" | "yellow" | "green" | "blue" | "purple" | "pink" | "red";
+/** A project's tag: a named, colored label that groups datasets, scorers and experiments. */
+export interface Tag {
+    /** Tag ID. */
+    id: string;
+    /** Name, unique in the project without case. */
+    name: string;
+    /** Display color. */
+    color: TagColor;
+}
 /** A dataset and its versions. */
 export interface Dataset extends Identity {
     /** Dataset ID. */
     id: string;
     /** Archive timestamp on current servers; absent on older compatible responses. */
     archivedAt?: string | null;
+    /** The dataset's tags on servers that support tags; absent on older responses. */
+    tags?: Tag[];
     /** Versions of this dataset. */
     versions: DatasetVersion[];
 }
@@ -139,11 +179,15 @@ export type MetricDefinition = {
     name: string;
     /** Boolean or free-text metric. */
     type: "boolean" | "text";
+    /** What the metric checks, shown with its check in Hue's run view (1–500 characters). */
+    description?: string;
 } | {
     /** Metric name. */
     name: string;
     /** Numeric metric. */
     type: "number";
+    /** What the metric checks, shown with its check in Hue's run view (1–500 characters). */
+    description?: string;
     /** Inclusive lower bound. */
     min?: number;
     /** Inclusive upper bound. */
@@ -153,6 +197,8 @@ export type MetricDefinition = {
     name: string;
     /** Categorical metric. */
     type: "category";
+    /** What the metric checks, shown with its check in Hue's run view (1–500 characters). */
+    description?: string;
     /** Allowed values. */
     categories: string[];
 };
@@ -397,6 +443,8 @@ export interface Scorer extends Identity {
     id: string;
     /** Archive timestamp on current servers; absent on older compatible responses. */
     archivedAt?: string | null;
+    /** The scorer's tags on servers that support tags; absent on older responses. */
+    tags?: Tag[];
     /** Published versions, when included in the response. */
     versions?: ScorerVersion[];
 }
@@ -486,6 +534,10 @@ export interface Experiment {
     name: string;
     /** Frozen dataset version under test. */
     datasetVersionId: string;
+    /** The experiment's own tags on servers that support tags; absent on older responses. */
+    tags?: Tag[];
+    /** Its dataset's tags, which tag filters match as well; absent on older responses. */
+    evalSetTags?: Tag[];
     /** Configuration handed to the target. */
     config: JsonValue;
     /** Digest of `config`. */
@@ -546,6 +598,12 @@ export interface TypedError {
      * (`agent`, `infrastructure`, `configuration`, `telemetry` or `unsupported`), which Hue reads
      * only for a name it does not know. The SDK's own types need none. */
     class?: "agent" | "infrastructure" | "configuration" | "telemetry" | "unsupported";
+    /** For a service failure (`ServiceRefused`, `ConnectionFailed`, `TimedOut`,
+     * `ConfigurationRejected`): the service's own word for what failed, when its answer names one,
+     * such as Hue's gateway diagnostic (`authorization_unavailable`) from the answer's
+     * `x-hue-diagnostic` header or its JSON body's `diagnostic`. A short lower-case token; absent
+     * when the answer names none. */
+    cause?: string;
 }
 /** Input for {@link EvaluationClient.startExecution}. */
 export interface StartExecution {

package/dist/privacy.d.ts

@@ -2,8 +2,26 @@ import type { ReadableSpan } from "@opentelemetry/sdk-trace";
 import type { ReadableLogRecord } from "@opentelemetry/sdk-logs";
 import { type Resource } from "@opentelemetry/resources";
 import type { HueOptions } from "./types.js";
+import { truncateUtf8 } from "./safety.js";
+export { truncateUtf8 };
 /** Attribute keys (and their dotted children) removed in metadata-only mode. */
 export declare const contentPrefixes: string[];
+export declare function isContentKey(key: string): boolean;
+/** The record attribute listing the keys whose values were cut to fit Hue's 256 KiB value cap:
+ * Hue's receiver writes it for the values it cuts, and the SDK writes it for the values it cuts
+ * before export, under the same name, so a reader learns of a cut value from one place. */
+export declare const TRUNCATED_KEY = "hue.truncated";
+/** The size of a structured value the marker replaced. */
+export declare const TRUNCATED_BYTES_KEY = "hue.truncated_bytes";
+/** The root span attribute counting this trace's records the SDK could not export at all (a
+ * record over the request limit even with every content value shed, or one its queue had no
+ * room for): what Hue stores of the trace is incomplete by that many records. */
+export declare const DROPPED_RECORDS_KEY = "hue.sdk.dropped_records";
+/** Hue's receiver's own marker for a structured value it replaced: nothing of the value, its
+ * size, and the flag a reader tests. */
+export declare function truncatedMarker(bytes: number): Record<string, unknown>;
+/** `listed` with `keys` added, each once, in order. */
+export declare function withTruncatedKeys(listed: unknown, keys: readonly string[]): string[];
 export type ResourceCache = WeakMap<Resource, Resource>;
 export declare function redactSpan(span: ReadableSpan, options: HueOptions, cache: ResourceCache): ReadableSpan;
 export declare function redactLog(log: ReadableLogRecord, options: HueOptions, cache: ResourceCache): ReadableLogRecord;