@hue-run/sdk 0.5.1 → 0.7.0

package/dist/evals/client.js

@@ -12,6 +12,85 @@ export class HueApiError extends Error {
         this.name = "HueApiError";
     }
 }
+const registryFieldAliases = [
+    ["datasetId", "evalSetId"],
+    ["datasetVersionId", "evalSetVersionId"],
+    ["scorerId", "evaluatorId"],
+    ["scorerVersionId", "evaluatorVersionId"],
+];
+const registryEnvelopes = new Set(["items", "item", "versions", "version"]);
+// Only Hue response envelopes are traversed. Case inputs, metadata, and evaluator
+// definitions are customer JSON and must retain their original field names.
+function productRegistryFields(value) {
+    if (Array.isArray(value))
+        return value.map((item) => productRegistryFields(item));
+    if (value === null || typeof value !== "object")
+        return value;
+    const result = { ...value };
+    for (const key of registryEnvelopes) {
+        if (Object.hasOwn(result, key))
+            result[key] = productRegistryFields(result[key]);
+    }
+    for (const [legacy, product] of registryFieldAliases) {
+        if (Object.hasOwn(result, legacy)) {
+            if (Object.hasOwn(result, product) && result[legacy] !== result[product])
+                throw new HueApiError();
+            result[product] = result[legacy];
+        }
+    }
+    return result;
+}
+const runResponseAliases = [
+    ["datasetId", "evalSetId"],
+    ["datasetName", "evalSetName"],
+    ["datasetDisplayName", "evalSetDisplayName"],
+    ["datasetVersion", "evalSetVersion"],
+    ["datasetVersionId", "evalSetVersionId"],
+    ["datasetVersionIds", "evalSetVersionIds"],
+    ["scorerId", "evaluatorId"],
+    ["scorerName", "evaluatorName"],
+    ["scorerVersion", "evaluatorVersion"],
+    ["scorerVersionId", "evaluatorVersionId"],
+    ["scorerVersionIds", "evaluatorVersionIds"],
+    ["scorerVersions", "evaluatorVersions"],
+    ["evaluationRunId", "scoringId"],
+];
+const runResponseEnvelopes = new Set([
+    "items",
+    "item",
+    "versions",
+    "version",
+    "scorerVersions",
+    "evaluatorVersions",
+]);
+function productRunFields(value, kind) {
+    if (Array.isArray(value))
+        return value.map((item) => productRunFields(item, kind));
+    if (value === null || typeof value !== "object")
+        return value;
+    const result = { ...value };
+    for (const key of runResponseEnvelopes) {
+        if (Object.hasOwn(result, key))
+            result[key] = productRunFields(result[key], kind);
+    }
+    for (const key of ["evaluation", "scoring"]) {
+        if (Object.hasOwn(result, key))
+            result[key] = productRunFields(result[key], "scoring");
+    }
+    const identityAlias = kind === "result" ? ["runId", "scoringId"] : ["experimentId", "runId"];
+    const aliases = [...runResponseAliases, identityAlias];
+    for (const [legacy, product] of aliases) {
+        if (Object.hasOwn(result, legacy)) {
+            if (Object.hasOwn(result, product) &&
+                JSON.stringify(result[legacy]) !== JSON.stringify(result[product]))
+                throw new HueApiError();
+            result[product] = result[legacy];
+        }
+    }
+    if (Object.hasOwn(result, "evaluation") && !Object.hasOwn(result, "scoring"))
+        result.scoring = result.evaluation;
+    return result;
+}
 /**
  * Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
  * results and hosted judge jobs. No implicit mutation retry: callers retain stable idempotency keys
@@ -225,6 +304,58 @@ export class EvaluationClient {
     getScorerVersion(id) {
         return this.request("GET", `/scorer-versions/${uuid(id)}`);
     }
+    /** Creates an eval set using the existing v1 registry path. */
+    async createEvalSet(input) {
+        return productRegistryFields(await this.createDataset(input));
+    }
+    /** Reads an eval set and its versions. */
+    async getEvalSet(id) {
+        return productRegistryFields(await this.getDataset(id));
+    }
+    /** Lists eval sets. */
+    async listEvalSets(page) {
+        return productRegistryFields(await this.listDatasets(page));
+    }
+    /** Creates a draft eval set version, optionally copying cases from another version. */
+    async createEvalSetVersion(id, input = {}) {
+        return productRegistryFields(await this.createDatasetVersion(id, input));
+    }
+    /** Reads an eval set version. */
+    async getEvalSetVersion(id) {
+        return productRegistryFields(await this.getDatasetVersion(id));
+    }
+    /** Lists cases in an eval set version. */
+    async listEvalSetCases(id, page) {
+        return productRegistryFields(await this.listCases(id, page));
+    }
+    /** Adds a case to a draft eval set version at its expected revision. */
+    async addEvalSetCase(id, input) {
+        return productRegistryFields(await this.addCase(id, input));
+    }
+    /** Freezes a draft eval set version at its expected revision. */
+    async freezeEvalSetVersion(id, expectedRevision) {
+        return productRegistryFields(await this.freezeDatasetVersion(id, expectedRevision));
+    }
+    /** Creates an evaluator identity. */
+    async createEvaluator(input) {
+        return productRegistryFields(await this.createScorer(input));
+    }
+    /** Reads an evaluator and its published versions. */
+    async getEvaluator(id) {
+        return productRegistryFields(await this.getScorer(id));
+    }
+    /** Lists evaluators. */
+    async listEvaluators(page) {
+        return productRegistryFields(await this.listScorers(page));
+    }
+    /** Publishes an immutable evaluator version. */
+    async publishEvaluatorVersion(id, definition) {
+        return productRegistryFields(await this.publishScorerVersion(id, definition));
+    }
+    /** Reads a published evaluator version. */
+    async getEvaluatorVersion(id) {
+        return productRegistryFields(await this.getScorerVersion(id));
+    }
     /** Creates an experiment over a frozen dataset version with pinned scorer versions and a configuration. */
     createExperiment(input) {
         return this.request("POST", "/experiments", input);
@@ -376,6 +507,70 @@ export class EvaluationClient {
     getResult(id) {
         return this.request("GET", `/evaluation-results/${uuid(id)}`);
     }
+    /** Creates a run using product request fields on the existing v1 path. */
+    async createRun(input) {
+        return productRunFields(await this.request("POST", "/experiments", input), "run");
+    }
+    /** Reads a run with its scoring progress. */
+    async getRun(id) {
+        return productRunFields(await this.getExperiment(id), "run");
+    }
+    /** Lists a run's cases with their latest executions. */
+    async listRunItems(id, page) {
+        return productRunFields(await this.listExperimentItems(id, page), "run");
+    }
+    /** Reads one frozen case of a run. */
+    async getRunCase(id, caseId) {
+        return productRunFields(await this.getExperimentCase(id, caseId), "run");
+    }
+    /** Starts or replays a target execution for a run case. */
+    async startRunExecution(id, caseId, input) {
+        return productRunFields(await this.startExecution(id, caseId, input), "run");
+    }
+    /** Reads one run execution. */
+    async getRunExecution(id) {
+        return productRunFields(await this.getExecution(id), "run");
+    }
+    /** Saves a run execution's outcome. */
+    async completeRunExecution(id, input) {
+        return productRunFields(await this.completeExecution(id, input), "run");
+    }
+    /** Marks a run finished. */
+    async finishRun(id, idempotencyKey) {
+        return productRunFields(await this.finishExperiment(id, idempotencyKey), "run");
+    }
+    /** Creates a standalone scoring pass over saved subjects. */
+    async createScoring(input) {
+        return productRunFields(await this.request("POST", "/evaluation-runs", input), "scoring");
+    }
+    /** Reads a scoring pass and its pinned evaluators. */
+    async getScoring(id) {
+        return productRunFields(await this.getEvaluationRun(id), "scoring");
+    }
+    /** Lists scoring passes in the project. */
+    async listScorings(page) {
+        return productRunFields(await this.listEvaluationRuns(page), "scoring");
+    }
+    /** Lists the subjects of a scoring pass. */
+    async listScoringItems(id, page) {
+        return productRunFields(await this.listEvaluationItems(id, page), "scoring");
+    }
+    /** Reads a saved subject with product-named source fields. */
+    async getScoringSubject(id) {
+        return productRunFields(await this.getSubject(id), "run");
+    }
+    /** Uploads evaluator results for a scoring pass using product request fields. */
+    async submitScoringResults(id, input) {
+        return productRunFields(await this.request("POST", `/evaluation-runs/${uuid(id)}/results`, input), "result");
+    }
+    /** Lists result summaries for a scoring pass. */
+    async listScoringResults(id, page) {
+        return productRunFields(await this.listResults(id, page), "result");
+    }
+    /** Reads a stored evaluator result. */
+    async getScoringResult(id) {
+        return productRunFields(await this.getResult(id), "result");
+    }
     /** Dispatches hosted judge jobs for `llm_judge` pins; check {@link getJudgeBudget} first. */
     createJudgeJobs(id, input) {
         return this.request("POST", `/evaluation-runs/${uuid(id)}/judge-jobs`, input);

package/dist/evals/types.d.ts

@@ -76,6 +76,21 @@ export interface DatasetCase {
     /** Immutable input-file manifest identity, when files are attached. */
     artifactManifestId?: string | null;
 }
+/** An eval set and its versions. Both field names remain available during v1 compatibility. */
+export type EvalSet = Omit<Dataset, "versions"> & {
+    /** Versions of this eval set. */
+    versions: EvalSetVersion[];
+};
+/** A version of an eval set, with its product field name. */
+export type EvalSetVersion = DatasetVersion & {
+    /** Owning eval set ID. */
+    evalSetId: string;
+};
+/** A stored case in an eval set version. */
+export type EvalSetCase = DatasetCase & {
+    /** Eval set version containing this case. */
+    evalSetVersionId: string;
+};
 /** One pinned input file of a case or subject, as recorded in Hue's immutable manifest. */
 export interface CaseFile {
     /** Hue artifact identity of the pinned bytes. */
@@ -346,6 +361,16 @@ export interface ScorerVersion {
     /** The pinned definition. */
     definition: ScorerDefinition;
 }
+/** An evaluator and its published versions. */
+export type Evaluator = Omit<Scorer, "versions"> & {
+    /** Published versions, when included in the response. */
+    versions?: EvaluatorVersion[];
+};
+/** An immutable published evaluator definition. */
+export type EvaluatorVersion = ScorerVersion & {
+    /** Owning evaluator ID when the server supplies it; older v1 responses may omit it. */
+    evaluatorId?: string;
+};
 /** Final state of a target execution. */
 export type TerminalState = "succeeded" | "error" | "cancelled";
 /** One attempt to run the target for a case. */
@@ -439,6 +464,30 @@ export interface Experiment {
         cancelled: number;
     };
 }
+/** A run over a frozen eval set version and configuration. */
+export type Run = Experiment & {
+    /** Frozen eval set version under test. */
+    evalSetVersionId: string;
+    /** Scoring pass created for this run. */
+    scoring: Scoring;
+};
+/** One frozen case in a run. */
+export type RunCase = ExperimentCase & {
+    /** Source eval set version ID. */
+    evalSetVersionId: string;
+};
+/** A scoring pass over saved subjects with pinned evaluators. */
+export type Scoring = EvaluationRun & {
+    /** Evaluator versions pinned to this scoring pass. */
+    evaluatorVersions: EvaluatorVersion[];
+    /** Linked run ID, or null for standalone scoring. */
+    runId?: string | null;
+};
+/** One row in the project's scoring list. */
+export type ScoringSummary = EvaluationRunSummary & {
+    /** Linked run ID, or null for standalone scoring. */
+    runId: string | null;
+};
 /** A sanitized error type with an optional bounded message. */
 export interface TypedError {
     /** Stable error type. */
@@ -549,6 +598,13 @@ export interface Subject {
     /** The target's declared primary generated artifact, or `null`. */
     primaryArtifactId?: string | null;
 }
+/** An immutable saved subject with product-named source fields. */
+export type ScoringSubject = Subject & {
+    /** Source eval set version ID. */
+    evalSetVersionId: string;
+    /** Source run ID. */
+    runId: string;
+};
 /** A reported metric value. */
 export interface Metric {
     /** Declared metric name. */
@@ -588,6 +644,27 @@ export type Result = Score & {
     /** Source digest of the local scorer, for `local_code` pins. */
     sourceDigest?: string;
 };
+/** A score uploaded through the product-named scoring method. */
+export type ScoringResultInput = Score & {
+    /** Scoring item this result belongs to. */
+    evaluationItemId: string;
+    /** Evaluator version that produced the score. */
+    evaluatorVersionId: string;
+    /** Source digest for a local code evaluator. */
+    sourceDigest?: string;
+};
+/** A result as listed by {@link EvaluationClient.listScoringResults}. */
+export type ScoringResultSummary = ResultSummary & {
+    /** Evaluator version that produced this result. */
+    evaluatorVersionId: string;
+};
+/** A full stored result from {@link EvaluationClient.getScoringResult}. */
+export type StoredScoringResult = StoredResult & {
+    /** Scoring pass containing this result. */
+    scoringId: string;
+    /** Evaluator version that produced this result. */
+    evaluatorVersionId: string;
+};
 /** What a local scorer callback receives. */
 export interface ScoreContext {
     /** Case inputs. */

package/dist/live-spans.d.ts

@@ -0,0 +1,25 @@
+import { type Attributes } from "@opentelemetry/api";
+import { type ReadableSpan } from "@opentelemetry/sdk-trace";
+/**
+ * A finished span never carries the placeholder markers, whatever an application set: Hue would
+ * read such a span as a malformed placeholder and reject it.
+ */
+export declare function withoutPlaceholderMarkers(attributes: Attributes): Attributes;
+/** Open spans tracked for announcement at once; later starts are not announced. */
+export declare const MAX_LIVE_SPANS = 1024;
+export declare const LIVE_SPAN_INTERVAL_MILLIS = 500;
+/**
+ * Response header on every trace acknowledgement from a Hue that accepts placeholders. Without
+ * it the receiver predates them and rejects each one by its zero end time.
+ */
+export declare const PLACEHOLDERS_HEADER = "hue-pending-spans";
+/** Hue's own spans and recognizable AI spans, judged from what is known when the span starts. */
+export declare function announcesLiveSpan(span: ReadableSpan): boolean;
+/**
+ * A normal OTLP span announcing `span` while it runs: a new identity whose parent is the real
+ * span, the real start time and an end time of 0. The markers are added after redaction.
+ */
+export declare function pendingPlaceholder(span: ReadableSpan): {
+    record: ReadableSpan;
+    markers: Attributes;
+};

package/dist/live-spans.js

@@ -0,0 +1,115 @@
+import { SpanStatusCode } from "@opentelemetry/api";
+import { RandomIdGenerator } from "@opentelemetry/sdk-trace";
+import { HUE_SCOPE } from "./config.js";
+/** Marks a placeholder for a span that has started but not ended. The value versions the shape. */
+const PENDING_SPAN_TYPE_KEY = "hue.span_type";
+const PENDING_SPAN_TYPE = "pending_span";
+/** The real span's own parent. Absent when the real span is a root. */
+const PENDING_PARENT_KEY = "hue.pending_parent_id";
+const markerKeys = [PENDING_SPAN_TYPE_KEY, PENDING_PARENT_KEY];
+/**
+ * A finished span never carries the placeholder markers, whatever an application set: Hue would
+ * read such a span as a malformed placeholder and reject it.
+ */
+export function withoutPlaceholderMarkers(attributes) {
+    if (!markerKeys.some((key) => Object.hasOwn(attributes, key)))
+        return attributes;
+    const kept = { ...attributes };
+    for (const key of markerKeys)
+        delete kept[key];
+    return kept;
+}
+/** Open spans tracked for announcement at once; later starts are not announced. */
+export const MAX_LIVE_SPANS = 1024;
+export const LIVE_SPAN_INTERVAL_MILLIS = 500;
+/**
+ * Response header on every trace acknowledgement from a Hue that accepts placeholders. Without
+ * it the receiver predates them and rejects each one by its zero end time.
+ */
+export const PLACEHOLDERS_HEADER = "hue-pending-spans";
+const MAX_PLACEHOLDER_VALUE_BYTES = 64 * 1024;
+// Kept narrow on purpose: common application processors export only these spans, and a
+// placeholder whose real span is filtered later would read as running until the trace stalls.
+const livePrefixes = ["gen_ai.", "ai.", "llm.", "traceloop."];
+// Definitions and instructions are large and rarely useful while a span runs; the real span
+// still carries them. Copied markers would misplace the placeholder.
+const omittedKeys = [
+    "gen_ai.tool.definitions",
+    "gen_ai.system_instructions",
+    PENDING_SPAN_TYPE_KEY,
+    PENDING_PARENT_KEY,
+];
+const ids = new RandomIdGenerator();
+/** Hue's own spans and recognizable AI spans, judged from what is known when the span starts. */
+export function announcesLiveSpan(span) {
+    if (span.instrumentationScope.name === HUE_SCOPE || span.name.startsWith("ai."))
+        return true;
+    for (const key in span.attributes)
+        if (livePrefixes.some((prefix) => key.startsWith(prefix)))
+            return true;
+    return false;
+}
+function valueBytes(value) {
+    if (typeof value === "string")
+        return Buffer.byteLength(value);
+    if (!Array.isArray(value))
+        return 8;
+    let bytes = 0;
+    for (const item of value)
+        bytes += typeof item === "string" ? Buffer.byteLength(item) : 8;
+    return bytes;
+}
+function placeholderAttributes(source) {
+    const attributes = {};
+    for (const key in source) {
+        if (!Object.hasOwn(source, key))
+            continue;
+        if (omittedKeys.some((omitted) => key === omitted || key.startsWith(`${omitted}.`)))
+            continue;
+        const value = source[key];
+        if (valueBytes(value) <= MAX_PLACEHOLDER_VALUE_BYTES)
+            attributes[key] = value;
+    }
+    return attributes;
+}
+/**
+ * A normal OTLP span announcing `span` while it runs: a new identity whose parent is the real
+ * span, the real start time and an end time of 0. The markers are added after redaction.
+ */
+export function pendingPlaceholder(span) {
+    const real = span.spanContext();
+    const own = {
+        traceId: real.traceId,
+        spanId: ids.generateSpanId(),
+        traceFlags: real.traceFlags,
+        ...(real.traceState ? { traceState: real.traceState } : {}),
+    };
+    const parent = span.parentSpanContext?.spanId;
+    const record = {
+        name: span.name,
+        kind: span.kind,
+        spanContext: () => own,
+        // Hue re-keys a placeholder as its real span, so the flags describe the real span's parent.
+        parentSpanContext: { ...real, isRemote: span.parentSpanContext?.isRemote === true },
+        startTime: span.startTime,
+        endTime: [0, 0],
+        duration: [0, 0],
+        ended: true,
+        status: { code: SpanStatusCode.UNSET },
+        attributes: placeholderAttributes(span.attributes),
+        links: [],
+        events: [],
+        resource: span.resource,
+        instrumentationScope: span.instrumentationScope,
+        droppedAttributesCount: 0,
+        droppedEventsCount: 0,
+        droppedLinksCount: 0,
+    };
+    return {
+        record,
+        markers: {
+            [PENDING_SPAN_TYPE_KEY]: PENDING_SPAN_TYPE,
+            ...(parent ? { [PENDING_PARENT_KEY]: parent } : {}),
+        },
+    };
+}

package/dist/transport.d.ts

@@ -50,8 +50,25 @@ export declare class HueTransport {
     private closed;
     private shutdownPromise?;
     private flushPromise?;
+    private live;
+    private liveTimer?;
+    private liveSpans;
+    private placeholdersRejected;
+    private placeholders;
+    private placeholderSources;
+    private batchSpans?;
     constructor(options: HueOptions);
+    /**
+     * Advisory records (placeholders) are admitted only while the queue is under a quarter of its
+     * record and byte budgets, so they never take more than a quarter from real records. They are
+     * skipped silently.
+     */
     private enqueue;
+    private track;
+    /** Placeholders are built lazily, so input set right after a span starts is included. */
+    private announceLiveSpans;
+    private stopLiveTimer;
+    private stopLiveSpans;
     /** Cumulative counters and current queue gauges. */
     getReport(): ExportReport;
     /** Copies of the latest 128 sanitized issues, oldest first. */