@hue-run/sdk 0.6.0 → 0.8.0

package/dist/evals/runner.d.ts

@@ -50,7 +50,7 @@ interface RunnerOptions {
      * (for example a Hue-operated grading worker that owns the evaluator source) instead of
      * refusing the run. Their IDs are reported in `deferredScorerVersionIds`. */
     deferUnboundLocalScorers?: boolean;
-    /** Cases in flight at once, 1–16. Default 1. */
+    /** Cases in flight at once, 1–64. Default 1. */
     concurrency?: number;
     /** Deadline for JSON Schema scoring in its worker, 100–60000 ms. Default 2000. */
     schemaTimeoutMillis?: number;

package/dist/evals/runner.js

@@ -80,8 +80,8 @@ function settings(options) {
     if (typeof options.persistResultContent !== "boolean")
         throw new TypeError("Choose persistResultContent explicitly: true or false");
     const concurrency = options.concurrency ?? 1;
-    if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 16)
-        throw new RangeError("concurrency must be 1–16");
+    if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 64)
+        throw new RangeError("concurrency must be 1–64");
     const timeout = options.schemaTimeoutMillis ?? 2000;
     if (!Number.isInteger(timeout) || timeout < 100 || timeout > 60_000)
         throw new RangeError("schemaTimeoutMillis must be 100–60000");

package/dist/evals/simulation.d.ts

@@ -1,7 +1,7 @@
 import type { HueClient } from "../client.js";
 import { type EnvironmentClient } from "../environment/client.js";
 import type { EnvironmentTool } from "../environment/tools.js";
-import type { EnvironmentIdentity, PublishableEnvironmentDefinition } from "../environment/types.js";
+import type { EnvironmentIdentity, PublishableEnvironmentDefinition, WorldHandoff } from "../environment/types.js";
 import { type EvaluationClient } from "./client.js";
 import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
 import { type RunnerReport } from "./runner.js";
@@ -109,10 +109,15 @@ export interface SimulationTargetContext {
     executionId: string;
     /** Stable world identity for adapter control operations such as coverage reporting. */
     environmentRunId: string;
-    /** Framework-neutral local callables backed by this attempt's isolated world. */
+    /** Framework-neutral local callables backed by this attempt's isolated world; empty for a
+     * gateway world, whose calls go to the provider mirrors in `world`. */
     tools: Record<string, EnvironmentTool>;
-    /** Short-lived capability for providers that execute MCP remotely. */
-    mcp: SimulationMcpCapability;
+    /** The mirror URLs, world token, environment carriers and MCP configuration of a gateway
+     * world; absent for a world created while the gateway was off. */
+    world?: WorldHandoff;
+    /** One MCP endpoint and bearer: the gateway world's first MCP mirror with the world token, or
+     * the deprecated execution-scoped `hue_sim_` capability of a legacy world. */
+    mcp?: SimulationMcpCapability;
     /** Credential-bearing provider connections for this callback only. Hue never
      * checkpoints, logs or adds this response to parity digests. */
     connectionBundle?: AttemptConnectionBundleV2;
@@ -149,7 +154,7 @@ export interface RunSimulationOptions {
     };
     /** Local scorer callbacks bound by their declared source digests. */
     localScorers?: LocalScorer[];
-    /** Cases in flight, 1–16; defaults to 1. */
+    /** Cases in flight, 1–64; defaults to 1. */
     concurrency?: number;
     /** JSON Schema worker deadline in milliseconds. */
     schemaTimeoutMillis?: number;
@@ -157,6 +162,10 @@ export interface RunSimulationOptions {
     maxSteps?: number;
     /** Per-world lease in seconds, 1–86400. */
     ttlSeconds?: number;
+    /** The agent revision under test, sent on world create for the world's fingerprint. */
+    agentRevision?: string;
+    /** Emit a one-time `DeprecationWarning` when the deployment serves a legacy world; on by default. */
+    deprecationWarnings?: boolean;
     /** Cooperative caller cancellation signal. */
     signal?: AbortSignal;
     /** Optional display name for the fresh experiment. */

package/dist/evals/simulation.js

@@ -378,10 +378,21 @@ export async function runSimulation(options) {
         baseUrl: options.client.baseUrl,
     });
     try {
-        const scenarioDigest = digest(definitionIdentity(definition));
+        // The agent revision is part of the attempt: a resume after it changed would otherwise
+        // finish the remaining cases under a different revision than the completed ones. A
+        // checkpoint written before revisions were tracked carries the definition digest alone and
+        // still matches its definition.
+        const scenarioDigest = digest({
+            definition: definitionIdentity(definition),
+            agentRevision: options.agentRevision ?? null,
+        });
         let attempt = await store.read("active-attempt");
-        if (attempt && attempt.stage !== "completed" && attempt.scenarioDigest !== scenarioDigest)
-            throw new Error("Recover the unfinished simulation before running a changed scenario");
+        const previousDigest = digest(definitionIdentity(definition));
+        if (attempt &&
+            attempt.stage !== "completed" &&
+            attempt.scenarioDigest !== scenarioDigest &&
+            attempt.scenarioDigest !== previousDigest)
+            throw new Error("Recover the unfinished simulation before running a changed scenario or agent revision");
         if (!attempt || attempt.stage === "completed") {
             attempt = {
                 scenarioDigest,
@@ -432,6 +443,8 @@ export async function runSimulation(options) {
                 requested,
                 maxSteps: options.maxSteps,
                 ttlSeconds: options.ttlSeconds,
+                agentRevision: options.agentRevision,
+                deprecationWarnings: options.deprecationWarnings,
                 signal: options.signal,
                 onProgress: (event) => options.onProgress?.({
                     ...event,
@@ -448,11 +461,16 @@ export async function runSimulation(options) {
                     executionId: targetContext.executionId,
                     environmentRunId: targetContext.environmentRunId,
                     tools: targetContext.tools,
-                    mcp: {
-                        url: targetContext.mcp.url,
-                        token: targetContext.mcp.token,
-                        expiresAt: targetContext.mcp.expiresAt,
-                    },
+                    ...(targetContext.world ? { world: structuredClone(targetContext.world) } : {}),
+                    ...(targetContext.mcp
+                        ? {
+                            mcp: {
+                                url: targetContext.mcp.url,
+                                token: targetContext.mcp.token,
+                                expiresAt: targetContext.mcp.expiresAt,
+                            },
+                        }
+                        : {}),
                     ...(targetContext.connectionBundle
                         ? { connectionBundle: structuredClone(targetContext.connectionBundle) }
                         : {}),

package/dist/live-spans.d.ts

@@ -0,0 +1,25 @@
+import { type Attributes } from "@opentelemetry/api";
+import { type ReadableSpan } from "@opentelemetry/sdk-trace";
+/**
+ * A finished span never carries the placeholder markers, whatever an application set: Hue would
+ * read such a span as a malformed placeholder and reject it.
+ */
+export declare function withoutPlaceholderMarkers(attributes: Attributes): Attributes;
+/** Open spans tracked for announcement at once; later starts are not announced. */
+export declare const MAX_LIVE_SPANS = 1024;
+export declare const LIVE_SPAN_INTERVAL_MILLIS = 500;
+/**
+ * Response header on every trace acknowledgement from a Hue that accepts placeholders. Without
+ * it the receiver predates them and rejects each one by its zero end time.
+ */
+export declare const PLACEHOLDERS_HEADER = "hue-pending-spans";
+/** Hue's own spans and recognizable AI spans, judged from what is known when the span starts. */
+export declare function announcesLiveSpan(span: ReadableSpan): boolean;
+/**
+ * A normal OTLP span announcing `span` while it runs: a new identity whose parent is the real
+ * span, the real start time and an end time of 0. The markers are added after redaction.
+ */
+export declare function pendingPlaceholder(span: ReadableSpan): {
+    record: ReadableSpan;
+    markers: Attributes;
+};

package/dist/live-spans.js

@@ -0,0 +1,115 @@
+import { SpanStatusCode } from "@opentelemetry/api";
+import { RandomIdGenerator } from "@opentelemetry/sdk-trace";
+import { HUE_SCOPE } from "./config.js";
+/** Marks a placeholder for a span that has started but not ended. The value versions the shape. */
+const PENDING_SPAN_TYPE_KEY = "hue.span_type";
+const PENDING_SPAN_TYPE = "pending_span";
+/** The real span's own parent. Absent when the real span is a root. */
+const PENDING_PARENT_KEY = "hue.pending_parent_id";
+const markerKeys = [PENDING_SPAN_TYPE_KEY, PENDING_PARENT_KEY];
+/**
+ * A finished span never carries the placeholder markers, whatever an application set: Hue would
+ * read such a span as a malformed placeholder and reject it.
+ */
+export function withoutPlaceholderMarkers(attributes) {
+    if (!markerKeys.some((key) => Object.hasOwn(attributes, key)))
+        return attributes;
+    const kept = { ...attributes };
+    for (const key of markerKeys)
+        delete kept[key];
+    return kept;
+}
+/** Open spans tracked for announcement at once; later starts are not announced. */
+export const MAX_LIVE_SPANS = 1024;
+export const LIVE_SPAN_INTERVAL_MILLIS = 500;
+/**
+ * Response header on every trace acknowledgement from a Hue that accepts placeholders. Without
+ * it the receiver predates them and rejects each one by its zero end time.
+ */
+export const PLACEHOLDERS_HEADER = "hue-pending-spans";
+const MAX_PLACEHOLDER_VALUE_BYTES = 64 * 1024;
+// Kept narrow on purpose: common application processors export only these spans, and a
+// placeholder whose real span is filtered later would read as running until the trace stalls.
+const livePrefixes = ["gen_ai.", "ai.", "llm.", "traceloop."];
+// Definitions and instructions are large and rarely useful while a span runs; the real span
+// still carries them. Copied markers would misplace the placeholder.
+const omittedKeys = [
+    "gen_ai.tool.definitions",
+    "gen_ai.system_instructions",
+    PENDING_SPAN_TYPE_KEY,
+    PENDING_PARENT_KEY,
+];
+const ids = new RandomIdGenerator();
+/** Hue's own spans and recognizable AI spans, judged from what is known when the span starts. */
+export function announcesLiveSpan(span) {
+    if (span.instrumentationScope.name === HUE_SCOPE || span.name.startsWith("ai."))
+        return true;
+    for (const key in span.attributes)
+        if (livePrefixes.some((prefix) => key.startsWith(prefix)))
+            return true;
+    return false;
+}
+function valueBytes(value) {
+    if (typeof value === "string")
+        return Buffer.byteLength(value);
+    if (!Array.isArray(value))
+        return 8;
+    let bytes = 0;
+    for (const item of value)
+        bytes += typeof item === "string" ? Buffer.byteLength(item) : 8;
+    return bytes;
+}
+function placeholderAttributes(source) {
+    const attributes = {};
+    for (const key in source) {
+        if (!Object.hasOwn(source, key))
+            continue;
+        if (omittedKeys.some((omitted) => key === omitted || key.startsWith(`${omitted}.`)))
+            continue;
+        const value = source[key];
+        if (valueBytes(value) <= MAX_PLACEHOLDER_VALUE_BYTES)
+            attributes[key] = value;
+    }
+    return attributes;
+}
+/**
+ * A normal OTLP span announcing `span` while it runs: a new identity whose parent is the real
+ * span, the real start time and an end time of 0. The markers are added after redaction.
+ */
+export function pendingPlaceholder(span) {
+    const real = span.spanContext();
+    const own = {
+        traceId: real.traceId,
+        spanId: ids.generateSpanId(),
+        traceFlags: real.traceFlags,
+        ...(real.traceState ? { traceState: real.traceState } : {}),
+    };
+    const parent = span.parentSpanContext?.spanId;
+    const record = {
+        name: span.name,
+        kind: span.kind,
+        spanContext: () => own,
+        // Hue re-keys a placeholder as its real span, so the flags describe the real span's parent.
+        parentSpanContext: { ...real, isRemote: span.parentSpanContext?.isRemote === true },
+        startTime: span.startTime,
+        endTime: [0, 0],
+        duration: [0, 0],
+        ended: true,
+        status: { code: SpanStatusCode.UNSET },
+        attributes: placeholderAttributes(span.attributes),
+        links: [],
+        events: [],
+        resource: span.resource,
+        instrumentationScope: span.instrumentationScope,
+        droppedAttributesCount: 0,
+        droppedEventsCount: 0,
+        droppedLinksCount: 0,
+    };
+    return {
+        record,
+        markers: {
+            [PENDING_SPAN_TYPE_KEY]: PENDING_SPAN_TYPE,
+            ...(parent ? { [PENDING_PARENT_KEY]: parent } : {}),
+        },
+    };
+}

package/dist/transport.d.ts

@@ -50,8 +50,25 @@ export declare class HueTransport {
     private closed;
     private shutdownPromise?;
     private flushPromise?;
+    private live;
+    private liveTimer?;
+    private liveSpans;
+    private placeholdersRejected;
+    private placeholders;
+    private placeholderSources;
+    private batchSpans?;
     constructor(options: HueOptions);
+    /**
+     * Advisory records (placeholders) are admitted only while the queue is under a quarter of its
+     * record and byte budgets, so they never take more than a quarter from real records. They are
+     * skipped silently.
+     */
     private enqueue;
+    private track;
+    /** Placeholders are built lazily, so input set right after a span starts is included. */
+    private announceLiveSpans;
+    private stopLiveTimer;
+    private stopLiveSpans;
     /** Cumulative counters and current queue gauges. */
     getReport(): ExportReport;
     /** Copies of the latest 128 sanitized issues, oldest first. */