@hue-run/sdk 0.2.2 → 0.3.1

package/dist/evals/local-worker.d.ts

@@ -0,0 +1,88 @@
+import type { HueClient } from "../client.js";
+import type { EnvironmentClient } from "../environment/client.js";
+import type { EnvironmentTool } from "../environment/tools.js";
+import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
+import type { EvaluationClient } from "./client.js";
+import { type RunnerReport } from "./runner.js";
+import type { ExperimentCase, JsonValue, LocalAgentRegistration, LocalScorer } from "./types.js";
+/** Candidate-visible context for one queued local agent execution. */
+export interface LocalAgentTargetContext {
+    /** Frozen candidate configuration, cloned before invocation. */
+    config: JsonValue;
+    /** Identity only. Expected outcomes, metadata and world definitions are evaluator-private. */
+    item: Pick<ExperimentCase, "id" | "externalKey">;
+    /** Identity of this target execution. */
+    executionId: string;
+    /** Stable world identity for adapter control operations such as coverage reporting. */
+    environmentRunId: string;
+    /** Trace identities without mutable span or grading data. */
+    trace: {
+        /** OpenTelemetry trace identifier. */
+        traceId: string;
+        /** Root execution span identifier. */
+        spanId: string;
+    };
+    /** Short-lived, execution-scoped hosted tools for model providers that execute MCP remotely. */
+    mcp: {
+        /** Execution-scoped MCP endpoint. */
+        url: string;
+        /** Short-lived bearer, never the project service key. */
+        token: string;
+        /** Capability expiry as an ISO timestamp. */
+        expiresAt: string;
+    };
+    /** Credential-bearing provider connections for this callback only. Hue never
+     * checkpoints, logs or adds this response to parity digests. */
+    connectionBundle?: AttemptConnectionBundleV2;
+}
+/** Fixed local callback, clients and durable queue-worker settings. */
+export interface RunLocalAgentOptions {
+    /** Evaluation client for the worker project. */
+    client: EvaluationClient;
+    /** Environment client for the same origin and project. */
+    environmentClient: EnvironmentClient;
+    /** Application telemetry client; required trace exports are acknowledged. */
+    hue: HueClient;
+    /** Private durable directory for worker identity and execution checkpoints. */
+    checkpointDirectory: string;
+    /** Fixed agent key and revision registered for queued runs. */
+    agent: LocalAgentRegistration;
+    /** Local scorer bindings matching the published source digests. */
+    scorers?: LocalScorer[];
+    /** Cases in flight, between 1 and 16; defaults to 1. */
+    concurrency?: number;
+    /** Polling interval in milliseconds, 250–60000; defaults to 2000. */
+    pollIntervalMillis?: number;
+    /** Stops polling cooperatively; does not cancel an active callback. */
+    signal?: AbortSignal;
+    /** Useful for one-shot jobs and deterministic acceptance. Omit to keep polling. */
+    maxRuns?: number;
+    /** Opt into the experiment's immutable V2 provider profile. These three values are
+     * validated together before the worker polls; no endpoint or credential is supplied here. */
+    actualAgentManifest?: ActualAgentManifestInputV2 | ((context: {
+        /** Frozen experiment configuration. */
+        config: JsonValue;
+        /** Full frozen case for resolving actual nonsecret evidence before candidate projection. */
+        item: ExperimentCase;
+        /** Cooperative worker stop signal. */
+        signal?: AbortSignal;
+    }) => ActualAgentManifestInputV2 | Promise<ActualAgentManifestInputV2>);
+    /** Exact provider instances and ordered surfaces asserted for strict preflight. */
+    requestedProviders?: RequestedAttemptProviderV2[];
+    /** Requested MCP surface projected to the backwards-compatible `context.mcp`. */
+    mcpSurface?: {
+        /** Provider instance selected from `requestedProviders`. */
+        providerInstanceKey: string;
+        /** Selected MCP surface. */
+        surfaceKey: "google.gmail/mcp" | "slack/mcp";
+    };
+    /** Invokes the existing agent against isolated tools and candidate-safe context. */
+    target(inputs: JsonValue, tools: Record<string, EnvironmentTool>, context: LocalAgentTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
+    /** Called after the experiment and queue completion are acknowledged. */
+    onCompleted?(report: RunnerReport): void | Promise<void>;
+}
+/**
+ * Starts an outbound-only worker for one fixed local agent entry point. Hue chooses
+ * only the registered key/revision; no command or source is received from the cloud.
+ */
+export declare function runLocalAgent(options: RunLocalAgentOptions): Promise<void>;

package/dist/evals/local-worker.js

@@ -0,0 +1,171 @@
+import { join, resolve } from "node:path";
+import { randomUUID } from "node:crypto";
+import { CheckpointStore } from "./checkpoint.js";
+import { pinRequestedAttemptV2, requestedAttemptV2, runEnvironmentTarget, } from "./environment-target.js";
+import { runExperiment, OutcomeSerializationError, TargetOutcomeUncertainError, UncertainExecutionError, } from "./runner.js";
+/** Allowlist the candidate surface instead of forwarding the generic evaluation context. */
+function localAgentTargetContext(context) {
+    return {
+        config: structuredClone(context.config),
+        item: { id: context.item.id, externalKey: context.item.externalKey },
+        executionId: context.executionId,
+        environmentRunId: context.environmentRunId,
+        trace: { traceId: context.trace.traceId, spanId: context.trace.spanId },
+        mcp: {
+            url: context.mcp.url,
+            token: context.mcp.token,
+            expiresAt: context.mcp.expiresAt,
+        },
+        ...(context.connectionBundle
+            ? { connectionBundle: structuredClone(context.connectionBundle) }
+            : {}),
+    };
+}
+function validInterval(value) {
+    const interval = value ?? 2_000;
+    if (!Number.isInteger(interval) || interval < 250 || interval > 60_000)
+        throw new RangeError("pollIntervalMillis must be 250–60000");
+    return interval;
+}
+function stopReason(signal) {
+    return signal?.reason instanceof Error
+        ? signal.reason
+        : new Error("Local worker stopped", { cause: signal?.reason });
+}
+function wait(milliseconds, signal) {
+    if (signal?.aborted)
+        return Promise.reject(stopReason(signal));
+    return new Promise((resolve, reject) => {
+        const timeout = setTimeout(done, milliseconds);
+        const aborted = () => {
+            clearTimeout(timeout);
+            reject(stopReason(signal));
+        };
+        function done() {
+            signal?.removeEventListener("abort", aborted);
+            resolve();
+        }
+        signal?.addEventListener("abort", aborted, { once: true });
+    });
+}
+function needsAttention(error, seen = new Set()) {
+    if (error instanceof TargetOutcomeUncertainError ||
+        error instanceof UncertainExecutionError ||
+        error instanceof OutcomeSerializationError)
+        return true;
+    if (!(error instanceof AggregateError) || seen.has(error))
+        return false;
+    seen.add(error);
+    return error.errors.some((nested) => needsAttention(nested, seen));
+}
+/**
+ * Starts an outbound-only worker for one fixed local agent entry point. Hue chooses
+ * only the registered key/revision; no command or source is received from the cloud.
+ */
+export async function runLocalAgent(options) {
+    const requestedConfiguration = requestedAttemptV2(options);
+    const interval = validInterval(options.pollIntervalMillis);
+    const maxRuns = options.maxRuns ?? Number.POSITIVE_INFINITY;
+    if (!(maxRuns === Number.POSITIVE_INFINITY || (Number.isInteger(maxRuns) && maxRuns > 0)))
+        throw new RangeError("maxRuns must be a positive integer");
+    if (options.environmentClient.baseUrl !== options.client.baseUrl)
+        throw new Error("Environments and evaluations must use the same Hue origin");
+    const directory = resolve(options.checkpointDirectory);
+    const project = await options.client.checkConnection();
+    const store = await CheckpointStore.acquire(directory, {
+        kind: "local-agent-worker",
+        projectId: project.id,
+        baseUrl: options.client.baseUrl,
+        agent: { key: options.agent.key, revision: options.agent.revision },
+    });
+    try {
+        let workerId = await store.read("worker-id");
+        if (workerId !== undefined &&
+            !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(workerId))
+            throw new Error("Invalid local worker identity");
+        if (workerId === undefined) {
+            workerId = randomUUID();
+            await store.write("worker-id", workerId);
+        }
+        let completed = 0;
+        while (completed < maxRuns && !options.signal?.aborted) {
+            const agent = await options.client.registerLocalAgent({
+                ...options.agent,
+                capabilities: options.agent.capabilities ?? ["environment:v1"],
+                scorerDigests: options.agent.scorerDigests ??
+                    options.scorers?.map((item) => item.definition.sourceDigest) ??
+                    [],
+            });
+            const claim = await options.client.claimLocalAgentRun({ agentId: agent.id, workerId });
+            if (!claim) {
+                await wait(interval, options.signal);
+                continue;
+            }
+            const heartbeat = setInterval(() => {
+                void options.client
+                    .heartbeatLocalAgentRun({ runId: claim.runId, workerId })
+                    .catch(() => undefined);
+            }, Math.min(15_000, Math.max(1_000, interval)));
+            let experimentFinished = false;
+            try {
+                const requested = requestedConfiguration
+                    ? pinRequestedAttemptV2(requestedConfiguration, (await options.client.getExperiment(claim.experimentId)).config)
+                    : undefined;
+                const report = await runExperiment({
+                    client: options.client,
+                    hue: options.hue,
+                    experimentId: claim.experimentId,
+                    checkpointDirectory: join(directory, `experiment-${claim.experimentId}`),
+                    persistResultContent: true,
+                    environmentEvidence: "required",
+                    traceEvidence: { mode: "required" },
+                    scorers: options.scorers,
+                    concurrency: options.concurrency,
+                    target: (inputs, context) => runEnvironmentTarget({
+                        client: options.client,
+                        environmentClient: options.environmentClient,
+                        hue: options.hue,
+                        inputs,
+                        context,
+                        requested,
+                        signal: options.signal,
+                        target: (targetInputs, targetContext) => options.target(structuredClone(targetInputs), targetContext.tools, localAgentTargetContext(targetContext)),
+                    }),
+                });
+                // From this point onward the experiment outcome is authoritative. If reporting the
+                // queue completion fails, leave the claim intact for checkpointed recovery instead of
+                // rewriting a successful experiment as an agent failure.
+                experimentFinished = true;
+                await options.client.completeLocalAgentRun({
+                    runId: claim.runId,
+                    workerId,
+                    state: "completed",
+                });
+                await options.onCompleted?.(report);
+                completed++;
+            }
+            catch (error) {
+                // A durable outcome can still need completion/result uploads. Leave operational
+                // failures claimed so the same worker can resume them through its checkpoints.
+                // Attention is terminal in the queue and is reserved for explicit unsafe-to-resume
+                // outcomes that require operator intervention.
+                if (!experimentFinished && needsAttention(error))
+                    await options.client
+                        .completeLocalAgentRun({
+                        runId: claim.runId,
+                        workerId,
+                        state: "attention",
+                        failureType: error instanceof Error ? error.name.slice(0, 200) : "WorkerError",
+                    })
+                        .catch(() => undefined);
+                throw error;
+            }
+            finally {
+                clearInterval(heartbeat);
+            }
+        }
+    }
+    finally {
+        await store.release();
+    }
+}

package/dist/evals/runner.d.ts

@@ -90,7 +90,7 @@ export interface RunnerReport {
     subjectIds: string[];
     /** Result IDs uploaded by this call. */
     resultIds: string[];
-    /** `llm_judge` and `manual` pins left pending for hosted or human scoring. */
+    /** Pins without a local implementation, left pending for their authorized executor. */
     deferredScorerVersionIds: string[];
 }
 /**

package/dist/evals/runner.js

@@ -4,7 +4,7 @@ import { HueExportError } from "../transport.js";
 import { loadEnvironmentEvidence } from "./environment-evidence.js";
 import { CheckpointStore } from "./checkpoint.js";
 import { json, uuid } from "./json.js";
-import { persistedScore, scoreLocally, validateScorerBindings } from "./scorers.js";
+import { persistedScore, scoreLocally, validateScorerBindings, isLocallyExecutable, } from "./scorers.js";
 /**
  * Thrown when a case has a started attempt without a saved outcome. The runner never reruns the
  * target; inspect the execution and authorize a new attempt explicitly through `startExecution`.
@@ -105,7 +105,8 @@ async function pool(items, concurrency, execute) {
 async function scoresFor(versions, context, options, executionId) {
     const scores = [];
     let environmentUnavailable = false;
-    if (options.environmentEvidence === "required") {
+    if (options.environmentEvidence === "required" &&
+        versions.some((version) => isLocallyExecutable(version.definition))) {
         try {
             context = {
                 ...context,
@@ -117,8 +118,8 @@ async function scoresFor(versions, context, options, executionId) {
         }
     }
     for (const version of versions) {
-        // Hosted/manual pins remain pending for their authorized executor.
-        if (version.definition.kind === "llm_judge" || version.definition.kind === "manual")
+        // Every pin without a local implementation belongs to another executor.
+        if (!isLocallyExecutable(version.definition))
             continue;
         const score = persistedScore(environmentUnavailable && version.definition.kind === "local_code"
             ? { state: "error", error: { type: "EnvironmentEvidenceUnavailable" } }
@@ -136,8 +137,16 @@ async function scoresFor(versions, context, options, executionId) {
     }
     return scores;
 }
-async function uploadScores(options, runId, scores, save) {
-    for (const score of scores) {
+async function uploadScores(options, runId, scores, save, versions) {
+    // A previous SDK may have checkpointed a placeholder for an unknown kind.
+    // Keep its evidence intact, but never upload or report it as a local result.
+    const local = scores.filter((score) => {
+        const pin = versions.find((version) => version.id === score.payload.scorerVersionId);
+        if (!pin)
+            throw new Error("Saved result references an unpinned scorer version");
+        return isLocallyExecutable(pin.definition);
+    });
+    for (const score of local) {
         if (score.receipt)
             continue;
         if (!score.payload.evaluationItemId)
@@ -149,6 +158,7 @@ async function uploadScores(options, runId, scores, save) {
         score.receipt = result.ids;
         await save();
     }
+    return local.flatMap((score) => score.receipt ?? []);
 }
 /**
  * Runs every case of a frozen experiment through `target` on this machine, completes each
@@ -204,7 +214,7 @@ export async function runExperiment(options) {
         subjectIds: [],
         resultIds: [],
         deferredScorerVersionIds: versions
-            .filter((version) => ["llm_judge", "manual"].includes(version.definition.kind))
+            .filter((version) => !isLocallyExecutable(version.definition))
             .map((version) => version.id),
     };
     try {
@@ -352,9 +362,9 @@ export async function runExperiment(options) {
                     score.payload.evaluationItemId = prepared.completion.evaluationItemId;
                 await save();
             }
-            await uploadScores(options, experiment.evaluation.id, prepared.scores, save);
+            const results = await uploadScores(options, experiment.evaluation.id, prepared.scores, save, versions);
             report.subjectIds.push(prepared.completion.subjectId);
-            report.resultIds.push(...prepared.scores.flatMap((score) => score.receipt ?? []));
+            report.resultIds.push(...results);
         });
         let finish = await store.read("finish");
         if (!finish) {
@@ -395,7 +405,7 @@ export async function rescore(options) {
         subjectIds: [],
         resultIds: [],
         deferredScorerVersionIds: run.scorerVersions
-            .filter((version) => ["llm_judge", "manual"].includes(version.definition.kind))
+            .filter((version) => !isLocallyExecutable(version.definition))
             .map((version) => version.id),
     };
     try {
@@ -419,9 +429,9 @@ export async function rescore(options) {
                 await store.write(file, saved);
             }
             const current = saved;
-            await uploadScores(options, run.id, current.scores, () => store.write(file, current));
+            const results = await uploadScores(options, run.id, current.scores, () => store.write(file, current), run.scorerVersions);
             report.subjectIds.push(item.subjectId);
-            report.resultIds.push(...current.scores.flatMap((score) => score.receipt ?? []));
+            report.resultIds.push(...results);
         });
         return report;
     }

package/dist/evals/scorer-publication.js

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { json } from "./json.js";
+import { json, digest } from "./json.js";
 const metricName = z
     .string()
     .min(1)
@@ -21,6 +21,15 @@ const metric = z.discriminatedUnion("type", [
     }),
 ]);
 const metrics = z.array(metric).min(1);
+const worldOutcomeMetrics = [
+    "completed_run",
+    "saved_draft",
+    "correct_destination",
+    "content",
+    "unrelated_preserved",
+    "process_constraints",
+    "task_success",
+].map((name) => ({ name, type: "boolean" }));
 const jsonValue = z.custom((value) => {
     try {
         json(value);
@@ -73,6 +82,13 @@ const sdkScorerPublication = z.union([
         sourceDigest: z.string(),
         metrics,
     }),
+    z.strictObject({
+        kind: z.literal("world_outcome"),
+        entry: z.literal("hue.conversion_outcome.v1"),
+        metrics: metrics
+            .default(worldOutcomeMetrics)
+            .refine((value) => digest(value) === digest(worldOutcomeMetrics), "World outcome metrics are fixed by the pinned entry"),
+    }),
     z.strictObject({ kind: z.literal("manual"), metrics }),
     z.strictObject({ kind: z.literal("llm_judge"), config: judgeConfig, metrics }),
 ]);

package/dist/evals/scorers.d.ts

@@ -17,12 +17,18 @@ export declare function defineLocalScorer(options: {
 }): LocalScorer;
 /** Check local callbacks before target invocation; never execute downloaded source code. */
 export declare function validateScorerBindings(versions: ScorerVersion[], scorers?: LocalScorer[]): void;
+/** Only implementations this SDK owns may produce local results. Unknown pins are deferred. */
+export declare function isLocallyExecutable(definition: {
+    kind: string;
+    entry?: string;
+}): definition is Extract<ScorerDefinition, {
+    kind: "builtin" | "local_code";
+}>;
 /**
- * Scores one subject with a pinned scorer version on this machine: built-ins run here, `local_code`
- * pins run the matching callback from `options.scorers`, `manual` pins are skipped. Failures are
- * returned as sanitized error scores.
+ * Scores one subject locally using a known built-in or a bound `local_code` callback.
+ * Scorer failures are returned as sanitized error scores.
  *
- * @throws TypeError for an `llm_judge` pin, which must be dispatched through `createJudgeJobs`.
+ * @throws TypeError for pins without a local implementation; their authorized executor owns scoring.
  */
 export declare function scoreLocally(version: ScorerVersion, context: ScoreContext, options?: {
     scorers?: LocalScorer[];

package/dist/evals/scorers.js

@@ -99,17 +99,22 @@ export function validateScorerBindings(versions, scorers = []) {
             throw new Error("A pinned local scorer has no matching language/source/entrypoint/metric binding");
     }
 }
+/** Only implementations this SDK owns may produce local results. Unknown pins are deferred. */
+export function isLocallyExecutable(definition) {
+    return (definition.kind === "local_code" ||
+        (definition.kind === "builtin" &&
+            ["hue.exact_match.v1", "hue.includes.v1", "hue.json_schema.v1"].includes(definition.entry ?? "")));
+}
 /**
- * Scores one subject with a pinned scorer version on this machine: built-ins run here, `local_code`
- * pins run the matching callback from `options.scorers`, `manual` pins are skipped. Failures are
- * returned as sanitized error scores.
+ * Scores one subject locally using a known built-in or a bound `local_code` callback.
+ * Scorer failures are returned as sanitized error scores.
  *
- * @throws TypeError for an `llm_judge` pin, which must be dispatched through `createJudgeJobs`.
+ * @throws TypeError for pins without a local implementation; their authorized executor owns scoring.
  */
 export async function scoreLocally(version, context, options = {}) {
     const definition = version.definition;
-    if (definition.kind === "llm_judge")
-        throw new TypeError("Hosted judge pins require dispatch through createJudgeJobs; do not submit a local result");
+    if (!isLocallyExecutable(definition))
+        throw new TypeError("This scorer must be deferred to its authorized executor; do not submit a local result");
     const timeout = options.schemaTimeoutMillis ?? 2000;
     if (!Number.isInteger(timeout) || timeout < 100 || timeout > 60_000)
         throw new RangeError("schemaTimeoutMillis must be 100–60000");
@@ -133,8 +138,6 @@ export async function scoreLocally(version, context, options = {}) {
         if (environment !== undefined)
             validateEnvironmentEvidence(environment);
         const owned = structuredClone(context);
-        if (definition.kind === "manual")
-            return skip("Manual scoring requires a human session");
         if (definition.kind === "local_code") {
             const binding = options.scorers?.find((local) => digest(local.definition) === digest(definition));
             if (!binding)

package/dist/evals/simulation.d.ts

@@ -1,9 +1,9 @@
 import type { HueClient } from "../client.js";
 import { type EnvironmentClient } from "../environment/client.js";
-import { type EnvironmentTool } from "../environment/tools.js";
-import type { EnvironmentDefinition, EnvironmentIdentity } from "../environment/types.js";
+import type { EnvironmentTool } from "../environment/tools.js";
+import type { EnvironmentIdentity, PublishableEnvironmentDefinition } from "../environment/types.js";
 import { type EvaluationClient } from "./client.js";
-import { type ActualAgentManifestInputV2, type AttemptConnectionBundleV2, type RequestedAttemptProviderV2 } from "./attempt.js";
+import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
 import { type RunnerReport } from "./runner.js";
 import type { ExperimentCase, Identity, JsonValue, LocalScorer, ScorerDefinition, SimulationMcpCapability } from "./types.js";
 /** Repository-authored scorer identity and its public definition or local binding. */
@@ -40,7 +40,7 @@ export type SimulationScenario = {
     /** Environment identity and authored world definition. */
     environment: EnvironmentIdentity & {
         /** Definition normalized and published as an immutable version. */
-        definition: EnvironmentDefinition;
+        definition: PublishableEnvironmentDefinition;
     };
     /** Cases published into one frozen dataset version. */
     cases: RepositorySimulationCase[];
@@ -92,8 +92,8 @@ export type SimulationProgress = {
 export interface SimulationTargetContext {
     /** Frozen experiment configuration. */
     config: JsonValue;
-    /** Frozen case including its immutable environment-version pin. */
-    item: ExperimentCase;
+    /** Candidate-visible identity. References, metadata and source pins stay with grading. */
+    item: Pick<ExperimentCase, "id" | "externalKey">;
     /** Target execution identity. */
     executionId: string;
     /** Stable world identity for adapter control operations such as coverage reporting. */