@hue-run/sdk 0.13.1 → 0.13.2

package/CLI.md

@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
 The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
 `capture_content=False`. For a supported application, setup installs the dependency and adds the
 managed import and middleware registration to the existing entrypoint; an unreferenced helper is
-not a completed integration. TypeScript uses `@hue-run/sdk@0.13.1`, `@opentelemetry/api@1.9.1` and
+not a completed integration. TypeScript uses `@hue-run/sdk@0.13.2`, `@opentelemetry/api@1.9.1` and
 `@opentelemetry/context-async-hooks@2.11.0`; Python setup uses its separately tested package pin.
 Content capture requires an ordinary account-managed key and a later explicit application decision.
 
@@ -469,7 +469,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
 # Set project to an existing supported fixture; use the same directory on resume.
 project=/absolute/path/to/supported-fixture
 node packages/sdk-typescript/scripts/verify-setup-live.mjs \
-  --archive .artifacts/typescript/hue-run-sdk-0.13.1.tgz \
+  --archive .artifacts/typescript/hue-run-sdk-0.13.2.tgz \
   --origin https://STAGING_ORIGIN \
   --project "$project" --command setup \
   --evidence .context/setup-staging-before-claim.json
@@ -480,7 +480,7 @@ the private local handoff and finish the real browser claim, then reconcile the
 
 ```sh
 node packages/sdk-typescript/scripts/verify-setup-live.mjs \
-  --archive .artifacts/typescript/hue-run-sdk-0.13.1.tgz \
+  --archive .artifacts/typescript/hue-run-sdk-0.13.2.tgz \
   --origin https://STAGING_ORIGIN \
   --project "$project" --command claim \
   --evidence .context/setup-staging-after-claim.json
@@ -523,10 +523,10 @@ production agent lacks. Pin the SDK and bring its `zod` peer along, so the agent
 gains no dependency:
 
 ```sh
-npx --yes --package @hue-run/sdk@0.13.1 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" --command "node agent.js" --env-file .env.hue
-npx --yes --package @hue-run/sdk@0.13.1 --package "zod@^4.6.5" hue eval --case https://app.hue.run/projects/demo/scenarios/<id> --command "python agent.py" --timeout 120 --baseline <experiment id>
-npx --yes --package @hue-run/sdk@0.13.1 --package "zod@^4.6.5" hue eval --set "Billing regressions" --scorer-version <id> --command "node agent.js" --save-version
-npx --yes --package @hue-run/sdk@0.13.1 --package "zod@^4.6.5" hue eval --worker --command "node agent.js" --agent-key support-agent --env-file .env.hue
+npx --yes --package @hue-run/sdk@0.13.2 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" --command "node agent.js" --env-file .env.hue
+npx --yes --package @hue-run/sdk@0.13.2 --package "zod@^4.6.5" hue eval --case https://app.hue.run/projects/demo/scenarios/<id> --command "python agent.py" --timeout 120 --baseline <experiment id>
+npx --yes --package @hue-run/sdk@0.13.2 --package "zod@^4.6.5" hue eval --set "Billing regressions" --scorer-version <id> --command "node agent.js" --save-version
+npx --yes --package @hue-run/sdk@0.13.2 --package "zod@^4.6.5" hue eval --worker --command "node agent.js" --agent-key support-agent --revision <new-agent-revision> --env-file .env.hue
 ```
 
 Without the second `--package`, `hue eval` exits with `hue eval needs zod`, even for `--help`;
@@ -562,7 +562,7 @@ export default function runMyAgent(inputs: JsonValue, context: SimulationTargetC
 ```
 
 ```sh
-npx --yes --package @hue-run/sdk@0.13.1 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" ./hue-agent.ts --content --env-file .env.hue
+npx --yes --package @hue-run/sdk@0.13.2 --package "zod@^4.6.5" hue eval --case "Refund an eligible charge" ./hue-agent.ts --content --env-file .env.hue
 ```
 
 `--scenario` remains an alias for `--case` for existing scripts. Pass one selection flag.
@@ -581,9 +581,11 @@ Failing cases print the scorer explanation. An evaluator Hue records as not appl
 that case's columns and neither passes nor fails it, so in a mixed eval set a case is decided by
 the evaluators that apply to it; the pass line counts the not-applicable results. A case that no
 pinned evaluator applies to is an error that names what they need. A result of a pinned Hue judge
-(an evaluator of kind `world_judge`) that Hue marks advisory, as it does every judge's today, is
+(an evaluator of kind `world_judge`) that Hue marks advisory is
 shown in its column, marked `(advisory)`, but never decides a case or the exit code; the pass line
-counts advisory failures as not counted, and a case only advisory results ran for is an error. Any
+counts advisory failures as not counted, and a case only advisory results ran for is an error.
+The Answer outcome evaluator uses its required judge to decide answer-only cases; if that judge
+cannot run, the case is inconclusive. Any
 other evaluator's result, an error, or a metric that carries `passed` is never advisory, whatever
 its evidence says. A metric name that
 several evaluators report, such as two judges' `verdict`, gets a column per evaluator, labelled
@@ -629,8 +631,21 @@ script name; `--revision` is sent to Hue as the agent revision of every world.
 
 `--worker` registers the adapter through `runLocalAgent()` with key `--agent-key` (default: the
 adapter filename slug), name `--agent-name` (default: the key), revision `--revision` (default:
-`AGENT_REVISION`, then the Git `HEAD` short hash, then `dev`) and capability `environment:v1`
-plus any repeated `--capability <value>`, prints the registration and each claimed run with its
+`AGENT_REVISION`, then the Git `HEAD` short hash, then `dev`). Command workers register both
+`environment:v1` and `direct:v1`, plus any repeated `--capability <value>`: the same worker runs
+world cases, answer-only cases and mixed eval sets launched from Hue. Commands keep the launch
+directory for both case kinds, so relative application entry points and configuration work.
+Direct cases receive the task/config on stdin and private files/output through `HUE_CASE_*`,
+without a world token, MCP connection or `HUE_ENVIRONMENT_RUN_ID`. App calls that require a world
+must stay disabled for this handoff; never fall back to live app credentials.
+
+Adapter-file workers register `environment:v1` by default. Add `--capability direct:v1` only when
+the adapter also handles `context.mode === "direct"`; its direct context carries `config`, case
+identity, `executionId`, `files`, `outputDirectory` and `signal`, with no world or tools. These
+worker behaviors are added in TypeScript 0.13.2. Upgrading an existing command worker requires a
+new `--revision`, because adding a capability to a previously registered revision is refused.
+
+The worker prints the registration and each claimed run with its
 URL, executes runs launched from Hue until Ctrl+C or `--max-runs <n>`, and prints the verdict table
 after each run. Selection flags do not apply; Hue chooses the pinned experiment. The worker exits 0
 when it stops normally. Hue offers a case whose world comes with agent-visible files only to a

package/EVALUATIONS.md

@@ -297,12 +297,19 @@ acceptance remains a post-publication Fern gate.
 
 ## Direct cases and files
 
-Cases without a simulated world — document workflows whose inputs are a task plus pinned files —
+Cases without a simulated world — answer-only tasks, or document workflows with pinned files —
 run on the same runner under the same checkpoint rules. They shipped in `@hue-run/sdk` `0.5.0`
 and require a Hue deployment that serves case `inputFiles` on experiment
 items, subject `files`, and the artifact reservation, upload, completion and download APIs. The
 Python SDK has no equivalent.
 
+The TypeScript 0.13.2 `hue eval --worker --command "…"` wrapper supplies both callbacks, so one
+command worker accepts direct and world cases, including mixed eval sets. It keeps the launch
+directory for relative application entry points; private case inputs/output still use
+`HUE_CASE_*`. An adapter-file worker opts into the direct callback with `--capability direct:v1`
+and must handle `context.mode === "direct"`. Restart an existing environment-only command
+registration with a new `--revision` when upgrading.
+
 On the outbound worker, supply `directTarget` beside or instead of `target`. Supplying
 `directTarget` registers the `direct:v1` capability and supplying `target` registers
 `environment:v1` (both strings are exported as `localAgentCapabilities`). Declare the file

package/README.md

@@ -647,10 +647,12 @@ their agents with `runLocalAgent()`; setup does not register workers or launch s
 The `hue eval` command, shipped in TypeScript `0.5.0` (`--case` since `0.6.0`, `--scenario`
 before), wraps `runSimulation()` and `runLocalAgent()` for an adapter file or a shell command:
 `hue eval --case "<name>" ./hue-agent.ts --content` creates a fresh run from a published case's
-immutable pins, runs the agent in one isolated world per case, waits
+immutable pins, runs the agent with an isolated world when the case pins one, waits
 for Hue's outcome checks and prints the run URL and per-case PASS/FAIL verdicts with an exit code;
-`--worker` registers the same adapter for runs launched from Hue. It needs a Read and
-write key in `HUE_API_KEY` (never printed). Content capture stays off unless `--content` is
+`--worker` registers the same adapter for runs launched from Hue. In TypeScript 0.13.2, a
+command worker handles both world and answer-only cases; adapter workers opt into direct cases
+with `--capability direct:v1`. Upgrade an existing command registration under a new `--revision`.
+It needs a Read and write key in `HUE_API_KEY` (never printed). Content capture stays off unless `--content` is
 passed; the example passes it so case spans carry content. Case outputs, error messages and
 explanations are stored in Hue either way (`--no-output` keeps them out of a one-shot run's
 results); a command's stdout is its stored answer, with the credentials `hue eval` handed the case

package/dist/cli/eval.js

@@ -60,6 +60,8 @@ Agent (exactly one):
 
 Modes:
   --worker                        Register the agent and poll for runs launched from Hue
+                                  Commands support cases with or without a world; adapter files
+                                  opt into direct cases with --capability direct:v1
   --max-runs <n>                  Stop the worker after n completed runs
   --agent-key <key>               Agent key (default: slug of the adapter filename)
   --agent-name <name>             Agent display name (default: the key)
@@ -600,9 +602,11 @@ function redacting(adapter) {
 function parentEnvironment(allowHueCredentials) {
     // A world variable left in this process's environment belongs to another world: only the
     // current case's world sets them, whether or not the case has one.
-    return withoutWorldVariables(allowHueCredentials
+    const parent = withoutWorldVariables(allowHueCredentials
         ? Object.fromEntries(Object.entries(process.env).filter((entry) => entry[1] !== undefined))
         : stripHueControlPlaneCredentials(process.env));
+    delete parent.HUE_ENVIRONMENT_RUN_ID;
+    return parent;
 }
 function commandAdapter(command, timeoutSeconds, options) {
     return async (inputs, context) => {
@@ -683,7 +687,7 @@ function directCommandAdapter(command, timeoutSeconds, options) {
             files: context.files,
         });
         const stdout = await spawnAgentCommand(command, {
-            cwd: layout.caseDirectory,
+            cwd: options.workingDirectory ?? layout.caseDirectory,
             env: {
                 ...parentEnvironment(options.allowHueCredentials),
                 HUE_CASE_DIR: layout.caseDirectory,
@@ -1286,7 +1290,7 @@ async function runDirect(run, connection) {
         telemetryNotAccepted: report.telemetryNotAccepted,
     }, run.baselineId, output, signal);
 }
-async function runWorker(values, connection, adapter, agent, hue, output, signal) {
+async function runWorker(values, connection, agents, agent, hue, output, signal) {
     const client = new ObservedClient(connection);
     const environmentClient = createEnvironmentClient(connection);
     const wait = integer("wait", values.wait, 300, 0, 86_400);
@@ -1305,6 +1309,9 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
         output.log(`Claimed run ${claim.runId}: ${new URL(`/experiments/${claim.experimentId}`, connection.baseUrl).toString()}`);
     };
     const telemetry = telemetryReporter(output);
+    // A command can use the same stdin contract for both case kinds. Existing adapter files
+    // may require a world, so their direct context remains an explicit capability opt-in.
+    const direct = values.command !== undefined || values.capability?.includes("direct:v1");
     await runLocalAgent({
         client,
         environmentClient,
@@ -1339,7 +1346,7 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
         },
         target(inputs, tools, context) {
             output.log(`[${context.item.externalKey}] agent started`);
-            return adapter(inputs, {
+            return agents.simulation(inputs, {
                 config: context.config,
                 item: context.item,
                 executionId: context.executionId,
@@ -1355,6 +1362,22 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
                 signal: context.signal ?? signal,
             });
         },
+        ...(direct
+            ? {
+                directTarget(inputs, context) {
+                    output.log(`[${context.item.externalKey}] agent started`);
+                    return agents.direct(inputs, {
+                        mode: "direct",
+                        config: context.config,
+                        item: context.item,
+                        executionId: context.executionId,
+                        files: context.files,
+                        outputDirectory: context.outputDirectory,
+                        signal: context.signal ?? signal,
+                    });
+                },
+            }
+            : {}),
         async onCompleted(report) {
             output.log(`Run ${report.runId} completed: ${report.subjectIds.length} case${report.subjectIds.length === 1 ? "" : "s"}`);
             telemetry.rest(report.telemetryNotAccepted);
@@ -1490,6 +1513,9 @@ export async function runEvalCommand(argv) {
             direct: redacting(loaded ??
                 directCommandAdapter(values.command, timeout, {
                     allowHueCredentials: values["allow-hue-credentials"],
+                    // A worker's single command must resolve relative entry points the same way for
+                    // direct and world cases. One-shot direct runs keep their private-directory cwd.
+                    ...(values.worker ? { workingDirectory: process.cwd() } : {}),
                 })),
         };
         hue = createHue({
@@ -1500,7 +1526,7 @@ export async function runEvalCommand(argv) {
             timeoutMillis: CASE_TRACE_EXPORT_MILLIS,
         });
         return values.worker
-            ? await runWorker(values, connection, agents.simulation, agent, hue, output, controller.signal)
+            ? await runWorker(values, connection, agents, agent, hue, output, controller.signal)
             : await runOnce(values, connection, agents, agent, hue, output, controller.signal);
     }
     catch (error) {

package/dist/evals/client.d.ts

@@ -110,7 +110,7 @@ export declare class EvaluationClient {
     createScorer(input: TaggedIdentity): Promise<Scorer>;
     /** Renames a scorer or replaces its tags by name; the same values again change nothing. */
     updateScorer(id: string, input: IdentityUpdate): Promise<Scorer>;
-    /** Lists the project's tags in the order they were created. */
+    /** Lists the project's tags in its order: as people arranged them in Hue, then newer tags. */
     listTags(): Promise<{
         /** The project's tags. */
         items: Tag[];

package/dist/evals/client.js

@@ -451,7 +451,7 @@ export class EvaluationClient {
     updateScorer(id, input) {
         return this.request("PATCH", `/scorers/${uuid(id)}`, withTagNames(input), undefined, undefined, true);
     }
-    /** Lists the project's tags in the order they were created. */
+    /** Lists the project's tags in its order: as people arranged them in Hue, then newer tags. */
     listTags() {
         return this.request("GET", "/tags");
     }

package/dist/evals/scorer-publication.js

@@ -142,12 +142,17 @@ const sdkScorerPublication = z.union([
     }),
     z.strictObject({
         kind: z.literal("world_outcome"),
-        entry: z.literal("hue.outcome_assertions.v2"),
+        entry: z.enum([
+            "hue.outcome_assertions.v2",
+            "hue.outcome_assertions.v4",
+            "hue.outcome_assertions.v5",
+            "hue.outcome_assertions.v6",
+        ]),
         metrics: fixedMetrics(outcomeAssertionMetrics),
     }),
     z.strictObject({
         kind: z.literal("world_outcome"),
-        entry: z.literal("hue.outcome_assertions.v3"),
+        entry: z.enum(["hue.outcome_assertions.v3", "hue.answer_outcome.v1", "hue.answer_outcome.v2"]),
         metrics: fixedMetrics(outcomeAssertionV3Metrics),
         config: z.strictObject({ judge: outcomeJudgeConfig }),
     }),

package/dist/evals/types.d.ts

@@ -275,6 +275,25 @@ export type ScorerDefinition = {
         /** The judge every case scored by this version uses. */
         judge: OutcomeJudgeConfig;
     };
+} | {
+    /** A case's reviewed assertions, scored inside Hue from gateway evidence. */
+    kind: "world_outcome";
+    /** Versioned gateway-ledger assertion semantics. */
+    entry: "hue.outcome_assertions.v4" | "hue.outcome_assertions.v5" | "hue.outcome_assertions.v6";
+    /** `task_success` and the fixed assertion counts defined by the entry. */
+    metrics: MetricDefinition[];
+} | {
+    /** An answer-only case, scored inside Hue without world evidence. */
+    kind: "world_outcome";
+    /** Versioned answer semantics; the pinned judge is required to decide the case. */
+    entry: "hue.answer_outcome.v1" | "hue.answer_outcome.v2";
+    /** The assertion and judge counts defined by the entry. */
+    metrics: MetricDefinition[];
+    /** The judge pin used to score the answer. */
+    config: {
+        /** The required judge every case scored by this version uses. */
+        judge: OutcomeJudgeConfig;
+    };
 } | {
     /** Scored by a person in Hue; the local runner defers it. */
     kind: "manual";

package/dist/version.d.ts

@@ -1,2 +1,2 @@
 /** Package version shared by the instrumentation scope and the export User-Agent. */
-export declare const sdkVersion = "0.13.1";
+export declare const sdkVersion = "0.13.2";

package/dist/version.js

@@ -1,3 +1,3 @@
 // Generated by scripts/write-version.mjs from package.json; do not edit by hand.
 /** Package version shared by the instrumentation scope and the export User-Agent. */
-export const sdkVersion = "0.13.1";
+export const sdkVersion = "0.13.2";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hue-run/sdk",
-  "version": "0.13.1",
+  "version": "0.13.2",
   "private": false,
   "license": "MIT",
   "publishConfig": {