@hue-run/sdk 0.7.0 → 0.8.1

package/dist/cli/eval.js

@@ -6,6 +6,7 @@ import { parseArgs } from "node:util";
 import { randomUUID } from "node:crypto";
 import { createHue } from "../client.js";
 import { createEnvironmentClient } from "../environment/client.js";
+import { agentEnvironment, stripHueControlPlaneCredentials, writeMcpConfig, } from "../environment/world.js";
 import { EvaluationClient, HueApiError } from "../evals/client.js";
 import { TargetResult } from "../evals/types.js";
 import { CheckpointStore } from "../evals/checkpoint.js";
@@ -16,6 +17,7 @@ import { matchByName, parseScenarioSelector, resolveEvalSetPins, resolveScenario
 import { collectDirectOutputs, stageDirectCase } from "./eval-direct.js";
 import { runSimulation, } from "../evals/simulation.js";
 import { collectExperimentVerdicts, compareVerdicts, metricPassed, } from "../evals/verdicts.js";
+import { envFileArgument, envFileOptions } from "./env-file.js";
 const USAGE = `Usage: hue eval [adapter-file] [options]
 
 Run a local agent against a published case or eval set, then print Hue's verdicts.
@@ -32,9 +34,15 @@ Selection (exactly one, not used with --worker):
 
 Agent (exactly one):
   <adapter-file>                  Module exporting default or runMyAgent(inputs, context)
-  --command "<shell command>"     Simulation: spawned per case with HUE_MCP_URL, HUE_MCP_TOKEN,
-                                  HUE_MCP_EXPIRES_AT, HUE_EXECUTION_ID, HUE_ENVIRONMENT_RUN_ID,
-                                  HUE_CASE_ID and HUE_CASE_KEY set; {"inputs","config"} on stdin.
+  --command "<shell command>"     Simulation: spawned per case with the world's environment
+                                  (HUE_WORLD_ID, HUE_WORLD_TOKEN, one HUE_SIM_<SURFACE>_URL per
+                                  provider mirror, HUE_MCP_CONFIG, plus HUE_MCP_URL, HUE_MCP_TOKEN
+                                  and HUE_MCP_EXPIRES_AT for the first MCP mirror), HUE_EXECUTION_ID,
+                                  HUE_ENVIRONMENT_RUN_ID, HUE_CASE_ID and HUE_CASE_KEY set;
+                                  {"inputs","config"} on stdin. HUE_API_KEY and other Hue
+                                  control-plane credentials are removed from the child.
+  --allow-hue-credentials         Keep HUE_API_KEY and other Hue control-plane credentials in
+                                  the --command child (off by default)
                                   Direct: spawned in a private case directory with HUE_CASE_DIR,
                                   HUE_CASE_INPUTS, HUE_CASE_OUTPUT_DIR, HUE_CASE_ID, HUE_CASE_KEY
                                   and HUE_EXECUTION_ID set; files/<role>/ hold the pinned inputs
@@ -49,17 +57,18 @@ Modes:
 
 Connection:
   --env-file <path>               Load a dotenv file (HUE_API_KEY, HUE_BASE_URL) first
+  --env-path <path>               Same as --env-file
   --origin <url>                  Hue origin (default: HUE_BASE_URL or https://app.hue.run)
 
 Output and limits:
-  --name <run name>               Run name (default: <case> · <agent key> · <revision>)
+  --name <run name>               Run name (default: <agent key> @ <revision>, hashes shortened)
   --baseline <experiment id|url>  Compare verdicts with a previous experiment
   --json                          Print one JSON document on stdout; progress goes to stderr
   --content                       Capture telemetry content; one-shot also persists
                                   outputs/explanations (--worker always persists them)
   --save-version                  Freeze an unsaved eval-set version before running
   --checkpoint-dir <path>         Private checkpoint directory (default: .hue/eval/<agent-key>)
-  --concurrency <n>               Cases in flight, 1-16 (default: 1)
+  --concurrency <n>               Cases in flight, 1-64 (default: 1)
   --timeout <seconds>             Per-case --command timeout (default: 600)
   --wait <seconds>                Verdict wait after the run finishes (default: 300)
   -h, --help                      Show this help
@@ -92,12 +101,13 @@ function parse(argv) {
                 "scorer-version": { type: "string", multiple: true },
                 mode: { type: "string" },
                 command: { type: "string" },
+                "allow-hue-credentials": { type: "boolean", default: false },
                 worker: { type: "boolean", default: false },
                 "max-runs": { type: "string" },
                 "agent-key": { type: "string" },
                 "agent-name": { type: "string" },
                 revision: { type: "string" },
-                "env-file": { type: "string" },
+                ...envFileOptions,
                 origin: { type: "string" },
                 name: { type: "string" },
                 baseline: { type: "string" },
@@ -164,6 +174,12 @@ function derivedAgentKey(adapterFile, command) {
         "";
     return slug(basename(script, extname(script)));
 }
+function defaultRunName(agent) {
+    const revision = /^[0-9a-f]{12,64}$/i.test(agent.revision)
+        ? agent.revision.slice(0, 7)
+        : agent.revision;
+    return `${agent.key} @ ${revision}`;
+}
 function gitRevision() {
     try {
         const value = execFileSync("git", ["rev-parse", "--short=12", "HEAD"], {
@@ -296,29 +312,65 @@ function parseAnswer(text) {
         return text;
     }
 }
-/** Runs the shell command once per case; the MCP token travels only through the child's environment. */
-function commandAdapter(command, timeoutSeconds) {
-    return async (inputs, context) => parseAnswer(await spawnAgentCommand(command, {
-        env: {
-            ...process.env,
-            HUE_MCP_URL: context.mcp.url,
-            HUE_MCP_TOKEN: context.mcp.token,
-            HUE_MCP_EXPIRES_AT: context.mcp.expiresAt,
+/**
+ * Runs the shell command once per case. The world token and mirror URLs travel only through the
+ * child's environment and an owner-only MCP configuration file that is removed after the run;
+ * Hue control-plane credentials (`HUE_API_KEY`, `HUE_MCP_KEY`, any `hue_sk_` value) stay with
+ * the CLI unless `--allow-hue-credentials` is passed. A world created while the gateway is off
+ * still gets the `hue_sim_` capability under the same `HUE_MCP_*` names.
+ */
+/** The parent environment an agent child starts from: without Hue control-plane credentials
+ * unless `--allow-hue-credentials` was passed. */
+function parentEnvironment(allowHueCredentials) {
+    return allowHueCredentials
+        ? Object.fromEntries(Object.entries(process.env).filter((entry) => entry[1] !== undefined))
+        : stripHueControlPlaneCredentials(process.env);
+}
+function commandAdapter(command, timeoutSeconds, options) {
+    return async (inputs, context) => {
+        const parent = parentEnvironment(options.allowHueCredentials);
+        const identity = {
             HUE_EXECUTION_ID: context.executionId,
             HUE_ENVIRONMENT_RUN_ID: context.environmentRunId,
             HUE_CASE_ID: context.item.id,
             HUE_CASE_KEY: context.item.externalKey,
-        },
-        stdin: JSON.stringify({ inputs, config: context.config }),
-        timeoutSeconds,
-        ...(context.signal ? { signal: context.signal } : {}),
-    }));
+        };
+        const configFile = context.world ? await writeMcpConfig(context.world) : undefined;
+        try {
+            const env = context.world
+                ? {
+                    ...agentEnvironment(context.world, { parent, includeHueCredentials: true }),
+                    ...identity,
+                    HUE_MCP_CONFIG: configFile.path,
+                }
+                : {
+                    ...parent,
+                    ...(context.mcp
+                        ? {
+                            HUE_MCP_URL: context.mcp.url,
+                            HUE_MCP_TOKEN: context.mcp.token,
+                            HUE_MCP_EXPIRES_AT: context.mcp.expiresAt,
+                        }
+                        : {}),
+                    ...identity,
+                };
+            return parseAnswer(await spawnAgentCommand(command, {
+                env,
+                stdin: JSON.stringify({ inputs, config: context.config }),
+                timeoutSeconds,
+                ...(context.signal ? { signal: context.signal } : {}),
+            }));
+        }
+        finally {
+            await configFile?.dispose();
+        }
+    };
 }
 /**
  * Direct cases: the command works in a private case directory and writes its documents to
  * `output/`. Its stdout is only used as the JSON output when it wrote no result or summary file.
  */
-function directCommandAdapter(command, timeoutSeconds) {
+function directCommandAdapter(command, timeoutSeconds, options) {
     return async (inputs, context) => {
         const layout = await stageDirectCase(context.outputDirectory, {
             inputs,
@@ -330,7 +382,7 @@ function directCommandAdapter(command, timeoutSeconds) {
         const stdout = await spawnAgentCommand(command, {
             cwd: layout.caseDirectory,
             env: {
-                ...process.env,
+                ...parentEnvironment(options.allowHueCredentials),
                 HUE_CASE_DIR: layout.caseDirectory,
                 HUE_CASE_INPUTS: layout.inputsPath,
                 HUE_CASE_OUTPUT_DIR: layout.outputDirectory,
@@ -636,7 +688,7 @@ function parseBaseline(value) {
 }
 async function runOnce(values, connection, agents, agent, hue, output, signal) {
     const client = new EvaluationClient(connection);
-    const concurrency = integer("concurrency", values.concurrency, 1, 1, 16);
+    const concurrency = integer("concurrency", values.concurrency, 1, 1, 64);
     const baselineId = parseBaseline(values.baseline);
     const mode = parseMode(values.mode);
     const pins = await resolveSelection(client, values);
@@ -648,7 +700,7 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
         const frozen = await client.freezeDatasetVersion(pins.datasetVersionId, pins.revision);
         output.log(`Saved "${pins.name}" version ${frozen.version}.`);
     }
-    const runName = values.name ?? `${pins.name} · ${agent.key} · ${agent.revision}`;
+    const runName = values.name ?? defaultRunName(agent);
     const project = await client.checkConnection();
     if (await detectDirect(client, pins, mode))
         return runDirect({
@@ -683,6 +735,10 @@ async function runOnce(values, connection, agents, agent, hue, output, signal) {
         persistResultContent: values.content,
         traceEvidence: { mode: "required" },
         concurrency,
+        agentRevision: agent.revision,
+        // The CLI adapts to whatever the deployment serves; the library warning is for code that
+        // still reads the legacy capability itself.
+        deprecationWarnings: false,
         signal,
         target: agents.simulation,
         async onProgress(event) {
@@ -820,7 +876,7 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
     const client = new ObservedClient(connection);
     const environmentClient = createEnvironmentClient(connection);
     const wait = integer("wait", values.wait, 300, 0, 86_400);
-    const concurrency = integer("concurrency", values.concurrency, 1, 1, 16);
+    const concurrency = integer("concurrency", values.concurrency, 1, 1, 64);
     const maxRuns = values["max-runs"] === undefined
         ? undefined
         : integer("max-runs", values["max-runs"], 1, 1, 1_000_000);
@@ -847,6 +903,7 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
         },
         scorers: [],
         concurrency,
+        deprecationWarnings: false,
         signal,
         ...(maxRuns === undefined ? {} : { maxRuns }),
         target(inputs, tools, context) {
@@ -857,7 +914,8 @@ async function runWorker(values, connection, adapter, agent, hue, output, signal
                 executionId: context.executionId,
                 environmentRunId: context.environmentRunId,
                 tools,
-                mcp: context.mcp,
+                ...(context.world ? { world: context.world } : {}),
+                ...(context.mcp ? { mcp: context.mcp } : {}),
                 ...(context.connectionBundle ? { connectionBundle: context.connectionBundle } : {}),
                 signal,
             });
@@ -926,12 +984,19 @@ export async function runEvalCommand(argv) {
             throw new UsageError("--worker takes no selection; Hue chooses the run to execute");
         if (!values.worker && selections !== 1)
             throw new UsageError("Pass exactly one of --case, --set or --dataset-version");
-        if (values["env-file"]) {
+        let envFile;
+        try {
+            envFile = envFileArgument(values, process.cwd());
+        }
+        catch (error) {
+            throw new UsageError(error.message);
+        }
+        if (envFile) {
             try {
-                process.loadEnvFile(resolve(values["env-file"]));
+                process.loadEnvFile(resolve(envFile));
             }
             catch (error) {
-                throw new UsageError(`Unable to load ${values["env-file"]}: ${error instanceof Error ? error.message : String(error)}`);
+                throw new UsageError(`Unable to load ${envFile}: ${error instanceof Error ? error.message : String(error)}`);
             }
         }
         const apiKey = process.env.HUE_API_KEY?.trim();
@@ -959,8 +1024,13 @@ export async function runEvalCommand(argv) {
                         throw new Error("The adapter returned generated files for a simulated-world case");
                     return answer;
                 }
-                : commandAdapter(values.command, timeout),
-            direct: loaded ?? directCommandAdapter(values.command, timeout),
+                : commandAdapter(values.command, timeout, {
+                    allowHueCredentials: values["allow-hue-credentials"],
+                }),
+            direct: loaded ??
+                directCommandAdapter(values.command, timeout, {
+                    allowHueCredentials: values["allow-hue-credentials"],
+                }),
         };
         hue = createHue({ apiKey, baseUrl, serviceName: key, captureContent: values.content });
         return values.worker

package/dist/cli/login.d.ts

@@ -14,7 +14,7 @@ export interface LoginCommandIo {
     fetch?: typeof fetch;
     openBrowser?: (url: string) => Promise<boolean>;
 }
-export declare const LOGIN_USAGE = "Usage: hue login [--origin URL] [--env-file PATH] [--keys evaluations|coding-agent|both]\n                 [--no-browser] [--force] [--gitignore]\n\nStore the key you created in Hue in a private env file. By default one \"Read and write\" key serves\nboth evaluations (HUE_API_KEY) and your coding agent (HUE_MCP_KEY); it is validated against Hue\nbefore it is stored, and key values are never printed.\n\nOptions:\n  --origin URL     Hue origin (default https://app.hue.run)\n  --env-file PATH  Env file to write (default .env.hue in the current directory)\n  --keys KIND      evaluations (HUE_API_KEY), coding-agent (HUE_MCP_KEY) or both from one key\n                   (default both)\n  --no-browser     Do not open the key settings page in a browser\n  --force          Replace an existing different value in the env file\n  --gitignore      Add the env file to .gitignore when a git repository does not ignore it\n  -h, --help       Show this help";
+export declare const LOGIN_USAGE = "Usage: hue login [--origin URL] [--env-path PATH] [--keys evaluations|coding-agent|both]\n                 [--no-browser] [--force] [--gitignore]\n\nStore the key you created in Hue in a private env file. By default one \"Read and write\" key serves\nboth evaluations (HUE_API_KEY) and your coding agent (HUE_MCP_KEY); it is validated against Hue\nbefore it is stored, and key values are never printed.\n\nOptions:\n  --origin URL     Hue origin (default https://app.hue.run)\n  --env-path PATH  Env file to write (default .env.hue in the current directory); it is\n                   created when missing. --env-file also works, but Node itself exits\n                   before this command runs when that file does not exist yet\n  --keys KIND      evaluations (HUE_API_KEY), coding-agent (HUE_MCP_KEY) or both from one key\n                   (default both)\n  --no-browser     Do not open the key settings page in a browser\n  --force          Replace an existing different value in the env file\n  --gitignore      Add the env file to .gitignore when a git repository does not ignore it\n  -h, --help       Show this help";
 /** Hue MCP endpoint that pairs with an application origin. */
 export declare function mcpUrlForOrigin(origin: string): string;
 /** Normalizes a Hue origin: HTTPS, or HTTP for loopback only; no credentials, path, query or hash. */

package/dist/cli/login.js

@@ -7,6 +7,7 @@ import { createInterface } from "node:readline";
 import { Writable } from "node:stream";
 import { parseArgs } from "node:util";
 import { isLoopbackHost } from "../config.js";
+import { envFileArgument, envFileOptions } from "./env-file.js";
 const DEFAULT_ORIGIN = "https://app.hue.run";
 const DEFAULT_ENV_FILE = ".env.hue";
 /** Settings section that lists and creates project service keys. */
@@ -27,7 +28,7 @@ const KEY_KINDS = {
 };
 /** Settings preset that authorizes both evaluations and the coding agent's MCP reads and writes. */
 const KEY_PRESET = "Read and write";
-export const LOGIN_USAGE = `Usage: hue login [--origin URL] [--env-file PATH] [--keys evaluations|coding-agent|both]
+export const LOGIN_USAGE = `Usage: hue login [--origin URL] [--env-path PATH] [--keys evaluations|coding-agent|both]
                  [--no-browser] [--force] [--gitignore]
 
 Store the key you created in Hue in a private env file. By default one "Read and write" key serves
@@ -36,7 +37,9 @@ before it is stored, and key values are never printed.
 
 Options:
   --origin URL     Hue origin (default ${DEFAULT_ORIGIN})
-  --env-file PATH  Env file to write (default ${DEFAULT_ENV_FILE} in the current directory)
+  --env-path PATH  Env file to write (default ${DEFAULT_ENV_FILE} in the current directory); it is
+                   created when missing. --env-file also works, but Node itself exits
+                   before this command runs when that file does not exist yet
   --keys KIND      evaluations (HUE_API_KEY), coding-agent (HUE_MCP_KEY) or both from one key
                    (default both)
   --no-browser     Do not open the key settings page in a browser
@@ -558,7 +561,7 @@ function parseLoginArguments(argv) {
         strict: true,
         options: {
             origin: { type: "string" },
-            "env-file": { type: "string" },
+            ...envFileOptions,
             keys: { type: "string" },
             "no-browser": { type: "boolean", default: false },
             force: { type: "boolean", default: false },
@@ -591,8 +594,10 @@ export async function runLoginCommand(argv, io = {}) {
         return code;
     };
     let parsed;
+    let envFileOption;
     try {
         parsed = parseLoginArguments(argv);
+        envFileOption = envFileArgument(parsed.values, cwd);
     }
     catch (error) {
         return fail(`${error.message}\n\n${LOGIN_USAGE}`, 2);
@@ -615,7 +620,7 @@ export async function runLoginCommand(argv, io = {}) {
     if (!origin)
         return fail("--origin must be an HTTPS origin such as https://app.hue.run (plain HTTP is accepted for loopback test servers only).", 2);
     const mcpUrl = mcpUrlForOrigin(origin);
-    const envPath = resolve(cwd, parsed.values["env-file"] ?? DEFAULT_ENV_FILE);
+    const envPath = resolve(cwd, envFileOption ?? DEFAULT_ENV_FILE);
     const envDisplay = displayPath(cwd, envPath);
     let envFile;
     try {

package/dist/client.d.ts

@@ -1,6 +1,6 @@
 import { type Context, type Span, type Tracer } from "@opentelemetry/api";
 import { HueTransport } from "./transport.js";
-import type { ExportReport, FlushableLoggerProvider, FlushableTracerProvider, HueOptions, HueSpan, ModelOptions, ProjectConnection, SpanOptions, ToolOptions, VerifyTraceOptions, TraceVerification, SafeLifecycleOptions, SafeLifecycleResult } from "./types.js";
+import type { ExportReport, FileRecord, FlushableLoggerProvider, FlushableTracerProvider, HueOptions, HueSpan, ModelOptions, ProjectConnection, SpanOptions, ProviderToolCallOptions, ToolOptions, VerifyTraceOptions, TraceVerification, SafeLifecycleOptions, SafeLifecycleResult } from "./types.js";
 /**
  * Attach mode: the application owns its OpenTelemetry providers and passes the transport whose
  * processors it attached to them. The client flushes these providers but never shuts them down.
@@ -75,7 +75,8 @@ export declare class HueClient {
      * `options.callId` is recorded as `gen_ai.tool.call.id`, like the Python `call_id=` keyword.
      * `options.mcp` records the MCP `initialize` `serverInfo` as `mcp.server.name` /
      * `mcp.server.version` so a generic tool name can be attributed to the server that
-     * handled it. Pass `client.getServerVersion()`.
+     * handled it. Pass `client.getServerVersion()`. `mcp.provider` / `mcp.surface` record the Hue
+     * provider and surface as `hue.mcp.provider` / `hue.mcp.surface`.
      */
     tool<T>(name: string, input: unknown, execute: () => Promise<T> | T, options?: ToolOptions): Promise<T>;
     /**
@@ -84,7 +85,8 @@ export declare class HueClient {
      * `gen_ai.request.model` and `gen_ai.provider.name`. The argument order matches `withSpan`. The
      * handle's `setInput`/`setOutput` record `gen_ai.input.messages` / `gen_ai.output.messages`,
      * which should use the GenAI semantic-convention message shape; `recordMessages` inside the
-     * callback inherits the request metadata.
+     * callback inherits the request metadata. `options.systemInstructions` and `options.tools` are
+     * recorded as `gen_ai.system_instructions` and `gen_ai.tool.definitions`, content like `input`.
      */
     model<T>(model: string, callback: (span: HueSpan) => Promise<T> | T, options: ModelOptions): Promise<T>;
     private setUsage;
@@ -113,6 +115,8 @@ export declare class HueClient {
         input?: unknown;
         /** Output messages, ideally in the GenAI semantic-convention shape; any JSON-encodable value. */
         output?: unknown;
+        /** System instructions sent separately from the messages, as `gen_ai.system_instructions`. */
+        systemInstructions?: unknown;
         /** `gen_ai.operation.name` for the record; defaults to the enclosing `model()` span's value. */
         operation?: string;
         /** `gen_ai.provider.name` for the record; defaults to the enclosing `model()` span's value. */
@@ -120,6 +124,30 @@ export declare class HueClient {
         /** `gen_ai.request.model` for the record; defaults to the enclosing `model()` span's value. */
         model?: string;
     }, explicitContext?: Context): void;
+    /**
+     * Records the tools a model provider executed itself while producing `response`, which no
+     * `hue.tool()` call saw: OpenAI Responses `mcp_call`, `web_search_call`, `file_search_call` and
+     * `code_interpreter_call` items, and Anthropic Messages `mcp_tool_use` / `server_tool_use`
+     * blocks with their result blocks. Each becomes an `execute_tool {name}` child span of the active
+     * (or given) context with `gen_ai.tool.type` `extension` and `gen_ai.tool.call.id`; MCP calls add
+     * `mcp.server.name` (the provider's label, or the `servers` entry for it). Arguments and results
+     * follow `captureContent`; a failed call carries `error.type` and ERROR status. An OpenAI
+     * `mcp_list_tools` item becomes a `tools/list` child span carrying that server's tools as
+     * `gen_ai.tool.definitions`. Call it inside `hue.model()` so the spans nest under the model call
+     * and `provider` defaults to its provider; pass `request` to record each server's host as
+     * `server.address`. The spans have no duration of their own: the provider ran the tools inside
+     * the model request. Unreadable items are skipped and counted; nothing is thrown.
+     */
+    recordProviderToolCalls(response: unknown, options?: ProviderToolCallOptions): void;
+    /**
+     * Adds a `hue.file` event to the active (or given) span for a file the work read, received or
+     * produced: `hue.file.sha256`, `hue.file.role`, `hue.file.media_type`, `hue.file.size` when known
+     * and, when `captureContent` is true, `hue.file.name`. `data` is hashed and measured locally and
+     * never exported. The event is metadata, so it is recorded in both capture modes. An invalid
+     * record, or one without an active span, is omitted and counted, never thrown; an invalid name
+     * alone is omitted and counted while the rest is recorded.
+     */
+    recordFile(file: FileRecord, explicitContext?: Context): void;
     private setContent;
     /**
      * Confirms the key and origin by reading the current project; a setup and CI diagnostic, not a