@hue-run/sdk 0.5.0 → 0.6.0

package/CLI.md

@@ -42,7 +42,7 @@ project manifest are refused because managers can update ancestor locks; Python
 The generated `hue.setup.mjs` or `hue_setup.py` always selects `captureContent: false` /
 `capture_content=False`. For a supported application, setup installs the dependency and adds the
 managed import and middleware registration to the existing entrypoint; an unreferenced helper is
-not a completed integration. TypeScript uses `@hue-run/sdk@0.5.0`, `@opentelemetry/api@1.9.1` and
+not a completed integration. TypeScript uses `@hue-run/sdk@0.6.0`, `@opentelemetry/api@1.9.1` and
 `@opentelemetry/context-async-hooks@2.11.0`; Python uses published `hue-run==0.2.2`.
 Content capture requires an ordinary account-managed key and a later explicit application decision.
 
@@ -145,14 +145,17 @@ handoff does not create a trial, reset quota or rerun business work.
 `hue login` stores keys that a person creates in Hue; it never mints one, because setup
 credentials are deliberately isolated from ordinary project keys. It prints the key settings page
 (`<origin>/settings/integrations`, opened in a browser only when a terminal is attached and
-`--no-browser` is absent), then reads each requested key from stdin without echo. A
-**Tracing and evaluations** key is validated with `GET /api/v1/projects/current` and stored as
-`HUE_API_KEY` with `HUE_BASE_URL`; a **Coding agent (read + evaluations)** key is validated with an
-MCP `tools/list` request and stored as `HUE_MCP_KEY` with `HUE_MCP_URL`. A rejected key (`401` or
-`403`) exits `1` and stores nothing for that key.
+`--no-browser` is absent), then reads one **Read and write** key from stdin without echo. By
+default that single key serves both uses: it is checked with `GET /api/v1/projects/current` and
+`GET /api/v1/datasets` (a **Read** or **Tracing only** key is refused because it cannot use
+evaluations) and with an MCP `tools/list` request, then stored as `HUE_API_KEY` with
+`HUE_BASE_URL` and `HUE_MCP_KEY` with `HUE_MCP_URL` in one write. A rejected key (`401` or `403`)
+exits `1` and stores nothing. `--keys evaluations` stores only `HUE_API_KEY`; `--keys coding-agent`
+stores only `HUE_MCP_KEY` and also accepts a **Read** key, for an agent that should only inspect the
+project. Run them separately with different keys if you want to revoke either use alone.
 
 ```sh
-hue login                                  # both keys into ./.env.hue
+hue login                                  # one key for both uses into ./.env.hue
 hue login --keys coding-agent --gitignore  # only HUE_MCP_KEY; add .env.hue to .gitignore
 hue login --origin https://staging.hue.run --env-file .env.staging
 ```
@@ -163,7 +166,7 @@ replaced only with `--force`. Empty values, whitespace and URLs are refused befo
 The MCP endpoint is `https://mcp.hue.run/mcp` for `https://app.hue.run`,
 `https://mcp.staging.hue.run/mcp` for `https://staging.hue.run` and `<origin>/api/mcp` otherwise;
 plain HTTP origins are accepted for loopback test servers only. Output names variables and lengths
-(`Stored HUE_API_KEY (NN chars)`), never values. When git does not ignore the env file, the command
+(`Stored the key (NN chars) as HUE_API_KEY and HUE_MCP_KEY`), never values. When git does not ignore the env file, the command
 warns; `--gitignore` appends the file name to the `.gitignore` next to it. Exit codes: `0` stored,
 `1` failed, `2` usage error, `130` interrupted.
 
@@ -297,7 +300,7 @@ node packages/sdk-typescript/scripts/verify-package.mjs --artifacts-dir .artifac
 # Set project to an existing supported fixture; use the same directory on resume.
 project=/absolute/path/to/supported-fixture
 node packages/sdk-typescript/scripts/verify-setup-live.mjs \
-  --archive .artifacts/typescript/hue-run-sdk-0.5.0.tgz \
+  --archive .artifacts/typescript/hue-run-sdk-0.6.0.tgz \
   --origin https://STAGING_ORIGIN \
   --project "$project" --command setup \
   --evidence .context/setup-staging-before-claim.json
@@ -308,7 +311,7 @@ the private local handoff and finish the real browser claim, then reconcile the
 
 ```sh
 node packages/sdk-typescript/scripts/verify-setup-live.mjs \
-  --archive .artifacts/typescript/hue-run-sdk-0.5.0.tgz \
+  --archive .artifacts/typescript/hue-run-sdk-0.6.0.tgz \
   --origin https://STAGING_ORIGIN \
   --project "$project" --command claim \
   --evidence .context/setup-staging-after-claim.json
@@ -330,14 +333,14 @@ Publication under a candidate dist-tag, registry acceptance, production activati
 the same version to `latest`, and clean-project literal `@latest` smoke remain separate gates; see
 [RELEASING.md](../../RELEASING.md).
 
-## Evaluate an agent against a Scenario
+## Evaluate an agent against a case
 
-`hue eval` runs a developer's local agent against a published Hue Scenario or a saved eval set
+`hue eval` runs a developer's local agent against a published Hue case or a saved eval set
 and prints Hue's verdicts. It is the command-line form of `runSimulation()` (one-shot) and
 `runLocalAgent()` (worker): the agent, its prompts and its provider credentials stay in the local
 process, Hue creates one isolated simulated world per case, and Hue-owned outcome checks grade
-the sealed world. Hue never executes the agent. `HUE_API_KEY` must be a **Tracing and
-evaluations** project key (a **Tracing only** key cannot read Scenarios or create experiments);
+the sealed world. Hue never executes the agent. `HUE_API_KEY` must be a **Read and
+write** project key (a **Read** or **Tracing only** key cannot read cases or create runs);
 the CLI never prints it. The optional `zod` peer of `@hue-run/sdk/evals` must be installed.
 
 Write an adapter module that hands the case inputs and the world's tools or MCP connection to
@@ -359,16 +362,18 @@ export default function runMyAgent(inputs: JsonValue, context: SimulationTargetC
 ```
 
 ```sh
-hue eval --scenario "Refund an eligible charge" ./hue-agent.ts --env-file .env.hue
-hue eval --scenario https://app.hue.run/projects/demo/scenarios/<id> ./hue-agent.ts --baseline <experiment id>
+hue eval --case "Refund an eligible charge" ./hue-agent.ts --env-file .env.hue
+hue eval --case https://app.hue.run/projects/demo/scenarios/<id> ./hue-agent.ts --baseline <run id>
 hue eval --set "Billing regressions" --scorer-version <id> ./hue-agent.ts --save-version
-hue eval --scenario "Refund an eligible charge" --command "python agent.py" --timeout 120 --content
+hue eval --case "Refund an eligible charge" --command "python agent.py" --timeout 120 --content
 hue eval --worker ./hue-agent.ts --agent-key support-agent --env-file .env.hue
 ```
 
-The one-shot mode resolves the selection (`--scenario` by name, ID or URL; `--set` by name, ID or
+`--scenario` remains an alias for `--case` for existing scripts. Pass one selection flag.
+
+The one-shot mode resolves the selection (`--case` by name, ID or URL; `--set` by name, ID or
 URL with explicit `--scorer-version` pins; or `--dataset-version` with `--scorer-version`),
-creates a fresh experiment from those immutable pins named `<scenario> · <agent key> · <revision>`
+creates a fresh run from those immutable pins named `<case> · <agent key> · <revision>`
 (`--name` overrides), prints `Run: <url>` and `Experiment: <id>` as soon as the experiment exists,
 one line per case event (world created, agent started, world sealed), then
 `Waiting for Hue checks...` and a table with one row per case: boolean metrics as `PASS`/`FAIL`,
@@ -418,7 +423,7 @@ on any Node.js version.
 
 Eval sets whose cases pin no simulated world — a task plus pinned input files, answered with
 generated documents — run as **direct** cases through `runExperiment()`. `hue eval` detects this
-from the saved version (`--mode direct|simulation` overrides the detection; `--scenario` is always
+from the saved version (`--mode direct|simulation` overrides the detection; `--case` is always
 a simulation). `--set` accepts the eval set's slug, name, ID or URL; `--set-version <n>` pins a
 saved version other than the latest; `--scorer <slug|name|id>` pins an evaluator at its newest
 published version, beside or instead of explicit `--scorer-version` IDs.

package/ENVIRONMENTS.md

@@ -66,7 +66,7 @@ to the callback and is never written to checkpoints.
 published immutable pins, such as a Scenario's frozen case and Hue-owned outcome checks or a saved
 eval set with explicitly chosen scorer versions. `resolveScenarioPins(client, selector)` reads
 those pins from a Scenario ID, its Hue URL or its name (`listScenarios` and `getScenario` expose
-the underlying reads; a Tracing and evaluations key is required), and `resolveEvalSetPins`
+the underlying reads; a Read and write key is required), and `resolveEvalSetPins`
 resolves an eval set to its latest saved version. `runSimulation` creates the experiment directly
 (`name` defaults to the dataset name, `config` to `{}`) and binds the pins and configuration into
 the checkpoint identity, so resuming with different pins is refused like the other kinds.
@@ -79,7 +79,7 @@ until every item has a terminal result for every pin or the budget elapses (`com
 `summarizeVerdicts` turns results into per-case rows with `passed` and totals; `compareVerdicts`
 diffs two summaries by case key; and `collectExperimentVerdicts` combines those reads for one
 experiment. The `hue eval` command uses the same path; see
-[Evaluate an agent against a Scenario](CLI.md#evaluate-an-agent-against-a-scenario).
+[Evaluate an agent against a case](CLI.md#evaluate-an-agent-against-a-case).
 
 ### Pinned provider-profile preflight
 

package/EVALUATIONS.md

@@ -9,7 +9,7 @@ npm install @hue-run/sdk zod
 
 The SDK executes targets and scorers on your machine. Hue stores pinned definitions, experiment progress and results. It does not execute uploaded source code. Follow the [installation guide](https://docs.hue.run/installation) to add `@hue-run/sdk` to your application.
 
-Create a **Tracing and evaluations** project service key under **Settings → Integrations & API keys** and expose it to this server-side process as `HUE_API_KEY`. A **Tracing only** key cannot author datasets or evaluation runs.
+Create a **Read and write** project service key under **Settings → Integrations & API keys** and expose it to this server-side process as `HUE_API_KEY`. A **Tracing only** key cannot author datasets or evaluation runs.
 
 ```ts
 import { randomUUID } from "node:crypto";
@@ -70,6 +70,25 @@ try {
 }
 ```
 
+For new registry code, use `createEvalSet`, `getEvalSet`, `listEvalSets`,
+`createEvalSetVersion`, `getEvalSetVersion`, `listEvalSetCases`, `addEvalSetCase`,
+and `freezeEvalSetVersion`. Evaluators use `createEvaluator`, `getEvaluator`,
+`listEvaluators`, `publishEvaluatorVersion`, and `getEvaluatorVersion`. These
+methods return product fields such as `evalSetId` and `evalSetVersionId` alongside
+the existing v1 fields. Evaluator versions include `evaluatorId` when the server
+supplies their owning identity; older v1 responses may omit it. They use the existing v1 paths;
+the earlier method names remain callable for existing integrations.
+
+For new run and scoring code, use `createRun`, `getRun`, `listRunItems`,
+`getRunCase`, `startRunExecution`, `getRunExecution`,
+`completeRunExecution`, and `finishRun`. Use `createScoring`, `getScoring`,
+`listScorings`, `listScoringItems`, `getScoringSubject`,
+`submitScoringResults`, `listScoringResults`, and `getScoringResult` to score
+saved subjects. `createRun` accepts `evalSetVersionId` and
+`evaluatorVersionIds`; `createScoring` and `submitScoringResults` use evaluator
+version IDs. A run ID and a scoring ID identify different records. The methods
+use the existing v1 paths and leave existing runner entry points callable.
+
 Create another experiment with the same frozen version and different `config` to compare configurations. The runner reads the exact experiment case/version and scorer definitions; it never resolves a mutable latest version. `rescore` accepts an existing evaluation-run ID and has no target callback. Subject IDs refer to immutable saved outputs and trace evidence.
 
 `rescore` preserves terminal scores already recorded for each item and evaluator version,

package/README.md

@@ -445,12 +445,12 @@ their agents with `runLocalAgent()`; setup does not register workers or launch s
 ### Command-line evaluation
 
 The unreleased `hue eval` command wraps `runSimulation()` and `runLocalAgent()` for an adapter
-file or a shell command: `hue eval --scenario "<name>" ./hue-agent.ts` creates a fresh experiment
-from a published Scenario's immutable pins, runs the agent in one isolated world per case, waits
+file or a shell command: `hue eval --case "<name>" ./hue-agent.ts` creates a fresh run
+from a published case's immutable pins, runs the agent in one isolated world per case, waits
 for Hue's outcome checks and prints the run URL and per-case PASS/FAIL verdicts with an exit code;
-`--worker` registers the same adapter for runs launched from Hue. It needs a Tracing and
-evaluations key in `HUE_API_KEY` (never printed) and keeps content capture off unless `--content`
-is passed. See [Evaluate an agent against a Scenario](CLI.md#evaluate-an-agent-against-a-scenario).
+`--worker` registers the same adapter for runs launched from Hue. It needs a Read and
+write key in `HUE_API_KEY` (never printed) and keeps content capture off unless `--content`
+is passed. See [Evaluate an agent against a case](CLI.md#evaluate-an-agent-against-a-case).
 Eval sets whose cases pin files instead of a world run as direct cases through `runExperiment()`:
 `hue eval --set <slug> --scorer <slug> --command "…"` hands the agent each case's pinned files in a
 private directory, uploads the documents it writes and waits for Hue's grading executor to score

package/dist/cli/eval.js

@@ -18,10 +18,11 @@ import { runSimulation, } from "../evals/simulation.js";
 import { collectExperimentVerdicts, compareVerdicts, metricPassed, } from "../evals/verdicts.js";
 const USAGE = `Usage: hue eval [adapter-file] [options]
 
-Run a local agent against a Hue Scenario or eval set, then print Hue's verdicts.
+Run a local agent against a published case or eval set, then print Hue's verdicts.
 
 Selection (exactly one, not used with --worker):
-  --scenario <name|id|url>        Published Scenario to run
+  --case <name|id|url>            Published case to run
+  --scenario <name|id|url>        Legacy alias for --case
   --set <name|id|url>             Saved eval set; requires --scorer or --scorer-version
   --set-version <n>               Saved version number of the eval set (default: latest saved)
   --dataset-version <id>          Frozen dataset version; requires --scorer or --scorer-version
@@ -51,7 +52,7 @@ Connection:
   --origin <url>                  Hue origin (default: HUE_BASE_URL or https://app.hue.run)
 
 Output and limits:
-  --name <run name>               Experiment name (default: <scenario> · <agent key> · <revision>)
+  --name <run name>               Run name (default: <case> · <agent key> · <revision>)
   --baseline <experiment id|url>  Compare verdicts with a previous experiment
   --json                          Print one JSON document on stdout; progress goes to stderr
   --content                       Capture telemetry content; one-shot also persists
@@ -63,7 +64,7 @@ Output and limits:
   --wait <seconds>                Verdict wait after the run finishes (default: 300)
   -h, --help                      Show this help
 
-HUE_API_KEY must be a "Tracing and evaluations" project key; it is never printed.
+HUE_API_KEY must be a "Read and write" project key; it is never printed.
 Code evaluators pinned to a direct run are graded by Hue's executor after the upload; the wait
 covers them. Exit codes: 0 every case passed, 1 a case failed, errored or is incomplete,
 2 usage error, 130 interrupted.
@@ -82,6 +83,7 @@ function parse(argv) {
             allowPositionals: true,
             strict: true,
             options: {
+                case: { type: "string" },
                 scenario: { type: "string" },
                 set: { type: "string" },
                 "set-version": { type: "string" },
@@ -414,7 +416,7 @@ function redact(message, secrets) {
 }
 function explain(error) {
     if (error instanceof HueApiError && (error.status === 401 || error.status === 403))
-        return `${error.message}. Check that HUE_API_KEY is a "Tracing and evaluations" project key for this origin.`;
+        return `${error.message}. Check that HUE_API_KEY is a "Read and write" project key for this origin.`;
     if (error instanceof Error) {
         const causes = [];
         let cause = error.cause;
@@ -508,15 +510,16 @@ async function resolveSelection(client, values) {
     const extra = [...(values["scorer-version"] ?? [])];
     for (const selector of values.scorer ?? [])
         extra.push(await resolveScorerVersion(client, selector));
-    if (values.scenario) {
+    const caseSelector = values.case ?? values.scenario;
+    if (caseSelector) {
         if (values["set-version"])
             throw new UsageError("--set-version applies to --set only");
-        const pins = await resolveScenarioPins(client, values.scenario);
+        const pins = await resolveScenarioPins(client, caseSelector);
         pins.scorerVersionIds = [...new Set([...pins.scorerVersionIds, ...extra])];
         return pins;
     }
     if (!extra.length)
-        throw new UsageError(`${values.set ? "--set" : "--dataset-version"} needs at least one --scorer or --scorer-version; use --scenario for published pins`);
+        throw new UsageError(`${values.set ? "--set" : "--dataset-version"} needs at least one --scorer or --scorer-version; use --case for published pins`);
     if (values.set) {
         const pins = await resolveEvalSetPins(client, values.set, { scorerVersionIds: extra });
         if (values["set-version"] === undefined)
@@ -918,11 +921,11 @@ export async function runEvalCommand(argv) {
         const adapterFile = positionals[0];
         if ((adapterFile ? 1 : 0) + (values.command ? 1 : 0) !== 1)
             throw new UsageError("Pass exactly one agent: an adapter file or --command");
-        const selections = [values.scenario, values.set, values["dataset-version"]].filter((value) => value !== undefined).length;
+        const selections = [values.case, values.scenario, values.set, values["dataset-version"]].filter((value) => value !== undefined).length;
         if (values.worker && selections)
             throw new UsageError("--worker takes no selection; Hue chooses the run to execute");
         if (!values.worker && selections !== 1)
-            throw new UsageError("Pass exactly one of --scenario, --set or --dataset-version");
+            throw new UsageError("Pass exactly one of --case, --set or --dataset-version");
         if (values["env-file"]) {
             try {
                 process.loadEnvFile(resolve(values["env-file"]));
@@ -933,7 +936,7 @@ export async function runEvalCommand(argv) {
         }
         const apiKey = process.env.HUE_API_KEY?.trim();
         if (!apiKey)
-            throw new UsageError('HUE_API_KEY is required: a "Tracing and evaluations" project key, set in the environment or an ignored --env-file');
+            throw new UsageError('HUE_API_KEY is required: a "Read and write" project key, set in the environment or an ignored --env-file');
         secrets.push(apiKey);
         const baseUrl = values.origin ?? process.env.HUE_BASE_URL?.trim() ?? "https://app.hue.run";
         const connection = { apiKey, baseUrl };

package/dist/cli/login.d.ts

@@ -14,7 +14,7 @@ export interface LoginCommandIo {
     fetch?: typeof fetch;
     openBrowser?: (url: string) => Promise<boolean>;
 }
-export declare const LOGIN_USAGE = "Usage: hue login [--origin URL] [--env-file PATH] [--keys evaluations|coding-agent|both]\n                 [--no-browser] [--force] [--gitignore]\n\nStore keys you created in Hue in a private env file. Each key is validated against Hue before it\nis stored; key values are never printed.\n\nOptions:\n  --origin URL     Hue origin (default https://app.hue.run)\n  --env-file PATH  Env file to write (default .env.hue in the current directory)\n  --keys KIND      evaluations (HUE_API_KEY), coding-agent (HUE_MCP_KEY) or both (default both)\n  --no-browser     Do not open the key settings page in a browser\n  --force          Replace an existing different value in the env file\n  --gitignore      Add the env file to .gitignore when a git repository does not ignore it\n  -h, --help       Show this help";
+export declare const LOGIN_USAGE = "Usage: hue login [--origin URL] [--env-file PATH] [--keys evaluations|coding-agent|both]\n                 [--no-browser] [--force] [--gitignore]\n\nStore the key you created in Hue in a private env file. By default one \"Read and write\" key serves\nboth evaluations (HUE_API_KEY) and your coding agent (HUE_MCP_KEY); it is validated against Hue\nbefore it is stored, and key values are never printed.\n\nOptions:\n  --origin URL     Hue origin (default https://app.hue.run)\n  --env-file PATH  Env file to write (default .env.hue in the current directory)\n  --keys KIND      evaluations (HUE_API_KEY), coding-agent (HUE_MCP_KEY) or both from one key\n                   (default both)\n  --no-browser     Do not open the key settings page in a browser\n  --force          Replace an existing different value in the env file\n  --gitignore      Add the env file to .gitignore when a git repository does not ignore it\n  -h, --help       Show this help";
 /** Hue MCP endpoint that pairs with an application origin. */
 export declare function mcpUrlForOrigin(origin: string): string;
 /** Normalizes a Hue origin: HTTPS, or HTTP for loopback only; no credentials, path, query or hash. */

package/dist/cli/login.js

@@ -19,24 +19,26 @@ const KEY_KINDS = {
     evaluations: {
         variable: "HUE_API_KEY",
         urlVariable: "HUE_BASE_URL",
-        preset: "Tracing and evaluations",
     },
     "coding-agent": {
         variable: "HUE_MCP_KEY",
         urlVariable: "HUE_MCP_URL",
-        preset: "Coding agent (read + evaluations)",
     },
 };
+/** Settings preset that authorizes both evaluations and the coding agent's MCP reads and writes. */
+const KEY_PRESET = "Read and write";
 export const LOGIN_USAGE = `Usage: hue login [--origin URL] [--env-file PATH] [--keys evaluations|coding-agent|both]
                  [--no-browser] [--force] [--gitignore]
 
-Store keys you created in Hue in a private env file. Each key is validated against Hue before it
-is stored; key values are never printed.
+Store the key you created in Hue in a private env file. By default one "Read and write" key serves
+both evaluations (HUE_API_KEY) and your coding agent (HUE_MCP_KEY); it is validated against Hue
+before it is stored, and key values are never printed.
 
 Options:
   --origin URL     Hue origin (default ${DEFAULT_ORIGIN})
   --env-file PATH  Env file to write (default ${DEFAULT_ENV_FILE} in the current directory)
-  --keys KIND      evaluations (HUE_API_KEY), coding-agent (HUE_MCP_KEY) or both (default both)
+  --keys KIND      evaluations (HUE_API_KEY), coding-agent (HUE_MCP_KEY) or both from one key
+                   (default both)
   --no-browser     Do not open the key settings page in a browser
   --force          Replace an existing different value in the env file
   --gitignore      Add the env file to .gitignore when a git repository does not ignore it
@@ -216,7 +218,10 @@ async function readBoundedText(response) {
 function isRecord(value) {
     return typeof value === "object" && value !== null && !Array.isArray(value);
 }
-/** Mirrors `checkConnection()`: `GET /api/v1/projects/current` with the key, no redirects. */
+/**
+ * Mirrors `checkConnection()` with `GET /api/v1/projects/current`, then confirms evaluation access
+ * with `GET /api/v1/datasets`. No redirects are followed.
+ */
 async function checkEvaluationsKey(fetchImpl, origin, apiKey) {
     let response;
     try {
@@ -246,15 +251,44 @@ async function checkEvaluationsKey(fetchImpl, origin, apiKey) {
             detail: `Hue answered HTTP ${response.status} while checking the evaluations key.`,
         };
     }
+    let projectName;
     try {
         const project = JSON.parse(await readBoundedText(response));
         if (!isRecord(project) || typeof project.name !== "string")
             throw new Error("Invalid project");
-        return { ok: true, detail: project.name };
+        projectName = project.name;
     }
     catch {
         return { ok: false, rejected: false, detail: "Hue returned an unexpected project response." };
     }
+    // Every valid key reaches the project check; only a key with write access can list eval sets,
+    // so a Read or Tracing only key is refused here instead of failing later in `hue eval`.
+    let evaluations;
+    try {
+        evaluations = await fetchImpl(`${origin}/api/v1/datasets`, {
+            method: "GET",
+            headers: { authorization: `Bearer ${apiKey}`, accept: "application/json" },
+            redirect: "error",
+            signal: AbortSignal.timeout(REQUEST_TIMEOUT_MILLIS),
+        });
+    }
+    catch {
+        return { ok: false, rejected: false, detail: `Could not reach ${origin}.` };
+    }
+    await evaluations.body?.cancel();
+    if (evaluations.status === 401 || evaluations.status === 403)
+        return {
+            ok: false,
+            rejected: true,
+            detail: `This key cannot use evaluations (HTTP ${evaluations.status}); it is a Read or Tracing only key.`,
+        };
+    if (!evaluations.ok)
+        return {
+            ok: false,
+            rejected: false,
+            detail: `Hue answered HTTP ${evaluations.status} while checking evaluation access.`,
+        };
+    return { ok: true, detail: projectName };
 }
 /** Reads JSON-RPC messages from a JSON body or a `text/event-stream` body. */
 function parseJsonRpcMessages(text, contentType) {
@@ -574,6 +608,9 @@ export async function runLoginCommand(argv, io = {}) {
     if (keysOption !== "evaluations" && keysOption !== "coding-agent" && keysOption !== "both")
         return fail(`--keys must be evaluations, coding-agent or both.\n\n${LOGIN_USAGE}`, 2);
     const kinds = keysOption === "both" ? ["evaluations", "coding-agent"] : [keysOption];
+    // Every requested variable comes from one pasted key: the same "Read and write" preset serves
+    // evaluations and the coding agent, so asking twice would only add a step.
+    const variables = kinds.map((kind) => KEY_KINDS[kind].variable).join(" and ");
     const origin = parseHueOrigin(parsed.values.origin ?? DEFAULT_ORIGIN);
     if (!origin)
         return fail("--origin must be an HTTPS origin such as https://app.hue.run (plain HTTP is accepted for loopback test servers only).", 2);
@@ -588,10 +625,9 @@ export async function runLoginCommand(argv, io = {}) {
         return fail(error.message);
     }
     const settingsUrl = `${origin}${KEY_SETTINGS_PATH}`;
-    out("Hue keys are created in the app; this command validates and stores them locally.");
-    out(`Create keys at: ${settingsUrl}`);
-    for (const kind of kinds)
-        out(`  ${KEY_KINDS[kind].variable}: a "${KEY_KINDS[kind].preset}" key`);
+    out("Hue keys are created in the app; this command validates and stores one locally.");
+    out(`Create a "${KEY_PRESET}" key at: ${settingsUrl}`);
+    out(`  It is stored as ${variables}.`);
     if (!parsed.values["no-browser"] && isTTY(stdout)) {
         const opened = await openBrowser(settingsUrl).catch(() => false);
         if (opened)
@@ -603,75 +639,71 @@ export async function runLoginCommand(argv, io = {}) {
             out(`${variable} is already stored in ${envDisplay}; a different value requires --force.`);
     }
     const stored = [];
-    // A key written before a later prompt fails still has to reach the ignore protection below, so
-    // the loop records why it stopped instead of returning past it.
+    // Validation and the write record why they stopped instead of returning, so a key that did land
+    // in the file still reaches the ignore protection below.
     let failure;
     const prompter = isTTY(stdin)
         ? createTerminalPrompter(stdin, stdout)
         : createLinePrompter(stdin, stdout);
-    const nothingElse = () => (stored.length ? "No further key was stored." : "Nothing was stored.");
     try {
+        const answer = await prompter.ask(`Paste the "${KEY_PRESET}" key (${variables}): `);
+        const value = answer?.trim() ?? "";
+        const reason = answer === null ? null : invalidKeyReason(value);
+        const updates = {};
         for (const kind of kinds) {
-            const { variable, urlVariable, preset } = KEY_KINDS[kind];
-            const urlValue = kind === "evaluations" ? origin : mcpUrl;
-            const answer = await prompter.ask(`Paste the "${preset}" key (${variable}): `);
-            if (answer === null) {
-                failure = { message: `No ${variable} was entered; input ended.` };
-                break;
-            }
-            const value = answer.trim();
-            const reason = invalidKeyReason(value);
-            if (reason) {
-                failure = {
-                    message: `${reason} Create a "${preset}" key at ${settingsUrl} and paste it.`,
-                };
-                break;
-            }
-            if (!parsed.values.force) {
-                for (const [name, next] of [
-                    [variable, value],
-                    [urlVariable, urlValue],
-                ]) {
-                    const current = readEnvValue(envFile.text, name);
-                    if (current !== undefined && current !== next)
-                        failure ??= {
-                            message: `${name} in ${envDisplay} already has a different value; rerun with --force to replace it.`,
-                        };
-                }
-                if (failure)
-                    break;
+            const { variable, urlVariable } = KEY_KINDS[kind];
+            updates[variable] = value;
+            updates[urlVariable] = kind === "evaluations" ? origin : mcpUrl;
+        }
+        if (answer === null)
+            failure = { message: `No key was entered; input ended.` };
+        else if (reason)
+            failure = {
+                message: `${reason} Create a "${KEY_PRESET}" key at ${settingsUrl} and paste it.`,
+            };
+        else if (!parsed.values.force) {
+            for (const [name, next] of Object.entries(updates)) {
+                const current = readEnvValue(envFile.text, name);
+                if (current !== undefined && current !== next)
+                    failure ??= {
+                        message: `${name} in ${envDisplay} already has a different value; rerun with --force to replace it.`,
+                    };
             }
+        }
+        for (const kind of failure ? [] : kinds) {
             const check = kind === "evaluations"
                 ? await checkEvaluationsKey(fetchImpl, origin, value)
                 : await checkCodingAgentKey(fetchImpl, mcpUrl, value);
             if (!check.ok) {
                 failure = {
                     message: check.rejected
-                        ? `${check.detail} Create a "${preset}" key at ${settingsUrl} and try again.`
-                        : `${check.detail} ${nothingElse()}`,
+                        ? `${check.detail} Create a "${KEY_PRESET}" key at ${settingsUrl} and try again. Nothing was stored.`
+                        : `${check.detail} Nothing was stored.`,
                 };
                 break;
             }
             out(kind === "evaluations"
-                ? `Evaluations key accepted for project "${check.detail}".`
-                : `Coding-agent key accepted; the Hue MCP server lists ${check.detail} tools.`);
-            const text = mergeEnvText(envFile.text, { [variable]: value, [urlVariable]: urlValue });
+                ? `Evaluations access accepted for project "${check.detail}".`
+                : `Coding-agent access accepted; the Hue MCP server lists ${check.detail} tools.`);
+        }
+        if (!failure) {
+            const text = mergeEnvText(envFile.text, updates);
             try {
                 await writePrivateFile(envPath, text, 0o600);
+                envFile = { text, exists: true };
+                stored.push(...kinds);
+                const urls = kinds.map((kind) => KEY_KINDS[kind].urlVariable).join(" and ");
+                out(`Stored the key (${value.length} chars) as ${variables}, with ${urls}, in ${envDisplay}.`);
             }
             catch (error) {
                 failure = { message: `Could not write ${envDisplay}: ${error.message}` };
-                break;
             }
-            envFile = { text, exists: true };
-            stored.push(kind);
-            out(`Stored ${variable} (${value.length} chars) and ${urlVariable} in ${envDisplay}.`);
         }
     }
     catch (error) {
         failure =
             error instanceof PromptInterrupted
-                ? { message: `Interrupted; ${nothingElse().toLowerCase()}`, code: 130 }
+                ? { message: "Interrupted; nothing was stored.", code: 130 }
                 : { message: `hue login failed: ${error.message}` };
     }
     finally {
@@ -707,6 +739,6 @@ export async function runLoginCommand(argv, io = {}) {
         // `hue mcp install` defaults to production; a non-default origin needs its own endpoint.
         out(`  hue mcp install --client claude-code${mcpUrl === mcpUrlForOrigin(DEFAULT_ORIGIN) ? "" : ` --url ${mcpUrl}`}`);
     if (stored.includes("evaluations"))
-        out(`  hue eval --scenario "<name>" ./hue-agent.ts --env-file ${envDisplay}`);
+        out(`  hue eval --case "<name>" ./hue-agent.ts --env-file ${envDisplay}`);
     return 0;
 }

package/dist/evals/client.d.ts

@@ -1,6 +1,6 @@
 import type { ProjectConnection } from "../types.js";
 import { type AttemptConnectionBundleV2, type PrepareAttemptRequestV2 } from "./attempt.js";
-import type { ArtifactReservation, ArtifactUpload, CaseConversion, CaseConversionSummary, CaseWrite, CompleteExecution, Completion, Dataset, DatasetCase, DatasetVersion, EvaluationItem, EvaluationRun, EvaluationRunSummary, EnvironmentEvidenceSnapshot, Execution, Experiment, ExperimentCase, ExperimentItem, Identity, JsonValue, LocalAgentClaim, LocalAgentRegistration, RegisteredLocalAgent, JudgeBudget, JudgeJob, Page, PageOptions, RegistryPageOptions, Result, ResultSummary, Scorer, ScorerDefinition, ScorerVersion, SimulationMcpCapability, StartExecution, Subject, StoredResult } from "./types.js";
+import type { ArtifactReservation, ArtifactUpload, CaseConversion, CaseConversionSummary, CaseWrite, CompleteExecution, Completion, Dataset, DatasetCase, DatasetVersion, EvalSet, EvalSetCase, EvalSetVersion, EvaluationItem, EvaluationRun, EvaluationRunSummary, EnvironmentEvidenceSnapshot, Execution, Experiment, ExperimentCase, ExperimentItem, Evaluator, EvaluatorVersion, Identity, JsonValue, LocalAgentClaim, LocalAgentRegistration, RegisteredLocalAgent, JudgeBudget, JudgeJob, Page, PageOptions, RegistryPageOptions, Result, ResultSummary, Run, RunCase, Scorer, ScorerDefinition, ScorerVersion, Scoring, ScoringResultInput, ScoringResultSummary, ScoringSubject, ScoringSummary, SimulationMcpCapability, StartExecution, Subject, StoredResult, StoredScoringResult } from "./types.js";
 /** Connection options for {@link createEvaluationClient}. */
 export interface EvaluationClientOptions {
     /** Project service key sent as a Bearer token; server side only. */
@@ -71,6 +71,39 @@ export declare class EvaluationClient {
     publishScorerVersion(id: string, definition: ScorerDefinition): Promise<ScorerVersion>;
     /** Reads a published scorer version. */
     getScorerVersion(id: string): Promise<ScorerVersion>;
+    /** Creates an eval set using the existing v1 registry path. */
+    createEvalSet(input: Identity): Promise<EvalSet>;
+    /** Reads an eval set and its versions. */
+    getEvalSet(id: string): Promise<EvalSet>;
+    /** Lists eval sets. */
+    listEvalSets(page?: RegistryPageOptions): Promise<Page<Omit<EvalSet, "versions">>>;
+    /** Creates a draft eval set version, optionally copying cases from another version. */
+    createEvalSetVersion(id: string, input?: {
+        fromVersionId?: string;
+    }): Promise<EvalSetVersion>;
+    /** Reads an eval set version. */
+    getEvalSetVersion(id: string): Promise<EvalSetVersion>;
+    /** Lists cases in an eval set version. */
+    listEvalSetCases(id: string, page?: PageOptions): Promise<Page<EvalSetCase>>;
+    /** Adds a case to a draft eval set version at its expected revision. */
+    addEvalSetCase(id: string, input: CaseWrite): Promise<{
+        /** The stored case. */
+        item: EvalSetCase;
+        /** The version with its new revision. */
+        version: EvalSetVersion;
+    }>;
+    /** Freezes a draft eval set version at its expected revision. */
+    freezeEvalSetVersion(id: string, expectedRevision: number): Promise<EvalSetVersion>;
+    /** Creates an evaluator identity. */
+    createEvaluator(input: Identity): Promise<Evaluator>;
+    /** Reads an evaluator and its published versions. */
+    getEvaluator(id: string): Promise<Evaluator>;
+    /** Lists evaluators. */
+    listEvaluators(page?: RegistryPageOptions): Promise<Page<Evaluator>>;
+    /** Publishes an immutable evaluator version. */
+    publishEvaluatorVersion(id: string, definition: ScorerDefinition): Promise<EvaluatorVersion>;
+    /** Reads a published evaluator version. */
+    getEvaluatorVersion(id: string): Promise<EvaluatorVersion>;
     /** Creates an experiment over a frozen dataset version with pinned scorer versions and a configuration. */
     createExperiment(input: {
         idempotencyKey: string;
@@ -175,6 +208,70 @@ export declare class EvaluationClient {
     listResults(id: string, page?: PageOptions): Promise<Page<ResultSummary>>;
     /** Reads a full stored result. */
     getResult(id: string): Promise<StoredResult>;
+    /** Creates a run using product request fields on the existing v1 path. */
+    createRun(input: {
+        idempotencyKey: string;
+        name: string;
+        evalSetVersionId: string;
+        evaluatorVersionIds: string[];
+        config: JsonValue;
+    }): Promise<{
+        /** Run ID. */
+        id: string;
+        /** ID of the run's scoring pass. */
+        scoringId: string;
+        /** Existing v1 field for the scoring pass ID. */
+        evaluationRunId: string;
+    }>;
+    /** Reads a run with its scoring progress. */
+    getRun(id: string): Promise<Run>;
+    /** Lists a run's cases with their latest executions. */
+    listRunItems(id: string, page?: PageOptions): Promise<Page<ExperimentItem>>;
+    /** Reads one frozen case of a run. */
+    getRunCase(id: string, caseId: string): Promise<RunCase>;
+    /** Starts or replays a target execution for a run case. */
+    startRunExecution(id: string, caseId: string, input: StartExecution): Promise<Execution>;
+    /** Reads one run execution. */
+    getRunExecution(id: string): Promise<Execution>;
+    /** Saves a run execution's outcome. */
+    completeRunExecution(id: string, input: CompleteExecution): Promise<Completion>;
+    /** Marks a run finished. */
+    finishRun(id: string, idempotencyKey: string): Promise<{
+        /** Run ID. */
+        id: string;
+        /** When the run finished. */
+        finishedAt: string;
+    }>;
+    /** Creates a standalone scoring pass over saved subjects. */
+    createScoring(input: {
+        idempotencyKey: string;
+        name: string;
+        subjectIds: string[];
+        evaluatorVersionIds: string[];
+    }): Promise<{
+        /** Scoring pass ID. */
+        id: string;
+    }>;
+    /** Reads a scoring pass and its pinned evaluators. */
+    getScoring(id: string): Promise<Scoring>;
+    /** Lists scoring passes in the project. */
+    listScorings(page?: PageOptions): Promise<Page<ScoringSummary>>;
+    /** Lists the subjects of a scoring pass. */
+    listScoringItems(id: string, page?: PageOptions): Promise<Page<EvaluationItem>>;
+    /** Reads a saved subject with product-named source fields. */
+    getScoringSubject(id: string): Promise<ScoringSubject>;
+    /** Uploads evaluator results for a scoring pass using product request fields. */
+    submitScoringResults(id: string, input: {
+        idempotencyKey: string;
+        results: ScoringResultInput[];
+    }): Promise<{
+        /** Stored result IDs, in input order. */
+        ids: string[];
+    }>;
+    /** Lists result summaries for a scoring pass. */
+    listScoringResults(id: string, page?: PageOptions): Promise<Page<ScoringResultSummary>>;
+    /** Reads a stored evaluator result. */
+    getScoringResult(id: string): Promise<StoredScoringResult>;
     /** Dispatches hosted judge jobs for `llm_judge` pins; check {@link getJudgeBudget} first. */
     createJudgeJobs(id: string, input: {
         idempotencyKey: string;
@@ -230,7 +327,7 @@ export declare class EvaluationClient {
         /** Acknowledged terminal queue state. */
         state: "completed" | "attention";
     }>;
-    /** Lists Scenarios (draft and published) of the project; requires a Tracing and evaluations key. */
+    /** Lists Scenarios (draft and published) of the project; requires a Read and write key. */
     listCaseConversions(page?: PageOptions): Promise<Page<CaseConversionSummary>>;
     /** Reads one Scenario with its immutable publication pins. */
     getCaseConversion(id: string): Promise<CaseConversion>;

package/dist/evals/client.js

@@ -12,6 +12,85 @@ export class HueApiError extends Error {
         this.name = "HueApiError";
     }
 }
+const registryFieldAliases = [
+    ["datasetId", "evalSetId"],
+    ["datasetVersionId", "evalSetVersionId"],
+    ["scorerId", "evaluatorId"],
+    ["scorerVersionId", "evaluatorVersionId"],
+];
+const registryEnvelopes = new Set(["items", "item", "versions", "version"]);
+// Only Hue response envelopes are traversed. Case inputs, metadata, and evaluator
+// definitions are customer JSON and must retain their original field names.
+function productRegistryFields(value) {
+    if (Array.isArray(value))
+        return value.map((item) => productRegistryFields(item));
+    if (value === null || typeof value !== "object")
+        return value;
+    const result = { ...value };
+    for (const key of registryEnvelopes) {
+        if (Object.hasOwn(result, key))
+            result[key] = productRegistryFields(result[key]);
+    }
+    for (const [legacy, product] of registryFieldAliases) {
+        if (Object.hasOwn(result, legacy)) {
+            if (Object.hasOwn(result, product) && result[legacy] !== result[product])
+                throw new HueApiError();
+            result[product] = result[legacy];
+        }
+    }
+    return result;
+}
+const runResponseAliases = [
+    ["datasetId", "evalSetId"],
+    ["datasetName", "evalSetName"],
+    ["datasetDisplayName", "evalSetDisplayName"],
+    ["datasetVersion", "evalSetVersion"],
+    ["datasetVersionId", "evalSetVersionId"],
+    ["datasetVersionIds", "evalSetVersionIds"],
+    ["scorerId", "evaluatorId"],
+    ["scorerName", "evaluatorName"],
+    ["scorerVersion", "evaluatorVersion"],
+    ["scorerVersionId", "evaluatorVersionId"],
+    ["scorerVersionIds", "evaluatorVersionIds"],
+    ["scorerVersions", "evaluatorVersions"],
+    ["evaluationRunId", "scoringId"],
+];
+const runResponseEnvelopes = new Set([
+    "items",
+    "item",
+    "versions",
+    "version",
+    "scorerVersions",
+    "evaluatorVersions",
+]);
+function productRunFields(value, kind) {
+    if (Array.isArray(value))
+        return value.map((item) => productRunFields(item, kind));
+    if (value === null || typeof value !== "object")
+        return value;
+    const result = { ...value };
+    for (const key of runResponseEnvelopes) {
+        if (Object.hasOwn(result, key))
+            result[key] = productRunFields(result[key], kind);
+    }
+    for (const key of ["evaluation", "scoring"]) {
+        if (Object.hasOwn(result, key))
+            result[key] = productRunFields(result[key], "scoring");
+    }
+    const identityAlias = kind === "result" ? ["runId", "scoringId"] : ["experimentId", "runId"];
+    const aliases = [...runResponseAliases, identityAlias];
+    for (const [legacy, product] of aliases) {
+        if (Object.hasOwn(result, legacy)) {
+            if (Object.hasOwn(result, product) &&
+                JSON.stringify(result[legacy]) !== JSON.stringify(result[product]))
+                throw new HueApiError();
+            result[product] = result[legacy];
+        }
+    }
+    if (Object.hasOwn(result, "evaluation") && !Object.hasOwn(result, "scoring"))
+        result.scoring = result.evaluation;
+    return result;
+}
 /**
  * Typed client for Hue's evaluation REST API: datasets, scorers, experiments, executions, runs,
  * results and hosted judge jobs. No implicit mutation retry: callers retain stable idempotency keys
@@ -225,6 +304,58 @@ export class EvaluationClient {
     getScorerVersion(id) {
         return this.request("GET", `/scorer-versions/${uuid(id)}`);
     }
+    /** Creates an eval set using the existing v1 registry path. */
+    async createEvalSet(input) {
+        return productRegistryFields(await this.createDataset(input));
+    }
+    /** Reads an eval set and its versions. */
+    async getEvalSet(id) {
+        return productRegistryFields(await this.getDataset(id));
+    }
+    /** Lists eval sets. */
+    async listEvalSets(page) {
+        return productRegistryFields(await this.listDatasets(page));
+    }
+    /** Creates a draft eval set version, optionally copying cases from another version. */
+    async createEvalSetVersion(id, input = {}) {
+        return productRegistryFields(await this.createDatasetVersion(id, input));
+    }
+    /** Reads an eval set version. */
+    async getEvalSetVersion(id) {
+        return productRegistryFields(await this.getDatasetVersion(id));
+    }
+    /** Lists cases in an eval set version. */
+    async listEvalSetCases(id, page) {
+        return productRegistryFields(await this.listCases(id, page));
+    }
+    /** Adds a case to a draft eval set version at its expected revision. */
+    async addEvalSetCase(id, input) {
+        return productRegistryFields(await this.addCase(id, input));
+    }
+    /** Freezes a draft eval set version at its expected revision. */
+    async freezeEvalSetVersion(id, expectedRevision) {
+        return productRegistryFields(await this.freezeDatasetVersion(id, expectedRevision));
+    }
+    /** Creates an evaluator identity. */
+    async createEvaluator(input) {
+        return productRegistryFields(await this.createScorer(input));
+    }
+    /** Reads an evaluator and its published versions. */
+    async getEvaluator(id) {
+        return productRegistryFields(await this.getScorer(id));
+    }
+    /** Lists evaluators. */
+    async listEvaluators(page) {
+        return productRegistryFields(await this.listScorers(page));
+    }
+    /** Publishes an immutable evaluator version. */
+    async publishEvaluatorVersion(id, definition) {
+        return productRegistryFields(await this.publishScorerVersion(id, definition));
+    }
+    /** Reads a published evaluator version. */
+    async getEvaluatorVersion(id) {
+        return productRegistryFields(await this.getScorerVersion(id));
+    }
     /** Creates an experiment over a frozen dataset version with pinned scorer versions and a configuration. */
     createExperiment(input) {
         return this.request("POST", "/experiments", input);
@@ -376,6 +507,70 @@ export class EvaluationClient {
     getResult(id) {
         return this.request("GET", `/evaluation-results/${uuid(id)}`);
     }
+    /** Creates a run using product request fields on the existing v1 path. */
+    async createRun(input) {
+        return productRunFields(await this.request("POST", "/experiments", input), "run");
+    }
+    /** Reads a run with its scoring progress. */
+    async getRun(id) {
+        return productRunFields(await this.getExperiment(id), "run");
+    }
+    /** Lists a run's cases with their latest executions. */
+    async listRunItems(id, page) {
+        return productRunFields(await this.listExperimentItems(id, page), "run");
+    }
+    /** Reads one frozen case of a run. */
+    async getRunCase(id, caseId) {
+        return productRunFields(await this.getExperimentCase(id, caseId), "run");
+    }
+    /** Starts or replays a target execution for a run case. */
+    async startRunExecution(id, caseId, input) {
+        return productRunFields(await this.startExecution(id, caseId, input), "run");
+    }
+    /** Reads one run execution. */
+    async getRunExecution(id) {
+        return productRunFields(await this.getExecution(id), "run");
+    }
+    /** Saves a run execution's outcome. */
+    async completeRunExecution(id, input) {
+        return productRunFields(await this.completeExecution(id, input), "run");
+    }
+    /** Marks a run finished. */
+    async finishRun(id, idempotencyKey) {
+        return productRunFields(await this.finishExperiment(id, idempotencyKey), "run");
+    }
+    /** Creates a standalone scoring pass over saved subjects. */
+    async createScoring(input) {
+        return productRunFields(await this.request("POST", "/evaluation-runs", input), "scoring");
+    }
+    /** Reads a scoring pass and its pinned evaluators. */
+    async getScoring(id) {
+        return productRunFields(await this.getEvaluationRun(id), "scoring");
+    }
+    /** Lists scoring passes in the project. */
+    async listScorings(page) {
+        return productRunFields(await this.listEvaluationRuns(page), "scoring");
+    }
+    /** Lists the subjects of a scoring pass. */
+    async listScoringItems(id, page) {
+        return productRunFields(await this.listEvaluationItems(id, page), "scoring");
+    }
+    /** Reads a saved subject with product-named source fields. */
+    async getScoringSubject(id) {
+        return productRunFields(await this.getSubject(id), "run");
+    }
+    /** Uploads evaluator results for a scoring pass using product request fields. */
+    async submitScoringResults(id, input) {
+        return productRunFields(await this.request("POST", `/evaluation-runs/${uuid(id)}/results`, input), "result");
+    }
+    /** Lists result summaries for a scoring pass. */
+    async listScoringResults(id, page) {
+        return productRunFields(await this.listResults(id, page), "result");
+    }
+    /** Reads a stored evaluator result. */
+    async getScoringResult(id) {
+        return productRunFields(await this.getResult(id), "result");
+    }
     /** Dispatches hosted judge jobs for `llm_judge` pins; check {@link getJudgeBudget} first. */
     createJudgeJobs(id, input) {
         return this.request("POST", `/evaluation-runs/${uuid(id)}/judge-jobs`, input);
@@ -412,7 +607,7 @@ export class EvaluationClient {
     completeLocalAgentRun(input) {
         return this.request("POST", "/local-agent-worker/runs/complete", input);
     }
-    /** Lists Scenarios (draft and published) of the project; requires a Tracing and evaluations key. */
+    /** Lists Scenarios (draft and published) of the project; requires a Read and write key. */
     listCaseConversions(page) {
         return this.request("GET", `/case-conversions${this.page(page)}`);
     }

package/dist/evals/scenarios.d.ts

@@ -21,9 +21,9 @@ export interface ScenarioPins {
 }
 /** Subset of {@link EvaluationClient} used to resolve Scenario pins. */
 export type ScenarioClient = Pick<EvaluationClient, "listCaseConversions" | "getCaseConversion" | "getDataset" | "getDatasetVersion" | "listDatasets">;
-/** Lists Scenarios of the project; requires a Tracing and evaluations key. */
+/** Lists Scenarios of the project; requires a Read and write key. */
 export declare function listScenarios(client: Pick<EvaluationClient, "listCaseConversions">, page?: PageOptions): Promise<Page<CaseConversionSummary>>;
-/** Reads one Scenario with its publication pins; requires a Tracing and evaluations key. */
+/** Reads one Scenario with its publication pins; requires a Read and write key. */
 export declare function getScenario(client: Pick<EvaluationClient, "getCaseConversion">, id: string): Promise<CaseConversion>;
 /** How a selector was interpreted: a UUID, a Hue URL or a display name. */
 export type ScenarioSelector = {

package/dist/evals/scenarios.js

@@ -1,11 +1,11 @@
 const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 /** Scenarios listed while resolving a name; bounds the registry reads of one selection. */
 const MAX_LISTED = 200;
-/** Lists Scenarios of the project; requires a Tracing and evaluations key. */
+/** Lists Scenarios of the project; requires a Read and write key. */
 export function listScenarios(client, page) {
     return client.listCaseConversions(page);
 }
-/** Reads one Scenario with its publication pins; requires a Tracing and evaluations key. */
+/** Reads one Scenario with its publication pins; requires a Read and write key. */
 export function getScenario(client, id) {
     return client.getCaseConversion(id);
 }

package/dist/evals/types.d.ts

@@ -76,6 +76,21 @@ export interface DatasetCase {
     /** Immutable input-file manifest identity, when files are attached. */
     artifactManifestId?: string | null;
 }
+/** An eval set and its versions. Both field names remain available during v1 compatibility. */
+export type EvalSet = Omit<Dataset, "versions"> & {
+    /** Versions of this eval set. */
+    versions: EvalSetVersion[];
+};
+/** A version of an eval set, with its product field name. */
+export type EvalSetVersion = DatasetVersion & {
+    /** Owning eval set ID. */
+    evalSetId: string;
+};
+/** A stored case in an eval set version. */
+export type EvalSetCase = DatasetCase & {
+    /** Eval set version containing this case. */
+    evalSetVersionId: string;
+};
 /** One pinned input file of a case or subject, as recorded in Hue's immutable manifest. */
 export interface CaseFile {
     /** Hue artifact identity of the pinned bytes. */
@@ -346,6 +361,16 @@ export interface ScorerVersion {
     /** The pinned definition. */
     definition: ScorerDefinition;
 }
+/** An evaluator and its published versions. */
+export type Evaluator = Omit<Scorer, "versions"> & {
+    /** Published versions, when included in the response. */
+    versions?: EvaluatorVersion[];
+};
+/** An immutable published evaluator definition. */
+export type EvaluatorVersion = ScorerVersion & {
+    /** Owning evaluator ID when the server supplies it; older v1 responses may omit it. */
+    evaluatorId?: string;
+};
 /** Final state of a target execution. */
 export type TerminalState = "succeeded" | "error" | "cancelled";
 /** One attempt to run the target for a case. */
@@ -439,6 +464,30 @@ export interface Experiment {
         cancelled: number;
     };
 }
+/** A run over a frozen eval set version and configuration. */
+export type Run = Experiment & {
+    /** Frozen eval set version under test. */
+    evalSetVersionId: string;
+    /** Scoring pass created for this run. */
+    scoring: Scoring;
+};
+/** One frozen case in a run. */
+export type RunCase = ExperimentCase & {
+    /** Source eval set version ID. */
+    evalSetVersionId: string;
+};
+/** A scoring pass over saved subjects with pinned evaluators. */
+export type Scoring = EvaluationRun & {
+    /** Evaluator versions pinned to this scoring pass. */
+    evaluatorVersions: EvaluatorVersion[];
+    /** Linked run ID, or null for standalone scoring. */
+    runId?: string | null;
+};
+/** One row in the project's scoring list. */
+export type ScoringSummary = EvaluationRunSummary & {
+    /** Linked run ID, or null for standalone scoring. */
+    runId: string | null;
+};
 /** A sanitized error type with an optional bounded message. */
 export interface TypedError {
     /** Stable error type. */
@@ -549,6 +598,13 @@ export interface Subject {
     /** The target's declared primary generated artifact, or `null`. */
     primaryArtifactId?: string | null;
 }
+/** An immutable saved subject with product-named source fields. */
+export type ScoringSubject = Subject & {
+    /** Source eval set version ID. */
+    evalSetVersionId: string;
+    /** Source run ID. */
+    runId: string;
+};
 /** A reported metric value. */
 export interface Metric {
     /** Declared metric name. */
@@ -588,6 +644,27 @@ export type Result = Score & {
     /** Source digest of the local scorer, for `local_code` pins. */
     sourceDigest?: string;
 };
+/** A score uploaded through the product-named scoring method. */
+export type ScoringResultInput = Score & {
+    /** Scoring item this result belongs to. */
+    evaluationItemId: string;
+    /** Evaluator version that produced the score. */
+    evaluatorVersionId: string;
+    /** Source digest for a local code evaluator. */
+    sourceDigest?: string;
+};
+/** A result as listed by {@link EvaluationClient.listScoringResults}. */
+export type ScoringResultSummary = ResultSummary & {
+    /** Evaluator version that produced this result. */
+    evaluatorVersionId: string;
+};
+/** A full stored result from {@link EvaluationClient.getScoringResult}. */
+export type StoredScoringResult = StoredResult & {
+    /** Scoring pass containing this result. */
+    scoringId: string;
+    /** Evaluator version that produced this result. */
+    evaluatorVersionId: string;
+};
 /** What a local scorer callback receives. */
 export interface ScoreContext {
     /** Case inputs. */

package/dist/version.d.ts

@@ -1,2 +1,2 @@
 /** Package version shared by the instrumentation scope and the export User-Agent. */
-export declare const sdkVersion = "0.5.0";
+export declare const sdkVersion = "0.6.0";

package/dist/version.js

@@ -1,3 +1,3 @@
 // Generated by scripts/write-version.mjs from package.json; do not edit by hand.
 /** Package version shared by the instrumentation scope and the export User-Agent. */
-export const sdkVersion = "0.5.0";
+export const sdkVersion = "0.6.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hue-run/sdk",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "private": false,
   "license": "MIT",
   "publishConfig": {