@hue-run/sdk 0.4.2 → 0.5.1

package/dist/evals/runner.d.ts

@@ -1,7 +1,7 @@
 import type { HueClient } from "../client.js";
 import type { HueSpan } from "../types.js";
 import { EvaluationClient } from "./client.js";
-import type { ExperimentCase, JsonValue, LocalScorer } from "./types.js";
+import { TargetResult, type ExperimentCase, type JsonValue, type LocalFile, type LocalScorer } from "./types.js";
 /**
  * Thrown when a case has a started attempt without a saved outcome. The runner never reruns the
  * target; inspect the execution and authorize a new attempt explicitly through `startExecution`.
@@ -46,12 +46,36 @@ interface RunnerOptions {
     persistResultContent: boolean;
     /** Local callbacks bound to `local_code` scorer pins by digest. */
     scorers?: LocalScorer[];
+    /** Leave pinned `local_code` versions this process has no binding for to another executor
+     * (for example a Hue-operated grading worker that owns the evaluator source) instead of
+     * refusing the run. Their IDs are reported in `deferredScorerVersionIds`. */
+    deferUnboundLocalScorers?: boolean;
     /** Cases in flight at once, 1–16. Default 1. */
     concurrency?: number;
     /** Deadline for JSON Schema scoring in its worker, 100–60000 ms. Default 2000. */
     schemaTimeoutMillis?: number;
-    /** Resolve sealed world evidence for local scoring and historical rescoring. */
-    environmentEvidence?: "required";
+    /** Resolve sealed evidence by execution identity. when_pinned skips known direct cases. */
+    environmentEvidence?: "required" | "when_pinned";
+    /** Verified input copies and generated files live here; defaults to `<checkpointDirectory>/files`.
+     * Generated files are always saved and uploaded: they are the execution's evidence. */
+    filesDirectory?: string;
+}
+/** Immutable case context passed to a direct experiment target. */
+export interface RunExperimentTargetContext {
+    /** Frozen experiment configuration, validated as JSON. */
+    config: JsonValue;
+    /** The frozen case, cloned before invocation. */
+    item: ExperimentCase;
+    /** The `hue.experiment.case` span this attempt runs inside. */
+    span: HueSpan;
+    /** `executionId` identifies this attempt. Deriving an environment run's idempotency
+     * key from it keeps a resumed upload bound to the same world. */
+    executionId: string;
+    /** Verified copies of the case's pinned input files meant for the agent. Evaluator-only
+     * organization templates are withheld, as in the managed protocol. */
+    files: LocalFile[];
+    /** A private scratch directory for this case; return generated files with `withFiles`. */
+    outputDirectory: string;
 }
 /** Options for {@link runExperiment}. */
 export interface RunExperimentOptions extends RunnerOptions {
@@ -69,13 +93,9 @@ export interface RunExperimentOptions extends RunnerOptions {
         /** Why evidence is omitted, up to 4000 characters. */
         reason: string;
     };
-    /** Runs the application for one frozen case; return the output, or `undefined` when unavailable. */
-    target(inputs: JsonValue, context: {
-        config: JsonValue;
-        item: ExperimentCase;
-        span: HueSpan;
-        executionId: string;
-    }): JsonValue | undefined | Promise<JsonValue | undefined>;
+    /** Runs the application for one frozen case; return the output, `withFiles(output, files)`
+     * when it generated files, or `undefined` when unavailable. */
+    target(inputs: JsonValue, context: RunExperimentTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
 }
 /** Options for {@link rescore}. */
 export interface RescoreOptions extends RunnerOptions {

package/dist/evals/runner.js

@@ -1,10 +1,15 @@
 import { randomUUID } from "node:crypto";
+import { mkdir } from "node:fs/promises";
+import { join, resolve } from "node:path";
 import { ROOT_CONTEXT } from "@opentelemetry/api";
 import { HueExportError } from "../transport.js";
+import { HueApiError } from "./client.js";
 import { loadEnvironmentEvidence } from "./environment-evidence.js";
 import { CheckpointStore } from "./checkpoint.js";
+import { downloadCaseFiles, localOutputFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
 import { json, uuid } from "./json.js";
-import { persistedScore, scoreLocally, validateScorerBindings, isLocallyExecutable, } from "./scorers.js";
+import { executableHere, persistedScore, scoreLocally, validateScorerBindings } from "./scorers.js";
+import { TargetResult, } from "./types.js";
 /**
  * Thrown when a case has a started attempt without a saved outcome. The runner never reruns the
  * target; inspect the execution and authorize a new attempt explicitly through `startExecution`.
@@ -52,9 +57,26 @@ export class TargetOutcomeUncertainError extends Error {
         this.name = "TargetOutcomeUncertainError";
     }
 }
+/** What {@link RunExperimentOptions.target} receives for one frozen case. */
+/**
+ * Pinned inputs this process must download. The target receives the agent-visible roles;
+ * scorer-only roles (organization templates, evaluator references such as a legal corpus) are
+ * fetched only when a bound code evaluator will grade here. A customer running `hue eval` with
+ * grading deferred to Hue never receives them.
+ */
+function neededInputFiles(files, versions, options) {
+    if (!files?.length)
+        return [];
+    const codeEvaluatorRunsHere = versions.some((version) => version.definition.kind === "local_code" && executableHere(version.definition, options));
+    return codeEvaluatorRunsHere
+        ? files
+        : files.filter((file) => targetFileRoles.includes(file.role));
+}
 function settings(options) {
-    if (options.environmentEvidence !== undefined && options.environmentEvidence !== "required")
-        throw new TypeError("environmentEvidence must be required when supplied");
+    if (options.environmentEvidence !== undefined &&
+        options.environmentEvidence !== "required" &&
+        options.environmentEvidence !== "when_pinned")
+        throw new TypeError("environmentEvidence must be required or when_pinned when supplied");
     if (typeof options.persistResultContent !== "boolean")
         throw new TypeError("Choose persistResultContent explicitly: true or false");
     const concurrency = options.concurrency ?? 1;
@@ -65,15 +87,15 @@ function settings(options) {
         throw new RangeError("schemaTimeoutMillis must be 100–60000");
     return concurrency;
 }
-async function allPages(page) {
+async function allPages(page, maximum = 5000) {
     const items = [];
     const cursors = new Set();
     let after;
     do {
         const response = await page(after);
         items.push(...response.items);
-        if (items.length > 5000)
-            throw new RangeError("Runner supports at most 5000 items");
+        if (items.length > maximum)
+            throw new RangeError(`Runner supports at most ${maximum} items`);
         if (response.nextCursor === null)
             break;
         after = uuid(response.nextCursor);
@@ -102,24 +124,28 @@ async function pool(items, concurrency, execute) {
     if (failures.length)
         throw new AggregateError(failures, "Multiple case operations failed; resume uses saved outcomes");
 }
-async function scoresFor(versions, context, options, executionId) {
+async function scoresFor(versions, context, options, executionId, hasEnvironment = true) {
     const scores = [];
     let environmentUnavailable = false;
-    if (options.environmentEvidence === "required" &&
-        versions.some((version) => isLocallyExecutable(version.definition))) {
+    if ((options.environmentEvidence === "required" ||
+        (options.environmentEvidence === "when_pinned" && hasEnvironment)) &&
+        versions.some((version) => executableHere(version.definition, options))) {
         try {
             context = {
                 ...context,
                 environment: await loadEnvironmentEvidence(options.client, executionId),
             };
         }
-        catch {
-            environmentUnavailable = true;
+        catch (error) {
+            // Generic targets may attach a world independently of the case pin. Preserve
+            // required evidence lookups, including the optional 404 for an unpinned case.
+            if (!(error instanceof HueApiError && error.status === 404 && !hasEnvironment))
+                environmentUnavailable = true;
         }
     }
     for (const version of versions) {
         // Every pin without a local implementation belongs to another executor.
-        if (!isLocallyExecutable(version.definition))
+        if (!executableHere(version.definition, options))
             continue;
         const score = persistedScore(environmentUnavailable && version.definition.kind === "local_code"
             ? { state: "error", error: { type: "EnvironmentEvidenceUnavailable" } }
@@ -137,25 +163,35 @@ async function scoresFor(versions, context, options, executionId) {
     }
     return scores;
 }
-async function uploadScores(options, runId, scores, save, versions) {
+async function uploadScores(options, runId, scores, save, versions, resolveConflict) {
     // A previous SDK may have checkpointed a placeholder for an unknown kind.
     // Keep its evidence intact, but never upload or report it as a local result.
     const local = scores.filter((score) => {
         const pin = versions.find((version) => version.id === score.payload.scorerVersionId);
         if (!pin)
             throw new Error("Saved result references an unpinned scorer version");
-        return isLocallyExecutable(pin.definition);
+        return executableHere(pin.definition, options);
     });
     for (const score of local) {
         if (score.receipt)
             continue;
         if (!score.payload.evaluationItemId)
             throw new Error("Scoring requires the acknowledged evaluation item identity");
-        const result = await options.client.submitResults(runId, {
-            idempotencyKey: score.key,
-            results: [score.payload],
-        });
-        score.receipt = result.ids;
+        try {
+            const result = await options.client.submitResults(runId, {
+                idempotencyKey: score.key,
+                results: [score.payload],
+            });
+            score.receipt = result.ids;
+        }
+        catch (error) {
+            const receipt = error instanceof HueApiError && error.status === 409
+                ? await resolveConflict?.(score)
+                : undefined;
+            if (!receipt)
+                throw error;
+            score.receipt = receipt;
+        }
         await save();
     }
     return local.flatMap((score) => score.receipt ?? []);
@@ -189,7 +225,8 @@ export async function runExperiment(options) {
     if (!version.frozenAt || !version.contentDigest)
         throw new Error("Experiment dataset must be frozen");
     const versions = experiment.evaluation.scorerVersions;
-    validateScorerBindings(versions, options.scorers);
+    if (!options.deferUnboundLocalScorers)
+        validateScorerBindings(versions, options.scorers);
     const items = await allPages((after) => options.client.listExperimentItems(experiment.id, { after }));
     if (items.length !== experiment.caseCount)
         throw new Error("Frozen experiment case count differs from API items");
@@ -214,14 +251,70 @@ export async function runExperiment(options) {
         subjectIds: [],
         resultIds: [],
         deferredScorerVersionIds: versions
-            .filter((version) => !isLocallyExecutable(version.definition))
+            .filter((version) => !executableHere(version.definition, options))
             .map((version) => version.id),
     };
     try {
+        const filesRoot = resolve(options.filesDirectory ?? join(options.checkpointDirectory, "files"));
+        const sanitize = (message) => message.slice(0, 4000).toWellFormed().replaceAll("\u0000", "");
+        const errorPayload = (error) => ({
+            type: "TargetError",
+            ...(options.persistResultContent && error instanceof Error
+                ? { message: sanitize(error.message) }
+                : {}),
+        });
+        /** Publish staged files, score with every verified file on disk and save the completion. */
+        async function prepare(file, saved, frozenCase, caseDirectory, 
+        /** The target's output, in memory when result content is not persisted. */
+        output) {
+            const needed = neededInputFiles(frozenCase.inputFiles, versions, options);
+            const inputs = needed.length
+                ? await downloadCaseFiles(options.client, needed, join(caseDirectory, "inputs"))
+                : [];
+            await uploadOutputFiles(options.client, saved.executionId, saved.files, () => store.write(file, saved));
+            const outputs = localOutputFiles(saved.files);
+            const primary = outputs.find((item) => item.primary);
+            const scores = await scoresFor(versions, {
+                inputs: frozenCase.inputs,
+                hasExpected: frozenCase.hasExpected,
+                ...(frozenCase.hasExpected ? { expected: frozenCase.expected } : {}),
+                metadata: frozenCase.metadata,
+                hasOutput: saved.hasOutput,
+                ...(saved.hasOutput ? { output: output } : {}),
+                executionState: saved.state,
+                ...(inputs.length || outputs.length ? { files: [...inputs, ...outputs] } : {}),
+            }, options, saved.executionId, Boolean(frozenCase.environmentVersionId));
+            const complete = {
+                idempotencyKey: randomUUID(),
+                state: saved.state,
+                ...(options.persistResultContent && saved.hasOutput ? { output: output } : {}),
+                ...(saved.error ? { error: saved.error } : {}),
+                ...(outputs.length
+                    ? {
+                        artifactIds: outputs.map((item) => item.artifactId),
+                        ...(primary ? { primaryArtifactId: primary.artifactId } : {}),
+                    }
+                    : {}),
+                traceEvidence: options.traceEvidence.mode,
+                ...(options.traceEvidence.mode === "omit"
+                    ? { omissionReason: options.traceEvidence.reason }
+                    : {}),
+            };
+            const prepared = {
+                stage: "prepared",
+                executionId: saved.executionId,
+                complete,
+                scores,
+                exportState: "pending",
+            };
+            await store.write(file, prepared);
+            return prepared;
+        }
         await pool(items, concurrency, async (item) => {
             const file = `case-${uuid(item.id)}`;
+            const caseDirectory = join(filesRoot, `case-${uuid(item.id)}`);
             let checkpoint = await store.read(file);
-            if (checkpoint && checkpoint.stage !== "prepared") {
+            if (checkpoint && checkpoint.stage !== "prepared" && checkpoint.stage !== "uploading") {
                 if (checkpoint.stage === "serialization_failed")
                     throw new OutcomeSerializationError(checkpoint.executionId);
                 const execution = checkpoint.stage === "starting"
@@ -232,14 +325,31 @@ export async function runExperiment(options) {
                     : await options.client.getExecution(checkpoint.executionId);
                 throw new UncertainExecutionError(item.id, execution.id);
             }
+            if (checkpoint?.stage === "uploading") {
+                // The target finished and its files are staged: publish and score them without a
+                // second invocation. Metadata-only mode discarded the output, so its outcome is lost.
+                if (checkpoint.hasOutput && checkpoint.output === undefined)
+                    throw new UncertainExecutionError(item.id, checkpoint.executionId);
+                const frozenCase = await options.client.getExperimentCase(experiment.id, item.id);
+                checkpoint = await prepare(file, checkpoint, frozenCase, caseDirectory, checkpoint.output);
+            }
             if (!checkpoint) {
                 if (item.execution)
                     throw new UncertainExecutionError(item.id, item.execution.id);
                 const frozenCase = await options.client.getExperimentCase(experiment.id, item.id);
                 if (frozenCase.datasetVersionId !== version.id)
                     throw new Error("Case is not from the pinned dataset version");
+                // Validate before creating a remote execution. SDK/input failures are not
+                // target failures and cannot consume a case's execution slot.
                 const targetInputs = json(frozenCase.inputs);
                 const targetConfig = json(experiment.config);
+                // Pinned input files are verified on disk before an execution exists for the same reason.
+                const needed = neededInputFiles(frozenCase.inputFiles, versions, options);
+                const inputFiles = needed.length
+                    ? await downloadCaseFiles(options.client, needed, join(caseDirectory, "inputs"))
+                    : [];
+                const outputDirectory = join(caseDirectory, "work");
+                await mkdir(outputDirectory, { recursive: true, mode: 0o700 });
                 const failureSequenceBefore = options.hue.transport.getFailureSequence();
                 checkpoint = await options.hue.withSpan("hue.experiment.case", async (span) => {
                     const start = {
@@ -255,14 +365,23 @@ export async function runExperiment(options) {
                     await store.write(file, { stage: "running", executionId: execution.id });
                     let state = "succeeded";
                     let output;
+                    let generated;
                     let targetError;
                     try {
-                        output = await options.target(targetInputs, {
+                        const result = await options.target(targetInputs, {
                             config: targetConfig,
                             item: structuredClone(frozenCase),
                             span,
                             executionId: execution.id,
+                            files: structuredClone(inputFiles.filter((entry) => targetFileRoles.includes(entry.role))),
+                            outputDirectory,
                         });
+                        if (result instanceof TargetResult) {
+                            output = result.output;
+                            generated = result.files;
+                        }
+                        else
+                            output = result;
                     }
                     catch (error) {
                         if (error instanceof TargetOutcomeUncertainError)
@@ -286,48 +405,36 @@ export async function runExperiment(options) {
                     }
                     if (output !== undefined)
                         span.setOutput(output);
-                    const scores = await scoresFor(versions, {
-                        inputs: frozenCase.inputs,
-                        hasExpected: frozenCase.hasExpected,
-                        ...(frozenCase.hasExpected ? { expected: frozenCase.expected } : {}),
-                        metadata: frozenCase.metadata,
-                        hasOutput: output !== undefined,
-                        ...(output !== undefined ? { output } : {}),
-                        executionState: state,
-                    }, options, execution.id);
-                    const complete = {
-                        idempotencyKey: randomUUID(),
+                    let staged = [];
+                    // An empty declared list (a direct case answered only through stdout, or
+                    // `withFiles(output, [])`) means no generated files, not a missing-files error.
+                    if (generated !== undefined && generated.length > 0 && state === "succeeded") {
+                        try {
+                            staged = await stageOutputFiles(generated, join(caseDirectory, "outputs"));
+                        }
+                        catch (error) {
+                            // Files the target declared but did not deliver are its own failure; the
+                            // outcome is still saved instead of leaving the execution uncertain.
+                            if (!(error instanceof OutputFileError))
+                                throw error;
+                            state = "error";
+                            targetError = error;
+                            options.hue.recordError(span.span, error);
+                        }
+                    }
+                    const uploading = {
+                        stage: "uploading",
+                        executionId: execution.id,
                         state,
+                        hasOutput: output !== undefined,
                         ...(options.persistResultContent && output !== undefined ? { output } : {}),
-                        ...(state === "error"
-                            ? {
-                                error: {
-                                    type: "TargetError",
-                                    ...(options.persistResultContent && targetError instanceof Error
-                                        ? {
-                                            message: targetError.message
-                                                .slice(0, 4000)
-                                                .toWellFormed()
-                                                .replaceAll("\u0000", ""),
-                                        }
-                                        : {}),
-                                },
-                            }
-                            : {}),
-                        traceEvidence: options.traceEvidence.mode,
-                        ...(options.traceEvidence.mode === "omit"
-                            ? { omissionReason: options.traceEvidence.reason }
-                            : {}),
+                        ...(state === "error" ? { error: errorPayload(targetError) } : {}),
+                        files: staged,
                     };
-                    const prepared = {
-                        stage: "prepared",
-                        executionId: execution.id,
-                        complete,
-                        scores,
-                        exportState: "pending",
-                    };
-                    await store.write(file, prepared);
-                    return prepared;
+                    // Without persisted result content a restart cannot reconstruct the outcome; the
+                    // saved stage then reports the execution as uncertain instead of guessing.
+                    await store.write(file, uploading);
+                    return prepare(file, uploading, frozenCase, caseDirectory, output);
                 }, {
                     parentContext: ROOT_CONTEXT,
                     input: frozenCase.inputs,
@@ -385,7 +492,8 @@ export async function rescore(options) {
         options.client.checkConnection(),
         options.client.getEvaluationRun(options.runId),
     ]);
-    validateScorerBindings(run.scorerVersions, options.scorers);
+    if (!options.deferUnboundLocalScorers)
+        validateScorerBindings(run.scorerVersions, options.scorers);
     const items = await allPages((after) => options.client.listEvaluationItems(run.id, { after }));
     if (items.length !== run.itemCount)
         throw new Error("Frozen evaluation run item count differs from API items");
@@ -405,16 +513,40 @@ export async function rescore(options) {
         subjectIds: [],
         resultIds: [],
         deferredScorerVersionIds: run.scorerVersions
-            .filter((version) => !isLocallyExecutable(version.definition))
+            .filter((version) => !executableHere(version.definition, options))
             .map((version) => version.id),
     };
+    const filesRoot = resolve(options.filesDirectory ?? join(options.checkpointDirectory, "files"));
     try {
+        // Grade again can also schedule Hue's built-in checks. Terminal results are immutable:
+        // preserve their receipts, including when another executor wins during local scoring.
+        const resultKey = (itemId, scorerVersionId) => `${itemId}:${scorerVersionId}`;
+        const receipts = new Map();
+        const refreshReceipts = async () => {
+            const results = await allPages((after) => options.client.listResults(run.id, { after }), 5000 * 32);
+            for (const result of results)
+                receipts.set(resultKey(result.itemId, result.scorerVersionId), result.id);
+        };
+        await refreshReceipts();
         await pool(items, concurrency, async (item) => {
             const file = `item-${uuid(item.id)}`;
+            const pending = run.scorerVersions.filter((version) => !receipts.has(resultKey(item.id, version.id)));
             let saved = await store.read(file);
+            if (!saved && !pending.some((version) => executableHere(version.definition, options)))
+                saved = { scores: [] };
             if (!saved) {
                 const subject = await options.client.getSubject(item.subjectId);
-                const scores = await scoresFor(run.scorerVersions, {
+                // The frozen manifest holds the case inputs and the target's documents; a code
+                // evaluator grades the saved bytes, verified against their pinned identities.
+                // Built-ins grade the JSON output alone, so no file — least of all a scorer-only
+                // organization template or evaluator reference — is fetched onto this machine for them.
+                const codeEvaluatorRunsHere = pending.some((version) => version.definition.kind === "local_code" && executableHere(version.definition, options));
+                const files = codeEvaluatorRunsHere && subject.files?.length
+                    ? await downloadCaseFiles(options.client, subject.files, join(filesRoot, `subject-${uuid(item.subjectId)}`), subject.primaryArtifactId)
+                    : [];
+                // Older servers omit the world pin; keep their previous behaviour.
+                const hasEnvironment = subject.environmentVersionId === undefined ? true : subject.environmentVersionId !== null;
+                const scores = await scoresFor(pending, {
                     inputs: subject.inputs,
                     hasOutput: subject.hasOutput,
                     hasExpected: subject.hasExpected,
@@ -422,16 +554,32 @@ export async function rescore(options) {
                     ...(subject.hasExpected ? { expected: subject.expected } : {}),
                     metadata: subject.metadata,
                     executionState: subject.executionState,
-                }, options, subject.executionId);
+                    ...(files.length ? { files } : {}),
+                }, options, subject.executionId, hasEnvironment);
                 for (const score of scores)
                     score.payload.evaluationItemId = item.id;
                 saved = { scores };
                 await store.write(file, saved);
             }
             const current = saved;
-            const results = await uploadScores(options, run.id, current.scores, () => store.write(file, current), run.scorerVersions);
+            for (const score of current.scores) {
+                const receipt = receipts.get(resultKey(item.id, score.payload.scorerVersionId));
+                if (receipt)
+                    score.receipt = [receipt];
+            }
+            const results = await uploadScores(options, run.id, current.scores, () => store.write(file, current), run.scorerVersions, async (score) => {
+                await refreshReceipts();
+                const receipt = receipts.get(resultKey(item.id, score.payload.scorerVersionId));
+                return receipt ? [receipt] : undefined;
+            });
             report.subjectIds.push(item.subjectId);
-            report.resultIds.push(...results);
+            report.resultIds.push(...new Set([
+                ...results,
+                ...run.scorerVersions.flatMap((version) => {
+                    const receipt = receipts.get(resultKey(item.id, version.id));
+                    return receipt && executableHere(version.definition, options) ? [receipt] : [];
+                }),
+            ]));
         });
         return report;
     }

package/dist/evals/scenarios.d.ts

@@ -0,0 +1,79 @@
+import type { EvaluationClient } from "./client.js";
+import type { CaseConversion, CaseConversionSummary, Page, PageOptions } from "./types.js";
+/** Immutable pins resolved from a published Scenario or a saved eval set. */
+export interface ScenarioPins {
+    /** Scenario ID, or `null` when the pins came from an eval set. */
+    scenarioId: string | null;
+    /** Display name of the dataset behind the pins. */
+    name: string;
+    /** Dataset holding the pinned version. */
+    datasetId: string;
+    /** Pinned dataset version; frozen only when `saved` is true. */
+    datasetVersionId: string;
+    /** Pinned scorer versions; a Scenario pins exactly one. */
+    scorerVersionIds: string[];
+    /** Pinned simulated-world version, or `null` when the selection does not pin one. */
+    environmentVersionId: string | null;
+    /** Whether the dataset version is frozen (`frozenAt` is set); experiments require a saved version. */
+    saved: boolean;
+    /** Current optimistic revision of the dataset version, needed to freeze an unsaved draft. */
+    revision: number;
+}
+/** Subset of {@link EvaluationClient} used to resolve Scenario pins. */
+export type ScenarioClient = Pick<EvaluationClient, "listCaseConversions" | "getCaseConversion" | "getDataset" | "getDatasetVersion" | "listDatasets">;
+/** Lists Scenarios of the project; requires a Read and write key. */
+export declare function listScenarios(client: Pick<EvaluationClient, "listCaseConversions">, page?: PageOptions): Promise<Page<CaseConversionSummary>>;
+/** Reads one Scenario with its publication pins; requires a Read and write key. */
+export declare function getScenario(client: Pick<EvaluationClient, "getCaseConversion">, id: string): Promise<CaseConversion>;
+/** How a selector was interpreted: a UUID, a Hue URL or a display name. */
+export type ScenarioSelector = {
+    /** The selector is an ID or a URL naming one. */
+    kind: "id";
+    /** Lowercase UUID. */
+    id: string;
+} | {
+    /** The selector is a display name to match. */
+    kind: "name";
+    /** Trimmed name. */
+    name: string;
+};
+/**
+ * Interprets a selector as a UUID, a Hue URL containing `/<segment>/<uuid>` (for example
+ * `/scenarios/<uuid>`, query parameters ignored) or a display name.
+ *
+ * @throws TypeError for an empty selector or a URL without the expected segment.
+ */
+export declare function parseScenarioSelector(selector: string, segments?: string[]): ScenarioSelector;
+/** A candidate with a display name, such as a dataset or Scenario. */
+export interface NamedCandidate {
+    /** Display name compared case-insensitively. */
+    name: string;
+}
+/** Outcome of {@link matchByName}. */
+export interface NameMatch<T extends NamedCandidate> {
+    /** Candidates that matched; one means an unambiguous selection. */
+    matches: T[];
+    /** Whether the matches are exact (case-insensitive) rather than prefix or substring matches. */
+    exact: boolean;
+}
+/** Case-insensitive exact matches first, then unique prefix/substring matches. */
+export declare function matchByName<T extends NamedCandidate>(candidates: T[], name: string): NameMatch<T>;
+/**
+ * Resolves a published Scenario's immutable pins from its ID, its Hue URL or its name. A name
+ * matches the dataset name of published Scenarios case-insensitively: exact matches first, then
+ * a unique prefix or substring.
+ *
+ * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft.
+ */
+export declare function resolveScenarioPins(client: ScenarioClient, selector: string): Promise<ScenarioPins>;
+/**
+ * Resolves an eval set (dataset) by ID, Hue URL or name to its latest saved version. When the
+ * set has no saved version, the latest draft is returned with `saved: false` so a caller can
+ * freeze it explicitly. Scorer versions are not pinned by a set; supply them separately.
+ *
+ * @throws Error when no set matches, several match, or the set has no versions.
+ */
+export declare function resolveEvalSetPins(client: ScenarioClient, selector: string, options?: {
+    /** Scorer versions to pin alongside the dataset version. */
+    scorerVersionIds?: string[];
+}): Promise<ScenarioPins>;