@hue-run/sdk 0.4.2 → 0.5.1

package/dist/evals/scenarios.js

@@ -0,0 +1,169 @@
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+/** Scenarios listed while resolving a name; bounds the registry reads of one selection. */
+const MAX_LISTED = 200;
+/** Lists Scenarios of the project; requires a Read and write key. */
+export function listScenarios(client, page) {
+    return client.listCaseConversions(page);
+}
+/** Reads one Scenario with its publication pins; requires a Read and write key. */
+export function getScenario(client, id) {
+    return client.getCaseConversion(id);
+}
+/**
+ * Interprets a selector as a UUID, a Hue URL containing `/<segment>/<uuid>` (for example
+ * `/scenarios/<uuid>`, query parameters ignored) or a display name.
+ *
+ * @throws TypeError for an empty selector or a URL without the expected segment.
+ */
+export function parseScenarioSelector(selector, segments = ["scenarios"]) {
+    const value = selector.trim();
+    if (!value)
+        throw new TypeError("A Scenario selector is required");
+    if (UUID.test(value))
+        return { kind: "id", id: value.toLowerCase() };
+    if (/^https?:\/\//i.test(value)) {
+        let url;
+        try {
+            url = new URL(value);
+        }
+        catch {
+            throw new TypeError("The selector is not a valid URL");
+        }
+        const parts = url.pathname.split("/");
+        for (let index = 0; index < parts.length - 1; index++) {
+            const next = parts[index + 1];
+            if (segments.includes(parts[index]) && UUID.test(next))
+                return { kind: "id", id: next.toLowerCase() };
+        }
+        throw new TypeError(`The URL does not contain /${segments.join("|")}/<id>; paste the page URL or the ID`);
+    }
+    return { kind: "name", name: value };
+}
+/** Case-insensitive exact matches first, then unique prefix/substring matches. */
+export function matchByName(candidates, name) {
+    const wanted = name.trim().toLowerCase();
+    const exact = candidates.filter((candidate) => candidate.name.trim().toLowerCase() === wanted);
+    if (exact.length)
+        return { matches: exact, exact: true };
+    const prefix = candidates.filter((candidate) => candidate.name.trim().toLowerCase().startsWith(wanted));
+    if (prefix.length)
+        return { matches: prefix, exact: false };
+    return {
+        matches: candidates.filter((candidate) => candidate.name.toLowerCase().includes(wanted)),
+        exact: false,
+    };
+}
+async function listPublishedScenarios(client) {
+    const items = [];
+    let after;
+    for (;;) {
+        const page = await client.listCaseConversions({ after, limit: 100 });
+        items.push(...page.items.filter((item) => item.status === "published"));
+        if (!page.nextCursor || items.length >= MAX_LISTED)
+            return items.slice(0, MAX_LISTED);
+        after = page.nextCursor;
+    }
+}
+const describe = (candidates) => candidates.map((candidate) => `${candidate.name} (${candidate.id})`).join(", ");
+async function pinsFromScenario(client, scenario, dataset) {
+    if (!scenario.publication)
+        throw new Error(`Scenario ${scenario.id} is a draft without published pins; publish it in Hue first`);
+    const version = await client.getDatasetVersion(scenario.publication.datasetVersionId);
+    return {
+        scenarioId: scenario.id,
+        name: dataset.name,
+        datasetId: scenario.publication.datasetId,
+        datasetVersionId: version.id,
+        scorerVersionIds: [scenario.publication.scorerVersionId],
+        environmentVersionId: scenario.publication.environmentVersionId,
+        saved: version.frozenAt !== null,
+        revision: version.revision,
+    };
+}
+/**
+ * Resolves a published Scenario's immutable pins from its ID, its Hue URL or its name. A name
+ * matches the dataset name of published Scenarios case-insensitively: exact matches first, then
+ * a unique prefix or substring.
+ *
+ * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft.
+ */
+export async function resolveScenarioPins(client, selector) {
+    const parsed = parseScenarioSelector(selector);
+    if (parsed.kind === "id") {
+        const scenario = await client.getCaseConversion(parsed.id);
+        if (!scenario.publication)
+            throw new Error(`Scenario ${scenario.id} is a draft without published pins; publish it in Hue first`);
+        return pinsFromScenario(client, scenario, await client.getDataset(scenario.publication.datasetId));
+    }
+    const published = await listPublishedScenarios(client);
+    const datasets = new Map();
+    const candidates = [];
+    for (const summary of published) {
+        const scenario = await client.getCaseConversion(summary.id);
+        if (!scenario.publication)
+            continue;
+        let dataset = datasets.get(scenario.publication.datasetId);
+        if (!dataset) {
+            dataset = await client.getDataset(scenario.publication.datasetId);
+            datasets.set(dataset.id, dataset);
+        }
+        candidates.push({ name: dataset.name, id: scenario.id, scenario, dataset });
+    }
+    const { matches } = matchByName(candidates, parsed.name);
+    if (matches.length === 1)
+        return pinsFromScenario(client, matches[0].scenario, matches[0].dataset);
+    if (matches.length > 1)
+        throw new Error(`Several published Scenarios match "${parsed.name}"; pass an ID or URL instead: ${describe(matches)}`);
+    throw new Error(candidates.length
+        ? `No published Scenario matches "${parsed.name}". Published Scenarios: ${describe(candidates)}`
+        : `No published Scenario matches "${parsed.name}"; publish one in Hue first`);
+}
+/**
+ * Resolves an eval set (dataset) by ID, Hue URL or name to its latest saved version. When the
+ * set has no saved version, the latest draft is returned with `saved: false` so a caller can
+ * freeze it explicitly. Scorer versions are not pinned by a set; supply them separately.
+ *
+ * @throws Error when no set matches, several match, or the set has no versions.
+ */
+export async function resolveEvalSetPins(client, selector, options = {}) {
+    const parsed = parseScenarioSelector(selector, ["datasets", "eval-sets", "evalsets", "sets"]);
+    let dataset;
+    if (parsed.kind === "id")
+        dataset = await client.getDataset(parsed.id);
+    else {
+        const candidates = [];
+        let after;
+        for (;;) {
+            const page = await client.listDatasets({ after, limit: 100 });
+            candidates.push(...page.items.filter((item) => !item.archivedAt));
+            if (!page.nextCursor || candidates.length >= MAX_LISTED)
+                break;
+            after = page.nextCursor;
+        }
+        // An exact slug is the stable handle scripts and agents pass; names are matched after it.
+        const wanted = parsed.name.trim().toLowerCase();
+        const bySlug = candidates.filter((item) => item.slug?.toLowerCase() === wanted);
+        const { matches } = bySlug.length ? { matches: bySlug } : matchByName(candidates, parsed.name);
+        if (matches.length > 1)
+            throw new Error(`Several eval sets match "${parsed.name}"; pass an ID or URL instead: ${describe(matches)}`);
+        if (!matches.length)
+            throw new Error(candidates.length
+                ? `No eval set matches "${parsed.name}". Eval sets: ${describe(candidates)}`
+                : `No eval set matches "${parsed.name}"`);
+        dataset = await client.getDataset(matches[0].id);
+    }
+    const latest = (versions) => versions.reduce((best, version) => (best === undefined || version.version > best.version ? version : best), undefined);
+    const version = latest(dataset.versions.filter((item) => item.frozenAt !== null)) ?? latest(dataset.versions);
+    if (!version)
+        throw new Error(`Eval set "${dataset.name}" has no versions`);
+    return {
+        scenarioId: null,
+        name: dataset.name,
+        datasetId: dataset.id,
+        datasetVersionId: version.id,
+        scorerVersionIds: [...(options.scorerVersionIds ?? [])],
+        environmentVersionId: null,
+        saved: version.frozenAt !== null,
+        revision: version.revision,
+    };
+}

package/dist/evals/scorers.d.ts

@@ -16,7 +16,20 @@ export declare function defineLocalScorer(options: {
     score: LocalScorer["score"];
 }): LocalScorer;
 /** Check local callbacks before target invocation; never execute downloaded source code. */
+export declare function isBoundLocally(definition: ScorerDefinition, scorers?: LocalScorer[]): boolean;
 export declare function validateScorerBindings(versions: ScorerVersion[], scorers?: LocalScorer[]): void;
+/**
+ * Whether this process produces the result for a pinned version. Built-ins always run here; a
+ * `local_code` pin runs here when bound, and with `deferUnboundLocalScorers` an unbound pin is
+ * left to the executor that owns its source (for example Hue's grading worker) instead of
+ * failing the run.
+ */
+export declare function executableHere(definition: ScorerDefinition, options: {
+    scorers?: LocalScorer[];
+    deferUnboundLocalScorers?: boolean;
+}): definition is Extract<ScorerDefinition, {
+    kind: "builtin" | "local_code";
+}>;
 /** Only implementations this SDK owns may produce local results. Unknown pins are deferred. */
 export declare function isLocallyExecutable(definition: {
     kind: string;

package/dist/evals/scorers.js

@@ -92,13 +92,28 @@ function skip(explanation) {
     return { state: "skipped", explanation };
 }
 /** Check local callbacks before target invocation; never execute downloaded source code. */
+export function isBoundLocally(definition, scorers = []) {
+    return scorers.some((local) => digest(local.definition) === digest(definition));
+}
 export function validateScorerBindings(versions, scorers = []) {
     for (const { definition } of versions) {
-        if (definition.kind === "local_code" &&
-            !scorers.some((local) => digest(local.definition) === digest(definition)))
+        if (definition.kind === "local_code" && !isBoundLocally(definition, scorers))
             throw new Error("A pinned local scorer has no matching language/source/entrypoint/metric binding");
     }
 }
+/**
+ * Whether this process produces the result for a pinned version. Built-ins always run here; a
+ * `local_code` pin runs here when bound, and with `deferUnboundLocalScorers` an unbound pin is
+ * left to the executor that owns its source (for example Hue's grading worker) instead of
+ * failing the run.
+ */
+export function executableHere(definition, options) {
+    if (!isLocallyExecutable(definition))
+        return false;
+    if (definition.kind === "local_code" && options.deferUnboundLocalScorers)
+        return isBoundLocally(definition, options.scorers);
+    return true;
+}
 /** Only implementations this SDK owns may produce local results. Unknown pins are deferred. */
 export function isLocallyExecutable(definition) {
     return (definition.kind === "local_code" ||
@@ -127,7 +142,10 @@ export async function scoreLocally(version, context, options = {}) {
             return { state: "error", error: { type: "LocalScorerError" } };
         }
     }
-    if (!context.hasOutput && !(definition.kind === "local_code" && context.environment))
+    // A code evaluator can grade a sealed world or generated files without a JSON output.
+    const generatedFiles = context.files?.some((file) => file.role === "output") ?? false;
+    if (!context.hasOutput &&
+        !(definition.kind === "local_code" && (context.environment || generatedFiles)))
         return skip("Output evidence is unavailable");
     if (context.hasOutput && context.output === undefined)
         throw new TypeError("hasOutput requires a present JSON output");

package/dist/evals/simulation.d.ts

@@ -22,12 +22,23 @@ export interface RepositorySimulationCase {
     /** Optional caller-owned case metadata. */
     metadata?: Record<string, JsonValue>;
 }
-/** Immutable app-authored experiment reference or repository-authored simulation definition. */
+/** Immutable app-authored experiment reference, published pins or repository-authored definition. */
 export type SimulationDefinition = {
     /** Select an existing app-authored immutable experiment template. */
     kind: "experiment";
     /** Experiment to clone into a fresh attempt. */
     experimentId: string;
+} | {
+    /** Run already published immutable pins, such as a Scenario's frozen case and outcome checks. */
+    kind: "pins";
+    /** Frozen dataset version whose cases pin their simulated-world versions. */
+    datasetVersionId: string;
+    /** Immutable scorer versions to pin; Hue-executed pins need no local callback. */
+    scorerVersionIds: string[];
+    /** JSON configuration passed to every target callback; defaults to `{}`. */
+    config?: JsonValue;
+    /** Experiment display name; defaults to the dataset name, then `Simulation`. */
+    name?: string;
 } | {
     /** Publish and resolve the repository-authored definition. */
     kind: "repository";

package/dist/evals/simulation.js

@@ -49,6 +49,14 @@ function normalizedEnvironmentDefinition(definition) {
 function definitionIdentity(definition) {
     if (definition.kind === "experiment")
         return definition;
+    // The display name is cosmetic; the pins and configuration define the immutable selection.
+    if (definition.kind === "pins")
+        return json({
+            kind: definition.kind,
+            datasetVersionId: definition.datasetVersionId,
+            scorerVersionIds: [...definition.scorerVersionIds].sort(),
+            config: definition.config ?? {},
+        });
     return json({
         kind: definition.kind,
         name: definition.name,
@@ -272,6 +280,16 @@ async function resolveDataset(client, scenario, environmentVersionId) {
         }, "Dataset freeze acknowledgement is unavailable")).id;
     }
 }
+/** Display name of the dataset owning a version, or undefined when it cannot be read. */
+async function datasetName(client, versionId) {
+    try {
+        const version = await client.getDatasetVersion(versionId);
+        return (await client.getDataset(version.datasetId)).name;
+    }
+    catch {
+        return undefined;
+    }
+}
 async function allCases(client, versionId) {
     const items = [];
     let after;
@@ -295,6 +313,22 @@ async function resolveExperiment(options, definition, idempotencyKey) {
         });
         return { experimentId: created.id, bindings: options.localScorers ?? [] };
     }
+    if (definition.kind === "pins") {
+        const { datasetVersionId, scorerVersionIds } = definition;
+        if (!scorerVersionIds.length)
+            throw new TypeError("Pinned scenarios require a scorer version");
+        const created = await options.client.createExperiment({
+            idempotencyKey,
+            name: options.runName ??
+                definition.name ??
+                (await datasetName(options.client, datasetVersionId)) ??
+                "Simulation",
+            datasetVersionId,
+            scorerVersionIds: [...scorerVersionIds],
+            config: definition.config ?? {},
+        });
+        return { experimentId: created.id, bindings: options.localScorers ?? [] };
+    }
     const environmentVersionId = await resolveEnvironment(options.environmentClient, definition.environment);
     const datasetVersionId = await resolveDataset(options.client, definition, environmentVersionId);
     const scorers = await resolveScorers(options.client, definition.scorers);
@@ -320,6 +354,8 @@ export async function runSimulation(options) {
         throw new TypeError("runSimulation requires a definition");
     if (options.definition && options.scenario && options.definition !== options.scenario)
         throw new TypeError("Pass either definition or scenario, not both");
+    if (definition.kind === "pins" && !definition.scorerVersionIds.length)
+        throw new TypeError("Pinned scenarios require a scorer version");
     if (!options.definition && !scenarioDeprecationWarned) {
         scenarioDeprecationWarned = true;
         process.emitWarning("runSimulation option scenario is deprecated; use definition", "DeprecationWarning");

package/dist/evals/types.d.ts

@@ -76,10 +76,32 @@ export interface DatasetCase {
     /** Immutable input-file manifest identity, when files are attached. */
     artifactManifestId?: string | null;
 }
+/** One pinned input file of a case or subject, as recorded in Hue's immutable manifest. */
+export interface CaseFile {
+    /** Hue artifact identity of the pinned bytes. */
+    artifactId: string;
+    /** How the file relates to the case; `org_template` is evaluator-only. */
+    role: "source" | "attached_template" | "attached_reference" | "original" | "org_template" | "evaluator_reference";
+    /** Declared file name. */
+    filename: string;
+    /** Declared content type. */
+    contentType: string;
+    /** Verified size in bytes. */
+    byteSize: number;
+    /** Verified SHA-256, hex encoded. */
+    sha256: string;
+}
+/** A frozen manifest entry of a subject: the case inputs plus the outputs the target produced. */
+export type SubjectFile = Omit<CaseFile, "role"> & {
+    /** Input role, or `output` for a file the target generated. */
+    role: CaseFile["role"] | "output";
+};
 /** A frozen case as an experiment sees it. */
 export interface ExperimentCase extends DatasetCase {
     /** Whether a reference output is stored; JSON `null` counts as present. */
     hasExpected: boolean;
+    /** Pinned input files, present on current servers when the case has a manifest. */
+    inputFiles?: CaseFile[];
 }
 /** Input for {@link EvaluationClient.addCase}. */
 export interface CaseWrite {
@@ -372,6 +394,17 @@ export interface EvaluationRun {
         pending: number;
     };
 }
+/** One row of the project's evaluation-run listing; `getEvaluationRun` reads the pins and score counts. */
+export interface EvaluationRunSummary {
+    /** Run ID. */
+    id: string;
+    /** Display name. */
+    name: string;
+    /** Experiment whose default run this is, or `null` for a standalone scoring run. */
+    experimentId: string | null;
+    /** Creation time. */
+    createdAt: string;
+}
 /** An experiment: a frozen dataset version, a configuration and pinned scorers. */
 export interface Experiment {
     /** Experiment ID. */
@@ -432,6 +465,10 @@ export interface CompleteExecution {
     state: TerminalState;
     /** Target output; omit when unavailable. */
     output?: JsonValue;
+    /** Verified artifacts the target generated; the manifest freezes them with the case inputs. */
+    artifactIds?: string[];
+    /** The declared primary generated artifact, one of `artifactIds`. */
+    primaryArtifactId?: string;
     /** Sanitized failure for `state: "error"`. */
     error?: TypedError;
     /** Trace revision the stored snapshot must have reached. */
@@ -505,6 +542,12 @@ export interface Subject {
     traceExternalId: string | null;
     /** Reason trace evidence was omitted, or `null`. */
     omissionReason: string | null;
+    /** Present on current servers: the case's pinned world, or null for an ordinary case. */
+    environmentVersionId?: string | null;
+    /** Present on current servers: the frozen input and output files of this subject. */
+    files?: SubjectFile[];
+    /** The target's declared primary generated artifact, or `null`. */
+    primaryArtifactId?: string | null;
 }
 /** A reported metric value. */
 export interface Metric {
@@ -563,7 +606,62 @@ export interface ScoreContext {
     executionState: TerminalState;
     /** Authoritative sealed world and complete journal, when required by the runner. */
     environment?: EnvironmentEvidence;
-}
+    /** Pinned input files and the target's generated files, verified and saved on this machine.
+     * Present only when the runner handled files for this execution. */
+    files?: LocalFile[];
+}
+/** A verified copy of a case input or generated output on the runner's disk. */
+export interface LocalFile {
+    /** Hue artifact identity; generated files receive theirs after upload. */
+    artifactId: string;
+    /** Input role, or `output` for a file the target generated. */
+    role: CaseFile["role"] | "output";
+    /** File name. */
+    filename: string;
+    /** Content type. */
+    contentType: string;
+    /** Verified size in bytes. */
+    byteSize: number;
+    /** Verified SHA-256, hex encoded. */
+    sha256: string;
+    /** Absolute path of the verified bytes on this machine. */
+    path: string;
+    /** Whether this is the execution's primary generated document. */
+    primary?: boolean;
+}
+/** A file the target generated for one case. Bytes are read from `path` or taken from `bytes`. */
+export type OutputFile = {
+    /** File name Hue stores; sanitized to one path segment. */
+    filename: string;
+    /** One of the accepted generated content types. */
+    contentType: string;
+    /** The declared primary document; at most one per execution. */
+    primary?: boolean;
+} & ({
+    /** Path of the generated file on this machine. */
+    path: string;
+    /** Not used when `path` is given. */
+    bytes?: undefined;
+} | {
+    /** Generated bytes held in memory. */
+    bytes: Uint8Array;
+    /** Not used when `bytes` is given. */
+    path?: undefined;
+});
+/** A target's saved outcome when it produced files. Create it with `withFiles`. */
+export declare class TargetResult {
+    /** JSON output of the target, or `undefined` when it produced only files. */
+    readonly output: JsonValue | undefined;
+    /** Generated files to save with the execution. */
+    readonly files: OutputFile[];
+    constructor(
+    /** JSON output of the target, or `undefined` when it produced only files. */
+    output: JsonValue | undefined, 
+    /** Generated files to save with the execution. */
+    files: OutputFile[]);
+}
+/** Return this from a target to save generated files with the execution. */
+export declare function withFiles(output: JsonValue | undefined, files: OutputFile[]): TargetResult;
 /** Sealed environment evidence resolved through one target execution. */
 export interface EnvironmentEvidenceSnapshot extends EnvironmentCoverage {
     /** Environment-run identity. */
@@ -601,6 +699,40 @@ export interface LocalScorer {
     /** Trusted local code. There is no callback timeout or side-effect cancellation. */
     score(context: ScoreContext): Score | Promise<Score>;
 }
+/** An artifact reservation as Hue reports it through the reserve, upload and complete steps. */
+export interface ArtifactReservation {
+    /** Artifact ID. */
+    id: string;
+    /** Declared file name. */
+    filename: string;
+    /** Declared content type. */
+    declaredContentType: string;
+    /** Declared size in bytes. */
+    declaredBytes: number;
+    /** Declared SHA-256, hex encoded. */
+    declaredSha256: string;
+    /** Lifecycle state; only `ready` artifacts are verified and downloadable. */
+    state: "reserved" | "verifying" | "ready" | "rejected" | "cancelled" | "abandoned";
+    /** Progress of the byte copy into Hue storage. */
+    copyState: "none" | "started" | "acknowledged";
+    /** Verified size once ready, otherwise `null`. */
+    verifiedBytes: number | null;
+    /** Verified SHA-256 once ready, otherwise `null`. */
+    verifiedSha256: string | null;
+    /** Why verification failed, or `null`. */
+    failureCode: string | null;
+}
+/** A short-lived storage capability for staging one artifact's bytes. */
+export interface ArtifactUpload {
+    /** Storage URL that accepts the bytes; the Hue key is never sent there. */
+    uploadUrl: string;
+    /** HTTP method the capability accepts. */
+    method: "PUT";
+    /** Headers to send with the bytes; omitted or null means the file content-type only. */
+    headers?: Record<string, string> | null;
+    /** Capability expiry as an ISO timestamp. */
+    expiresAt: string;
+}
 /** Short-lived execution-scoped MCP connection for one simulated world. */
 export interface SimulationMcpCapability {
     /** HTTPS MCP endpoint. */
@@ -641,3 +773,44 @@ export interface LocalAgentClaim {
     /** Pinned experiment to execute. */
     experimentId: string;
 }
+/** A Scenario as listed by {@link EvaluationClient.listCaseConversions}. Extra server fields are ignored. */
+export interface CaseConversionSummary {
+    /** Scenario ID. */
+    id: string;
+    /** Scenario domain label. */
+    domain: string;
+    /** Whether the Scenario has immutable published pins. */
+    status: "draft" | "published";
+    /** Optimistic-concurrency revision of the Scenario. */
+    revision: number;
+    /** Creation timestamp. */
+    createdAt: string;
+    /** Source trace the Scenario was converted from. */
+    traceId: string;
+}
+/** Immutable pins created when a Scenario is published. */
+export interface CaseConversionPublication {
+    /** Published dataset case. */
+    caseId: string;
+    /** Dataset holding the published case. */
+    datasetId: string;
+    /** Frozen dataset version holding the published case. */
+    datasetVersionId: string;
+    /** Environment identity of the simulated world. */
+    environmentId: string;
+    /** Immutable environment version the case pins. */
+    environmentVersionId: string;
+    /** Scorer identity of the published outcome checks. */
+    scorerId: string;
+    /** Immutable scorer version pinned by the Scenario. */
+    scorerVersionId: string;
+}
+/** A Scenario read by {@link EvaluationClient.getCaseConversion}. Extra server fields are ignored. */
+export interface CaseConversion extends Partial<Omit<CaseConversionSummary, "id" | "status">> {
+    /** Scenario ID. */
+    id: string;
+    /** Whether the Scenario has immutable published pins. */
+    status: "draft" | "published";
+    /** Published pins, or `null` while the Scenario is a draft. */
+    publication: CaseConversionPublication | null;
+}

package/dist/evals/types.js

@@ -1 +1,17 @@
-export {};
+/** A target's saved outcome when it produced files. Create it with `withFiles`. */
+export class TargetResult {
+    output;
+    files;
+    constructor(
+    /** JSON output of the target, or `undefined` when it produced only files. */
+    output, 
+    /** Generated files to save with the execution. */
+    files) {
+        this.output = output;
+        this.files = files;
+    }
+}
+/** Return this from a target to save generated files with the execution. */
+export function withFiles(output, files) {
+    return new TargetResult(output, files);
+}