@hue-run/sdk 0.9.0 → 0.10.0

package/dist/evals/runner.js

@@ -1,12 +1,14 @@
 import { randomUUID } from "node:crypto";
-import { mkdir } from "node:fs/promises";
+import { rmSync } from "node:fs";
+import { mkdir, rm } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { ROOT_CONTEXT } from "@opentelemetry/api";
 import { HueExportError } from "../transport.js";
 import { HueApiError } from "./client.js";
 import { loadEnvironmentEvidence } from "./environment-evidence.js";
 import { CheckpointStore } from "./checkpoint.js";
-import { downloadCaseFiles, localOutputFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
+import { onForcedExit } from "./exit-cleanup.js";
+import { assertSafeFileNames, downloadCaseFiles, downloadCaseInputs, localOutputFiles, verifyCaseFiles, OutputFileError, stageOutputFiles, targetFileRoles, uploadOutputFiles, } from "./files.js";
 import { json, uuid } from "./json.js";
 import { executableHere, persistedScore, scoreLocally, validateScorerBindings } from "./scorers.js";
 import { TargetResult, } from "./types.js";
@@ -72,6 +74,59 @@ function neededInputFiles(files, versions, options) {
         ? files
         : files.filter((file) => targetFileRoles.includes(file.role));
 }
+/** Remove a case's files; a failure is reported as a warning and never fails the case. */
+async function removeCaseFiles(directory) {
+    try {
+        await rm(directory, { recursive: true, force: true });
+    }
+    catch {
+        process.emitWarning(`Could not remove the case files in ${directory}`, {
+            code: "HUE_CASE_FILES_NOT_REMOVED",
+        });
+    }
+}
+/** Everything of a world case's directory except the staged outputs. */
+const WORK_PARTS = ["inputs", "evaluator-inputs", "work"];
+async function releaseWorldFiles(directory, keepOutputs) {
+    if (!keepOutputs)
+        return removeCaseFiles(directory);
+    for (const part of WORK_PARTS)
+        await removeCaseFiles(join(directory, part));
+}
+function releaseWorldFilesSync(directory, keepOutputs) {
+    for (const path of keepOutputs ? WORK_PARTS.map((part) => join(directory, part)) : [directory])
+        rmSync(path, { recursive: true, force: true });
+}
+/** Stable reason, and first word of the omission reason, of a case failed for its telemetry. */
+export const TELEMETRY_NOT_ACCEPTED = "telemetry_not_accepted";
+/** The issues of a failed export, summed by signal, kind and status. */
+export function telemetryIssueCounts(error) {
+    if (!(error instanceof HueExportError))
+        return [];
+    const counts = new Map();
+    for (const issue of error.issues) {
+        if (issue.kind === "warning")
+            continue;
+        const key = `${issue.signal}:${issue.kind}:${issue.status ?? ""}`;
+        const entry = counts.get(key) ?? {
+            signal: issue.signal,
+            kind: issue.kind,
+            ...(issue.status === undefined ? {} : { status: issue.status }),
+            count: 0,
+        };
+        entry.count += issue.count;
+        counts.set(key, entry);
+    }
+    // A zero-count entry, such as the processor's own flush failure, only repeats a signal that
+    // already has counted records.
+    const counted = new Set([...counts.values()].filter((entry) => entry.count).map((e) => e.signal));
+    return [...counts.values()].filter((entry) => entry.count || !counted.has(entry.signal));
+}
+/** `telemetry_not_accepted`, then the issue counts, for example `traces failed 1 (HTTP 400)`. */
+export function describeTelemetryIssues(issues) {
+    const parts = issues.map((issue) => `${issue.signal} ${issue.kind} ${issue.count}${issue.status === undefined ? "" : ` (HTTP ${issue.status})`}`);
+    return parts.length ? `${TELEMETRY_NOT_ACCEPTED}: ${parts.join(", ")}` : TELEMETRY_NOT_ACCEPTED;
+}
 function settings(options) {
     if (options.environmentEvidence !== undefined &&
         options.environmentEvidence !== "required" &&
@@ -269,7 +324,7 @@ export async function runExperiment(options) {
         output) {
             const needed = neededInputFiles(frozenCase.inputFiles, versions, options);
             const inputs = needed.length
-                ? await downloadCaseFiles(options.client, needed, join(caseDirectory, "inputs"))
+                ? (await downloadCaseInputs(options.client, needed, caseDirectory)).all
                 : [];
             await uploadOutputFiles(options.client, saved.executionId, saved.files, () => store.write(file, saved));
             const outputs = localOutputFiles(saved.files);
@@ -310,10 +365,25 @@ export async function runExperiment(options) {
             await store.write(file, prepared);
             return prepared;
         }
-        await pool(items, concurrency, async (item) => {
+        // A case pinned to a world keeps its files apart and removes them when it ends: the agent's
+        // copies of its inputs, its work, the evaluator's downloads and the staged outputs.
+        const worldDirectoryOf = (itemId) => join(filesRoot, `world-case-${uuid(itemId)}`);
+        const runCase = async (item, files) => {
             const file = `case-${uuid(item.id)}`;
-            const caseDirectory = join(filesRoot, `case-${uuid(item.id)}`);
-            let checkpoint = await store.read(file);
+            const worldDirectory = worldDirectoryOf(item.id);
+            const caseDirectoryOf = (frozen) => frozen.environmentVersionId ? worldDirectory : join(filesRoot, `case-${uuid(item.id)}`);
+            let checkpoint;
+            try {
+                checkpoint = await store.read(file);
+            }
+            catch (error) {
+                // A checkpoint that is unsafe or fails its integrity check is never resumed from, so its
+                // staged outputs are not kept; an I/O failure may pass, so they are.
+                if (!error.code)
+                    files.keepOutputs = false;
+                throw error;
+            }
+            files.keepOutputs = checkpoint?.stage === "uploading";
             if (checkpoint && checkpoint.stage !== "prepared" && checkpoint.stage !== "uploading") {
                 if (checkpoint.stage === "serialization_failed")
                     throw new OutcomeSerializationError(checkpoint.executionId);
@@ -331,7 +401,8 @@ export async function runExperiment(options) {
                 if (checkpoint.hasOutput && checkpoint.output === undefined)
                     throw new UncertainExecutionError(item.id, checkpoint.executionId);
                 const frozenCase = await options.client.getExperimentCase(experiment.id, item.id);
-                checkpoint = await prepare(file, checkpoint, frozenCase, caseDirectory, checkpoint.output);
+                checkpoint = await prepare(file, checkpoint, frozenCase, caseDirectoryOf(frozenCase), checkpoint.output);
+                files.keepOutputs = false;
             }
             if (!checkpoint) {
                 if (item.execution)
@@ -343,10 +414,20 @@ export async function runExperiment(options) {
                 // target failures and cannot consume a case's execution slot.
                 const targetInputs = json(frozenCase.inputs);
                 const targetConfig = json(experiment.config);
-                // Pinned input files are verified on disk before an execution exists for the same reason.
+                // Pinned input files are verified before an execution exists for the same reason. Only
+                // the agent-visible ones are saved now: evaluator-only files a local code evaluator needs
+                // are checked and dropped, and saved for scoring after the target finished, so they are
+                // not on disk while it runs.
+                const caseDirectory = caseDirectoryOf(frozenCase);
+                const isTargetFile = (entry) => targetFileRoles.includes(entry.role);
                 const needed = neededInputFiles(frozenCase.inputFiles, versions, options);
-                const inputFiles = needed.length
-                    ? await downloadCaseFiles(options.client, needed, join(caseDirectory, "inputs"))
+                const agentFiles = needed.filter(isTargetFile);
+                // An agent working in a world receives its files by name: each must be one safe name.
+                if (frozenCase.environmentVersionId)
+                    assertSafeFileNames(agentFiles);
+                await verifyCaseFiles(options.client, needed.filter((entry) => !isTargetFile(entry)));
+                const inputFiles = agentFiles.length
+                    ? (await downloadCaseInputs(options.client, agentFiles, caseDirectory)).target
                     : [];
                 const outputDirectory = join(caseDirectory, "work");
                 await mkdir(outputDirectory, { recursive: true, mode: 0o700 });
@@ -373,7 +454,7 @@ export async function runExperiment(options) {
                             item: structuredClone(frozenCase),
                             span,
                             executionId: execution.id,
-                            files: structuredClone(inputFiles.filter((entry) => targetFileRoles.includes(entry.role))),
+                            files: structuredClone(inputFiles),
                             outputDirectory,
                         });
                         if (result instanceof TargetResult) {
@@ -433,6 +514,7 @@ export async function runExperiment(options) {
                     };
                     // Without persisted result content a restart cannot reconstruct the outcome; the
                     // saved stage then reports the execution as uncertain instead of guessing.
+                    files.keepOutputs = true;
                     await store.write(file, uploading);
                     return prepare(file, uploading, frozenCase, caseDirectory, output);
                 }, {
@@ -440,7 +522,9 @@ export async function runExperiment(options) {
                     input: frozenCase.inputs,
                     attributes: { "hue.experiment.id": experiment.id, "hue.dataset.case.id": item.id },
                 });
+                files.keepOutputs = false;
                 // End root before waiting for both OTLP signals. Export failure leaves the prepared checkpoint intact.
+                let exportError;
                 try {
                     await options.hue.flush();
                     // Another concurrent flush may already have surfaced this failure. OTLP
@@ -449,29 +533,96 @@ export async function runExperiment(options) {
                         throw new HueExportError(options.hue.transport
                             .getIssues()
                             .filter((issue) => issue.sequence > failureSequenceBefore && issue.kind !== "warning"), options.hue.transport.getReport());
-                    checkpoint.exportState = "accepted";
-                    await store.write(file, checkpoint);
+                }
+                catch (error) {
+                    exportError = error;
+                }
+                try {
+                    if (exportError === undefined) {
+                        checkpoint.exportState = "accepted";
+                        await store.write(file, checkpoint);
+                    }
                 }
                 catch (error) {
                     // The explicitly chosen omission policy can complete without acknowledged telemetry.
                     if (options.traceEvidence.mode !== "omit")
                         throw error;
                 }
+                if (exportError !== undefined && options.traceEvidence.mode !== "omit") {
+                    if (options.traceNotAccepted !== "fail_case")
+                        throw exportError;
+                    // Required evidence that Hue did not accept fails the case rather than leaving its
+                    // execution started: the outcome is kept, the evidence is declared omitted. Only an
+                    // export failure takes this path; a checkpoint that cannot be saved is raised above.
+                    const failed = checkpoint;
+                    const issues = telemetryIssueCounts(exportError);
+                    // Without its evidence the case fails whatever it answered: the output and generated
+                    // files are not attached, so no scorer can pass it, and local scores taken of them go.
+                    const { traceEvidence: _required, output: _output, artifactIds: _artifacts, primaryArtifactId: _primary, ...complete } = failed.complete;
+                    failed.scores = [];
+                    if (complete.state === "succeeded") {
+                        complete.state = "error";
+                        complete.error = {
+                            type: "TelemetryNotAccepted",
+                            ...(options.persistResultContent ? { message: describeTelemetryIssues(issues) } : {}),
+                        };
+                    }
+                    failed.complete = {
+                        ...complete,
+                        traceEvidence: "omit",
+                        omissionReason: describeTelemetryIssues(issues).slice(0, 4000),
+                    };
+                    failed.exportState = "not_accepted";
+                    failed.telemetryIssues = issues;
+                    await store.write(file, failed);
+                }
             }
             const prepared = checkpoint;
             await options.client.getExecution(prepared.executionId);
             if (prepared.exportState !== "accepted" && prepared.complete.traceEvidence !== "omit")
                 throw new Error("Target outcome is saved but trace export acknowledgement is unavailable. Restore/export the trace or explicitly complete with omitted evidence through the client; never rerun the target.");
             const save = () => store.write(file, prepared);
+            const notAccepted = prepared.exportState === "not_accepted"
+                ? {
+                    caseId: item.id,
+                    caseKey: item.externalKey,
+                    executionId: prepared.executionId,
+                    issues: prepared.telemetryIssues ?? [],
+                }
+                : undefined;
             if (!prepared.completion) {
                 prepared.completion = await options.client.completeExecution(prepared.executionId, prepared.complete);
                 for (const score of prepared.scores)
                     score.payload.evaluationItemId = prepared.completion.evaluationItemId;
                 await save();
+                // Once per failed case: a resumed call finds the completion and only reports it.
+                if (notAccepted)
+                    await options.onTelemetryNotAccepted?.(notAccepted);
             }
+            if (notAccepted)
+                (report.telemetryNotAccepted ??= []).push(notAccepted);
             const results = await uploadScores(options, experiment.evaluation.id, prepared.scores, save, versions);
             report.subjectIds.push(prepared.completion.subjectId);
             report.resultIds.push(...results);
+        };
+        await pool(items, concurrency, async (item) => {
+            // A world case's directory goes however the case ends, a forced exit included. An
+            // interrupted upload keeps only the staged outputs it resumes from; inputs are downloaded
+            // anew for scoring. A direct case has no such directory, so nothing is removed for it.
+            const worldDirectory = worldDirectoryOf(item.id);
+            const files = { keepOutputs: true };
+            const untrack = onForcedExit(() => releaseWorldFilesSync(worldDirectory, files.keepOutputs));
+            try {
+                await runCase(item, files);
+                await releaseWorldFiles(worldDirectory, false);
+            }
+            catch (error) {
+                await releaseWorldFiles(worldDirectory, files.keepOutputs);
+                throw error;
+            }
+            finally {
+                untrack();
+            }
         });
         let finish = await store.read("finish");
         if (!finish) {

package/dist/evals/scenarios.d.ts

@@ -1,4 +1,4 @@
-import type { EvaluationClient } from "./client.js";
+import { type EvaluationClient } from "./client.js";
 import type { CaseConversion, CaseConversionSummary, Page, PageOptions } from "./types.js";
 /** Immutable pins resolved from a published Scenario or a saved eval set. */
 export interface ScenarioPins {
@@ -59,11 +59,13 @@ export interface NameMatch<T extends NamedCandidate> {
 /** Case-insensitive exact matches first, then unique prefix/substring matches. */
 export declare function matchByName<T extends NamedCandidate>(candidates: T[], name: string): NameMatch<T>;
 /**
- * Resolves a published Scenario's immutable pins from its ID, its Hue URL or its name. A name
- * matches the dataset name of published Scenarios case-insensitively: exact matches first, then
- * a unique prefix or substring.
+ * Resolves a published Scenario's immutable pins from its ID, the ID of the eval set case it
+ * published, a Hue URL naming either (`/scenarios/<id>`, `/case-conversions/<id>` or
+ * `/cases/<id>`) or its name. A name matches the dataset name of published Scenarios
+ * case-insensitively: exact matches first, then a unique prefix or substring.
  *
- * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft.
+ * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft;
+ * {@link HueApiError} with status 404 when neither a Scenario nor a published case has the ID.
  */
 export declare function resolveScenarioPins(client: ScenarioClient, selector: string): Promise<ScenarioPins>;
 /**

package/dist/evals/scenarios.js

@@ -1,6 +1,10 @@
+import { HueApiError } from "./client.js";
 const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
-/** Scenarios listed while resolving a name; bounds the registry reads of one selection. */
+/** Published Scenarios a name is matched against; bounds the registry reads of one selection. */
 const MAX_LISTED = 200;
+/** Scenarios, drafts included, one selection lists at most: a case ID is found among them, and a
+ * mistyped one ends after a bounded number of reads. */
+const MAX_SCANNED = 1000;
 /** Lists Scenarios of the project; requires a Read and write key. */
 export function listScenarios(client, page) {
     return client.listCaseConversions(page);
@@ -53,17 +57,57 @@ export function matchByName(candidates, name) {
         exact: false,
     };
 }
-async function listPublishedScenarios(client) {
-    const items = [];
+/** Published Scenario summaries in listing order, page by page, ending after
+ * {@link MAX_SCANNED} Scenarios, at an empty page or when the listing repeats a cursor. */
+async function* publishedScenarios(client) {
+    const cursors = new Set();
+    let listed = 0;
     let after;
     for (;;) {
         const page = await client.listCaseConversions({ after, limit: 100 });
-        items.push(...page.items.filter((item) => item.status === "published"));
-        if (!page.nextCursor || items.length >= MAX_LISTED)
-            return items.slice(0, MAX_LISTED);
+        for (const item of page.items) {
+            if (++listed > MAX_SCANNED)
+                return;
+            if (item.status === "published")
+                yield item;
+        }
+        if (!page.items.length || !page.nextCursor || cursors.has(page.nextCursor))
+            return;
+        cursors.add(page.nextCursor);
         after = page.nextCursor;
     }
 }
+async function listPublishedScenarios(client) {
+    const items = [];
+    for await (const item of publishedScenarios(client))
+        if (items.push(item) >= MAX_LISTED)
+            break;
+    return items;
+}
+/** A listed Scenario, or undefined when it can no longer be read (removed since the listing). */
+async function readListed(client, id) {
+    try {
+        return await client.getCaseConversion(id);
+    }
+    catch (error) {
+        if (error instanceof HueApiError && error.status === 404)
+            return undefined;
+        throw error;
+    }
+}
+/**
+ * The published Scenario whose eval set case has this ID, the one Hue shows on the case page.
+ * Published Scenarios are read in listing order until one matches, past the bound a name search
+ * uses; undefined when none does within {@link MAX_SCANNED}.
+ */
+async function scenarioOfPublishedCase(client, caseId) {
+    for await (const summary of publishedScenarios(client)) {
+        const scenario = await readListed(client, summary.id);
+        if (scenario?.publication?.caseId.toLowerCase() === caseId)
+            return scenario;
+    }
+    return undefined;
+}
 const describe = (candidates) => candidates.map((candidate) => `${candidate.name} (${candidate.id})`).join(", ");
 async function pinsFromScenario(client, scenario, dataset) {
     if (!scenario.publication)
@@ -84,16 +128,29 @@ async function pinsFromScenario(client, scenario, dataset) {
     };
 }
 /**
- * Resolves a published Scenario's immutable pins from its ID, its Hue URL or its name. A name
- * matches the dataset name of published Scenarios case-insensitively: exact matches first, then
- * a unique prefix or substring.
+ * Resolves a published Scenario's immutable pins from its ID, the ID of the eval set case it
+ * published, a Hue URL naming either (`/scenarios/<id>`, `/case-conversions/<id>` or
+ * `/cases/<id>`) or its name. A name matches the dataset name of published Scenarios
+ * case-insensitively: exact matches first, then a unique prefix or substring.
  *
- * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft.
+ * @throws Error when no Scenario matches, several match, or the Scenario is an unpublished draft;
+ * {@link HueApiError} with status 404 when neither a Scenario nor a published case has the ID.
  */
 export async function resolveScenarioPins(client, selector) {
-    const parsed = parseScenarioSelector(selector);
+    const parsed = parseScenarioSelector(selector, ["scenarios", "case-conversions", "cases"]);
     if (parsed.kind === "id") {
-        const scenario = await client.getCaseConversion(parsed.id);
+        let scenario;
+        try {
+            scenario = await client.getCaseConversion(parsed.id);
+        }
+        catch (error) {
+            if (!(error instanceof HueApiError && error.status === 404))
+                throw error;
+            const published = await scenarioOfPublishedCase(client, parsed.id);
+            if (!published)
+                throw error;
+            scenario = published;
+        }
         if (!scenario.publication)
             throw new Error(`Scenario ${scenario.id} is a draft without published pins; publish it in Hue first`);
         return pinsFromScenario(client, scenario, await client.getDataset(scenario.publication.datasetId));
@@ -102,8 +159,8 @@ export async function resolveScenarioPins(client, selector) {
     const datasets = new Map();
     const candidates = [];
     for (const summary of published) {
-        const scenario = await client.getCaseConversion(summary.id);
-        if (!scenario.publication)
+        const scenario = await readListed(client, summary.id);
+        if (!scenario?.publication)
             continue;
         let dataset = datasets.get(scenario.publication.datasetId);
         if (!dataset) {

package/dist/evals/simulation.d.ts

@@ -4,8 +4,8 @@ import type { EnvironmentTool } from "../environment/tools.js";
 import type { EnvironmentIdentity, PublishableEnvironmentDefinition, WorldHandoff } from "../environment/types.js";
 import { type EvaluationClient } from "./client.js";
 import type { ActualAgentManifestInputV2, AttemptConnectionBundleV2, RequestedAttemptProviderV2 } from "./attempt.js";
-import { type RunnerReport } from "./runner.js";
-import type { ExperimentCase, Identity, JsonValue, LocalScorer, ScorerDefinition, SimulationMcpCapability } from "./types.js";
+import { type RunnerReport, type TelemetryNotAccepted } from "./runner.js";
+import type { ExperimentCase, Identity, JsonValue, LocalFile, LocalScorer, ScorerDefinition, SimulationMcpCapability, TargetResult } from "./types.js";
 /** Repository-authored scorer identity and its public definition or local binding. */
 export interface RepositorySimulationScorer extends Identity {
     /** Scorer definition to publish, or a local callback bound to its source digest. */
@@ -121,6 +121,12 @@ export interface SimulationTargetContext {
     /** Credential-bearing provider connections for this callback only. Hue never
      * checkpoints, logs or adds this response to parity digests. */
     connectionBundle?: AttemptConnectionBundleV2;
+    /** Verified copies of the case's agent-visible input files (`source`, templates,
+     * references, originals); evaluator-only files are withheld. Empty when the case has none. */
+    files: LocalFile[];
+    /** Private directory for this case, removed after it; return generated files with
+     * `withFiles`. */
+    outputDirectory: string;
     /** Cancellation is cooperative: pass this signal into the real agent/provider call. */
     signal?: AbortSignal;
 }
@@ -152,6 +158,12 @@ export interface RunSimulationOptions {
         /** Bounded explanation for the omission. */
         reason: string;
     };
+    /** What a case does when required telemetry is not accepted; see
+     * `traceNotAccepted` of {@link runExperiment}. Defaults to `"stop"`. */
+    traceNotAccepted?: "stop" | "fail_case";
+    /** Called as each case is failed for its telemetry; see `onTelemetryNotAccepted` of
+     * {@link runExperiment}. */
+    onTelemetryNotAccepted?(entry: TelemetryNotAccepted): void | Promise<void>;
     /** Local scorer callbacks bound by their declared source digests. */
     localScorers?: LocalScorer[];
     /** Cases in flight, 1–64; defaults to 1. */
@@ -190,8 +202,9 @@ export interface RunSimulationOptions {
         /** Selected MCP surface. */
         surfaceKey: "google.gmail/mcp" | "slack/mcp";
     };
-    /** Invokes the existing local agent exactly once for this attempt. */
-    target(inputs: JsonValue, context: SimulationTargetContext): JsonValue | undefined | Promise<JsonValue | undefined>;
+    /** Invokes the existing local agent exactly once for this attempt; return the output, or
+     * `withFiles(output, files)` when it generated files. */
+    target(inputs: JsonValue, context: SimulationTargetContext): JsonValue | TargetResult | undefined | Promise<JsonValue | TargetResult | undefined>;
     /** Receives bounded nonsecret lifecycle progress. */
     onProgress?(event: SimulationProgress): void | Promise<void>;
 }

package/dist/evals/simulation.js

@@ -430,6 +430,12 @@ export async function runSimulation(options) {
             checkpointDirectory: join(store.directory, `experiment-${experimentId}`),
             persistResultContent: options.persistResultContent,
             traceEvidence: options.traceEvidence,
+            ...(options.traceNotAccepted ? { traceNotAccepted: options.traceNotAccepted } : {}),
+            ...(options.onTelemetryNotAccepted
+                ? {
+                    onTelemetryNotAccepted: (entry) => options.onTelemetryNotAccepted(entry),
+                }
+                : {}),
             environmentEvidence: "required",
             scorers: bindings,
             concurrency: options.concurrency,
@@ -474,6 +480,8 @@ export async function runSimulation(options) {
                     ...(targetContext.connectionBundle
                         ? { connectionBundle: structuredClone(targetContext.connectionBundle) }
                         : {}),
+                    files: structuredClone(targetContext.files),
+                    outputDirectory: targetContext.outputDirectory,
                     signal: targetContext.signal,
                 }),
             }),

package/dist/evals/types.d.ts

@@ -264,6 +264,10 @@ export interface StoredResult extends ResultSummary {
     error: TypedError | null;
     /** Source digest of the local scorer that produced it, when applicable. */
     sourceDigest: string | null;
+    /** True only for a `skipped` result Hue's own outcome scoring recorded because the evaluator
+     * does not apply to the case (it has no outcome criteria or conversion rubric to grade); a
+     * submitted result never reads true. Absent from servers that predate it. */
+    notApplicable?: boolean;
 }
 /** A hosted judge job and its charge accounting. */
 export interface JudgeJob {

package/dist/evals/verdicts.d.ts

@@ -35,6 +35,11 @@ export interface VerdictResult {
     explanation: string | null;
     /** Scorer failure for `state: "error"`, otherwise `null`. */
     error: TypedError | null;
+    /** Whether Hue recorded that the evaluator does not apply to the case; such a result counts as
+     * neither a pass nor a failure. Absent means false. */
+    notApplicable?: boolean;
+    /** What a not-applicable evaluator needs that the case lacks, such as `outcome_criteria`. */
+    requires?: string;
 }
 /** Outcome of {@link waitForResults}. */
 export interface VerdictResults {
@@ -65,10 +70,15 @@ export interface CaseVerdict {
     externalKey: string;
     /** Subject scored for the case, or `null` before completion. */
     subjectId: string | null;
-    /** `passed` and `failed` are scored verdicts; `error`, `skipped` and `pending` carry no verdict. */
+    /** `passed` and `failed` are scored verdicts; `error`, `skipped` and `pending` carry no verdict.
+     * Only the evaluators that apply to the case decide it; one where none applies is an error. */
     state: "passed" | "failed" | "error" | "skipped" | "pending";
-    /** True only for `state: "passed"`: every scored metric passed and no pin errored or is missing. */
+    /** True only for `state: "passed"`: every scored metric of an applicable pin passed and no
+     * applicable pin errored or is missing. */
     passed: boolean;
+    /** Pinned scorer versions Hue recorded as not applicable to the case; always set by
+     * {@link summarizeVerdicts}. */
+    notApplicable?: string[];
     /** Reported metrics across pinned scorers, in result order. */
     metrics: (Metric & {
         /** Scorer version that reported the metric. */
@@ -97,6 +107,9 @@ export interface VerdictSummary {
         skipped: number;
         /** Cases still missing a result. */
         pending: number;
+        /** Results of evaluators that do not apply to their case, a case can have several; always
+         * set by {@link summarizeVerdicts}. */
+        notApplicable?: number;
     };
 }
 /** Whether one reported metric counts as passing: an explicit `passed`, else a true boolean. */

package/dist/evals/verdicts.js

@@ -66,6 +66,10 @@ export async function waitForResults(client, options) {
                     if (!id)
                         continue;
                     const stored = await client.getResult(id);
+                    // Only Hue's own flag on a skipped result marks it not applicable; the evidence alone
+                    // never does, and a scored or errored result keeps its verdict whatever it carries.
+                    const notApplicable = stored.state === "skipped" && stored.notApplicable === true;
+                    const evidence = stored.evidence;
                     results.push({
                         id: stored.id,
                         itemId: stored.itemId,
@@ -75,6 +79,10 @@ export async function waitForResults(client, options) {
                         metrics: stored.state === "scored" ? stored.metrics : [],
                         explanation: stored.explanation ?? null,
                         error: stored.error ?? null,
+                        notApplicable,
+                        ...(notApplicable && typeof evidence?.requires === "string"
+                            ? { requires: evidence.requires }
+                            : {}),
                     });
                 }
             return { complete, items, results };
@@ -105,11 +113,16 @@ export function summarizeVerdicts(results, options) {
         const subjectId = subjectByCase.get(item.id) ?? null;
         const own = subjectId ? results.results.filter((result) => result.subjectId === subjectId) : [];
         const metrics = own.flatMap((result) => result.metrics.map((metric) => ({ ...metric, scorerVersionId: result.scorerVersionId })));
-        const errors = own.filter((result) => result.state === "error");
-        const scored = own.filter((result) => result.state === "scored");
+        // An evaluator that does not apply to the case neither passes nor fails it.
+        const skippedAsInapplicable = (result) => result.state === "skipped" && result.notApplicable === true;
+        const inapplicable = own.filter(skippedAsInapplicable);
+        const applicable = own.filter((result) => !skippedAsInapplicable(result));
+        const errors = applicable.filter((result) => result.state === "error");
+        const scored = applicable.filter((result) => result.state === "scored");
         const failing = scored.filter((result) => !result.metrics.every(metricPassed));
         const missing = [...pins].some((pin) => !own.some((result) => result.scorerVersionId === pin));
-        const state = errors.length
+        const noneApplies = !missing && own.length > 0 && !applicable.length;
+        const state = errors.length || noneApplies
             ? "error"
             : failing.length
                 ? "failed"
@@ -118,23 +131,44 @@ export function summarizeVerdicts(results, options) {
                     : scored.length
                         ? "passed"
                         : "skipped";
+        const requires = [
+            ...new Set(inapplicable.flatMap((result) => (result.requires ? [result.requires] : []))),
+        ];
         return {
             caseId: item.id,
             externalKey: item.externalKey,
             subjectId,
             state,
             passed: state === "passed",
+            notApplicable: inapplicable.map((result) => result.scorerVersionId),
             metrics,
-            explanations: own
-                .filter((result) => result.state !== "scored" || failing.includes(result))
-                .map((result) => result.explanation)
-                .filter((explanation) => !!explanation),
+            explanations: [
+                ...(noneApplies
+                    ? [
+                        `No pinned evaluator applies to this case${requires.length ? ` (they need ${requires.join(" or ")})` : ""}; pin one that grades it`,
+                    ]
+                    : []),
+                ...applicable
+                    .filter((result) => result.state !== "scored" || failing.includes(result))
+                    .map((result) => result.explanation)
+                    .filter((explanation) => !!explanation),
+            ],
             errors: errors.map((result) => result.error?.type ?? "ScorerError"),
         };
     });
-    const totals = { cases: cases.length, passed: 0, failed: 0, error: 0, skipped: 0, pending: 0 };
-    for (const item of cases)
+    const totals = {
+        cases: cases.length,
+        passed: 0,
+        failed: 0,
+        error: 0,
+        skipped: 0,
+        pending: 0,
+        notApplicable: 0,
+    };
+    for (const item of cases) {
         totals[item.state]++;
+        totals.notApplicable += item.notApplicable?.length ?? 0;
+    }
     return { cases, totals };
 }
 /** Diffs the current summary against a baseline by case key. */