@ultimat3/ai 1.0.0

package/src/errors.ts

@@ -0,0 +1,422 @@
+// The X_* codes owned by @ultimat3/ai. Budget and threshold failures name the exact knob to
+// change, because the caller is often a CI job with no human attached.
+
+import { registerErrorCodes, UltimateError } from '@ultimat3/core';
+
+export const AI_ERROR_CODES = [
+  'X_AI_PROVIDER_UNAVAILABLE',
+  'X_AI_KEY_MISSING',
+  'X_AI_REQUEST_INVALID',
+  'X_AI_BUDGET_EXCEEDED',
+  'X_AI_GATEWAY_MISSING',
+  'X_AI_PROMPT_VERSION',
+  'X_LLM_OUTPUT_INVALID',
+  'X_LLM_REFUSED',
+  'X_LLM_TRUNCATED',
+  'X_EVAL_THRESHOLD',
+  'X_EVAL_BASELINE_MISSING',
+  'X_EVAL_BASELINE_INVALID',
+  'X_EVAL_MISSING',
+  'X_EVAL_RECORDING',
+  'X_VECTOR_DIM_MISMATCH',
+  'X_VECTOR_SCOPE_WIDENED',
+  'X_AI_EMBEDDER_INVALID',
+] as const;
+
+export type AiErrorCode = (typeof AI_ERROR_CODES)[number];
+
+export const AI_ERROR_TITLES: Readonly<Record<AiErrorCode, string>> = {
+  X_AI_PROVIDER_UNAVAILABLE: 'the model provider is unreachable',
+  X_AI_KEY_MISSING: 'the provider API key is not set',
+  X_AI_REQUEST_INVALID: 'the provider would reject this request',
+  X_AI_BUDGET_EXCEEDED: 'a model call would exceed its budget',
+  X_AI_GATEWAY_MISSING: 'an llm() action ran with no gateway installed',
+  X_AI_PROMPT_VERSION: 'prompt version or slots are wrong',
+  X_LLM_OUTPUT_INVALID: 'structured output failed its schema on the answer and the repair turn',
+  X_LLM_REFUSED: 'the model declined the request',
+  X_LLM_TRUNCATED: 'the answer hit its maxTokens ceiling before it was complete',
+  X_EVAL_THRESHOLD: 'an eval scored below its tolerance',
+  X_EVAL_BASELINE_MISSING: 'an eval has no recorded baseline to gate against',
+  X_EVAL_BASELINE_INVALID: 'a recorded baseline cannot be read',
+  X_EVAL_MISSING: 'a prompt has no eval',
+  X_EVAL_RECORDING: 'the gate ran with baseline recording switched on',
+  X_VECTOR_DIM_MISMATCH: 'embedding dimensions differ from the store',
+  X_VECTOR_SCOPE_WIDENED: 'a derived vector scope tried to leave its tenant',
+  X_AI_EMBEDDER_INVALID: 'an Embedder returned fewer vectors than texts it was given',
+};
+
+// Titles must be registered for `format()` to render the contract's first line. Unconditional and
+// in one call: every code above is owned here, so a second package claiming one is a real conflict
+// that has to surface as X_ERROR_CODE_DUPLICATE rather than resolve to whoever imported first.
+// `X_FORBIDDEN` (policy's) and `X_CURRENCY_MISMATCH` (money's) are thrown here but never titled.
+registerErrorCodes(
+  Object.fromEntries(Object.entries(AI_ERROR_TITLES).map(([code, title]) => [code, { title }])),
+);
+
+const docsFor = (code: AiErrorCode): string => `https://ultimate.dev/errors/${code}`;
+
+/** Every configured provider refused or errored. Carries what each one said. */
+export class AiProviderUnavailableError extends UltimateError {
+  constructor(input: { model: string; attempts: readonly string[] }) {
+    super({
+      code: 'X_AI_PROVIDER_UNAVAILABLE',
+      cause: `no provider could serve model "${input.model}" (${input.attempts.join(' | ')})`,
+      fix: 'check ai.providers in app.config.ts and the provider API key env var',
+      docs: docsFor('X_AI_PROVIDER_UNAVAILABLE'),
+    });
+  }
+}
+
+/**
+ * A request would exceed its token budget. Thrown BEFORE the call, so nothing is spent and
+ * nothing is silently truncated — a truncated prompt produces a confidently wrong answer,
+ * which is worse than a refusal.
+ */
+export class AiBudgetExceededError extends UltimateError {
+  constructor(input: {
+    scope: string;
+    requested: number;
+    remaining: number;
+    limit: number;
+    /** What the numbers count. Money scopes pass minor units so nothing reads as a float. */
+    unit?: string;
+  }) {
+    super({
+      code: 'X_AI_BUDGET_EXCEEDED',
+      cause:
+        `request needs ${input.requested} ${input.unit ?? 'tokens'} but scope ` +
+        `"${input.scope}" has ${input.remaining} of ${input.limit} left`,
+      fix: `raise ai.budget for "${input.scope}" in app.config.ts, or shorten the prompt`,
+      docs: docsFor('X_AI_BUDGET_EXCEEDED'),
+    });
+  }
+}
+
+/**
+ * An `llm()` action ran with no gateway installed. Ambient rather than injected because a
+ * declaration is authored at module scope, long before a provider exists — so the miss is a
+ * boot-order fault, and naming the boot call is the whole fix.
+ */
+export class AiGatewayMissingError extends UltimateError {
+  constructor(input: { prompt: string }) {
+    super({
+      code: 'X_AI_GATEWAY_MISSING',
+      cause: `an llm action on prompt "${input.prompt}" ran before any gateway was configured`,
+      fix: 'configureAi({ gateway: createGateway({ providers: [new AnthropicProvider()] }) }) at boot',
+      docs: docsFor('X_AI_GATEWAY_MISSING'),
+    });
+  }
+}
+
+/**
+ * The model's answer failed the action's `output` schema on the first turn AND on the repair
+ * turn that followed. Two failures is a disagreement between the prompt and the schema, not a
+ * bad roll — a third attempt only spends money, so this throws instead of looping.
+ */
+export class LlmOutputInvalidError extends UltimateError {
+  constructor(input: { prompt: string; attempts: number; issues: string }) {
+    super({
+      code: 'X_LLM_OUTPUT_INVALID',
+      cause:
+        `prompt "${input.prompt}" returned output failing its schema on all ` +
+        `${input.attempts} attempts: ${input.issues}`,
+      fix: 'describe the output shape in the prompt template and bump its version, or widen `output` in the llm() declaration',
+      docs: docsFor('X_LLM_OUTPUT_INVALID'),
+    });
+  }
+}
+
+/**
+ * The provider's safety classifiers declined the request. A refusal is a 200 with no answer in
+ * it, so it has to become an error HERE or it becomes an empty string somewhere downstream that
+ * reads exactly like a model with nothing to say. Distinct from `X_LLM_OUTPUT_INVALID` because
+ * the fix is different: nothing about the schema is wrong, and a repair turn buys a second
+ * refusal at full price.
+ */
+export class LlmRefusedError extends UltimateError {
+  constructor(input: {
+    prompt: string;
+    model: string;
+    /** A blessed model that is NOT the one that refused — the fix has to be pasteable. */
+    alternative: string;
+    category: string | undefined;
+    explanation: string | undefined;
+  }) {
+    super({
+      code: 'X_LLM_REFUSED',
+      cause:
+        `model "${input.model}" declined prompt "${input.prompt}"` +
+        `${input.category === undefined ? '' : ` (${input.category})`}` +
+        `${input.explanation === undefined ? '' : `: ${input.explanation}`}`,
+      fix: `set model: '${input.alternative}' on the llm() declaration, or edit the template in definePrompt('${input.prompt}') and bump its version`,
+      docs: docsFor('X_LLM_REFUSED'),
+      meta: { model: input.model, category: input.category },
+    });
+  }
+}
+
+/**
+ * The answer was cut off at the enforced ceiling and what arrived does not satisfy the schema.
+ * Thrown instead of the repair turn on purpose: the ceiling does not move between attempts, so
+ * a second answer truncates at exactly the same place and only spends money.
+ */
+export class LlmTruncatedError extends UltimateError {
+  constructor(input: { prompt: string; maxTokens: number }) {
+    super({
+      code: 'X_LLM_TRUNCATED',
+      cause: `prompt "${input.prompt}" was cut off at its ${input.maxTokens}-token ceiling`,
+      fix: `set maxTokens: ${input.maxTokens * 2} on the llm() declaration, or drop fields from its output schema`,
+      docs: docsFor('X_LLM_TRUNCATED'),
+    });
+  }
+}
+
+/** A prompt was requested at a version whose content hash does not match the registry. */
+export class AiPromptVersionError extends UltimateError {
+  constructor(input: { id: string; requested: string; available: readonly string[] }) {
+    super({
+      code: 'X_AI_PROMPT_VERSION',
+      cause: `prompt "${input.id}" has no version "${input.requested}" (have: ${
+        input.available.length > 0 ? input.available.join(', ') : 'none'
+      })`,
+      fix: 'bump the version in definePrompt after editing the template, then x manifest',
+      docs: docsFor('X_AI_PROMPT_VERSION'),
+    });
+  }
+}
+
+/**
+ * A prompt was rendered with variables its template does not accept. Shares
+ * `X_AI_PROMPT_VERSION` because it is the same class of fault: the prompt artifact and the
+ * call site disagree about the prompt's contract.
+ */
+export class AiPromptRenderError extends UltimateError {
+  constructor(input: { ref: string; missing: readonly string[] }) {
+    super({
+      code: 'X_AI_PROMPT_VERSION',
+      cause: `prompt "${input.ref}" was rendered without: ${input.missing.join(', ')}`,
+      fix: 'pass every {{variable}} the template declares, or remove it from the template',
+      docs: docsFor('X_AI_PROMPT_VERSION'),
+    });
+  }
+}
+
+/**
+ * An eval scored further below its recorded baseline than its tolerance allows. The gate is the
+ * DROP, not an absolute number — a model that got marginally worse everywhere is not the same
+ * event as a prompt edit that broke one case, and only the second one is anybody's fault.
+ *
+ * This is a test failure, not a warning.
+ */
+export class EvalThresholdError extends UltimateError {
+  constructor(input: {
+    eval: string;
+    score: number;
+    baseline: number;
+    tolerance: number;
+    promptVersion: string;
+    regressed: readonly string[];
+  }) {
+    super({
+      code: 'X_EVAL_THRESHOLD',
+      cause:
+        `eval "${input.eval}" scored ${input.score.toFixed(3)} against a recorded baseline of ` +
+        `${input.baseline.toFixed(3)} (tolerance ${input.tolerance.toFixed(3)}) on prompt ` +
+        `version ${input.promptVersion}; regressed: ${input.regressed.join(', ')}`,
+      fix: `x test ${input.eval} to see per-case scores, then fix the prompt — or ULTIMATE_EVAL_RECORD=1 x test eval to accept the new numbers as a reviewed diff`,
+      docs: docsFor('X_EVAL_THRESHOLD'),
+    });
+  }
+}
+
+/**
+ * An eval declared a baseline that has never been recorded. Not a pass: an eval with nothing to
+ * compare against gates on nothing, and a step that cannot fail is a step that is not running.
+ */
+export class EvalBaselineMissingError extends UltimateError {
+  constructor(input: { eval: string; path: string; reason: string; fix?: string }) {
+    super({
+      code: 'X_EVAL_BASELINE_MISSING',
+      cause: `eval "${input.eval}" gates against ${input.path}, which ${input.reason}`,
+      fix: input.fix ?? `ULTIMATE_EVAL_RECORD=1 x test eval, then commit ${input.path}`,
+      docs: docsFor('X_EVAL_BASELINE_MISSING'),
+    });
+  }
+}
+
+/** A recorded baseline that cannot be read. Never treated as absent — that would erase a gate. */
+export class EvalBaselineInvalidError extends UltimateError {
+  constructor(input: { path: string; problem: string }) {
+    super({
+      code: 'X_EVAL_BASELINE_INVALID',
+      cause: `the recorded baseline ${input.path} ${input.problem}`,
+      fix: `ULTIMATE_EVAL_RECORD=1 x test eval to re-record ${input.path}`,
+      docs: docsFor('X_EVAL_BASELINE_INVALID'),
+    });
+  }
+}
+
+/**
+ * A registered prompt that no eval names. An unevaluated prompt is untested code that costs
+ * money and answers users, so the gate fails on it exactly like an untyped module.
+ */
+export class EvalMissingError extends UltimateError {
+  constructor(input: { prompt: string; id: string }) {
+    super({
+      code: 'X_EVAL_MISSING',
+      cause: `prompt "${input.prompt}" has no eval`,
+      fix: `defineEval({ name: '${input.id}', prompt, cases, scorers, tolerance, baseline }) beside the prompt, then ULTIMATE_EVAL_RECORD=1 x test eval`,
+      docs: docsFor('X_EVAL_MISSING'),
+    });
+  }
+}
+
+/**
+ * The gate ran with baseline recording switched on. Recording makes every eval write the numbers
+ * it just measured and pass, so a `x verify` that inherited the flag reports green over scores
+ * nothing compared — and rewrites the committed baselines on its way through, which is the half
+ * a red step alone would not undo. Recording is a deliberate, reviewable diff, never a gate run.
+ */
+export class EvalRecordingError extends UltimateError {
+  constructor(input: { env: string }) {
+    super({
+      code: 'X_EVAL_RECORDING',
+      cause: `${input.env} is set, so every eval would re-record its baseline instead of gating on it`,
+      fix: `env -u ${input.env} x verify`,
+      docs: docsFor('X_EVAL_RECORDING'),
+    });
+  }
+}
+
+/** A vector's length does not match the store's declared dimension. */
+export class VectorDimMismatchError extends UltimateError {
+  constructor(input: { store: string; expected: number; received: number }) {
+    super({
+      code: 'X_VECTOR_DIM_MISMATCH',
+      cause: `store "${input.store}" expects ${input.expected} dimensions, got ${input.received}`,
+      fix: 'use the same embedder that created the store, or x ai reindex to rebuild it',
+      docs: docsFor('X_VECTOR_DIM_MISMATCH'),
+    });
+  }
+}
+
+/**
+ * A derived vector scope tried to leave the tenant it was bound to. Scopes only ever tighten,
+ * so this is always the same mistake: a request handler re-scoping the store it was handed
+ * instead of deriving from the unscoped one. Widening silently would be a cross-tenant read.
+ */
+export class VectorScopeWidenedError extends UltimateError {
+  constructor(input: { store: string; held: string; requested: string }) {
+    super({
+      code: 'X_VECTOR_SCOPE_WIDENED',
+      cause:
+        `store "${input.store}" is bound to tenant "${input.held}" and cannot be re-scoped ` +
+        `to "${input.requested}"`,
+      fix: `derive from the unscoped store instead: vectorStore.scoped({ tenant: '${input.requested}' })`,
+      docs: docsFor('X_VECTOR_SCOPE_WIDENED'),
+    });
+  }
+}
+
+/**
+ * An embedder returned vectors of a length other than the one it declares. Separate from the
+ * store's mismatch because the fix is different: here the DECLARATION is wrong, and every
+ * vector written before this call is the wrong width in whatever store accepted them.
+ */
+export class EmbedderDimMismatchError extends UltimateError {
+  constructor(input: { embedder: string; expected: number; received: number }) {
+    super({
+      code: 'X_VECTOR_DIM_MISMATCH',
+      cause:
+        `embedder "${input.embedder}" is declared with ${input.expected} dimensions but the ` +
+        `provider returned ${input.received}`,
+      fix: `set dimension: ${input.received} on the embedder, then x ai reindex to rebuild the store`,
+      docs: docsFor('X_VECTOR_DIM_MISMATCH'),
+    });
+  }
+}
+
+/**
+ * `embedOne` asked an `Embedder` for one vector and got none back — a batch-size invariant the
+ * embedder itself broke, not a caller mistake. Distinct from `X_VECTOR_DIM_MISMATCH`: this fires
+ * before there is a vector at all, so there is nothing yet to measure the width of.
+ */
+export class AiEmbedderInvalidError extends UltimateError {
+  constructor(input: { embedder: string }) {
+    super({
+      code: 'X_AI_EMBEDDER_INVALID',
+      cause: `embedder "${input.embedder}" returned no vector for a batch of one text`,
+      // The `${…}` the fix used to carry is unreadable to the `errors` gate, which blanks every
+      // interpolation — so the literal half alone has to name the call. Which embedder broke the
+      // invariant is a fact of the failure, and the cause and `meta` are where facts live.
+      fix: 'return one vector per input text from embed(), in the order the texts arrived',
+      docs: docsFor('X_AI_EMBEDDER_INVALID'),
+      meta: { embedder: input.embedder },
+    });
+  }
+}
+
+/** No credential at call time. Named env var, because that is the whole fix. */
+export class AiKeyMissingError extends UltimateError {
+  constructor(input: { provider: string; envVar: string }) {
+    super({
+      code: 'X_AI_KEY_MISSING',
+      cause: `provider "${input.provider}" has no API key: ${input.envVar} is unset and none was passed to its constructor`,
+      fix: `export ${input.envVar}=<key>, or pass { apiKey } when constructing the provider`,
+      docs: docsFor('X_AI_KEY_MISSING'),
+      meta: { provider: input.provider, envVar: input.envVar },
+    });
+  }
+}
+
+/**
+ * A request the provider would answer with a 400, refused locally instead. Local because a
+ * round trip to learn a rule the framework already knows is a round trip that costs latency
+ * and teaches nothing — and the provider's own message names the field, not the fix.
+ */
+export class AiRequestInvalidError extends UltimateError {
+  constructor(input: { detail: string; fix: string }) {
+    super({
+      code: 'X_AI_REQUEST_INVALID',
+      cause: input.detail,
+      fix: input.fix,
+      docs: docsFor('X_AI_REQUEST_INVALID'),
+    });
+  }
+}
+
+/**
+ * The provider answered with a non-2xx, an in-band error event, or a body nothing can be read
+ * out of. Carries `status` as a real field so the gateway's retry policy can read it: a 429 or
+ * a 503 is momentary, a 400 is the same rejection forever and retrying only burns the budget.
+ */
+export class AiTransportError extends UltimateError {
+  readonly status: number | undefined;
+
+  constructor(input: { provider: string; status?: number | undefined; detail: string }) {
+    super({
+      code: 'X_AI_PROVIDER_UNAVAILABLE',
+      cause: `provider "${input.provider}" ${
+        input.status === undefined ? 'failed' : `returned ${input.status}`
+      }: ${input.detail}`,
+      fix: fixForStatus(input.status),
+      docs: docsFor('X_AI_PROVIDER_UNAVAILABLE'),
+      meta: { provider: input.provider, status: input.status },
+    });
+    this.status = input.status;
+  }
+}
+
+function fixForStatus(status: number | undefined): string {
+  if (status === 401 || status === 403) {
+    return 'export ANTHROPIC_API_KEY=<key> with a key that is active for this model';
+  }
+  if (status === 429) {
+    return 'lower concurrency or raise the provider rate limit; the gateway already backs off';
+  }
+  if (status !== undefined && status >= 400 && status < 500) {
+    return 'fix the request named in cause — shorten the prompt, or correct the tool schema';
+  }
+  if (status !== undefined) return 'retry; if it persists check the provider status page';
+  return 'check egress from this process (x doctor --json), then the provider status page';
+}

package/src/eval-baseline.ts

@@ -0,0 +1,140 @@
+// The recorded scores an eval gates against.
+//
+// The gate is the DELTA from the last accepted run, never an absolute number: models drift,
+// prompts should not. An absolute floor fails every eval at once the day a provider ships a
+// slightly different model, which trains everyone to lower thresholds until they mean nothing.
+//
+// A baseline is a committed file, so accepting a new number is a reviewable diff — and it
+// carries the prompt ref and hash that produced it, because a score with no prompt attached is
+// not a measurement.
+
+import { isAbsolute } from 'node:path';
+import { EvalBaselineInvalidError, EvalBaselineMissingError } from './errors';
+
+export interface EvalBaseline {
+  readonly eval: string;
+  /** `id@version` of the prompt that produced these numbers — usually the previous version. */
+  readonly prompt: string;
+  readonly promptHash: string;
+  readonly score: number;
+  /** Per-case scores, so the gate can name the case a prompt edit broke. */
+  readonly cases: Readonly<Record<string, number>>;
+}
+
+/** One score that fell further than the eval's declared tolerance allows. */
+export interface Regression {
+  /** A case name, or `overall` for the run's mean. */
+  readonly case: string;
+  readonly baseline: number;
+  readonly score: number;
+}
+
+/** The run-level entry, named so a failure reads the same whether one case or all of them fell. */
+export const OVERALL = 'overall';
+
+/** Set to re-record every baseline instead of gating on it: `ULTIMATE_EVAL_RECORD=1 x test eval`. */
+export const RECORD_ENV = 'ULTIMATE_EVAL_RECORD';
+
+export const recordingBaselines = (): boolean => (Bun.env[RECORD_ENV] ?? '') !== '';
+
+/**
+ * Where the baseline lives. Declarations write `import.meta.resolve('./x.baseline.json')`, which
+ * is stable in a checkout and in a container; a cwd-relative path would resolve to a different
+ * file depending on where the suite was started, which is how a gate silently stops gating.
+ */
+export function baselinePath(spec: string, evalName: string): string {
+  if (spec.startsWith('file://')) return Bun.fileURLToPath(spec);
+  if (isAbsolute(spec)) return spec;
+  throw new EvalBaselineMissingError({
+    eval: evalName,
+    path: spec,
+    reason: 'is neither an absolute path nor a file:// URL',
+    fix: `baseline: import.meta.resolve('${spec}') in defineEval({ name: '${evalName}' })`,
+  });
+}
+
+/** The recorded scores, or `undefined` when this eval has never been recorded. */
+export async function readBaseline(path: string): Promise<EvalBaseline | undefined> {
+  const file = Bun.file(path);
+  if (!(await file.exists())) return undefined;
+  const parsed: unknown = await file.json().catch(() => undefined);
+  return parseBaseline(parsed, path);
+}
+
+/** Fixed key order and a trailing newline: a re-record must diff as scores, not as formatting. */
+export async function writeBaseline(path: string, baseline: EvalBaseline): Promise<void> {
+  const ordered = {
+    eval: baseline.eval,
+    prompt: baseline.prompt,
+    promptHash: baseline.promptHash,
+    score: round(baseline.score),
+    cases: Object.fromEntries(
+      Object.keys(baseline.cases)
+        .sort()
+        .map((name) => [name, round(baseline.cases[name] ?? 0)]),
+    ),
+  };
+  await Bun.write(path, `${JSON.stringify(ordered, null, 2)}\n`);
+}
+
+/**
+ * Every score that dropped further than `tolerance`. The run mean AND each case, because a mean
+ * that holds while one case collapses is the regression an eval exists to catch.
+ *
+ * A case the baseline does not know is not compared — a new case has nothing to regress from,
+ * and the run mean already covers whatever it scores.
+ */
+export function regressionsAgainst(input: {
+  readonly baseline: EvalBaseline;
+  readonly score: number;
+  readonly cases: Readonly<Record<string, number>>;
+  readonly tolerance: number;
+}): readonly Regression[] {
+  const found: Regression[] = [];
+  const fell = (was: number, now: number): boolean => now < was - input.tolerance;
+  if (fell(input.baseline.score, input.score)) {
+    found.push({ case: OVERALL, baseline: input.baseline.score, score: input.score });
+  }
+  for (const [name, was] of Object.entries(input.baseline.cases)) {
+    const now = input.cases[name];
+    if (now !== undefined && fell(was, now)) found.push({ case: name, baseline: was, score: now });
+  }
+  return found;
+}
+
+/** `case 0.40 ← 0.95` — the two numbers a reader needs, in the order they happened. */
+export const describeRegression = (regression: Regression): string =>
+  `${regression.case} ${regression.score.toFixed(2)} ← ${regression.baseline.toFixed(2)}`;
+
+/** Three decimals: enough to see a real drift, few enough that noise is not a diff. */
+const round = (value: number): number => Math.round(value * 1000) / 1000;
+
+function parseBaseline(value: unknown, path: string): EvalBaseline {
+  const invalid = (problem: string): never => {
+    throw new EvalBaselineInvalidError({ path, problem });
+  };
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return invalid('is not a JSON object');
+  }
+  const record = value as Record<string, unknown>;
+  const text = (key: string): string =>
+    typeof record[key] === 'string' ? (record[key] as string) : invalid(`has no string "${key}"`);
+  const score = record['score'];
+  if (typeof score !== 'number') invalid('has no number "score"');
+  const cases = record['cases'];
+  if (typeof cases !== 'object' || cases === null || Array.isArray(cases)) {
+    invalid('has no object "cases"');
+  }
+  const scores: Record<string, number> = {};
+  for (const [name, entry] of Object.entries(cases as Record<string, unknown>)) {
+    if (typeof entry !== 'number') invalid(`has a non-numeric score for case "${name}"`);
+    scores[name] = entry as number;
+  }
+  return {
+    eval: text('eval'),
+    prompt: text('prompt'),
+    promptHash: text('promptHash'),
+    score: score as number,
+    cases: scores,
+  };
+}