@kindgi/api 0.1.4 → 0.1.5

package/src/routes/cost.ts

@@ -1,25 +1,32 @@
 // SPDX-License-Identifier: Apache-2.0
 // Copyright (C) 2026 Kindgi Inc.
 
-import { Hono } from 'hono';
+import { type Context, Hono } from 'hono';
 
 import type { Cursor, TenantId, Timestamp } from '@kindgi/types';
 
+import { type ResourceRef, denyPayload, ref } from '@kindgi/authz';
+import type { ProjectBinding } from '@kindgi/platform';
 import {
   COST_AGGREGATE_DEFAULT_LIMIT,
   COST_AGGREGATE_MAX_LIMIT,
   COST_GROUP_DIMENSIONS,
   type CostAggregateGroup,
+  type CostAggregateInput,
   type CostBinding,
   type CostGroupDimension,
   type CostRecord,
   type CostRecordFilter,
   type CostTokenTotals,
 } from '../cost-binding.js';
+
 import { statusFor, toWireError } from '../errors.js';
+import type { Authorizer } from '../middleware/authorize.js';
 import type { AppEnv } from '../types.js';
+import { deniedBy } from './denied.js';
 import { clampLimit } from './pagination.js';
-import { parseScopeParams } from './scope-params.js';
+import { projectIdsCallerMay } from './readable-projects.js';
+import { parseScopeParams, scopeResourceRef } from './scope-params.js';
 
 /**
  * Cost readback routes — part of the admin control plane. Three
@@ -38,8 +45,23 @@ import { parseScopeParams } from './scope-params.js';
  *
  * Budgets are not part of this surface.
  */
-export function costRouter(binding: CostBinding): Hono<AppEnv> {
+export function costRouter(
+  binding: CostBinding,
+  /**
+   * With one: a record needs `read` on its project (the tenant, for one
+   * with no project); an aggregate, `read` on the scope it's asked for,
+   * and across projects (no scope, or an org) it counts only the projects
+   * the caller may read, unless the caller is a tenant admin.
+   */
+  authorizer?: Authorizer,
+  /** The tenant's projects, to check one by one when the authorizer can't list them. */
+  projects?: Pick<ProjectBinding, 'list'>,
+): Hono<AppEnv> {
   const r = new Hono<AppEnv>();
+  const recordRef = (tenantId: TenantId, rec: { readonly projectId?: unknown }): ResourceRef =>
+    rec.projectId !== undefined && rec.projectId !== null
+      ? ref('project', rec.projectId as string)
+      : ref('tenant', tenantId as unknown as string);
 
   const DEFAULT_AGGREGATE_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
 
@@ -81,8 +103,12 @@ export function costRouter(binding: CostBinding): Hono<AppEnv> {
       ...(scopeParsed.inherit !== undefined && { inherit: scopeParsed.inherit }),
       ...(include.rawUsage && { includeRawUsage: true }),
     });
+    const visible =
+      authorizer === undefined
+        ? page.data
+        : await authorizer.filterByCan(c, 'read', page.data, (rec) => recordRef(tenantId, rec));
     return c.json({
-      data: page.data.map(serializeRecord),
+      data: visible.map(serializeRecord),
       hasMore: page.nextCursor !== undefined,
       ...(page.nextCursor !== undefined && { nextCursor: page.nextCursor as unknown as string }),
     });
@@ -117,6 +143,8 @@ export function costRouter(binding: CostBinding): Hono<AppEnv> {
         ),
       );
     }
+    const refused = await deniedBy(authorizer, c, 'read', recordRef(tenantId, record));
+    if (refused !== undefined) return refused;
     return c.json(serializeRecord(record));
   });
 
@@ -249,6 +277,24 @@ export function costRouter(binding: CostBinding): Hono<AppEnv> {
       );
     }
 
+    const refused = await deniedBy(
+      authorizer,
+      c,
+      'read',
+      scopeResourceRef(scopeParsed.scope, tenantId),
+    );
+    if (refused !== undefined) return refused;
+
+    // Across projects, only what the caller may read counts (`aggregateReach`).
+    const reach = await aggregateReach(c, {
+      binding,
+      tenantId,
+      authorizer,
+      projects,
+      scope: scopeParsed.scope,
+    });
+    if (reach.kind === 'refused') return reach.response;
+
     const result = await binding.aggregate({
       tenantId,
       groupBy,
@@ -257,6 +303,7 @@ export function costRouter(binding: CostBinding): Hono<AppEnv> {
       ...(Object.keys(filter.value).length > 0 && { filter: filter.value }),
       ...(scopeParsed.scope !== undefined && { scope: scopeParsed.scope }),
       ...(scopeParsed.inherit !== undefined && { inherit: scopeParsed.inherit }),
+      ...(reach.kind === 'projects' && { readableProjectIds: reach.ids }),
       limit,
     });
 
@@ -285,6 +332,65 @@ export function costRouter(binding: CostBinding): Hono<AppEnv> {
 
 // -------- helpers --------
 
+/**
+ * What an aggregate may count for this caller. Across projects (no scope,
+ * the tenant, or an org), `read` on the scope lets a member ask, not see
+ * every project's spend: only the projects they may read count, applied in
+ * the binding's query (`readableProjectIds`). A tenant admin's, a project
+ * scope's (already checked) and one without authorization count everything
+ * the scope holds. When the readable projects can't be worked out, or the
+ * binding can't apply them, the aggregate is refused rather than
+ * over-counted.
+ */
+async function aggregateReach(
+  c: Context<AppEnv>,
+  args: {
+    readonly binding: CostBinding;
+    readonly tenantId: TenantId;
+    readonly authorizer: Authorizer | undefined;
+    readonly projects: Pick<ProjectBinding, 'list'> | undefined;
+    readonly scope: CostAggregateInput['scope'] | undefined;
+  },
+): Promise<
+  | { readonly kind: 'all' }
+  | { readonly kind: 'projects'; readonly ids: readonly string[] }
+  | { readonly kind: 'refused'; readonly response: Response }
+> {
+  const { binding, tenantId, authorizer, projects, scope } = args;
+  if (authorizer === undefined || scope?.kind === 'project') return { kind: 'all' };
+  if (await authorizer.can(c, 'admin', ref('tenant', tenantId as unknown as string))) {
+    return { kind: 'all' };
+  }
+  const ids = await projectIdsCallerMay(c, authorizer, projects, 'read');
+  if (ids !== undefined && binding.aggregatesReadableProjects === true) {
+    return { kind: 'projects', ids };
+  }
+  const deny = denyPayload(
+    'read',
+    scope?.kind === 'org' ? 'org' : 'tenant',
+    scope?.kind === 'org' ? (scope.orgId as unknown as string) : (tenantId as unknown as string),
+    ids === undefined
+      ? "the projects you may read can't be listed here, so an aggregate across projects can't be limited to them; ask per project (scopeKind=project)"
+      : "this deployment's cost store can't limit an aggregate to the projects you may read; ask per project (scopeKind=project)",
+  );
+  c.status(403);
+  return {
+    kind: 'refused',
+    response: c.json(
+      toWireError(
+        {
+          code: deny.code,
+          message: `Permission denied: ${deny.reason}`,
+          action: deny.action,
+          resource: deny.resource,
+          reason: deny.reason,
+        },
+        c.get('requestId'),
+      ),
+    ),
+  };
+}
+
 function parseRecordFilter(
   query: Readonly<Record<string, string | undefined>>,
   opts: { skipTime?: boolean } = {},

package/src/routes/denied.ts

@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Kindgi Inc.
+
+import type { Context } from 'hono';
+
+import type { Action, ResourceRef } from '@kindgi/authz';
+
+import type { Authorizer } from '../middleware/authorize.js';
+import type { AppEnv } from '../types.js';
+
+/**
+ * An in-handler check, for a resource known only once the request is
+ * read: nothing when the caller may take `action` on `resource` (or there
+ * is no authorizer), else the authorizer's own 403, to return as is.
+ */
+export async function deniedBy(
+  authorizer: Authorizer | undefined,
+  c: Context<AppEnv>,
+  action: Action,
+  resource: ResourceRef,
+): Promise<Response | undefined> {
+  if (authorizer === undefined) return undefined;
+  let allowed = false;
+  const answer = await authorizer.authorize(action, () => resource)(c, async () => {
+    allowed = true;
+  });
+  return allowed ? undefined : (answer as Response);
+}

package/src/routes/deployments.ts

@@ -9,7 +9,12 @@ import type { Agent, AgentId, AgentPins } from '@kindgi/agents';
 import { tuplesForCreate } from '@kindgi/authz';
 import { parsePublicKeyPem, verifyEd25519 } from '@kindgi/crypto';
 import type { Flow, FlowPins } from '@kindgi/flow';
-import { type Guardrail, validateGuardrailSpec } from '@kindgi/guardrails';
+import {
+  GUARDRAIL_SPEC_KEYS,
+  type Guardrail,
+  createCheckRegistry,
+  validateGuardrailSpec,
+} from '@kindgi/guardrails';
 import type { ProjectBinding, Scope } from '@kindgi/platform';
 import { validateToolManifest } from '@kindgi/tools';
 import type {
@@ -42,6 +47,7 @@ import { type FlowPinsLive, publishDeployedFlow, resolveFlowPins } from '../flow
 import type { GuardrailRegistryBinding } from '../guardrail-binding.js';
 import type { ImageRegistryBinding } from '../image-registry-binding.js';
 import type { LiveVersionBinding } from '../live-version-binding.js';
+import type { Authorizer } from '../middleware/authorize.js';
 import { PublishRefused } from '../publish-refused.js';
 import { type RegistryReadOnly, refuseReadOnly } from '../registry-read-only.js';
 import type { SecretBinding } from '../secrets-binding.js';
@@ -51,6 +57,7 @@ import type { AppEnv } from '../types.js';
 import { hasCapability, requireEnvName } from './env.js';
 import type { GuardrailWriteHook } from './guardrails.js';
 import { clampLimit } from './pagination.js';
+import { tenantResourceAccess } from './tenant-access.js';
 import type { ToolWriteHook } from './tools.js';
 
 /**
@@ -167,8 +174,12 @@ export interface DeploymentsRouterBindings {
   readonly onGuardrailWrite?: GuardrailWriteHook;
 }
 
-export function deploymentsRouter(bindings: DeploymentsRouterBindings): Hono<AppEnv> {
+export function deploymentsRouter(
+  bindings: DeploymentsRouterBindings,
+  authorizer?: Authorizer,
+): Hono<AppEnv> {
   const r = new Hono<AppEnv>();
+  r.use('*', tenantResourceAccess(authorizer));
 
   // ---------- GET / (list, cursor-paginated) ----------
   r.get('/', async (c) => {
@@ -476,10 +487,11 @@ export function deploymentsRouter(bindings: DeploymentsRouterBindings): Hono<App
     }
 
     // ---- Manifest validation ----
-    const validation = validatePrimitives(imageIndex, {
-      imageRef: wire.imageRef,
-      artifactVersion: wire.artifactVersion,
-    });
+    const validation = validatePrimitives(
+      imageIndex,
+      { imageRef: wire.imageRef, artifactVersion: wire.artifactVersion },
+      c.get('log'),
+    );
     if (validation.kind === 'err') {
       c.status(statusFor('deployment-validation-failed') as never);
       return c.json(
@@ -580,7 +592,7 @@ export function deploymentsRouter(bindings: DeploymentsRouterBindings): Hono<App
               await bindings.toolRegistry?.unregister({ tenantId, toolId, version });
             });
           } else if (outcome.kind !== 'already-registered') {
-            throw new PublishRefused('tool', `${tool.id}@${tool.version}`, outcome.kind);
+            throw new PublishRefused('tool', `${tool.id}@${tool.version}`, outcome);
           }
         }
       }
@@ -604,8 +616,22 @@ export function deploymentsRouter(bindings: DeploymentsRouterBindings): Hono<App
             rolled.push(async () => {
               await bindings.guardrailRegistry?.unregister({ tenantId, guardrailId });
             });
-          } else if (outcome.kind !== 'already-registered') {
-            throw new PublishRefused('guardrail', guardrail.id, outcome.kind);
+          } else if (outcome.kind === 'already-registered') {
+            const kept = await keepRegisteredGuardrail(
+              bindings.guardrailRegistry,
+              tenantId,
+              projectIdForGuardrail,
+              guardrail,
+            );
+            // Unregistered since the registry answered: registered afresh.
+            if (kept === 'registered') {
+              const guardrailId = guardrail.id;
+              rolled.push(async () => {
+                await bindings.guardrailRegistry?.unregister({ tenantId, guardrailId });
+              });
+            }
+          } else {
+            throw new PublishRefused('guardrail', guardrail.id, outcome);
           }
         }
       }
@@ -645,6 +671,13 @@ export function deploymentsRouter(bindings: DeploymentsRouterBindings): Hono<App
       // A primitive refused with a typed outcome is the caller's to fix:
       // that outcome's own status and code. Anything thrown is a 500.
       if (cause instanceof PublishRefused) {
+        if (cause.projectId !== undefined) {
+          c.get('log').info(`${cause.message}: deploy refused`, {
+            primitive: cause.primitive,
+            id: cause.id,
+            ownerProjectId: cause.projectId as unknown as string,
+          });
+        }
         c.status(statusFor(cause.code) as never);
         return c.json(
           toWireError(
@@ -1215,6 +1248,139 @@ interface DeployedImage {
   readonly artifactVersion: string;
 }
 
+/**
+ * A deploy keeps a guardrail id that's already live only when it's the
+ * deploy's own: in the project the deploy registers into, with the same
+ * definition (`sameGuardrailDefinition`, what its author declares). An id live in another project
+ * is refused (`guardrail-project-mismatch`, its project never named): the
+ * pack's agents would otherwise run that project's guardrail. One in this
+ * project with another definition is refused too
+ * (`guardrail-already-registered`): a deploy never changes a guardrail,
+ * and keeping the old one would run what the pack no longer says.
+ *
+ * `get` answers a guardrail without its project, so whether the id is in
+ * this project comes from the project's list. When the row is gone by the
+ * time it's looked at (unregistered meanwhile), the guardrail is
+ * registered again: `'registered'`, for the deploy to roll back.
+ */
+async function keepRegisteredGuardrail(
+  registry: GuardrailRegistryBinding,
+  tenantId: TenantId,
+  projectId: ProjectId,
+  guardrail: Guardrail,
+): Promise<'kept' | 'registered'> {
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const existing = await registry.get({ tenantId, guardrailId: guardrail.id });
+    if (existing !== null) {
+      if (!(await guardrailInProject(registry, tenantId, projectId, guardrail.id))) {
+        throw new PublishRefused('guardrail', guardrail.id, {
+          code: 'guardrail-project-mismatch',
+          reason: 'it belongs to another project',
+        });
+      }
+      if (!sameGuardrailDefinition(existing, guardrail)) {
+        throw new PublishRefused('guardrail', guardrail.id, {
+          code: 'guardrail-already-registered',
+          reason: `it is already registered with a different definition; unregister it (\`kindgi guardrails unregister ${guardrail.id}\`) and deploy again`,
+        });
+      }
+      return 'kept';
+    }
+    const again = await registry.register({
+      tenantId,
+      projectId,
+      guardrail,
+      enqueueTuples: (guardrailId) =>
+        tuplesForCreate({
+          kind: 'guardrail',
+          id: guardrailId as GuardrailId,
+          tenantId,
+          projectId,
+        }),
+    });
+    if (again.kind === 'ok') return 'registered';
+    if (again.kind !== 'already-registered') {
+      throw new PublishRefused('guardrail', guardrail.id, again);
+    }
+  }
+  throw new Error(`guardrail ${guardrail.id}: registered and unregistered while deploying`);
+}
+
+/** Whether the live guardrail `id` is in `projectId`, by that project's list. */
+async function guardrailInProject(
+  registry: GuardrailRegistryBinding,
+  tenantId: TenantId,
+  projectId: ProjectId,
+  id: GuardrailId,
+): Promise<boolean> {
+  let cursor: Cursor | undefined;
+  do {
+    const page = await registry.list({
+      tenantId,
+      limit: 100,
+      nameFilter: id,
+      scope: { kind: 'project', tenantId, projectId },
+      ...(cursor !== undefined && { cursor }),
+    });
+    if (page.data.some((g) => g.id === id)) return true;
+    cursor = page.nextCursor;
+  } while (cursor !== undefined);
+  return false;
+}
+
+/**
+ * The fields of a guardrail its author declares, which a deploy compares.
+ * Never compared: what a deploy or a release derives. That includes
+ * `configSchema`, which the deploy route dropped before 0.1.5, so a row a
+ * 0.1.4 deploy stored has none; the image and artifact version a code
+ * pointer names, which every new image changes; and any field a later
+ * release adds. So an unchanged pack redeploys across releases.
+ */
+const DECLARED_GUARDRAIL_FIELDS = [
+  'name',
+  'description',
+  'kind',
+  'check',
+  'config',
+  'action',
+  'severity',
+  'scope',
+  'budget',
+  'judgeCapabilities',
+  'sandbox',
+  'limits',
+  'network',
+  'needsSpec',
+] as const;
+
+/**
+ * The same guardrail definition: equal in what its author declares
+ * (`DECLARED_GUARDRAIL_FIELDS`, and of where its code lives only the
+ * module path), as JSON with keys in any order (a registry may store it as
+ * JSONB) and an absent field the same as an `undefined` one.
+ */
+export function sameGuardrailDefinition(a: Guardrail, b: Guardrail): boolean {
+  return canonicalJson(definitionOf(a)) === canonicalJson(definitionOf(b));
+}
+
+function definitionOf(guardrail: Guardrail): Record<string, unknown> {
+  const declared: Record<string, unknown> = {};
+  for (const field of DECLARED_GUARDRAIL_FIELDS) {
+    declared[field] = (guardrail as unknown as Record<string, unknown>)[field];
+  }
+  declared.modulePath = guardrail.codeArtifactRef?.modulePath;
+  return declared;
+}
+
+/** JSON with every object's keys sorted, and absent and `undefined` alike. */
+function canonicalJson(value: unknown): string {
+  return JSON.stringify(JSON.parse(JSON.stringify(value) ?? 'null'), (_key, v: unknown) =>
+    v !== null && typeof v === 'object' && !Array.isArray(v)
+      ? Object.fromEntries(Object.entries(v).sort(([x], [y]) => (x < y ? -1 : x > y ? 1 : 0)))
+      : v,
+  );
+}
+
 /** The pointer to a tool's or guardrail's module inside the deployed image. */
 function ociCodeArtifactRef(image: DeployedImage, modulePath: unknown) {
   return typeof modulePath === 'string' && modulePath.length > 0
@@ -1236,9 +1402,13 @@ function ociCodeArtifactRef(image: DeployedImage, modulePath: unknown) {
  * a guardrail's `checkId` becomes its `check`, as `kindgi dev`'s disk
  * registry maps it.
  */
+/** The built-in checks, for the config of a guardrail naming one. */
+const builtInChecks = createCheckRegistry();
+
 function validatePrimitives(
   index: Readonly<Record<string, unknown>>,
   image: DeployedImage,
+  log?: { debug(message: string): void },
 ): ValidateResult {
   const details: ValidationDetail[] = [];
 
@@ -1286,12 +1456,20 @@ function validatePrimitives(
       details.push({ primitive: 'guardrail', index: i, path: '', message: 'must be an object' });
       return;
     }
-    const {
-      checkModulePath,
-      checkId,
-      configSchema: _configSchema,
-      ...rest
-    } = raw as Record<string, unknown>;
+    // The rest is the guardrail as registered, its `configSchema` included:
+    // the runtime checks a guardrail naming this check against it. A field
+    // the guardrail spec doesn't have (an index from a newer CLI) is dropped,
+    // never refused, so a newer CLI never breaks a deployment.
+    const { checkModulePath, checkId, ...indexed } = raw as Record<string, unknown>;
+    const dropped = Object.keys(indexed).filter((k) => !GUARDRAIL_SPEC_KEYS.includes(k));
+    const rest = Object.fromEntries(
+      Object.entries(indexed).filter(([k]) => GUARDRAIL_SPEC_KEYS.includes(k)),
+    );
+    if (dropped.length > 0) {
+      log?.debug(
+        `deployment: guardrail ${String(indexed.id)}: ignored index field(s) this runtime doesn't know: ${dropped.join(', ')}`,
+      );
+    }
     const r = validateGuardrailSpec({
       ...rest,
       check: checkId ?? rest.check ?? rest.id,
@@ -1325,6 +1503,21 @@ function validatePrimitives(
       }
       return;
     }
+    // A guardrail naming a built-in check: its config against the built-in's.
+    const builtIn = builtInChecks.get(r.value.check);
+    const configProblems = builtIn?.configProblems?.(r.value.config) ?? [];
+    if (configProblems.length > 0) {
+      for (const problem of configProblems) {
+        details.push({
+          primitive: 'guardrail',
+          index: i,
+          id: r.value.id as unknown as string,
+          path: problem.path,
+          message: problem.message,
+        });
+      }
+      return;
+    }
     validatedGuardrails.push(r.value);
   });
 
@@ -1524,7 +1717,7 @@ async function registerAgents(input: {
       const outcome = await agents.publish({ tenantId, projectId, agent, enqueueTuples });
       if (outcome.kind === 'ok') written(outcome.agentId, outcome.version);
       else if (outcome.kind !== 'already-registered') {
-        throw new PublishRefused('agent', `${agent.id}@${agent.version}`, outcome.kind);
+        throw new PublishRefused('agent', `${agent.id}@${agent.version}`, outcome);
       }
     }
     return input.defined.map(deployedPrimitive);
@@ -1588,7 +1781,7 @@ async function registerFlows(input: {
       const outcome = await flows.publish({ tenantId, projectId, flow, enqueueTuples });
       if (outcome.kind === 'ok') written(outcome.flowId, outcome.version as unknown as string);
       else if (outcome.kind !== 'already-registered') {
-        throw new PublishRefused('flow', `${flow.id}@${flow.version}`, outcome.kind);
+        throw new PublishRefused('flow', `${flow.id}@${flow.version}`, outcome);
       }
     }
     return input.defined.map(deployedPrimitive);

package/src/routes/eval-comparison.ts

@@ -5,12 +5,18 @@ import type { AgentId } from '@kindgi/agents';
 import type { FlowVersionOverrides } from '@kindgi/flow';
 import type { ProjectId, Semver } from '@kindgi/types';
 
-import type { EvalBaseline, EvalComparison } from '../eval-run-binding.js';
+import type {
+  EvalBaseline,
+  EvalComparison,
+  EvalOverrides,
+  EvalSample,
+} from '../eval-run-binding.js';
 import { DEFAULT_COMPARISON } from '../judged-dispatcher.js';
 import { parseSegmentsBody } from './segments.js';
 
 const MAX_REPETITIONS = 10;
 const MAX_K = 100;
+const MAX_OVERRIDDEN_BLOCKS = 20;
 
 function obj(value: unknown): Record<string, unknown> | undefined {
   return typeof value === 'object' && value !== null && !Array.isArray(value)
@@ -84,6 +90,71 @@ function integerIn(raw: unknown, name: string, max: number): number | string {
     : `\`${name}\` must be an integer from 1 to ${max}`;
 }
 
+/** `overrides: { settings?, prompts? }`; `undefined` when it names nothing. */
+function parseOverrides(raw: unknown): EvalOverrides | undefined | string {
+  const o = obj(raw);
+  if (o === undefined) return '`overrides` must be an object: { settings?, prompts? }';
+  const extra = Object.keys(o).filter((key) => key !== 'settings' && key !== 'prompts');
+  if (extra.length > 0) {
+    return `\`overrides\` takes \`settings\` and \`prompts\`, not \`${extra[0]}\``;
+  }
+  const settings = o.settings === undefined ? {} : obj(o.settings);
+  const prompts = o.prompts === undefined ? {} : obj(o.prompts);
+  const settingsBlocks = settings === undefined ? [] : Object.entries(settings);
+  const promptBlocks = prompts === undefined ? [] : Object.entries(prompts);
+  if (
+    settings === undefined ||
+    settingsBlocks.length > MAX_OVERRIDDEN_BLOCKS ||
+    settingsBlocks.some(([id, values]) => id === '' || obj(values) === undefined)
+  ) {
+    return `\`overrides.settings\` must be an object of { blockId: { …values } } (at most ${MAX_OVERRIDDEN_BLOCKS} blocks)`;
+  }
+  if (
+    prompts === undefined ||
+    promptBlocks.length > 1 ||
+    promptBlocks.some(([id, content]) => {
+      const c = obj(content);
+      return (
+        id === '' ||
+        c === undefined ||
+        typeof c.template !== 'string' ||
+        Object.keys(c).length !== 1
+      );
+    })
+  ) {
+    return '`overrides.prompts` must be { blockId: { template } } for the prompt block (one)';
+  }
+  if (settingsBlocks.length === 0 && promptBlocks.length === 0) return undefined;
+  return {
+    ...(settingsBlocks.length > 0 && {
+      settings: settings as Readonly<Record<string, Readonly<Record<string, unknown>>>>,
+    }),
+    ...(promptBlocks.length > 0 && {
+      prompts: prompts as Readonly<Record<string, { readonly template: string }>>,
+    }),
+  };
+}
+
+/** `sample: { part, seed, holdOutShare }`, or an error message. */
+function parseSample(raw: unknown): EvalSample | string {
+  const o = obj(raw);
+  const share = o?.holdOutShare;
+  if (
+    o === undefined ||
+    (o.part !== 'search' && o.part !== 'hold-out') ||
+    typeof o.seed !== 'string' ||
+    o.seed === '' ||
+    o.seed.length > 200 ||
+    typeof share !== 'number' ||
+    share < 0.1 ||
+    share > 0.9 ||
+    Object.keys(o).some((key) => !['part', 'seed', 'holdOutShare'].includes(key))
+  ) {
+    return "`sample` must be { part: 'search' | 'hold-out', seed, holdOutShare } with holdOutShare from 0.1 to 0.9";
+  }
+  return { part: o.part, seed: o.seed, holdOutShare: share };
+}
+
 /**
  * A comparison eval run's settings from the start body: `baseline`,
  * `reads`, `repetitions`, `k`, `classWeights` and a flow candidate's
@@ -99,13 +170,19 @@ export function parseComparison(
   const { baseline, reads, repetitions, k, classWeights } = b;
   const versions = b.versions === undefined ? undefined : parseVersions(b.versions);
   if (typeof versions === 'string') return { kind: 'err', message: versions };
+  const overrides = b.overrides === undefined ? undefined : parseOverrides(b.overrides);
+  if (typeof overrides === 'string') return { kind: 'err', message: overrides };
+  const sample = b.sample === undefined ? undefined : parseSample(b.sample);
+  if (typeof sample === 'string') return { kind: 'err', message: sample };
   if (
     baseline === undefined &&
     reads === undefined &&
     repetitions === undefined &&
     k === undefined &&
     versions === undefined &&
-    classWeights === undefined
+    classWeights === undefined &&
+    overrides === undefined &&
+    sample === undefined
   ) {
     return { kind: 'ok' };
   }
@@ -144,6 +221,8 @@ export function parseComparison(
       k: topK,
       ...(versions !== undefined && { versions }),
       ...(classWeights !== undefined && { classWeights }),
+      ...(overrides !== undefined && { overrides }),
+      ...(sample !== undefined && { sample }),
     },
   };
 }