@kindgi/api 0.1.4 → 0.1.5

package/src/block-publish.ts

@@ -0,0 +1,181 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Kindgi Inc.
+
+import { type BlockDefinition, settingsSchemaIssues, validateBlock } from '@kindgi/agents';
+import { latestVersion, nextVersion } from '@kindgi/tools';
+import type { Cursor, ProjectId, TenantId } from '@kindgi/types';
+
+import type { BlockRecord, BlockRegistryBinding } from './block-binding.js';
+
+/** One thing wrong with a block version, as `validation-failed` lists them. */
+export interface BlockPublishIssue {
+  readonly path: string;
+  readonly message: string;
+}
+
+/**
+ * A new version against the block's latest: the same kind, and a
+ * settings version without a schema of its own satisfies the schema it
+ * keeps (`carriedSchema`).
+ */
+export function continuityIssues(
+  latest: BlockRecord | null,
+  block: BlockDefinition,
+): BlockPublishIssue[] {
+  if (latest === null) return [];
+  if (latest.kind !== block.kind) {
+    return [
+      {
+        path: '/kind',
+        message: `block "${block.id}" is a ${latest.kind} block; a version can't change its kind`,
+      },
+    ];
+  }
+  const carried = carriedSchema(latest, block);
+  if (block.kind !== 'settings' || carried === undefined) return [];
+  return settingsSchemaIssues(block.content.values, carried).map((i) => ({
+    ...i,
+    message: `${i.message} (the schema of version ${latest.version})`,
+  }));
+}
+
+/**
+ * The schema a settings version that gives none keeps: the latest
+ * version's. A version that gives a schema replaces it (`{}` drops the
+ * check on purpose).
+ */
+function carriedSchema(
+  latest: BlockRecord | null,
+  block: BlockDefinition,
+): Readonly<Record<string, unknown>> | undefined {
+  if (latest?.kind !== 'settings' || block.kind !== 'settings') return undefined;
+  return block.content.schema === undefined ? latest.content.schema : undefined;
+}
+
+/** The version as stored: with the schema it keeps, if any. */
+export function withCarriedSchema(
+  latest: BlockRecord | null,
+  block: BlockDefinition,
+): BlockDefinition {
+  const schema = carriedSchema(latest, block);
+  if (schema === undefined || block.kind !== 'settings') return block;
+  return { ...block, content: { ...block.content, schema } };
+}
+
+/** New content for a block: settings values, or a prompt template. */
+export type BlockEdit =
+  | { readonly kind: 'settings'; readonly values: Readonly<Record<string, unknown>> }
+  | { readonly kind: 'prompt'; readonly template: string };
+
+export interface PublishBlockEditInput {
+  readonly blocks: BlockRegistryBinding;
+  readonly tenantId: TenantId;
+  /** The version the edit starts from: its schema or parameters and description carry over. */
+  readonly from: BlockRecord;
+  readonly edit: BlockEdit;
+}
+
+export type PublishBlockEditOutcome =
+  | { readonly kind: 'ok'; readonly version: string }
+  | { readonly kind: 'invalid'; readonly issues: readonly BlockPublishIssue[] };
+
+/** How many numbers after the highest a publish tries before it gives up. */
+const MAX_TRIES = 100;
+
+/** The block version an edit would publish, numbered `version`; issues when it isn't valid. */
+export async function blockEditDefinition(
+  input: PublishBlockEditInput,
+  version: string,
+): Promise<
+  | { readonly kind: 'ok'; readonly block: BlockDefinition; readonly latest: BlockRecord | null }
+  | { readonly kind: 'invalid'; readonly issues: readonly BlockPublishIssue[] }
+> {
+  const { from, edit } = input;
+  if (from.kind !== edit.kind) {
+    return {
+      kind: 'invalid',
+      issues: [
+        { path: '/kind', message: `block "${from.id}" is a ${from.kind} block, not ${edit.kind}` },
+      ],
+    };
+  }
+  const content =
+    from.kind === 'settings' && edit.kind === 'settings'
+      ? {
+          values: edit.values,
+          ...(from.content.schema !== undefined && { schema: from.content.schema }),
+        }
+      : from.kind === 'prompt' && edit.kind === 'prompt'
+        ? {
+            template: edit.template,
+            ...(from.content.parameters !== undefined && { parameters: from.content.parameters }),
+          }
+        : undefined;
+  const validated = validateBlock({
+    id: from.id,
+    version,
+    kind: from.kind,
+    ...(from.description !== undefined && { description: from.description }),
+    content,
+  });
+  if (validated.kind === 'err') return { kind: 'invalid', issues: validated.error.issues };
+  const latest = await input.blocks.get({ tenantId: input.tenantId, blockId: from.id });
+  const continuity = continuityIssues(latest, validated.value);
+  if (continuity.length > 0) return { kind: 'invalid', issues: continuity };
+  return { kind: 'ok', block: validated.value, latest };
+}
+
+/**
+ * Publish an edit of a block as its next free version (the first patch
+ * after its highest, unregistered versions included, since versions
+ * never change), into the block's own project. The edit is checked as
+ * `POST /v1/blocks` checks a version: it must be a valid block of the
+ * same kind, and settings values must satisfy the schema they keep.
+ */
+export async function publishBlockEdit(
+  input: PublishBlockEditInput,
+): Promise<PublishBlockEditOutcome> {
+  const { blocks, tenantId, from } = input;
+  const versions = await allVersions(blocks, tenantId, from.id);
+  const highest = latestVersion(versions) ?? from.version;
+  let candidate = nextVersion(highest);
+  for (let tries = 0; candidate !== undefined && tries < MAX_TRIES; tries++) {
+    const checked = await blockEditDefinition(input, candidate);
+    if (checked.kind === 'invalid') return checked;
+    const outcome = await blocks.publish({
+      tenantId,
+      projectId: from.projectId,
+      block: withCarriedSchema(checked.latest, checked.block),
+    });
+    if (outcome.kind === 'ok') return { kind: 'ok', version: outcome.version };
+    if (outcome.kind !== 'already-registered') {
+      throw new Error(
+        `block "${from.id}": publishing into its own project ${from.projectId as ProjectId} failed (${outcome.kind})`,
+      );
+    }
+    candidate = nextVersion(candidate);
+  }
+  throw new Error(`block "${from.id}": no free version after ${highest}`);
+}
+
+/** Every version number a block holds, unregistered ones included. */
+async function allVersions(
+  blocks: BlockRegistryBinding,
+  tenantId: TenantId,
+  blockId: string,
+): Promise<string[]> {
+  const out: string[] = [];
+  let cursor: Cursor | undefined;
+  do {
+    const page = await blocks.listVersions({
+      tenantId,
+      blockId,
+      limit: 100,
+      includeTombstoned: true,
+      ...(cursor !== undefined && { cursor }),
+    });
+    out.push(...page.data.map((b) => b.version));
+    cursor = page.nextCursor;
+  } while (cursor !== undefined);
+  return out;
+}

package/src/caller.ts

@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Kindgi Inc.
+
+import type { Context } from 'hono';
+
+import { ref } from '@kindgi/authz';
+
+import type { Authorizer } from './middleware/authorize.js';
+import type { TokenPrincipal } from './token-admin.js';
+import type { AppEnv } from './types.js';
+
+/**
+ * Whom the request acts for, as an API key names it: the person when the
+ * token names one; otherwise the service account an API key acts for,
+ * else the key itself (a key store built before principals).
+ */
+export function callerPrincipal(c: Context<AppEnv>): TokenPrincipal | undefined {
+  const userId = c.get('userId');
+  if (userId !== undefined) return { kind: 'user', userId: userId as unknown as string };
+  const serviceAccountId = c.get('serviceAccountId') ?? (c.get('tokenId') as string | undefined);
+  if (serviceAccountId !== undefined) return { kind: 'service-account', serviceAccountId };
+  return undefined;
+}
+
+/**
+ * Whether the caller is a tenant admin: `admin` on the tenant when
+ * authorization is on, otherwise the `tenant-admin` scope. Never through
+ * a `member` key or a key limited to a project (see the authorizer).
+ */
+export async function isTenantAdmin(
+  c: Context<AppEnv>,
+  authorizer: Authorizer | undefined,
+): Promise<boolean> {
+  if (authorizer !== undefined) {
+    const tenantId = c.get('tenantId') as unknown as string;
+    return authorizer.can(c, 'admin', ref('tenant', tenantId));
+  }
+  return (
+    (c.get('scopes') ?? []).includes('tenant-admin') &&
+    c.get('tokenRole') !== 'member' &&
+    c.get('tokenProjectId') === undefined
+  );
+}
+
+/** A principal on the wire: `{kind: 'user' | 'service-account', id}`. */
+export function principalToWire(p: TokenPrincipal): {
+  kind: 'user' | 'service-account';
+  id: string;
+} {
+  return p.kind === 'user'
+    ? { kind: 'user', id: p.userId }
+    : { kind: 'service-account', id: p.serviceAccountId };
+}
+
+/** The caller as a principal reference: `user:<id>` or `service_account:<id>`. */
+export function callerRef(c: Context<AppEnv>): string | undefined {
+  const caller = callerPrincipal(c);
+  if (caller === undefined) return undefined;
+  return caller.kind === 'user'
+    ? `user:${caller.userId}`
+    : `service_account:${caller.serviceAccountId}`;
+}

package/src/capability-binding.ts

@@ -62,6 +62,18 @@ export interface CapabilityDescriptor {
    * layer — the SDK surfaces it for tooling / doc generation.
    */
   readonly paramsSchema?: Readonly<Record<string, unknown>>;
+  /**
+   * The tenant's registered providers with a model that has the feature,
+   * and those models. Absent when the binding doesn't read the provider
+   * registry; `[]` when no provider of the tenant has one.
+   */
+  readonly providers?: readonly CapabilityProvider[];
+}
+
+/** One provider with a model that has a feature, and those models. */
+export interface CapabilityProvider {
+  readonly providerId: string;
+  readonly models: readonly string[];
 }
 
 export interface CapabilityListInput {

package/src/cost-binding.ts

@@ -50,6 +50,14 @@ export interface CostBinding {
    * every group is capped by the route.
    */
   aggregate(input: CostAggregateInput): Promise<CostAggregateResult>;
+  /**
+   * `true` when `aggregate` honours `CostAggregateInput.readableProjectIds`,
+   * counting only those projects' records. Without it, the route answers
+   * an aggregate across projects (no scope, or an org) only for a caller
+   * who may read every project in it, and refuses anyone else rather than
+   * count projects they can't read.
+   */
+  readonly aggregatesReadableProjects?: boolean;
 }
 
 // A binding throws when its store fails: the route answers 500. It never
@@ -198,6 +206,17 @@ export interface CostAggregateInput {
    * shape uniformity.
    */
   readonly inherit?: boolean;
+  /**
+   * Count only records in these projects, and records with no project
+   * (the tenant's own, which `/v1/cost/records` shows to anyone who may
+   * read the tenant), within `scope`. An org scope covers only the org's
+   * projects: a record with no project never counts there, and nor does a
+   * listed project outside the org. The route sets it when the caller
+   * may not read every project (`read` on each listed one); absent, every
+   * record in `scope` counts. Empty: no project's records count. A binding
+   * that applies it says so with `CostBinding.aggregatesReadableProjects`.
+   */
+  readonly readableProjectIds?: readonly string[];
   /**
    * The most groups the caller wants (`?limit=`, 1..`COST_AGGREGATE_MAX_LIMIT`,
    * default `COST_AGGREGATE_DEFAULT_LIMIT`): the most expensive ones. A

package/src/derive-agent-version.ts

@@ -36,6 +36,8 @@ export interface DeriveAgentVersionInput {
   readonly label?: string;
   /** Who derived it (`user:<id>`). */
   readonly by?: string;
+  /** The improvement proposal it's derived for, recorded in `derivedFrom`. */
+  readonly proposalId?: string;
   /** The project, when the store doesn't record one on the version. */
   readonly projectId?: ProjectId;
   /** The authorization tuples for the new version, in the project it lands in. */
@@ -55,7 +57,9 @@ export type DeriveAgentVersionOutcome =
   | { readonly kind: 'unpinned' }
   | { readonly kind: 'invalid'; readonly issues: readonly SwapIssue[] }
   | { readonly kind: 'no-project' }
-  | { readonly kind: 'project-not-found'; readonly projectId: ProjectId };
+  | { readonly kind: 'project-not-found'; readonly projectId: ProjectId }
+  /** The agent belongs to another project than the one given (`projectId`): nothing is written. */
+  | { readonly kind: 'project-mismatch'; readonly projectId: ProjectId };
 
 /** How many numbers after the highest a derive tries before it gives up. */
 const MAX_TRIES = 100;
@@ -108,6 +112,7 @@ export async function deriveAgentVersion(
       reason: 'edited',
       ...(input.label !== undefined && { label: input.label }),
       ...(input.by !== undefined && { by: input.by }),
+      ...(input.proposalId !== undefined && { proposalId: input.proposalId }),
     },
   });
 }
@@ -139,6 +144,9 @@ async function publishNextFree(
     });
     if (outcome.kind === 'ok') return { kind: 'ok', agent };
     if (outcome.kind === 'project-not-found') return { kind: 'project-not-found', projectId };
+    if (outcome.kind === 'project-mismatch') {
+      return { kind: 'project-mismatch', projectId: outcome.projectId };
+    }
     candidate = nextVersion(candidate);
   }
   throw new Error(`agent "${agentId as unknown as string}": no free version after ${highest}`);

package/src/errors.ts

@@ -48,6 +48,17 @@ export const ERROR_CODE_TO_STATUS: Readonly<Record<string, number>> = {
   // A deployment refuses tenant configuration that reaches its host
   // (KINDGI_TENANT_HOST_ACCESS): a stdio MCP endpoint.
   'host-access-denied': 403,
+  // A request signed in by the session cookie, from an origin the
+  // deployment doesn't allow (or with no Origin): cross-site request
+  // forgery protection for browser sessions.
+  'csrf-origin-mismatch': 403,
+  // `POST /v1/auth/token-sign-in` on a deployment that doesn't allow
+  // signing in to the console with an API token.
+  'token-sign-in-off': 403,
+  // `POST /v1/auth/token-sign-in` with a key that can't open a console
+  // session: a service account's, or a narrowed one (a `member` role or one
+  // project), which a session would widen to the person's full grants.
+  'token-sign-in-not-allowed': 403,
   // 404
   'not-found': 404,
   'run-not-found': 404,
@@ -59,23 +70,35 @@ export const ERROR_CODE_TO_STATUS: Readonly<Record<string, number>> = {
   'flow-not-found': 404,
   // 409 — conflict
   'already-terminal': 409,
+  'fact-changed': 409,
+  'legal-hold': 409,
+  'erasure-in-progress': 409,
   'run-already-terminal': 409,
   'run-lease-lost': 409,
   'idempotency-key-body-mismatch': 409,
+  'idempotency-key-in-flight': 409,
+  'idempotency-key-replay-withheld': 409,
   'hitl-required': 409,
   'duplicate-node-id': 409,
   'duplicate-edge-id': 409,
   'agent-version-mismatch': 409,
   'agent-already-registered': 409,
+  'agent-project-mismatch': 409,
   'registry-read-only': 409,
   'agent-gone': 410,
+  'run-erased': 410,
   'flow-gone': 410,
   'policy-gone': 410,
   'eval-suite-gone': 410,
   'tool-gone': 410,
   'tool-already-registered': 409,
+  'tool-project-mismatch': 409,
   'guardrail-already-registered': 409,
+  // A deploy's guardrail id is live in another project: a deploy never takes it.
+  'guardrail-project-mismatch': 409,
+  'guardrail-config-invalid': 422,
   'flow-already-registered': 409,
+  'flow-project-mismatch': 409,
   'conversation-closed': 409,
   'invalid-agent': 400,
   'invalid-tool-definition': 400,
@@ -105,6 +128,8 @@ export const ERROR_CODE_TO_STATUS: Readonly<Record<string, number>> = {
   // belongs to an approval (decided through the approvals routes, which
   // check the reviewer and record the decision) or to the runtime itself.
   'run-resume-not-supported': 422,
+  /** Memory was asked to search by meaning, and no embedding model is configured. */
+  'semantic-unavailable': 422,
   // 429 — rate limit. No route in this package emits it; a rate limiter
   // in front of the routes can.
   'rate-limit-exceeded': 429,
@@ -137,16 +162,16 @@ export const ERROR_CODE_TO_STATUS: Readonly<Record<string, number>> = {
   'signing-key-revoked': 409,
   'signing-key-algorithm-unsupported': 400,
   'signing-key-store-error': 500,
-  // Supervisor proposals lifecycle.
+  // Improvement proposals.
   'proposal-not-found': 404,
   'proposal-invalid-state-transition': 409,
-  'proposal-terminal': 409,
-  'baseline-mismatch': 422,
-  'apply-change-failed': 422,
-  'ground-layer-violation': 422,
-  'agent-not-in-registry': 422,
+  // Evaluating would make a version that serves every unpinned scope.
+  'proposal-needs-pin': 409,
+  // Improvement passes.
+  'improve-unsupported': 501,
+  'improvement-pass-not-found': 404,
+  'improvement-pass-finished': 409,
   'version-already-exists': 409,
-  'invalid-new-version': 400,
   'supervisor-header-missing': 400,
   // Artifacts.
   'blob-not-found': 404,
@@ -158,6 +183,7 @@ export const ERROR_CODE_TO_STATUS: Readonly<Record<string, number>> = {
   'provider-not-found': 404,
   'provider-already-registered': 409,
   'invalid-provider': 400,
+  'provider-config-invalid': 422,
   // Admin plane — cost readback.
   'cost-record-not-found': 404,
   // Admin plane — adapters.
@@ -173,6 +199,7 @@ export const ERROR_CODE_TO_STATUS: Readonly<Record<string, number>> = {
   'block-project-mismatch': 409,
   'eval-suite-not-found': 404,
   'eval-suite-already-registered': 409,
+  'eval-suite-project-mismatch': 409,
   // Admin plane — eval-run dispatch.
   'eval-run-not-found': 404,
   // Judgments (yes/no on a run's output items) and judge classes.
@@ -200,12 +227,41 @@ export const ERROR_CODE_TO_STATUS: Readonly<Record<string, number>> = {
   'gate-policy-scope-unpinned': 409,
   'gate-policy-needs-pin': 409,
   'gate-policy-descendant-unpinned': 409,
+  // API keys: whom a key acts for, and what it may do.
+  /** Mint: the person or service account named doesn't exist. */
+  'principal-not-found': 404,
+  /** Mint: an `admin` key for a principal that isn't a tenant admin. */
+  'role-exceeds-principal': 403,
+  /** A key limited to one project, on a request that names another. */
+  'key-project-mismatch': 403,
+  'service-account-not-found': 404,
+  /** An active service account of the tenant already has the name. */
+  'service-account-name-taken': 409,
+  /** A grant for an unregistered service account. */
+  'service-account-unregistered': 409,
+  /** Adding a person with an email another person of the tenant has. */
+  'identity-user-email-taken': 409,
+  /** Removing tenant admin from the only person who holds it: the tenant would have none. */
+  'last-tenant-admin': 409,
+  /** Removing tenant admin from the seed user, whom the runtime re-grants it at every boot. */
+  'seed-user-admin': 409,
+  /** Unregistering yourself, or the deployment's seed user (`details.reason`). */
+  'identity-user-unregister-refused': 409,
+  /** A grant or a key for a person who was unregistered. */
+  'identity-user-unregistered': 409,
+  /** A person's grants on a runtime without an authorization store. */
+  'person-grants-unsupported': 501,
   /** Unregister: the version is live in a scope; move that pin first. */
   'agent-version-live': 409,
+  /** An artifact upload over the runtime's cap (`KINDGI_ARTIFACT_MAX_BYTES`). */
+  'artifact-too-large': 413,
   'run-not-finished': 409,
   'item-not-found': 400,
   // The judgment binding can't list judged runs, so no test sets from judgments.
   'test-sets-not-supported': 501,
+  'memory-operation-unsupported': 501,
+  // The conversation binding can't unregister (a runtime built before it).
+  'conversation-unregister-unsupported': 501,
   // Authorization is enforced, but a membership change can't be kept in step with it.
   'authz-membership-unsupported': 501,
   'eval-run-already-terminal': 409,
@@ -231,12 +287,22 @@ export const ERROR_CODE_TO_STATUS: Readonly<Record<string, number>> = {
   'session-inactive': 401,
   'identity-provider-not-found': 404,
   'identity-provider-already-registered': 409,
+  // The deployment couldn't use a provider's configuration (its issuer's
+  // discovery failed, its SAML metadata didn't parse, a host it may not
+  // reach): 422 with what went wrong.
+  'identity-provider-invalid': 422,
   'oauth-state-invalid': 400,
   'oauth-code-exchange-failed': 422,
   'oauth-refresh-failed': 422,
   'oauth-refresh-not-supported': 422,
   'invalid-provider-config': 400,
   'auth-not-session-token': 400,
+  // `POST /v1/auth/refresh` with a browser session (cookie): refused, so a
+  // fresh session token never reaches page scripts.
+  'cookie-session-not-refreshable': 400,
+  // `POST /v1/auth/token-sign-in` signed in by a session (a cookie or a
+  // session token) rather than an API token: there's nothing to exchange.
+  'token-sign-in-needs-an-api-token': 400,
   // OAuth redirect URIs + refresh.
   'redirect-uri-not-allowed': 400,
   'redirect-uri-mismatch': 400,
@@ -302,6 +368,8 @@ export const ERROR_CODE_TO_STATUS: Readonly<Record<string, number>> = {
   'trigger-register-failed': 500,
   'trigger-update-failed': 500,
   'trigger-lifecycle-failed': 500,
+  /** The deployment's trigger registry can't do this yet (fire history, run-now, a new owner). */
+  'trigger-operation-unsupported': 501,
   'webhook-signature-invalid': 401,
   'webhook-inactive': 410,
   'webhook-secret-missing': 500,

package/src/eval-case-binding.ts

@@ -39,7 +39,17 @@ export interface JudgedItemSummary extends JudgedItem {
    */
   readonly restricted?: { readonly yesWeight: number; readonly totalWeight: number };
   /** The reasons given, newest first. */
-  readonly reasons: readonly { readonly verdict: 'yes' | 'no'; readonly reason: string }[];
+  readonly reasons: readonly JudgedReason[];
+}
+
+/** One judgment's reason, and whose judgment it was. */
+export interface JudgedReason {
+  readonly verdict: 'yes' | 'no';
+  readonly reason: string;
+  /** The judgment's class, when it had one. */
+  readonly judgeClassId?: string;
+  /** Set when the judgment was recorded while its class was restricted (`Judgment.restricted`). */
+  readonly restricted?: true;
 }
 
 /** One case of a `judged` suite: a copy of a judged run and what people said about it. */
@@ -54,6 +64,12 @@ export interface JudgedEvalCase {
   /** The output that was judged. */
   readonly output: unknown;
   readonly items: readonly JudgedItemSummary[];
+  /**
+   * An erasure cleared this case (a person's words were erased): its
+   * `input`, `output` and `context` are gone and its items empty. Eval
+   * runs leave it out and count it (`erased`).
+   */
+  readonly erased?: true;
 }
 
 export interface EvalCasePutInput {

package/src/eval-run-binding.ts

@@ -173,6 +173,37 @@ export interface EvalComparison {
   readonly versions?: FlowVersionOverrides;
   /** Which judgments count (absent: `as-recorded`). */
   readonly classWeights?: EvalClassWeights;
+  /**
+   * For an agent candidate: block content its replays run instead of the
+   * version's pinned content (settings values, a prompt template: an
+   * improvement pass's search). A comparison with overrides can't gate a
+   * promotion: it didn't run a published version.
+   */
+  readonly overrides?: EvalOverrides;
+  /** Only part of the test set's cases (absent: all of them). */
+  readonly sample?: EvalSample;
+}
+
+/** Block content that replaces a version's pinned content: settings values and prompt templates, by block id. */
+export interface EvalOverrides {
+  readonly settings?: Readonly<Record<string, Readonly<Record<string, unknown>>>>;
+  readonly prompts?: Readonly<Record<string, { readonly template: string }>>;
+}
+
+/**
+ * Part of a test set: its cases split once into a hold-out part (about
+ * `holdOutShare` of them) and a search part (the rest), stratified by
+ * judgment (the cases with a "no" and the others split on their own), in
+ * the order of a hash of each case id and `seed`. The same seed splits
+ * the same test set the same way, so an improvement pass searches on one
+ * part and proves its candidate on the other. A promotion gate refuses a
+ * comparison on the search part.
+ */
+export interface EvalSample {
+  readonly part: 'search' | 'hold-out';
+  readonly seed: string;
+  /** 0.1 to 0.9. */
+  readonly holdOutShare: number;
 }
 
 export interface EvalRunStartInput {

package/src/eval-run-dispatcher.ts

@@ -89,6 +89,13 @@ export interface EvalRunSubjectInvokeOutcome {
     readonly arguments: unknown;
     readonly reason?: string;
   };
+  /**
+   * The past run was erased after the case was read (a person's words
+   * were removed): nothing was replayed. Not an error: the dispatcher
+   * leaves the case out and counts it, like a case erased before. A
+   * runtime that predates erasure never sets it.
+   */
+  readonly erased?: true;
 }
 
 /**

package/src/eval-sample.ts

@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Kindgi Inc.
+
+import { createHash } from 'node:crypto';
+
+import type { JudgedEvalCase } from './eval-case-binding.js';
+import type { EvalSample } from './eval-run-binding.js';
+
+/** Where a case falls in `[0, 1)` for a seed: the same for the same seed and case, always. */
+function position(seed: string, caseId: string): number {
+  const hex = createHash('sha256').update(`${seed}:${caseId}`, 'utf8').digest('hex').slice(0, 8);
+  return Number.parseInt(hex, 16) / 0x1_0000_0000;
+}
+
+/** A case's stratum: whether anyone judged one of its items "no". */
+function stratum(c: JudgedEvalCase): 'no' | 'yes' {
+  return c.items.some((item) => item.yesWeight < item.totalWeight) ? 'no' : 'yes';
+}
+
+/**
+ * The cases of a sample's part. The split is stratified by judgment: the
+ * cases with a "no" and the cases without are each split on their own,
+ * about `holdOutShare` of each into the hold-out part, in the order of a
+ * hash of the case id and `seed` (a stratum of two or more cases puts at
+ * least one in each part). The same seed and test set split the same way.
+ */
+export function sampleCases(
+  cases: readonly JudgedEvalCase[],
+  sample: EvalSample,
+): JudgedEvalCase[] {
+  const holdOut = new Set<string>();
+  for (const s of ['no', 'yes'] as const) {
+    const ordered = cases
+      .filter((c) => stratum(c) === s)
+      .map((c) => ({ id: c.caseId, at: position(sample.seed, c.caseId) }))
+      .sort((a, b) => a.at - b.at || a.id.localeCompare(b.id));
+    const n = ordered.length;
+    const count = n < 2 ? 0 : Math.min(n - 1, Math.max(1, Math.round(n * sample.holdOutShare)));
+    for (const c of ordered.slice(0, count)) holdOut.add(c.id);
+  }
+  return cases.filter((c) => holdOut.has(c.caseId) === (sample.part === 'hold-out'));
+}

package/src/eval-suite-binding.ts

@@ -62,6 +62,8 @@ export interface EvalSuiteRegistryBinding {
    * before calling — the binding receives a well-formed `EvalSuite`.
    * Bindings MAY reject with `already-registered` when the same
    * `(suiteId, version)` is re-published; the route maps that to `409`.
+   * A version of an eval suite whose versions live in another project is
+   * `project-mismatch` (suites never move between projects; `409 eval-suite-project-mismatch`).
    */
   publish(input: EvalSuitePublishInput): Promise<EvalSuitePublishOutcome>;
   /**
@@ -239,6 +241,18 @@ export type EvalSuitePublishOutcome =
       readonly suiteId: string;
       readonly version: string;
       readonly projectId: ProjectId;
+    }
+  | {
+      /**
+       * The eval suite's versions live in another project: an eval suite belongs to
+       * the project its first version was published into, and never
+       * moves. Nothing is written (the route answers `409 eval-suite-project-mismatch`).
+       */
+      readonly kind: 'project-mismatch';
+      readonly suiteId: string;
+      readonly version: string;
+      /** The project the eval suite belongs to. */
+      readonly projectId: ProjectId;
     };
 
 export type EvalSuiteUnregisterOutcome = {