@kindgi/api 0.1.2 → 0.1.4-rc.0

package/src/judged-items.ts

@@ -0,0 +1,263 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Kindgi Inc.
+
+/**
+ * The items of a run's output that people judge, and how a new output is
+ * scored against the judgments of an old one. Pure, with no dependencies,
+ * so a console can derive items the same way.
+ *
+ * Items:
+ *   - an agent turn's answer (its last agent message) is `answer`;
+ *   - a flow run's whole output is `output`;
+ *   - each element of a list in the output (an agent's typed result sits
+ *     under `output`) is an item, keyed by its own `id` or `key` when it
+ *     has one, else by where it sits (a JSON Pointer), with its rank.
+ *
+ * A judgment carries over to a new output's item when it's the same item:
+ * an item keyed by its own id is the same item when the key matches; any
+ * other (the answer, an element keyed by its place) only when its content
+ * is the same too. So a changed answer is a new item, with no judgments.
+ */
+
+export interface OutputItem {
+  /** The key judgments use for this item. */
+  readonly key: string;
+  /** Where it sits in the output (JSON Pointer). */
+  readonly pointer: string;
+  /** Its place in its list (0 = first). */
+  readonly rank?: number;
+  /** `true` when the key is the item's own id, so it identifies the item by itself. */
+  readonly ownKey: boolean;
+  readonly value: unknown;
+}
+
+const MAX_DEPTH = 3;
+const MAX_LIST = 100;
+
+function obj(value: unknown): Record<string, unknown> | undefined {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+
+function escapeToken(token: string): string {
+  return token.replace(/~/g, '~0').replace(/\//g, '~1');
+}
+
+function unescapeToken(token: string): string {
+  return token.replace(/~1/g, '/').replace(/~0/g, '~');
+}
+
+/** An element's own id, if it has a usable one. */
+function ownId(element: unknown): string | undefined {
+  const o = obj(element);
+  for (const field of ['id', 'key']) {
+    const v = o?.[field];
+    if (typeof v === 'string' && v.length > 0) return v;
+    if (typeof v === 'number') return String(v);
+  }
+  return undefined;
+}
+
+function listItems(value: unknown, pointer: string, depth: number): OutputItem[] {
+  if (depth > MAX_DEPTH) return [];
+  if (Array.isArray(value)) {
+    if (value.length === 0 || value.length > MAX_LIST) return [];
+    const seen = new Set<string>();
+    return value.map((element, i) => {
+      const id = ownId(element);
+      // Duplicate ids within one list fall back to the element's place.
+      const own = id !== undefined && !seen.has(id);
+      const key = own ? id : `${pointer}/${i}`;
+      seen.add(key);
+      return { key, pointer: `${pointer}/${i}`, rank: i, ownKey: own, value: element };
+    });
+  }
+  const o = obj(value);
+  if (o === undefined) return [];
+  return Object.entries(o).flatMap(([field, child]) =>
+    listItems(child, `${pointer}/${escapeToken(field)}`, depth + 1),
+  );
+}
+
+function turnAnswer(output: Record<string, unknown>): OutputItem | undefined {
+  const appended = Array.isArray(output.appended) ? output.appended : [];
+  for (let i = appended.length - 1; i >= 0; i--) {
+    const m = obj(appended[i]);
+    if (m?.role === 'agent' && m.content !== undefined && !hasToolCalls(m.content)) {
+      return { key: 'answer', pointer: `/appended/${i}/content`, ownKey: false, value: m.content };
+    }
+  }
+  return undefined;
+}
+
+function hasToolCalls(content: unknown): boolean {
+  return Array.isArray(obj(content)?.toolCalls);
+}
+
+/** The items of a run's output (`agentTurn`: an agent turn's result). */
+export function outputItems(output: unknown, agentTurn: boolean): readonly OutputItem[] {
+  const o = obj(output);
+  if (agentTurn) {
+    if (o === undefined) return [];
+    const answer = turnAnswer(o);
+    return [
+      ...(answer !== undefined ? [answer] : []),
+      ...(o.output !== undefined ? listItems(o.output, '/output', 0) : []),
+    ];
+  }
+  if (output === undefined || output === null) return [];
+  return [
+    { key: 'output', pointer: '', ownKey: false, value: output },
+    ...listItems(output, '', 0),
+  ];
+}
+
+/** The value at a JSON Pointer in `doc`, or `undefined` when there's none. */
+export function valueAt(doc: unknown, pointer: string): unknown {
+  if (pointer === '') return doc;
+  if (!pointer.startsWith('/')) return undefined;
+  let current: unknown = doc;
+  for (const raw of pointer.slice(1).split('/')) {
+    const token = unescapeToken(raw);
+    if (Array.isArray(current)) {
+      const i = Number(token);
+      current = Number.isInteger(i) ? current[i] : undefined;
+    } else {
+      current = obj(current)?.[token];
+    }
+    if (current === undefined) return undefined;
+  }
+  return current;
+}
+
+/** JSON with sorted keys, so equal values compare equal. */
+function canonical(value: unknown): string {
+  if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
+  const o = obj(value);
+  if (o !== undefined) {
+    return `{${Object.keys(o)
+      .sort()
+      .filter((k) => o[k] !== undefined)
+      .map((k) => `${JSON.stringify(k)}:${canonical(o[k])}`)
+      .join(',')}}`;
+  }
+  return JSON.stringify(value) ?? 'null';
+}
+
+/** What people said about one judged item (as a judged test-set case keeps it). */
+export interface ItemJudgments {
+  readonly key: string;
+  readonly pointer?: string;
+  readonly rank?: number;
+  readonly yesWeight: number;
+  readonly totalWeight: number;
+}
+
+/** A new output's item and the judged item it is the same as, if any. */
+export interface MatchedItem {
+  readonly item: OutputItem;
+  readonly judged?: ItemJudgments;
+}
+
+/**
+ * Match a new output's items to the items judged in a past output
+ * (`judgedOutput`, where the judged items' pointers point).
+ */
+export function matchJudged(
+  items: readonly OutputItem[],
+  judged: readonly ItemJudgments[],
+  judgedOutput: unknown,
+): readonly MatchedItem[] {
+  const byKey = new Map(judged.map((j) => [j.key, j]));
+  return items.map((item) => {
+    const j = byKey.get(item.key);
+    if (j === undefined) return { item };
+    if (item.ownKey) return { item, judged: j };
+    const before = j.pointer === undefined ? undefined : valueAt(judgedOutput, j.pointer);
+    return before !== undefined && canonical(before) === canonical(item.value)
+      ? { item, judged: j }
+      : { item };
+  });
+}
+
+/** An output scored against judgments: the sums the metrics are made of. */
+export interface OutputScore {
+  /** Σ yesWeight and Σ totalWeight over the output's judged items. */
+  readonly yesWeight: number;
+  readonly totalWeight: number;
+  readonly items: number;
+  readonly judgedItems: number;
+  /** The same sums over the judged items among the first `k` ranked items. */
+  readonly topK: { readonly yesWeight: number; readonly totalWeight: number };
+}
+
+export function scoreItems(matched: readonly MatchedItem[], k: number): OutputScore {
+  let yesWeight = 0;
+  let totalWeight = 0;
+  let judgedItems = 0;
+  for (const m of matched) {
+    if (m.judged === undefined) continue;
+    judgedItems += 1;
+    yesWeight += m.judged.yesWeight;
+    totalWeight += m.judged.totalWeight;
+  }
+  const top = matched
+    .filter((m) => m.item.rank !== undefined)
+    .sort((a, b) => (a.item.rank ?? 0) - (b.item.rank ?? 0))
+    .slice(0, k);
+  const topK = { yesWeight: 0, totalWeight: 0 };
+  for (const m of top) {
+    if (m.judged === undefined) continue;
+    topK.yesWeight += m.judged.yesWeight;
+    topK.totalWeight += m.judged.totalWeight;
+  }
+  return { yesWeight, totalWeight, items: matched.length, judgedItems, topK };
+}
+
+/** How a new output's items moved against the judged ones. */
+export interface ItemChanges {
+  /** Judged items the new output still has, with their rank before and now. */
+  readonly kept: readonly {
+    readonly key: string;
+    readonly rankBefore?: number;
+    readonly rank?: number;
+  }[];
+  /** Judged items the new output no longer has. */
+  readonly dropped: readonly { readonly key: string; readonly rankBefore?: number }[];
+  /** The new output's items no one judged (for experts to judge). */
+  readonly new: readonly {
+    readonly key: string;
+    readonly pointer: string;
+    readonly rank?: number;
+  }[];
+}
+
+export function itemChanges(
+  matched: readonly MatchedItem[],
+  judged: readonly ItemJudgments[],
+): ItemChanges {
+  const keptKeys = new Set<string>();
+  const kept: ItemChanges['kept'][number][] = [];
+  const fresh: ItemChanges['new'][number][] = [];
+  for (const m of matched) {
+    if (m.judged !== undefined) {
+      keptKeys.add(m.judged.key);
+      kept.push({
+        key: m.judged.key,
+        ...(m.judged.rank !== undefined && { rankBefore: m.judged.rank }),
+        ...(m.item.rank !== undefined && { rank: m.item.rank }),
+      });
+    } else {
+      fresh.push({
+        key: m.item.key,
+        pointer: m.item.pointer,
+        ...(m.item.rank !== undefined && { rank: m.item.rank }),
+      });
+    }
+  }
+  const dropped = judged
+    .filter((j) => !keptKeys.has(j.key))
+    .map((j) => ({ key: j.key, ...(j.rank !== undefined && { rankBefore: j.rank }) }));
+  return { kept, dropped, new: fresh };
+}

package/src/judgment-binding.ts

@@ -0,0 +1,349 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Kindgi Inc.
+
+import type { Cursor, ListScope, ProjectId, TenantId } from '@kindgi/types';
+
+/**
+ * Caller-plugged surface for judgments: a person says yes or no, with an
+ * optional reason, about one item of a run's output. A judgment may name a
+ * **judge class** ("expert", "user", …: the deployment's own names) that
+ * carries a weight, so later metrics can weigh judgments. Classes are
+ * optional: an unclassified judgment counts with weight 1, so judging
+ * needs no setup.
+ *
+ * A judgment keeps **copies** of what was judged: the run's input and
+ * output (once per run) and the judged item. The copies outlive the run's
+ * own retention, so a set of judged cases stays stable when the run is
+ * purged.
+ *
+ * Who judged: `assertedBy` is the authenticated caller (a user, or a
+ * service token such as an app's API key). It is never taken from the
+ * request body. When an app judges on behalf of one of its own users, it
+ * passes that user's opaque id as `participantId`.
+ *
+ * One live judgment per (run, item key, asserting principal, participant):
+ * recording another supersedes the previous one, which stays as history.
+ *
+ * Every method is tenant-scoped. Deletes are soft (`unregister`); purging
+ * follows the retention policies for the `judgment` and `judge_class`
+ * domains.
+ */
+export interface JudgmentRegistryBinding {
+  /** Create a judge class. `name-taken` when a live class of that scope has the name. */
+  createClass(input: JudgeClassCreateInput): Promise<JudgeClassCreateOutcome>;
+  /** Live classes, newest first, cursor-paginated; `scope` narrows to one scope. */
+  listClasses(input: JudgeClassListInput): Promise<JudgeClassPage>;
+  /**
+   * One class, or `null` when unknown. `includeUnregistered` also returns
+   * a retired class (judgments keep naming theirs).
+   */
+  getClass(input: JudgeClassGetInput): Promise<JudgeClass | null>;
+  /** Change a live class's weight or description; `null` when unknown or retired. */
+  updateClass(input: JudgeClassUpdateInput): Promise<JudgeClass | null>;
+  /** Retire a class: no new judgments may name it. `false` when unknown or already retired. */
+  unregisterClass(input: JudgeClassGetInput): Promise<{ readonly unregistered: boolean }>;
+
+  /**
+   * Record a judgment. The first judgment of a run also stores the run's
+   * copy (`run`); later ones keep the stored copy. Supersedes the live
+   * judgment with the same run, item key, `assertedBy` and `participantId`.
+   */
+  record(input: JudgmentRecordInput): Promise<Judgment>;
+  /** Live judgments, newest first, cursor-paginated. */
+  list(input: JudgmentListInput): Promise<JudgmentPage>;
+  /** One judgment (live or not) with its copies, or `null` when unknown. */
+  get(input: JudgmentGetInput): Promise<JudgmentWithCopies | null>;
+  /** Soft-delete a live judgment. `false` when unknown or already removed. */
+  unregister(input: JudgmentGetInput): Promise<{ readonly unregistered: boolean }>;
+  /**
+   * Judged runs with their copies and live judgments, newest first:
+   * what a test set is built from. Optional (a binding without it can't
+   * build test sets from judgments).
+   */
+  listJudgedRuns?(input: JudgedRunListInput): Promise<JudgedRunPage>;
+}
+
+// ---------- judge classes ----------
+
+/**
+ * Where a class applies: the whole tenant, one project, or one agent in a
+ * project. A judgment may name a class whose scope covers its run.
+ */
+export type JudgeClassScope =
+  | { readonly kind: 'tenant' }
+  | { readonly kind: 'project'; readonly projectId: ProjectId }
+  | { readonly kind: 'agent'; readonly projectId: ProjectId; readonly agentId: string };
+
+export const JUDGE_CLASS_SCOPE_KINDS = ['tenant', 'project', 'agent'] as const;
+export type JudgeClassScopeKind = (typeof JUDGE_CLASS_SCOPE_KINDS)[number];
+
+export interface JudgeClass {
+  readonly id: string;
+  readonly tenantId: TenantId;
+  readonly scope: JudgeClassScope;
+  /** The deployment's own word for the class: "expert", "user", "arbitrator"… */
+  readonly name: string;
+  /** How much a judgment of this class counts (≥ 0); an unclassified judgment counts 1. */
+  readonly weight: number;
+  readonly description?: string;
+  readonly createdAt: string;
+  readonly updatedAt: string;
+  /** Set when the class was retired. */
+  readonly unregisteredAt?: string;
+}
+
+export interface JudgeClassCreateInput {
+  readonly tenantId: TenantId;
+  readonly scope: JudgeClassScope;
+  readonly name: string;
+  readonly weight: number;
+  readonly description?: string;
+}
+
+export type JudgeClassCreateOutcome =
+  | { readonly kind: 'created'; readonly judgeClass: JudgeClass }
+  | { readonly kind: 'name-taken' };
+
+export interface JudgeClassListInput {
+  readonly tenantId: TenantId;
+  readonly scope?: JudgeClassScope;
+  readonly cursor?: Cursor;
+  readonly limit: number;
+}
+
+export interface JudgeClassPage {
+  readonly data: readonly JudgeClass[];
+  readonly hasMore: boolean;
+  readonly nextCursor?: Cursor;
+}
+
+export interface JudgeClassGetInput {
+  readonly tenantId: TenantId;
+  readonly judgeClassId: string;
+  readonly includeUnregistered?: boolean;
+}
+
+export interface JudgeClassUpdateInput {
+  readonly tenantId: TenantId;
+  readonly judgeClassId: string;
+  readonly weight?: number;
+  readonly description?: string;
+}
+
+/** Whether a class's scope covers a run in `projectId` whose subject is `subject`. */
+export function judgeClassApplies(
+  scope: JudgeClassScope,
+  run: { readonly projectId: ProjectId; readonly subject: JudgedSubject },
+): boolean {
+  switch (scope.kind) {
+    case 'tenant':
+      return true;
+    case 'project':
+      return scope.projectId === run.projectId;
+    case 'agent':
+      return (
+        scope.projectId === run.projectId &&
+        run.subject.kind === 'agent' &&
+        run.subject.id === scope.agentId
+      );
+  }
+}
+
+// ---------- judgments ----------
+
+export const VERDICTS = ['yes', 'no'] as const;
+export type Verdict = (typeof VERDICTS)[number];
+
+/** What a judged run ran: an agent at a version, or a flow at a version. */
+export interface JudgedSubject {
+  readonly kind: 'agent' | 'flow';
+  readonly id: string;
+  readonly version: string;
+}
+
+/** The judged item of a run's output. */
+export interface JudgedItem {
+  /** The caller's stable id for the item, e.g. a matched case's id. */
+  readonly key: string;
+  /** Where the item is in the run's output, as a JSON Pointer (RFC 6901), e.g. `/matches/2`. */
+  readonly pointer?: string;
+  /** The item's position in a ranked list (0 = first). */
+  readonly rank?: number;
+}
+
+/** Who asserted a judgment: the authenticated caller, never a typed name. */
+export interface JudgmentAssertedBy {
+  readonly kind: 'user' | 'service';
+  /** A user id, or for a service token its token or session id. */
+  readonly id: string;
+}
+
+export interface Judgment {
+  readonly id: string;
+  readonly tenantId: TenantId;
+  readonly projectId: ProjectId;
+  readonly runId: string;
+  readonly subject: JudgedSubject;
+  readonly item: JudgedItem;
+  readonly verdict: Verdict;
+  readonly reason?: string;
+  /** The judge's class; absent for an unclassified judgment (weight 1). */
+  readonly judgeClassId?: string;
+  readonly assertedBy: JudgmentAssertedBy;
+  /** The app's opaque id for its end user who judged, when an app judged on their behalf. */
+  readonly participantId?: string;
+  readonly createdAt: string;
+  /** Set when the judgment was removed or superseded. */
+  readonly unregisteredAt?: string;
+  /** The judgment that replaced this one. */
+  readonly supersededBy?: string;
+}
+
+/**
+ * What a judged agent turn read besides its input, captured when it was
+ * first judged, so the turn can later be replayed faithfully: the
+ * conversation before it and the context its retrievals returned. (Its
+ * tool calls and results, provider and model are in its output.)
+ */
+export interface JudgedRunContext {
+  /** The conversation's messages before the turn, oldest first (at most the last 200). */
+  readonly history?: readonly unknown[];
+  /** Whether older messages were left out of `history`. */
+  readonly historyTruncated?: boolean;
+  /** What the turn's retrievals returned. */
+  readonly retrieved?: unknown;
+  /**
+   * The reviewer's decision at the turn's session approval gate, when the
+   * turn waited on one. A replay of the turn follows it.
+   */
+  readonly sessionApproval?: { readonly approved: boolean; readonly rationale?: string };
+  /** For a flow run: what it did (see `JudgedFlowContext`). */
+  readonly flow?: JudgedFlowContext;
+}
+
+/** One tool call a judged run made, and its result. */
+export interface JudgedToolCall {
+  /** The run that made it: the flow run, a sub-flow's run, or an agent step's turn. */
+  readonly runId: string;
+  /** The tool node that made it, or the agent step whose turn did. */
+  readonly nodeId?: string;
+  /** The loop iteration, when the node is in a loop body. */
+  readonly scope?: string;
+  readonly toolId: string;
+  readonly arguments: unknown;
+  readonly result: unknown;
+}
+
+/** One agent step of a judged flow run: the turn it started. */
+export interface JudgedFlowStep {
+  readonly runId: string;
+  readonly nodeId?: string;
+  readonly scope?: string;
+  readonly agentId: string;
+  readonly agentVersion: string;
+  /** What the turn's retrievals returned. */
+  readonly retrieved?: unknown;
+}
+
+/**
+ * What a judged flow run did, kept at its first judgment: every tool
+ * call it made with its result (at its tool nodes, in its agent steps'
+ * turns and in its sub-flows), at most 500, and its agent steps.
+ */
+export interface JudgedFlowContext {
+  readonly calls: readonly JudgedToolCall[];
+  readonly steps: readonly JudgedFlowStep[];
+  /** More calls were made than were kept. */
+  readonly truncated?: boolean;
+}
+
+/** The stored copies of a judged run. */
+export interface JudgedRunCopy {
+  readonly runId: string;
+  readonly subject: JudgedSubject;
+  readonly input: unknown;
+  readonly output: unknown;
+  /** Absent for runs judged before context was captured, and for flow runs. */
+  readonly context?: JudgedRunContext;
+  readonly capturedAt: string;
+}
+
+export interface JudgmentWithCopies extends Judgment {
+  /** The run's input and output as they were when it was first judged. */
+  readonly run: JudgedRunCopy;
+  /** The judged item's value, when the judgment pointed at it. */
+  readonly itemValue?: unknown;
+}
+
+export interface JudgmentRecordInput {
+  readonly tenantId: TenantId;
+  readonly projectId: ProjectId;
+  readonly runId: string;
+  /** The run's copy, stored on its first judgment. */
+  readonly run: {
+    readonly subject: JudgedSubject;
+    readonly input: unknown;
+    readonly output: unknown;
+    readonly context?: JudgedRunContext;
+  };
+  readonly item: JudgedItem;
+  /** The item's value at `item.pointer`, resolved by the route from the run's output. */
+  readonly itemValue?: unknown;
+  readonly verdict: Verdict;
+  readonly reason?: string;
+  readonly judgeClassId?: string;
+  readonly assertedBy: JudgmentAssertedBy;
+  readonly participantId?: string;
+}
+
+export interface JudgmentListInput {
+  readonly tenantId: TenantId;
+  readonly scope?: ListScope;
+  readonly runId?: string;
+  readonly agentId?: string;
+  readonly agentVersion?: string;
+  readonly flowId?: string;
+  readonly verdict?: Verdict;
+  readonly judgeClassId?: string;
+  readonly participantId?: string;
+  readonly cursor?: Cursor;
+  readonly limit: number;
+}
+
+export interface JudgmentPage {
+  readonly data: readonly Judgment[];
+  readonly hasMore: boolean;
+  readonly nextCursor?: Cursor;
+}
+
+export interface JudgmentGetInput {
+  readonly tenantId: TenantId;
+  readonly judgmentId: string;
+}
+
+export interface JudgedRunListInput {
+  readonly tenantId: TenantId;
+  readonly projectId?: ProjectId;
+  readonly agentId?: string;
+  readonly agentVersion?: string;
+  readonly flowId?: string;
+  /** Runs first judged at or after this time (ISO 8601). */
+  readonly since?: string;
+  /** Runs first judged before this time (ISO 8601). */
+  readonly until?: string;
+  readonly cursor?: Cursor;
+  readonly limit: number;
+}
+
+export interface JudgedRunWithJudgments {
+  readonly projectId: ProjectId;
+  readonly run: JudgedRunCopy;
+  /** The run's live judgments. */
+  readonly judgments: readonly Judgment[];
+}
+
+export interface JudgedRunPage {
+  readonly data: readonly JudgedRunWithJudgments[];
+  readonly hasMore: boolean;
+  readonly nextCursor?: Cursor;
+}

package/src/middleware/auth.ts

@@ -37,8 +37,9 @@ export interface TokenResolution {
    * middleware surfaces it as `c.get('reviewerRole')` and approvals
    * routes gate visibility / decisions on the `standard < senior < admin`
    * hierarchy. Absent for tokens that were never provisioned with a
-   * reviewer role — such tokens can still hit `/v1/runs` etc., but any
-   * approvals route returns `403 permission-denied`.
+   * reviewer role: the approvals routes then take the role the reviewer
+   * roster gives the token's user (`ReviewerBinding.resolveReviewerRole`),
+   * and return `403 permission-denied` when it gives none.
    */
   readonly reviewerRole?: ReviewerRole;
   /**

package/src/middleware/idempotency.ts

@@ -63,7 +63,11 @@ const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
  *   byte-identical (status, content-type, body).
  * - Cached hit with different body hash → 409
  *   `idempotency-key-body-mismatch`.
- * - Cache miss → run the handler, then store the response.
+ * - Cache miss → run the handler, then store the response when the
+ *   request took effect (a status below 400). A refusal (4xx) changed
+ *   nothing, and a failure (5xx) may be transient: neither is stored, so
+ *   a retry with the same key after fixing the cause runs again instead
+ *   of replaying the old error.
  *
  * The middleware sits AFTER auth so `tenantId` is available; the
  * cache key is namespaced by tenant so callers can't collide across
@@ -122,6 +126,8 @@ export function idempotencyMiddleware(
     await next();
 
     const res = c.res;
+    // Only what took effect is replayed; see the doc above.
+    if (res.status >= 400) return;
     const savedBody = await res.clone().text();
     const contentType = res.headers.get('Content-Type') ?? 'application/octet-stream';
     await store.set(cacheKey, {

package/src/openapi/generate.ts

@@ -80,7 +80,13 @@ const TAG_DESCRIPTIONS: Readonly<Record<string, string>> = {
   policies:
     'Tenant policy catalog (list, get, versions, publish, unregister) — part of the admin control plane. Full versioned CRUD, mirrors `flows` 1:1. Registry-only: enforcement is out of scope. `policyKind` selects the shape of `spec` — closed enum, extended additively (`access-control`, `model-routing`, `adapter-allowlist`, `rate-limit`, `retention`, `compliance`). Runtime consumers (e.g. model routing for `model-routing`, retention sweeps for `retention`) read policies from this store and apply them at their own boundary. Caller-plugged via `PolicyRegistryBinding`.',
   'eval-suites':
-    'Evaluation suite catalog (list, get, versions, publish, unregister) — part of the admin control plane. Full versioned CRUD, mirrors `policies` 1:1. Registry-only: eval-run execution + per-kind grader dispatch live on the `eval-runs` tag. `evalKind` selects the shape of `spec` — closed enum, extended additively (`accuracy`, `pairwise`, `regression`, `human-review`, `benchmark`, `custom`). Runs against these suites are started through the `eval-runs` tag. Caller-plugged via `EvalSuiteRegistryBinding`.',
+    'Evaluation suite catalog (list, get, versions, publish, unregister) — part of the admin control plane. Full versioned CRUD, mirrors `policies` 1:1. Registry-only: eval-run execution + per-kind grader dispatch live on the `eval-runs` tag. `evalKind` selects the shape of `spec` — closed enum, extended additively (`accuracy`, `pairwise`, `regression`, `human-review`, `benchmark`, `custom`, `judged`). The cases of a `judged` suite are copies of judged runs built from judgments (`POST .../versions/from-judgments`, listed at `GET .../versions/{version}/cases`; `EvalCaseStoreBinding`). Runs against these suites are started through the `eval-runs` tag. Caller-plugged via `EvalSuiteRegistryBinding`.',
+  blocks:
+    "Data blocks: versioned prompts and settings an agent version pins when it's published (list, get, versions, publish, unregister, reinstate). A version never changes, nor does a block's kind; a settings block's values satisfy its schema and the latest version's. A block belongs to one project and is authorized through it: `read` on the project to read, `write` to publish, unregister or reinstate. Caller-plugged via `BlockRegistryBinding`.",
+  judgments:
+    "Judgments: yes or no, with an optional reason, about one item of a finished run's output, optionally recorded under a judge class (list, get, create, unregister). Each judgment keeps copies of what was judged (the run's input and output, and the item) so they outlive the run's own retention. Who judged comes from the authenticated caller, never the body; an app judging for one of its users passes that user's opaque id as `participantId`. One live judgment per run, item key, caller and participant: judging again supersedes the earlier one, which stays as history. Caller-plugged via `JudgmentRegistryBinding`.",
+  'judge-classes':
+    'Judge classes (list, get, create, update, unregister): the deployment\'s named kinds of judge ("expert", "user", ...), each with a weight, scoped to the tenant, a project, or an agent in a project. A judgment may name a class; an unclassified judgment counts with weight 1. Caller-plugged via `JudgmentRegistryBinding`.',
   'eval-runs':
     'Eval-run data plane (start, list, get, cancel, events SSE). Dispatches an eval run against a registered suite. Dispatch is per-`EvalKind`; kinds without a registered dispatcher return `422 dispatcher-not-registered`. `result` is kind-specific opaque JSON on the wire — for `accuracy` it is `{ passCount, totalCount, meanScore, perCase[] }`. SSE events mirror run streaming: per-case progress followed by a terminal frame carrying the aggregate. Caller-plugged via `EvalRunBinding`.',
   auth: 'OAuth 2.0 / OIDC identity providers + session lifecycle. Layers browser-based auth on top of the static bearer-token surface: bearer tokens continue to work byte-shape-identical; session tokens use the `kgi_sk_*` prefix so the same middleware routes both flavors. Providers are caller-plugged via `IdentityProviderBinding` (no baked-in list). Sessions persist via `SessionStoreBinding`. Code exchange is caller-supplied via `exchangeCode`. PKCE (S256) is mandatory. `clientSecretRef` is a REFERENCE — the plaintext secret never crosses the wire.',