@kindgi/api 0.1.3 → 0.1.4-rc.0

package/src/eval-run-dispatcher.ts

@@ -3,10 +3,13 @@
 
 import { randomUUID } from 'node:crypto';
 
-import type { RunId, TenantId, Timestamp } from '@kindgi/types';
+import type { ReplayTurnReport } from '@kindgi/agents';
+import type { RunReplayRef } from '@kindgi/runtime';
+import type { ProjectId, RunId, TenantId, Timestamp } from '@kindgi/types';
 
 import type {
   AgentRef,
+  EvalComparison,
   EvalRun,
   EvalRunBinding,
   EvalRunCancelInput,
@@ -50,6 +53,16 @@ export interface EvalRunSubjectInvokeInput {
   readonly input: unknown;
   readonly dryRun: boolean;
   readonly abortSignal: AbortSignal;
+  /** The eval run's project: where the subject runs. */
+  readonly projectId?: ProjectId;
+  /**
+   * Set when the subject re-runs a past run (a comparison eval run): the
+   * run is a replay of `of` for the eval run, under the replay rules
+   * (`InvokeAgentInput.replay`).
+   */
+  readonly replay?: RunReplayRef;
+  /** The conversation before the past turn, oldest first: a replayed turn starts from it. */
+  readonly history?: readonly unknown[];
 }
 
 export interface EvalRunSubjectInvokeOutcome {
@@ -57,6 +70,22 @@ export interface EvalRunSubjectInvokeOutcome {
   readonly error?: string;
   readonly costUsd?: number;
   readonly durationMs?: number;
+  /** The run the subject ran in. */
+  readonly runId?: RunId;
+  /** A replay's report: each tool call and what happened to it (`AgentTurnResult.replay`). */
+  readonly replay?: ReplayTurnReport;
+  /** The provider and model that answered. */
+  readonly provider?: { readonly id: string; readonly model: string };
+  /**
+   * Set when a replayed flow stopped at a tool call the replay refused (a
+   * write the past run didn't make): no output, and what it would have
+   * done. Not an error: the replay did what it's meant to.
+   */
+  readonly stopped?: {
+    readonly toolId: string;
+    readonly arguments: unknown;
+    readonly reason?: string;
+  };
 }
 
 /**
@@ -100,6 +129,10 @@ export interface DispatchContext {
   readonly dryRun: boolean;
   readonly abortSignal: AbortSignal;
   readonly subject: EvalSubjectInvoker;
+  /** The eval run's project. */
+  readonly projectId?: ProjectId;
+  /** A comparison eval run's baseline, reads and repetitions. */
+  readonly comparison?: EvalComparison;
   onProgress(perCase: Readonly<Record<string, unknown>>): void;
 }
 
@@ -113,6 +146,7 @@ export interface EvalRunDispatcher {
   validate?(
     suite: EvalSuite,
     target: AgentRef | FlowRef,
+    comparison?: EvalComparison,
   ): { kind: 'ok' } | { kind: 'err'; message: string };
   dispatch(ctx: DispatchContext): Promise<DispatchResult>;
 }
@@ -311,26 +345,13 @@ export function createInProcessEvalRunBinding(
         };
       }
       if (dispatcher.validate !== undefined) {
-        const v = dispatcher.validate(suite, target);
+        const v = dispatcher.validate(suite, target, input.comparison);
         if (v.kind === 'err') {
           return { kind: 'dispatcher-input-invalid', message: v.message };
         }
       }
       const runId = randomUUID() as unknown as RunId;
-      const startedAt = now().toISOString() as unknown as Timestamp;
-      const record: EvalRun = {
-        runId,
-        tenantId: input.tenantId,
-        suiteId: suite.id,
-        suiteVersion: suite.version,
-        kind: suite.kind,
-        ...(input.agentRef !== undefined && { agentRef: input.agentRef }),
-        ...(input.flowRef !== undefined && { flowRef: input.flowRef }),
-        status: 'running' as EvalRunStatus,
-        dryRun: input.dryRun === true,
-        startedAt,
-        ...(input.correlationId !== undefined && { correlationId: input.correlationId }),
-      };
+      const record = newRunRecord(input, suite, runId, now());
       setRun(record);
       const controller = new AbortController();
       controllers.set(runId as unknown as string, controller);
@@ -349,6 +370,8 @@ export function createInProcessEvalRunBinding(
             dryRun: input.dryRun === true,
             abortSignal: controller.signal,
             subject: options.subject,
+            projectId: input.projectId,
+            ...(input.comparison !== undefined && { comparison: input.comparison }),
             onProgress: (perCaseEntry: Readonly<Record<string, unknown>>) => {
               const current = runs.get(runId as unknown as string);
               if (current === undefined) return;
@@ -489,6 +512,24 @@ export function createInProcessEvalRunBinding(
   };
 }
 
+/** A started run's row: `running`, with what it runs and how. */
+function newRunRecord(input: EvalRunStartInput, suite: EvalSuite, runId: RunId, at: Date): EvalRun {
+  return {
+    runId,
+    tenantId: input.tenantId,
+    suiteId: suite.id,
+    suiteVersion: suite.version,
+    kind: suite.kind,
+    ...(input.agentRef !== undefined && { agentRef: input.agentRef }),
+    ...(input.flowRef !== undefined && { flowRef: input.flowRef }),
+    status: 'running',
+    dryRun: input.dryRun === true,
+    startedAt: at.toISOString() as unknown as Timestamp,
+    ...(input.correlationId !== undefined && { correlationId: input.correlationId }),
+    ...(input.comparison !== undefined && { comparison: input.comparison }),
+  };
+}
+
 function isTerminal(status: EvalRunStatus): boolean {
   return status === 'completed' || status === 'failed' || status === 'cancelled';
 }

package/src/eval-suite-binding.ts

@@ -100,6 +100,8 @@ export const EVAL_KINDS = [
   'human-review',
   'benchmark',
   'custom',
+  /** Built from judgments of past runs: cases are copies of judged runs (`EvalCaseStoreBinding`). */
+  'judged',
 ] as const;
 export type EvalKind = (typeof EVAL_KINDS)[number];
 

package/src/flow-binding.ts

@@ -39,11 +39,12 @@ export interface FlowRegistryBinding {
    */
   get(input: FlowGetInput): Promise<Flow | null>;
   /**
-   * Specific `(flowId, version)` lookup, or `null` if unknown.
-   * Returns tombstoned versions too — provenance paths need to
-   * resolve historical run references.
+   * Specific `(flowId, version)` lookup, or `null` if unknown. Returns
+   * unregistered (tombstoned) versions too, with `unregisteredAt` set:
+   * a resumed run and provenance read them, while a new run that names
+   * one is refused.
    */
-  getVersion(input: FlowGetVersionInput): Promise<Flow | null>;
+  getVersion(input: FlowGetVersionInput): Promise<FlowVersionRecord | null>;
   /**
    * Head-row existence check. Lets `GET /v1/flows/{id}` distinguish
    * `410 gone` (identity exists, no active version) from `404 not-
@@ -79,6 +80,12 @@ export interface FlowRegistryBinding {
   reinstateVersion(input: FlowReinstateVersionInput): Promise<FlowReinstateVersionOutcome>;
 }
 
+/** A flow version as `getVersion` reads it: `unregisteredAt` is set when it's unregistered. */
+export type FlowVersionRecord = Flow & {
+  /** ISO-8601; present only on an unregistered version. */
+  readonly unregisteredAt?: string;
+};
+
 export interface FlowListInput {
   readonly tenantId: TenantId;
   readonly limit: number;

package/src/flow-pins.ts

@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2026 Kindgi Inc.
+
+import type { AgentId } from '@kindgi/agents';
+import type { TupleEnqueueHook } from '@kindgi/authz';
+import { type Flow, type FlowPins, flowPinsDigest, flowRefs } from '@kindgi/flow';
+import { latestVersion } from '@kindgi/tools';
+import type { Cursor, FlowId, ProjectId, TenantId, ToolId } from '@kindgi/types';
+
+import type { AgentRegistryBinding } from './agent-binding.js';
+import { type UnpinnableRef, activeToolVersions } from './agent-pins.js';
+import { type DeployedVersionOutcome, deployVersion } from './deploy-versions.js';
+import type { FlowRegistryBinding } from './flow-binding.js';
+import type { ToolRegistryBinding } from './tool-binding.js';
+
+export type FlowPinsOutcome =
+  | { readonly kind: 'ok'; readonly pins: FlowPins }
+  | { readonly kind: 'unpinnable'; readonly issues: readonly UnpinnableRef[] };
+
+/**
+ * Pin a flow version when it's published: each tool it runs, and each
+ * agent it runs at no named version, to its latest active version now
+ * (what a run would have bound), so every run of the version uses those.
+ * A tool or agent with no published version makes the flow unpinnable:
+ * the caller refuses the publish, naming each, rather than store a
+ * partly pinned version.
+ */
+export async function resolveFlowPins(
+  tools: ToolRegistryBinding,
+  agents: AgentRegistryBinding,
+  tenantId: TenantId,
+  flow: Flow,
+): Promise<FlowPinsOutcome> {
+  const refs = flowRefs(flow);
+  const pinned: { tools: Record<string, string>; agents: Record<string, string> } = {
+    tools: {},
+    agents: {},
+  };
+  const issues: UnpinnableRef[] = [];
+  for (const id of refs.tools) {
+    const latest = latestVersion(await activeToolVersions(tools, tenantId, id as ToolId));
+    if (latest !== undefined) pinned.tools[id] = latest;
+    else
+      issues.push({
+        path: '/nodes',
+        message: `tool "${id}" has no published version; publish the tool first`,
+      });
+  }
+  for (const id of refs.agents) {
+    const latest = await agents.get({ tenantId, agentId: id as AgentId });
+    if (latest !== null) pinned.agents[id] = latest.version as unknown as string;
+    else
+      issues.push({
+        path: '/nodes',
+        message: `agent "${id}" has no published version; publish the agent first`,
+      });
+  }
+  if (issues.length > 0) return { kind: 'unpinnable', issues };
+  return { kind: 'ok', pins: pinned };
+}
+
+export interface PublishDeployedFlowInput {
+  readonly flows: FlowRegistryBinding;
+  readonly tenantId: TenantId;
+  readonly projectId: ProjectId;
+  /** The flow as the pack defines it, with no pins. */
+  readonly flow: Flow;
+  readonly pins: FlowPins;
+  readonly enqueueTuples: TupleEnqueueHook;
+}
+
+/** Register a deployed flow with its pins, by the deploy rule (`deployVersion`). */
+export async function publishDeployedFlow(
+  input: PublishDeployedFlowInput,
+): Promise<DeployedVersionOutcome> {
+  const { flows, tenantId, projectId, flow, pins, enqueueTuples } = input;
+  return deployVersion<Flow>({
+    label: `flow "${flow.id as unknown as string}"`,
+    definition: flow,
+    pins,
+    pinsDigest: flowPinsDigest(pins),
+    existing: await allFlowVersions(flows, tenantId, flow.id),
+    publish: async (version) => {
+      const outcome = await flows.publish({ tenantId, projectId, flow: version, enqueueTuples });
+      if (outcome.kind === 'ok') return 'ok';
+      return outcome.kind === 'project-not-found' ? 'skipped' : 'taken';
+    },
+  });
+}
+
+/** Page size for reading a flow's versions. */
+const VERSIONS_PAGE = 200;
+
+/** Every active version of a flow. */
+async function allFlowVersions(
+  flows: FlowRegistryBinding,
+  tenantId: TenantId,
+  flowId: FlowId,
+): Promise<readonly Flow[]> {
+  const versions: Flow[] = [];
+  let cursor: Cursor | undefined;
+  do {
+    const page = await flows.listVersions({
+      tenantId,
+      flowId,
+      limit: VERSIONS_PAGE,
+      ...(cursor !== undefined && { cursor }),
+    });
+    versions.push(...page.data);
+    cursor = page.nextCursor;
+  } while (cursor !== undefined);
+  return versions;
+}

package/src/index.ts

@@ -233,6 +233,7 @@ export type {
   AgentPublishInput,
   AgentPublishOutcome,
   AgentRegistryBinding,
+  AgentVersionRecord,
   AgentReinstateVersionInput,
   AgentReinstateVersionOutcome,
   AgentUnregisterInput,
@@ -247,6 +248,7 @@ export type {
   FlowPublishInput,
   FlowPublishOutcome,
   FlowRegistryBinding,
+  FlowVersionRecord,
   FlowReinstateVersionInput,
   FlowReinstateVersionOutcome,
   FlowUnregisterInput,
@@ -309,7 +311,7 @@ export {
   stdioRefusal,
 } from './tenant-host-access.js';
 export type { HostReach, TenantHostAccess } from './tenant-host-access.js';
-export { POLICY_KINDS } from '@kindgi/policy-contract';
+export { APPLIED_POLICY_KINDS, POLICY_KINDS } from '@kindgi/policy-contract';
 export type {
   Policy,
   PolicyGetInput,
@@ -328,6 +330,61 @@ export type {
   PolicyVersionPage,
   PolicyVersionRow,
 } from '@kindgi/policy-contract';
+export { JUDGE_CLASS_SCOPE_KINDS, VERDICTS, judgeClassApplies } from './judgment-binding.js';
+export type {
+  EvalCaseListInput,
+  EvalCasePage,
+  EvalCasePutInput,
+  EvalCaseStoreBinding,
+  JudgedEvalCase,
+  JudgedItemSummary,
+} from './eval-case-binding.js';
+export { MAX_JUDGED_CASES } from './routes/judged-suites.js';
+export { MAX_JUDGED_HISTORY } from './routes/judgment-context.js';
+export type {
+  JudgeClass,
+  JudgeClassCreateInput,
+  JudgeClassCreateOutcome,
+  JudgeClassGetInput,
+  JudgeClassListInput,
+  JudgeClassPage,
+  JudgeClassScope,
+  JudgeClassScopeKind,
+  JudgeClassUpdateInput,
+  JudgedItem,
+  JudgedFlowContext,
+  JudgedFlowStep,
+  JudgedRunContext,
+  JudgedToolCall,
+  JudgedRunCopy,
+  JudgedRunListInput,
+  JudgedRunPage,
+  JudgedRunWithJudgments,
+  JudgedSubject,
+  Judgment,
+  JudgmentAssertedBy,
+  JudgmentGetInput,
+  JudgmentListInput,
+  JudgmentPage,
+  JudgmentRecordInput,
+  JudgmentRegistryBinding,
+  JudgmentWithCopies,
+  Verdict,
+} from './judgment-binding.js';
+export { resolvePointer } from './routes/judgments.js';
+export type {
+  BlockGetInput,
+  BlockGetVersionInput,
+  BlockListInput,
+  BlockListVersionsInput,
+  BlockPage,
+  BlockPublishInput,
+  BlockPublishOutcome,
+  BlockRecord,
+  BlockRegistryBinding,
+  BlockReinstateOutcome,
+  BlockVersionInput,
+} from './block-binding.js';
 export { EVAL_KINDS } from './eval-suite-binding.js';
 export type {
   EvalKind,
@@ -348,6 +405,9 @@ export type {
 export { EVAL_RUN_STATUSES } from './eval-run-binding.js';
 export type {
   AgentRef,
+  EvalBaseline,
+  EvalComparison,
+  EvalReads,
   EvalRun,
   EvalRunBinding,
   EvalRunCancelInput,
@@ -379,7 +439,27 @@ export type {
   EvalSubjectInvoker,
   InProcessEvalRunBindingOptions,
 } from './eval-run-dispatcher.js';
-export { COST_GROUP_DIMENSIONS } from './cost-binding.js';
+export { DEFAULT_COMPARISON, createJudgedDispatcher } from './judged-dispatcher.js';
+export type {
+  ComparisonBaselineSummary,
+  ComparisonMetric,
+  JudgedCaseResult,
+  JudgedComparisonSummary,
+  JudgedDispatcherOptions,
+} from './judged-dispatcher.js';
+export { itemChanges, matchJudged, outputItems, scoreItems, valueAt } from './judged-items.js';
+export type {
+  ItemChanges,
+  ItemJudgments,
+  MatchedItem,
+  OutputItem,
+  OutputScore,
+} from './judged-items.js';
+export {
+  COST_AGGREGATE_DEFAULT_LIMIT,
+  COST_AGGREGATE_MAX_LIMIT,
+  COST_GROUP_DIMENSIONS,
+} from './cost-binding.js';
 export type {
   CostAggregateGroup,
   CostAggregateInput,
@@ -477,7 +557,10 @@ export type {
   DeploymentGetInput,
   DeploymentListInput,
   DeploymentPage,
+  DeployedAgent,
+  DeployedFlow,
   DeployedPrimitive,
+  DeployedVersion,
   DeploymentContents,
   DeploymentPrimitiveCounts,
   DeploymentRegisterInput,