@kindgi/api 0.1.3 → 0.1.4-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (222) hide show
  1. package/dist/agent-binding.d.ts +26 -4
  2. package/dist/agent-binding.d.ts.map +1 -1
  3. package/dist/agent-pins.d.ts +48 -0
  4. package/dist/agent-pins.d.ts.map +1 -0
  5. package/dist/agent-pins.js +102 -0
  6. package/dist/agent-pins.js.map +1 -0
  7. package/dist/app.d.ts +22 -0
  8. package/dist/app.d.ts.map +1 -1
  9. package/dist/app.js +27 -3
  10. package/dist/app.js.map +1 -1
  11. package/dist/block-binding.d.ts +132 -0
  12. package/dist/block-binding.d.ts.map +1 -0
  13. package/dist/block-binding.js +4 -0
  14. package/dist/block-binding.js.map +1 -0
  15. package/dist/block-pins.d.ts +22 -0
  16. package/dist/block-pins.d.ts.map +1 -0
  17. package/dist/block-pins.js +112 -0
  18. package/dist/block-pins.js.map +1 -0
  19. package/dist/cost-binding.d.ts +20 -1
  20. package/dist/cost-binding.d.ts.map +1 -1
  21. package/dist/cost-binding.js +4 -0
  22. package/dist/cost-binding.js.map +1 -1
  23. package/dist/deploy-versions.d.ts +60 -0
  24. package/dist/deploy-versions.d.ts.map +1 -0
  25. package/dist/deploy-versions.js +91 -0
  26. package/dist/deploy-versions.js.map +1 -0
  27. package/dist/deployment-binding.d.ts +24 -3
  28. package/dist/deployment-binding.d.ts.map +1 -1
  29. package/dist/derive-agent-version.d.ts +77 -0
  30. package/dist/derive-agent-version.d.ts.map +1 -0
  31. package/dist/derive-agent-version.js +149 -0
  32. package/dist/derive-agent-version.js.map +1 -0
  33. package/dist/errors.d.ts.map +1 -1
  34. package/dist/errors.js +18 -0
  35. package/dist/errors.js.map +1 -1
  36. package/dist/eval-case-binding.d.ts +65 -0
  37. package/dist/eval-case-binding.d.ts.map +1 -0
  38. package/dist/eval-case-binding.js +4 -0
  39. package/dist/eval-case-binding.js.map +1 -0
  40. package/dist/eval-run-binding.d.ts +37 -0
  41. package/dist/eval-run-binding.d.ts.map +1 -1
  42. package/dist/eval-run-binding.js.map +1 -1
  43. package/dist/eval-run-dispatcher.d.ts +41 -3
  44. package/dist/eval-run-dispatcher.d.ts.map +1 -1
  45. package/dist/eval-run-dispatcher.js +21 -15
  46. package/dist/eval-run-dispatcher.js.map +1 -1
  47. package/dist/eval-suite-binding.d.ts +1 -1
  48. package/dist/eval-suite-binding.d.ts.map +1 -1
  49. package/dist/eval-suite-binding.js +2 -0
  50. package/dist/eval-suite-binding.js.map +1 -1
  51. package/dist/flow-binding.d.ts +18 -4
  52. package/dist/flow-binding.d.ts.map +1 -1
  53. package/dist/flow-pins.d.ts +36 -0
  54. package/dist/flow-pins.d.ts.map +1 -0
  55. package/dist/flow-pins.js +81 -0
  56. package/dist/flow-pins.js.map +1 -0
  57. package/dist/guardrail-binding.d.ts +8 -0
  58. package/dist/guardrail-binding.d.ts.map +1 -1
  59. package/dist/handler-binding.d.ts +3 -0
  60. package/dist/handler-binding.d.ts.map +1 -1
  61. package/dist/index.d.ts +18 -6
  62. package/dist/index.d.ts.map +1 -1
  63. package/dist/index.js +8 -2
  64. package/dist/index.js.map +1 -1
  65. package/dist/judged-dispatcher.d.ts +138 -0
  66. package/dist/judged-dispatcher.d.ts.map +1 -0
  67. package/dist/judged-dispatcher.js +308 -0
  68. package/dist/judged-dispatcher.js.map +1 -0
  69. package/dist/judged-items.d.ts +86 -0
  70. package/dist/judged-items.d.ts.map +1 -0
  71. package/dist/judged-items.js +184 -0
  72. package/dist/judged-items.js.map +1 -0
  73. package/dist/judgment-binding.d.ts +316 -0
  74. package/dist/judgment-binding.d.ts.map +1 -0
  75. package/dist/judgment-binding.js +19 -0
  76. package/dist/judgment-binding.js.map +1 -0
  77. package/dist/openapi/generate.d.ts.map +1 -1
  78. package/dist/openapi/generate.js +4 -1
  79. package/dist/openapi/generate.js.map +1 -1
  80. package/dist/openapi/operations.d.ts.map +1 -1
  81. package/dist/openapi/operations.js +482 -13
  82. package/dist/openapi/operations.js.map +1 -1
  83. package/dist/openapi/schemas.d.ts +46 -0
  84. package/dist/openapi/schemas.d.ts.map +1 -1
  85. package/dist/openapi/schemas.js +1350 -175
  86. package/dist/openapi/schemas.js.map +1 -1
  87. package/dist/provider-binding.d.ts +12 -7
  88. package/dist/provider-binding.d.ts.map +1 -1
  89. package/dist/registry-read-only.d.ts +32 -0
  90. package/dist/registry-read-only.d.ts.map +1 -0
  91. package/dist/registry-read-only.js +22 -0
  92. package/dist/registry-read-only.js.map +1 -0
  93. package/dist/routes/agents.d.ts +9 -1
  94. package/dist/routes/agents.d.ts.map +1 -1
  95. package/dist/routes/agents.js +181 -11
  96. package/dist/routes/agents.js.map +1 -1
  97. package/dist/routes/blocks.d.ts +19 -0
  98. package/dist/routes/blocks.d.ts.map +1 -0
  99. package/dist/routes/blocks.js +306 -0
  100. package/dist/routes/blocks.js.map +1 -0
  101. package/dist/routes/cost.d.ts.map +1 -1
  102. package/dist/routes/cost.js +47 -2
  103. package/dist/routes/cost.js.map +1 -1
  104. package/dist/routes/deployments.d.ts +3 -0
  105. package/dist/routes/deployments.d.ts.map +1 -1
  106. package/dist/routes/deployments.js +224 -55
  107. package/dist/routes/deployments.js.map +1 -1
  108. package/dist/routes/eval-comparison.d.ts +15 -0
  109. package/dist/routes/eval-comparison.d.ts.map +1 -0
  110. package/dist/routes/eval-comparison.js +123 -0
  111. package/dist/routes/eval-comparison.js.map +1 -0
  112. package/dist/routes/eval-runs.d.ts +7 -1
  113. package/dist/routes/eval-runs.d.ts.map +1 -1
  114. package/dist/routes/eval-runs.js +42 -3
  115. package/dist/routes/eval-runs.js.map +1 -1
  116. package/dist/routes/eval-versions.d.ts +25 -0
  117. package/dist/routes/eval-versions.d.ts.map +1 -0
  118. package/dist/routes/eval-versions.js +66 -0
  119. package/dist/routes/eval-versions.js.map +1 -0
  120. package/dist/routes/flows.d.ts +13 -1
  121. package/dist/routes/flows.d.ts.map +1 -1
  122. package/dist/routes/flows.js +48 -3
  123. package/dist/routes/flows.js.map +1 -1
  124. package/dist/routes/guardrails.d.ts.map +1 -1
  125. package/dist/routes/guardrails.js +4 -0
  126. package/dist/routes/guardrails.js.map +1 -1
  127. package/dist/routes/hierarchy-errors.d.ts +45 -0
  128. package/dist/routes/hierarchy-errors.d.ts.map +1 -0
  129. package/dist/routes/hierarchy-errors.js +47 -0
  130. package/dist/routes/hierarchy-errors.js.map +1 -0
  131. package/dist/routes/judged-suites.d.ts +20 -0
  132. package/dist/routes/judged-suites.d.ts.map +1 -0
  133. package/dist/routes/judged-suites.js +272 -0
  134. package/dist/routes/judged-suites.js.map +1 -0
  135. package/dist/routes/judgment-context.d.ts +22 -0
  136. package/dist/routes/judgment-context.d.ts.map +1 -0
  137. package/dist/routes/judgment-context.js +88 -0
  138. package/dist/routes/judgment-context.js.map +1 -0
  139. package/dist/routes/judgment-flow-context.d.ts +32 -0
  140. package/dist/routes/judgment-flow-context.d.ts.map +1 -0
  141. package/dist/routes/judgment-flow-context.js +195 -0
  142. package/dist/routes/judgment-flow-context.js.map +1 -0
  143. package/dist/routes/judgments.d.ts +41 -0
  144. package/dist/routes/judgments.d.ts.map +1 -0
  145. package/dist/routes/judgments.js +566 -0
  146. package/dist/routes/judgments.js.map +1 -0
  147. package/dist/routes/orgs.d.ts +5 -2
  148. package/dist/routes/orgs.d.ts.map +1 -1
  149. package/dist/routes/orgs.js +38 -22
  150. package/dist/routes/orgs.js.map +1 -1
  151. package/dist/routes/policies.d.ts.map +1 -1
  152. package/dist/routes/policies.js +12 -1
  153. package/dist/routes/policies.js.map +1 -1
  154. package/dist/routes/projects.d.ts +10 -2
  155. package/dist/routes/projects.d.ts.map +1 -1
  156. package/dist/routes/projects.js +94 -80
  157. package/dist/routes/projects.js.map +1 -1
  158. package/dist/routes/providers.d.ts.map +1 -1
  159. package/dist/routes/providers.js +6 -1
  160. package/dist/routes/providers.js.map +1 -1
  161. package/dist/routes/runs.js +25 -3
  162. package/dist/routes/runs.js.map +1 -1
  163. package/dist/routes/teams.d.ts +6 -2
  164. package/dist/routes/teams.d.ts.map +1 -1
  165. package/dist/routes/teams.js +83 -73
  166. package/dist/routes/teams.js.map +1 -1
  167. package/dist/routes/tools.d.ts.map +1 -1
  168. package/dist/routes/tools.js +4 -0
  169. package/dist/routes/tools.js.map +1 -1
  170. package/dist/tool-binding.d.ts +8 -0
  171. package/dist/tool-binding.d.ts.map +1 -1
  172. package/openapi.json +14253 -10099
  173. package/package.json +21 -21
  174. package/src/agent-binding.ts +28 -4
  175. package/src/agent-pins.ts +147 -0
  176. package/src/app.ts +81 -3
  177. package/src/block-binding.ts +137 -0
  178. package/src/block-pins.ts +148 -0
  179. package/src/cost-binding.ts +21 -1
  180. package/src/deploy-versions.ts +157 -0
  181. package/src/deployment-binding.ts +27 -3
  182. package/src/derive-agent-version.ts +217 -0
  183. package/src/errors.ts +18 -0
  184. package/src/eval-case-binding.ts +71 -0
  185. package/src/eval-run-binding.ts +40 -0
  186. package/src/eval-run-dispatcher.ts +60 -16
  187. package/src/eval-suite-binding.ts +2 -0
  188. package/src/flow-binding.ts +20 -4
  189. package/src/flow-pins.ts +113 -0
  190. package/src/guardrail-binding.ts +9 -0
  191. package/src/handler-binding.ts +3 -0
  192. package/src/index.ts +86 -2
  193. package/src/judged-dispatcher.ts +530 -0
  194. package/src/judged-items.ts +263 -0
  195. package/src/judgment-binding.ts +349 -0
  196. package/src/openapi/generate.ts +7 -1
  197. package/src/openapi/operations.ts +579 -13
  198. package/src/openapi/schemas.ts +1408 -128
  199. package/src/provider-binding.ts +12 -7
  200. package/src/registry-read-only.ts +43 -0
  201. package/src/routes/agents.ts +252 -19
  202. package/src/routes/blocks.ts +387 -0
  203. package/src/routes/cost.ts +55 -1
  204. package/src/routes/deployments.ts +291 -56
  205. package/src/routes/eval-comparison.ts +135 -0
  206. package/src/routes/eval-runs.ts +57 -3
  207. package/src/routes/eval-versions.ts +110 -0
  208. package/src/routes/flows.ts +70 -5
  209. package/src/routes/guardrails.ts +7 -0
  210. package/src/routes/hierarchy-errors.ts +60 -0
  211. package/src/routes/judged-suites.ts +363 -0
  212. package/src/routes/judgment-context.ts +128 -0
  213. package/src/routes/judgment-flow-context.ts +245 -0
  214. package/src/routes/judgments.ts +743 -0
  215. package/src/routes/orgs.ts +44 -27
  216. package/src/routes/policies.ts +19 -0
  217. package/src/routes/projects.ts +118 -96
  218. package/src/routes/providers.ts +5 -0
  219. package/src/routes/runs.ts +29 -3
  220. package/src/routes/teams.ts +104 -90
  221. package/src/routes/tools.ts +7 -0
  222. package/src/tool-binding.ts +9 -0
@@ -0,0 +1,71 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright (C) 2026 Kindgi Inc.
3
+
4
+ import type { Cursor, TenantId } from '@kindgi/types';
5
+
6
+ import type { JudgedItem, JudgedRunContext, JudgedSubject } from './judgment-binding.js';
7
+
8
+ /**
9
+ * Caller-plugged storage for the cases of an eval-suite version, kept
10
+ * apart from the version's `spec` because a set can hold hundreds of
11
+ * cases. A `judged` suite version's cases are built from judgments: each
12
+ * is a copy of one judged run (its input, what it read, the output that
13
+ * was judged) with the judgments of its items summed up. Copies, so a
14
+ * test set stays the same when judgments are removed or runs purged.
15
+ *
16
+ * Cases are written once, when the version is published, and removed
17
+ * with it.
18
+ */
19
+ export interface EvalCaseStoreBinding {
20
+ /** Store a version's cases. Called once, right after the version is published. */
21
+ putCases(input: EvalCasePutInput): Promise<void>;
22
+ /** A version's cases, in the order they were stored, cursor-paginated. */
23
+ listCases(input: EvalCaseListInput): Promise<EvalCasePage>;
24
+ }
25
+
26
+ /** The judgments of one item of a case's output, summed up. */
27
+ export interface JudgedItemSummary extends JudgedItem {
28
+ readonly yes: number;
29
+ readonly no: number;
30
+ /** The weight behind "yes" (an unclassified judgment counts 1). */
31
+ readonly yesWeight: number;
32
+ /** The weight behind all judgments of the item. */
33
+ readonly totalWeight: number;
34
+ /** The reasons given, newest first. */
35
+ readonly reasons: readonly { readonly verdict: 'yes' | 'no'; readonly reason: string }[];
36
+ }
37
+
38
+ /** One case of a `judged` suite: a copy of a judged run and what people said about it. */
39
+ export interface JudgedEvalCase {
40
+ /** The judged run's id. */
41
+ readonly caseId: string;
42
+ /** What ran: the agent or flow at a version (the baseline when comparing to the recording). */
43
+ readonly subject: JudgedSubject;
44
+ readonly input: unknown;
45
+ /** What the turn read besides its input; absent when it wasn't captured. */
46
+ readonly context?: JudgedRunContext;
47
+ /** The output that was judged. */
48
+ readonly output: unknown;
49
+ readonly items: readonly JudgedItemSummary[];
50
+ }
51
+
52
+ export interface EvalCasePutInput {
53
+ readonly tenantId: TenantId;
54
+ readonly suiteId: string;
55
+ readonly version: string;
56
+ readonly cases: readonly JudgedEvalCase[];
57
+ }
58
+
59
+ export interface EvalCaseListInput {
60
+ readonly tenantId: TenantId;
61
+ readonly suiteId: string;
62
+ readonly version: string;
63
+ readonly cursor?: Cursor;
64
+ readonly limit: number;
65
+ }
66
+
67
+ export interface EvalCasePage {
68
+ readonly data: readonly JudgedEvalCase[];
69
+ readonly hasMore: boolean;
70
+ readonly nextCursor?: Cursor;
71
+ }
@@ -2,6 +2,7 @@
2
2
  // Copyright (C) 2026 Kindgi Inc.
3
3
 
4
4
  import type { AgentId } from '@kindgi/agents';
5
+ import type { FlowVersionOverrides } from '@kindgi/flow';
5
6
  import type { Scope } from '@kindgi/platform';
6
7
  import type { Cursor, FlowId, ProjectId, RunId, Semver, TenantId, Timestamp } from '@kindgi/types';
7
8
 
@@ -105,6 +106,8 @@ export interface EvalRun {
105
106
  readonly result?: Readonly<Record<string, unknown>>;
106
107
  readonly error?: string;
107
108
  readonly correlationId?: string;
109
+ /** A comparison eval run's baseline, reads and repetitions. */
110
+ readonly comparison?: EvalComparison;
108
111
  }
109
112
 
110
113
  export interface EvalRunFilter {
@@ -116,6 +119,41 @@ export interface EvalRunFilter {
116
119
  readonly to?: Timestamp;
117
120
  }
118
121
 
122
+ /**
123
+ * What a comparison eval run compares the candidate against:
124
+ * - `'recorded'`: the output each case recorded (what was judged);
125
+ * - `{ agentId, version }`: that version, replayed under the same rules;
126
+ * - `{ live }`: the version live in a scope (a project, segments).
127
+ */
128
+ export type EvalBaseline =
129
+ | 'recorded'
130
+ | { readonly agentId: AgentId; readonly version: Semver }
131
+ | {
132
+ readonly live: {
133
+ readonly projectId?: ProjectId;
134
+ readonly segments?: Readonly<Record<string, string>>;
135
+ };
136
+ };
137
+
138
+ /** Whether replayed reads use the past run's results when it has them, or run live. */
139
+ export type EvalReads = 'recorded' | 'live';
140
+
141
+ /** How a comparison eval run runs its cases (absent: the run isn't a comparison). */
142
+ export interface EvalComparison {
143
+ readonly baseline: EvalBaseline;
144
+ readonly reads: EvalReads;
145
+ /** How many times each case runs (1–10); with more than 1, the summary shows the spread. */
146
+ readonly repetitions: number;
147
+ /** How many ranked items `weightedPrecisionAtK` looks at (1–100). */
148
+ readonly k: number;
149
+ /**
150
+ * For a flow candidate: agents and tools its replays run at other exact
151
+ * versions than the flow version's pins ("this flow, with `acme.scorer`
152
+ * at 0.4.0"), without publishing a new flow version.
153
+ */
154
+ readonly versions?: FlowVersionOverrides;
155
+ }
156
+
119
157
  export interface EvalRunStartInput {
120
158
  readonly tenantId: TenantId;
121
159
  /**
@@ -130,6 +168,8 @@ export interface EvalRunStartInput {
130
168
  readonly flowRef?: FlowRef;
131
169
  readonly dryRun?: boolean;
132
170
  readonly correlationId?: string;
171
+ /** Set for a comparison eval run (a `judged` suite): its baseline, reads and repetitions. */
172
+ readonly comparison?: EvalComparison;
133
173
  }
134
174
 
135
175
  /**
@@ -3,10 +3,14 @@
3
3
 
4
4
  import { randomUUID } from 'node:crypto';
5
5
 
6
- import type { RunId, TenantId, Timestamp } from '@kindgi/types';
6
+ import type { ReplayTurnReport } from '@kindgi/agents';
7
+ import type { FlowVersionOverrides } from '@kindgi/flow';
8
+ import type { RunReplayRef } from '@kindgi/runtime';
9
+ import type { ProjectId, RunId, TenantId, Timestamp } from '@kindgi/types';
7
10
 
8
11
  import type {
9
12
  AgentRef,
13
+ EvalComparison,
10
14
  EvalRun,
11
15
  EvalRunBinding,
12
16
  EvalRunCancelInput,
@@ -50,6 +54,18 @@ export interface EvalRunSubjectInvokeInput {
50
54
  readonly input: unknown;
51
55
  readonly dryRun: boolean;
52
56
  readonly abortSignal: AbortSignal;
57
+ /** The eval run's project: where the subject runs. */
58
+ readonly projectId?: ProjectId;
59
+ /**
60
+ * Set when the subject re-runs a past run (a comparison eval run): the
61
+ * run is a replay of `of` for the eval run, under the replay rules
62
+ * (`InvokeAgentInput.replay`).
63
+ */
64
+ readonly replay?: RunReplayRef;
65
+ /** The conversation before the past turn, oldest first: a replayed turn starts from it. */
66
+ readonly history?: readonly unknown[];
67
+ /** A flow target's agents and tools at other exact versions (`EvalComparison.versions`). */
68
+ readonly versions?: FlowVersionOverrides;
53
69
  }
54
70
 
55
71
  export interface EvalRunSubjectInvokeOutcome {
@@ -57,6 +73,22 @@ export interface EvalRunSubjectInvokeOutcome {
57
73
  readonly error?: string;
58
74
  readonly costUsd?: number;
59
75
  readonly durationMs?: number;
76
+ /** The run the subject ran in. */
77
+ readonly runId?: RunId;
78
+ /** A replay's report: each tool call and what happened to it (`AgentTurnResult.replay`). */
79
+ readonly replay?: ReplayTurnReport;
80
+ /** The provider and model that answered. */
81
+ readonly provider?: { readonly id: string; readonly model: string };
82
+ /**
83
+ * Set when a replayed flow stopped at a tool call the replay refused (a
84
+ * write the past run didn't make): no output, and what it would have
85
+ * done. Not an error: the replay did what it's meant to.
86
+ */
87
+ readonly stopped?: {
88
+ readonly toolId: string;
89
+ readonly arguments: unknown;
90
+ readonly reason?: string;
91
+ };
60
92
  }
61
93
 
62
94
  /**
@@ -100,6 +132,10 @@ export interface DispatchContext {
100
132
  readonly dryRun: boolean;
101
133
  readonly abortSignal: AbortSignal;
102
134
  readonly subject: EvalSubjectInvoker;
135
+ /** The eval run's project. */
136
+ readonly projectId?: ProjectId;
137
+ /** A comparison eval run's baseline, reads and repetitions. */
138
+ readonly comparison?: EvalComparison;
103
139
  onProgress(perCase: Readonly<Record<string, unknown>>): void;
104
140
  }
105
141
 
@@ -113,6 +149,7 @@ export interface EvalRunDispatcher {
113
149
  validate?(
114
150
  suite: EvalSuite,
115
151
  target: AgentRef | FlowRef,
152
+ comparison?: EvalComparison,
116
153
  ): { kind: 'ok' } | { kind: 'err'; message: string };
117
154
  dispatch(ctx: DispatchContext): Promise<DispatchResult>;
118
155
  }
@@ -311,26 +348,13 @@ export function createInProcessEvalRunBinding(
311
348
  };
312
349
  }
313
350
  if (dispatcher.validate !== undefined) {
314
- const v = dispatcher.validate(suite, target);
351
+ const v = dispatcher.validate(suite, target, input.comparison);
315
352
  if (v.kind === 'err') {
316
353
  return { kind: 'dispatcher-input-invalid', message: v.message };
317
354
  }
318
355
  }
319
356
  const runId = randomUUID() as unknown as RunId;
320
- const startedAt = now().toISOString() as unknown as Timestamp;
321
- const record: EvalRun = {
322
- runId,
323
- tenantId: input.tenantId,
324
- suiteId: suite.id,
325
- suiteVersion: suite.version,
326
- kind: suite.kind,
327
- ...(input.agentRef !== undefined && { agentRef: input.agentRef }),
328
- ...(input.flowRef !== undefined && { flowRef: input.flowRef }),
329
- status: 'running' as EvalRunStatus,
330
- dryRun: input.dryRun === true,
331
- startedAt,
332
- ...(input.correlationId !== undefined && { correlationId: input.correlationId }),
333
- };
357
+ const record = newRunRecord(input, suite, runId, now());
334
358
  setRun(record);
335
359
  const controller = new AbortController();
336
360
  controllers.set(runId as unknown as string, controller);
@@ -349,6 +373,8 @@ export function createInProcessEvalRunBinding(
349
373
  dryRun: input.dryRun === true,
350
374
  abortSignal: controller.signal,
351
375
  subject: options.subject,
376
+ projectId: input.projectId,
377
+ ...(input.comparison !== undefined && { comparison: input.comparison }),
352
378
  onProgress: (perCaseEntry: Readonly<Record<string, unknown>>) => {
353
379
  const current = runs.get(runId as unknown as string);
354
380
  if (current === undefined) return;
@@ -489,6 +515,24 @@ export function createInProcessEvalRunBinding(
489
515
  };
490
516
  }
491
517
 
518
+ /** A started run's row: `running`, with what it runs and how. */
519
+ function newRunRecord(input: EvalRunStartInput, suite: EvalSuite, runId: RunId, at: Date): EvalRun {
520
+ return {
521
+ runId,
522
+ tenantId: input.tenantId,
523
+ suiteId: suite.id,
524
+ suiteVersion: suite.version,
525
+ kind: suite.kind,
526
+ ...(input.agentRef !== undefined && { agentRef: input.agentRef }),
527
+ ...(input.flowRef !== undefined && { flowRef: input.flowRef }),
528
+ status: 'running',
529
+ dryRun: input.dryRun === true,
530
+ startedAt: at.toISOString() as unknown as Timestamp,
531
+ ...(input.correlationId !== undefined && { correlationId: input.correlationId }),
532
+ ...(input.comparison !== undefined && { comparison: input.comparison }),
533
+ };
534
+ }
535
+
492
536
  function isTerminal(status: EvalRunStatus): boolean {
493
537
  return status === 'completed' || status === 'failed' || status === 'cancelled';
494
538
  }
@@ -100,6 +100,8 @@ export const EVAL_KINDS = [
100
100
  'human-review',
101
101
  'benchmark',
102
102
  'custom',
103
+ /** Built from judgments of past runs: cases are copies of judged runs (`EvalCaseStoreBinding`). */
104
+ 'judged',
103
105
  ] as const;
104
106
  export type EvalKind = (typeof EVAL_KINDS)[number];
105
107
 
@@ -6,6 +6,8 @@ import type { Flow } from '@kindgi/flow';
6
6
  import type { Scope } from '@kindgi/platform';
7
7
  import type { Cursor, FlowId, ProjectId, TenantId } from '@kindgi/types';
8
8
 
9
+ import type { RegistryReadOnly } from './registry-read-only.js';
10
+
9
11
  /**
10
12
  * Caller-plugged surface for the flow catalog. Mirrors
11
13
  * `AgentRegistryBinding` 1:1 — the API package does NOT own registry
@@ -26,6 +28,13 @@ import type { Cursor, FlowId, ProjectId, TenantId } from '@kindgi/types';
26
28
  * cursor round-trips as a string; it never inspects the payload.
27
29
  */
28
30
  export interface FlowRegistryBinding {
31
+ /**
32
+ * Set when this registry takes no writes (under `kindgi dev`, the
33
+ * pack's files are the source of its flows): every write is refused
34
+ * with `409 registry-read-only` and this reason, before the binding is
35
+ * called. See `RegistryReadOnly`.
36
+ */
37
+ readonly readOnly?: RegistryReadOnly;
29
38
  /**
30
39
  * Cursor-paginated list of flows (latest version per id, sorted by
31
40
  * flow id ascending). Optional `nameFilter` is a prefix match on the
@@ -39,11 +48,12 @@ export interface FlowRegistryBinding {
39
48
  */
40
49
  get(input: FlowGetInput): Promise<Flow | null>;
41
50
  /**
42
- * Specific `(flowId, version)` lookup, or `null` if unknown.
43
- * Returns tombstoned versions too — provenance paths need to
44
- * resolve historical run references.
51
+ * Specific `(flowId, version)` lookup, or `null` if unknown. Returns
52
+ * unregistered (tombstoned) versions too, with `unregisteredAt` set:
53
+ * a resumed run and provenance read them, while a new run that names
54
+ * one is refused.
45
55
  */
46
- getVersion(input: FlowGetVersionInput): Promise<Flow | null>;
56
+ getVersion(input: FlowGetVersionInput): Promise<FlowVersionRecord | null>;
47
57
  /**
48
58
  * Head-row existence check. Lets `GET /v1/flows/{id}` distinguish
49
59
  * `410 gone` (identity exists, no active version) from `404 not-
@@ -79,6 +89,12 @@ export interface FlowRegistryBinding {
79
89
  reinstateVersion(input: FlowReinstateVersionInput): Promise<FlowReinstateVersionOutcome>;
80
90
  }
81
91
 
92
+ /** A flow version as `getVersion` reads it: `unregisteredAt` is set when it's unregistered. */
93
+ export type FlowVersionRecord = Flow & {
94
+ /** ISO-8601; present only on an unregistered version. */
95
+ readonly unregisteredAt?: string;
96
+ };
97
+
82
98
  export interface FlowListInput {
83
99
  readonly tenantId: TenantId;
84
100
  readonly limit: number;
@@ -0,0 +1,113 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright (C) 2026 Kindgi Inc.
3
+
4
+ import type { AgentId } from '@kindgi/agents';
5
+ import type { TupleEnqueueHook } from '@kindgi/authz';
6
+ import { type Flow, type FlowPins, flowPinsDigest, flowRefs } from '@kindgi/flow';
7
+ import { latestVersion } from '@kindgi/tools';
8
+ import type { Cursor, FlowId, ProjectId, TenantId, ToolId } from '@kindgi/types';
9
+
10
+ import type { AgentRegistryBinding } from './agent-binding.js';
11
+ import { type UnpinnableRef, activeToolVersions } from './agent-pins.js';
12
+ import { type DeployedVersionOutcome, deployVersion } from './deploy-versions.js';
13
+ import type { FlowRegistryBinding } from './flow-binding.js';
14
+ import type { ToolRegistryBinding } from './tool-binding.js';
15
+
16
+ export type FlowPinsOutcome =
17
+ | { readonly kind: 'ok'; readonly pins: FlowPins }
18
+ | { readonly kind: 'unpinnable'; readonly issues: readonly UnpinnableRef[] };
19
+
20
+ /**
21
+ * Pin a flow version when it's published: each tool it runs, and each
22
+ * agent it runs at no named version, to its latest active version now
23
+ * (what a run would have bound), so every run of the version uses those.
24
+ * A tool or agent with no published version makes the flow unpinnable:
25
+ * the caller refuses the publish, naming each, rather than store a
26
+ * partly pinned version.
27
+ */
28
+ export async function resolveFlowPins(
29
+ tools: ToolRegistryBinding,
30
+ agents: AgentRegistryBinding,
31
+ tenantId: TenantId,
32
+ flow: Flow,
33
+ ): Promise<FlowPinsOutcome> {
34
+ const refs = flowRefs(flow);
35
+ const pinned: { tools: Record<string, string>; agents: Record<string, string> } = {
36
+ tools: {},
37
+ agents: {},
38
+ };
39
+ const issues: UnpinnableRef[] = [];
40
+ for (const id of refs.tools) {
41
+ const latest = latestVersion(await activeToolVersions(tools, tenantId, id as ToolId));
42
+ if (latest !== undefined) pinned.tools[id] = latest;
43
+ else
44
+ issues.push({
45
+ path: '/nodes',
46
+ message: `tool "${id}" has no published version; publish the tool first`,
47
+ });
48
+ }
49
+ for (const id of refs.agents) {
50
+ const latest = await agents.get({ tenantId, agentId: id as AgentId });
51
+ if (latest !== null) pinned.agents[id] = latest.version as unknown as string;
52
+ else
53
+ issues.push({
54
+ path: '/nodes',
55
+ message: `agent "${id}" has no published version; publish the agent first`,
56
+ });
57
+ }
58
+ if (issues.length > 0) return { kind: 'unpinnable', issues };
59
+ return { kind: 'ok', pins: pinned };
60
+ }
61
+
62
+ export interface PublishDeployedFlowInput {
63
+ readonly flows: FlowRegistryBinding;
64
+ readonly tenantId: TenantId;
65
+ readonly projectId: ProjectId;
66
+ /** The flow as the pack defines it, with no pins. */
67
+ readonly flow: Flow;
68
+ readonly pins: FlowPins;
69
+ readonly enqueueTuples: TupleEnqueueHook;
70
+ }
71
+
72
+ /** Register a deployed flow with its pins, by the deploy rule (`deployVersion`). */
73
+ export async function publishDeployedFlow(
74
+ input: PublishDeployedFlowInput,
75
+ ): Promise<DeployedVersionOutcome> {
76
+ const { flows, tenantId, projectId, flow, pins, enqueueTuples } = input;
77
+ return deployVersion<Flow>({
78
+ label: `flow "${flow.id as unknown as string}"`,
79
+ definition: flow,
80
+ pins,
81
+ pinsDigest: flowPinsDigest(pins),
82
+ existing: await allFlowVersions(flows, tenantId, flow.id),
83
+ publish: async (version) => {
84
+ const outcome = await flows.publish({ tenantId, projectId, flow: version, enqueueTuples });
85
+ if (outcome.kind === 'ok') return 'ok';
86
+ return outcome.kind === 'project-not-found' ? 'skipped' : 'taken';
87
+ },
88
+ });
89
+ }
90
+
91
+ /** Page size for reading a flow's versions. */
92
+ const VERSIONS_PAGE = 200;
93
+
94
+ /** Every active version of a flow. */
95
+ async function allFlowVersions(
96
+ flows: FlowRegistryBinding,
97
+ tenantId: TenantId,
98
+ flowId: FlowId,
99
+ ): Promise<readonly Flow[]> {
100
+ const versions: Flow[] = [];
101
+ let cursor: Cursor | undefined;
102
+ do {
103
+ const page = await flows.listVersions({
104
+ tenantId,
105
+ flowId,
106
+ limit: VERSIONS_PAGE,
107
+ ...(cursor !== undefined && { cursor }),
108
+ });
109
+ versions.push(...page.data);
110
+ cursor = page.nextCursor;
111
+ } while (cursor !== undefined);
112
+ return versions;
113
+ }
@@ -6,6 +6,8 @@ import type { Guardrail } from '@kindgi/guardrails';
6
6
  import type { Scope } from '@kindgi/platform';
7
7
  import type { Cursor, GuardrailId, ProjectId, TenantId } from '@kindgi/types';
8
8
 
9
+ import type { RegistryReadOnly } from './registry-read-only.js';
10
+
9
11
  /**
10
12
  * Caller-plugged surface for the guardrail catalog. Same shape as
11
13
  * `AgentRegistryBinding` / `ToolRegistryBinding`: the API package does
@@ -27,6 +29,13 @@ import type { Cursor, GuardrailId, ProjectId, TenantId } from '@kindgi/types';
27
29
  * inspects the payload.
28
30
  */
29
31
  export interface GuardrailRegistryBinding {
32
+ /**
33
+ * Set when this registry takes no writes (under `kindgi dev`, the
34
+ * pack's files are the source of its guardrails): every write is refused
35
+ * with `409 registry-read-only` and this reason, before the binding is
36
+ * called. See `RegistryReadOnly`.
37
+ */
38
+ readonly readOnly?: RegistryReadOnly;
30
39
  /**
31
40
  * Cursor-paginated list of guardrails sorted by id ascending.
32
41
  * Optional `nameFilter` is a prefix match on the guardrail id —
@@ -2,6 +2,7 @@
2
2
  // Copyright (C) 2026 Kindgi Inc.
3
3
 
4
4
  import type { AgentId } from '@kindgi/agents';
5
+ import type { FlowVersionOverrides } from '@kindgi/flow';
5
6
  import type { FlowId, ProjectId, RunId, Semver, TenantId } from '@kindgi/types';
6
7
 
7
8
  /**
@@ -107,6 +108,8 @@ export interface InvokeFlowBindingInput {
107
108
  * can't run in the background may treat `false` like `true`.
108
109
  */
109
110
  readonly wait?: boolean;
111
+ /** Agents and tools to run at other exact versions than the flow version's pins (`RunFlowInput.versions`). */
112
+ readonly versions?: FlowVersionOverrides;
110
113
  }
111
114
 
112
115
  export type RunHandlerOutcome =
package/src/index.ts CHANGED
@@ -233,11 +233,13 @@ export type {
233
233
  AgentPublishInput,
234
234
  AgentPublishOutcome,
235
235
  AgentRegistryBinding,
236
+ AgentVersionRecord,
236
237
  AgentReinstateVersionInput,
237
238
  AgentReinstateVersionOutcome,
238
239
  AgentUnregisterInput,
239
240
  AgentUnregisterOutcome,
240
241
  } from './agent-binding.js';
242
+ export type { RegistryReadOnly } from './registry-read-only.js';
241
243
  export type {
242
244
  FlowGetInput,
243
245
  FlowGetVersionInput,
@@ -247,6 +249,7 @@ export type {
247
249
  FlowPublishInput,
248
250
  FlowPublishOutcome,
249
251
  FlowRegistryBinding,
252
+ FlowVersionRecord,
250
253
  FlowReinstateVersionInput,
251
254
  FlowReinstateVersionOutcome,
252
255
  FlowUnregisterInput,
@@ -309,7 +312,7 @@ export {
309
312
  stdioRefusal,
310
313
  } from './tenant-host-access.js';
311
314
  export type { HostReach, TenantHostAccess } from './tenant-host-access.js';
312
- export { POLICY_KINDS } from '@kindgi/policy-contract';
315
+ export { APPLIED_POLICY_KINDS, POLICY_KINDS } from '@kindgi/policy-contract';
313
316
  export type {
314
317
  Policy,
315
318
  PolicyGetInput,
@@ -328,6 +331,61 @@ export type {
328
331
  PolicyVersionPage,
329
332
  PolicyVersionRow,
330
333
  } from '@kindgi/policy-contract';
334
+ export { JUDGE_CLASS_SCOPE_KINDS, VERDICTS, judgeClassApplies } from './judgment-binding.js';
335
+ export type {
336
+ EvalCaseListInput,
337
+ EvalCasePage,
338
+ EvalCasePutInput,
339
+ EvalCaseStoreBinding,
340
+ JudgedEvalCase,
341
+ JudgedItemSummary,
342
+ } from './eval-case-binding.js';
343
+ export { MAX_JUDGED_CASES } from './routes/judged-suites.js';
344
+ export { MAX_JUDGED_HISTORY } from './routes/judgment-context.js';
345
+ export type {
346
+ JudgeClass,
347
+ JudgeClassCreateInput,
348
+ JudgeClassCreateOutcome,
349
+ JudgeClassGetInput,
350
+ JudgeClassListInput,
351
+ JudgeClassPage,
352
+ JudgeClassScope,
353
+ JudgeClassScopeKind,
354
+ JudgeClassUpdateInput,
355
+ JudgedItem,
356
+ JudgedFlowContext,
357
+ JudgedFlowStep,
358
+ JudgedRunContext,
359
+ JudgedToolCall,
360
+ JudgedRunCopy,
361
+ JudgedRunListInput,
362
+ JudgedRunPage,
363
+ JudgedRunWithJudgments,
364
+ JudgedSubject,
365
+ Judgment,
366
+ JudgmentAssertedBy,
367
+ JudgmentGetInput,
368
+ JudgmentListInput,
369
+ JudgmentPage,
370
+ JudgmentRecordInput,
371
+ JudgmentRegistryBinding,
372
+ JudgmentWithCopies,
373
+ Verdict,
374
+ } from './judgment-binding.js';
375
+ export { resolvePointer } from './routes/judgments.js';
376
+ export type {
377
+ BlockGetInput,
378
+ BlockGetVersionInput,
379
+ BlockListInput,
380
+ BlockListVersionsInput,
381
+ BlockPage,
382
+ BlockPublishInput,
383
+ BlockPublishOutcome,
384
+ BlockRecord,
385
+ BlockRegistryBinding,
386
+ BlockReinstateOutcome,
387
+ BlockVersionInput,
388
+ } from './block-binding.js';
331
389
  export { EVAL_KINDS } from './eval-suite-binding.js';
332
390
  export type {
333
391
  EvalKind,
@@ -348,6 +406,9 @@ export type {
348
406
  export { EVAL_RUN_STATUSES } from './eval-run-binding.js';
349
407
  export type {
350
408
  AgentRef,
409
+ EvalBaseline,
410
+ EvalComparison,
411
+ EvalReads,
351
412
  EvalRun,
352
413
  EvalRunBinding,
353
414
  EvalRunCancelInput,
@@ -379,7 +440,27 @@ export type {
379
440
  EvalSubjectInvoker,
380
441
  InProcessEvalRunBindingOptions,
381
442
  } from './eval-run-dispatcher.js';
382
- export { COST_GROUP_DIMENSIONS } from './cost-binding.js';
443
+ export { DEFAULT_COMPARISON, createJudgedDispatcher } from './judged-dispatcher.js';
444
+ export type {
445
+ ComparisonBaselineSummary,
446
+ ComparisonMetric,
447
+ JudgedCaseResult,
448
+ JudgedComparisonSummary,
449
+ JudgedDispatcherOptions,
450
+ } from './judged-dispatcher.js';
451
+ export { itemChanges, matchJudged, outputItems, scoreItems, valueAt } from './judged-items.js';
452
+ export type {
453
+ ItemChanges,
454
+ ItemJudgments,
455
+ MatchedItem,
456
+ OutputItem,
457
+ OutputScore,
458
+ } from './judged-items.js';
459
+ export {
460
+ COST_AGGREGATE_DEFAULT_LIMIT,
461
+ COST_AGGREGATE_MAX_LIMIT,
462
+ COST_GROUP_DIMENSIONS,
463
+ } from './cost-binding.js';
383
464
  export type {
384
465
  CostAggregateGroup,
385
466
  CostAggregateInput,
@@ -477,7 +558,10 @@ export type {
477
558
  DeploymentGetInput,
478
559
  DeploymentListInput,
479
560
  DeploymentPage,
561
+ DeployedAgent,
562
+ DeployedFlow,
480
563
  DeployedPrimitive,
564
+ DeployedVersion,
481
565
  DeploymentContents,
482
566
  DeploymentPrimitiveCounts,
483
567
  DeploymentRegisterInput,