@kindgi/api 0.1.4-rc.0 → 0.1.4-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (262) hide show
  1. package/dist/agent-binding.d.ts +20 -2
  2. package/dist/agent-binding.d.ts.map +1 -1
  3. package/dist/agent-pins.d.ts.map +1 -1
  4. package/dist/agent-pins.js +4 -1
  5. package/dist/agent-pins.js.map +1 -1
  6. package/dist/app.d.ts +8 -0
  7. package/dist/app.d.ts.map +1 -1
  8. package/dist/app.js +30 -5
  9. package/dist/app.js.map +1 -1
  10. package/dist/deploy-versions.d.ts +10 -5
  11. package/dist/deploy-versions.d.ts.map +1 -1
  12. package/dist/deploy-versions.js +4 -10
  13. package/dist/deploy-versions.js.map +1 -1
  14. package/dist/derive-agent-version.d.ts +9 -1
  15. package/dist/derive-agent-version.d.ts.map +1 -1
  16. package/dist/derive-agent-version.js +11 -1
  17. package/dist/derive-agent-version.js.map +1 -1
  18. package/dist/errors.d.ts.map +1 -1
  19. package/dist/errors.js +26 -0
  20. package/dist/errors.js.map +1 -1
  21. package/dist/eval-case-binding.d.ts +10 -0
  22. package/dist/eval-case-binding.d.ts.map +1 -1
  23. package/dist/eval-run-binding.d.ts +21 -3
  24. package/dist/eval-run-binding.d.ts.map +1 -1
  25. package/dist/eval-run-binding.js.map +1 -1
  26. package/dist/eval-run-dispatcher.d.ts +3 -0
  27. package/dist/eval-run-dispatcher.d.ts.map +1 -1
  28. package/dist/eval-run-dispatcher.js.map +1 -1
  29. package/dist/flow-binding.d.ts +8 -0
  30. package/dist/flow-binding.d.ts.map +1 -1
  31. package/dist/flow-pins.d.ts +20 -7
  32. package/dist/flow-pins.d.ts.map +1 -1
  33. package/dist/flow-pins.js +36 -11
  34. package/dist/flow-pins.js.map +1 -1
  35. package/dist/gate-policy-binding.d.ts +164 -0
  36. package/dist/gate-policy-binding.d.ts.map +1 -0
  37. package/dist/gate-policy-binding.js +12 -0
  38. package/dist/gate-policy-binding.js.map +1 -0
  39. package/dist/gate.d.ts +56 -0
  40. package/dist/gate.d.ts.map +1 -0
  41. package/dist/gate.js +359 -0
  42. package/dist/gate.js.map +1 -0
  43. package/dist/guardrail-binding.d.ts +8 -0
  44. package/dist/guardrail-binding.d.ts.map +1 -1
  45. package/dist/handler-binding.d.ts +17 -2
  46. package/dist/handler-binding.d.ts.map +1 -1
  47. package/dist/index.d.ts +11 -4
  48. package/dist/index.d.ts.map +1 -1
  49. package/dist/index.js +3 -1
  50. package/dist/index.js.map +1 -1
  51. package/dist/judged-dispatcher.d.ts +20 -1
  52. package/dist/judged-dispatcher.d.ts.map +1 -1
  53. package/dist/judged-dispatcher.js +57 -7
  54. package/dist/judged-dispatcher.js.map +1 -1
  55. package/dist/judgment-binding.d.ts +38 -0
  56. package/dist/judgment-binding.d.ts.map +1 -1
  57. package/dist/judgment-binding.js +19 -0
  58. package/dist/judgment-binding.js.map +1 -1
  59. package/dist/live-version-binding.d.ts +206 -0
  60. package/dist/live-version-binding.d.ts.map +1 -0
  61. package/dist/live-version-binding.js +4 -0
  62. package/dist/live-version-binding.js.map +1 -0
  63. package/dist/middleware/authorize.d.ts.map +1 -1
  64. package/dist/middleware/authorize.js +13 -2
  65. package/dist/middleware/authorize.js.map +1 -1
  66. package/dist/middleware/project-ref.d.ts +25 -0
  67. package/dist/middleware/project-ref.d.ts.map +1 -0
  68. package/dist/middleware/project-ref.js +72 -0
  69. package/dist/middleware/project-ref.js.map +1 -0
  70. package/dist/openapi/generate.d.ts.map +1 -1
  71. package/dist/openapi/generate.js +4 -0
  72. package/dist/openapi/generate.js.map +1 -1
  73. package/dist/openapi/operations.d.ts +6 -0
  74. package/dist/openapi/operations.d.ts.map +1 -1
  75. package/dist/openapi/operations.js +524 -27
  76. package/dist/openapi/operations.js.map +1 -1
  77. package/dist/openapi/schemas.d.ts +43 -0
  78. package/dist/openapi/schemas.d.ts.map +1 -1
  79. package/dist/openapi/schemas.js +1050 -7
  80. package/dist/openapi/schemas.js.map +1 -1
  81. package/dist/publish-refused.d.ts +18 -0
  82. package/dist/publish-refused.d.ts.map +1 -0
  83. package/dist/publish-refused.js +22 -0
  84. package/dist/publish-refused.js.map +1 -0
  85. package/dist/registry-read-only.d.ts +32 -0
  86. package/dist/registry-read-only.d.ts.map +1 -0
  87. package/dist/registry-read-only.js +22 -0
  88. package/dist/registry-read-only.js.map +1 -0
  89. package/dist/retention-binding.d.ts +29 -0
  90. package/dist/retention-binding.d.ts.map +1 -1
  91. package/dist/routes/agent-releases.d.ts +40 -0
  92. package/dist/routes/agent-releases.d.ts.map +1 -0
  93. package/dist/routes/agent-releases.js +462 -0
  94. package/dist/routes/agent-releases.js.map +1 -0
  95. package/dist/routes/agents.d.ts +7 -1
  96. package/dist/routes/agents.d.ts.map +1 -1
  97. package/dist/routes/agents.js +38 -4
  98. package/dist/routes/agents.js.map +1 -1
  99. package/dist/routes/approvals.d.ts.map +1 -1
  100. package/dist/routes/approvals.js +27 -9
  101. package/dist/routes/approvals.js.map +1 -1
  102. package/dist/routes/audit.d.ts.map +1 -1
  103. package/dist/routes/audit.js +7 -0
  104. package/dist/routes/audit.js.map +1 -1
  105. package/dist/routes/auth.js +1 -1
  106. package/dist/routes/auth.js.map +1 -1
  107. package/dist/routes/blocks.d.ts.map +1 -1
  108. package/dist/routes/blocks.js +37 -12
  109. package/dist/routes/blocks.js.map +1 -1
  110. package/dist/routes/conversations.d.ts +0 -7
  111. package/dist/routes/conversations.d.ts.map +1 -1
  112. package/dist/routes/conversations.js +19 -3
  113. package/dist/routes/conversations.js.map +1 -1
  114. package/dist/routes/deployments.d.ts +6 -0
  115. package/dist/routes/deployments.d.ts.map +1 -1
  116. package/dist/routes/deployments.js +49 -5
  117. package/dist/routes/deployments.js.map +1 -1
  118. package/dist/routes/env.d.ts.map +1 -1
  119. package/dist/routes/env.js +1 -0
  120. package/dist/routes/env.js.map +1 -1
  121. package/dist/routes/eval-comparison.d.ts +4 -2
  122. package/dist/routes/eval-comparison.d.ts.map +1 -1
  123. package/dist/routes/eval-comparison.js +59 -10
  124. package/dist/routes/eval-comparison.js.map +1 -1
  125. package/dist/routes/eval-runs.d.ts +7 -1
  126. package/dist/routes/eval-runs.d.ts.map +1 -1
  127. package/dist/routes/eval-runs.js +34 -3
  128. package/dist/routes/eval-runs.js.map +1 -1
  129. package/dist/routes/eval-versions.d.ts +25 -0
  130. package/dist/routes/eval-versions.d.ts.map +1 -0
  131. package/dist/routes/eval-versions.js +66 -0
  132. package/dist/routes/eval-versions.js.map +1 -0
  133. package/dist/routes/flows.d.ts +14 -7
  134. package/dist/routes/flows.d.ts.map +1 -1
  135. package/dist/routes/flows.js +14 -8
  136. package/dist/routes/flows.js.map +1 -1
  137. package/dist/routes/gate-policies.d.ts +19 -0
  138. package/dist/routes/gate-policies.d.ts.map +1 -0
  139. package/dist/routes/gate-policies.js +191 -0
  140. package/dist/routes/gate-policies.js.map +1 -0
  141. package/dist/routes/gate-policy-spec.d.ts +17 -0
  142. package/dist/routes/gate-policy-spec.d.ts.map +1 -0
  143. package/dist/routes/gate-policy-spec.js +192 -0
  144. package/dist/routes/gate-policy-spec.js.map +1 -0
  145. package/dist/routes/gate-policy-wire.d.ts +3 -0
  146. package/dist/routes/gate-policy-wire.d.ts.map +1 -0
  147. package/dist/routes/gate-policy-wire.js +18 -0
  148. package/dist/routes/gate-policy-wire.js.map +1 -0
  149. package/dist/routes/guardrails.d.ts.map +1 -1
  150. package/dist/routes/guardrails.js +4 -0
  151. package/dist/routes/guardrails.js.map +1 -1
  152. package/dist/routes/hierarchy-errors.d.ts +21 -0
  153. package/dist/routes/hierarchy-errors.d.ts.map +1 -1
  154. package/dist/routes/hierarchy-errors.js +21 -0
  155. package/dist/routes/hierarchy-errors.js.map +1 -1
  156. package/dist/routes/judged-suites.js +7 -0
  157. package/dist/routes/judged-suites.js.map +1 -1
  158. package/dist/routes/judgments.d.ts +4 -1
  159. package/dist/routes/judgments.d.ts.map +1 -1
  160. package/dist/routes/judgments.js +92 -19
  161. package/dist/routes/judgments.js.map +1 -1
  162. package/dist/routes/live-scope-wire.d.ts +32 -0
  163. package/dist/routes/live-scope-wire.d.ts.map +1 -0
  164. package/dist/routes/live-scope-wire.js +71 -0
  165. package/dist/routes/live-scope-wire.js.map +1 -0
  166. package/dist/routes/policies.d.ts +6 -3
  167. package/dist/routes/policies.d.ts.map +1 -1
  168. package/dist/routes/policies.js +55 -3
  169. package/dist/routes/policies.js.map +1 -1
  170. package/dist/routes/projects.d.ts.map +1 -1
  171. package/dist/routes/projects.js +59 -9
  172. package/dist/routes/projects.js.map +1 -1
  173. package/dist/routes/retention.d.ts.map +1 -1
  174. package/dist/routes/retention.js +18 -2
  175. package/dist/routes/retention.js.map +1 -1
  176. package/dist/routes/reviewers.d.ts +8 -1
  177. package/dist/routes/reviewers.d.ts.map +1 -1
  178. package/dist/routes/reviewers.js +20 -3
  179. package/dist/routes/reviewers.js.map +1 -1
  180. package/dist/routes/runs.d.ts.map +1 -1
  181. package/dist/routes/runs.js +23 -13
  182. package/dist/routes/runs.js.map +1 -1
  183. package/dist/routes/secrets.d.ts.map +1 -1
  184. package/dist/routes/secrets.js +2 -0
  185. package/dist/routes/secrets.js.map +1 -1
  186. package/dist/routes/segments.d.ts +17 -0
  187. package/dist/routes/segments.d.ts.map +1 -0
  188. package/dist/routes/segments.js +68 -0
  189. package/dist/routes/segments.js.map +1 -0
  190. package/dist/routes/teams.d.ts.map +1 -1
  191. package/dist/routes/teams.js +53 -8
  192. package/dist/routes/teams.js.map +1 -1
  193. package/dist/routes/tools.d.ts.map +1 -1
  194. package/dist/routes/tools.js +4 -0
  195. package/dist/routes/tools.js.map +1 -1
  196. package/dist/routes/uuid-param.d.ts +11 -0
  197. package/dist/routes/uuid-param.d.ts.map +1 -0
  198. package/dist/routes/uuid-param.js +19 -0
  199. package/dist/routes/uuid-param.js.map +1 -0
  200. package/dist/tool-binding.d.ts +8 -0
  201. package/dist/tool-binding.d.ts.map +1 -1
  202. package/openapi.json +6484 -2402
  203. package/package.json +21 -21
  204. package/src/agent-binding.ts +21 -2
  205. package/src/agent-pins.ts +3 -1
  206. package/src/app.ts +47 -4
  207. package/src/deploy-versions.ts +12 -15
  208. package/src/derive-agent-version.ts +12 -1
  209. package/src/errors.ts +26 -0
  210. package/src/eval-case-binding.ts +7 -0
  211. package/src/eval-run-binding.ts +31 -3
  212. package/src/eval-run-dispatcher.ts +3 -0
  213. package/src/flow-binding.ts +9 -0
  214. package/src/flow-pins.ts +49 -9
  215. package/src/gate-policy-binding.ts +171 -0
  216. package/src/gate.ts +467 -0
  217. package/src/guardrail-binding.ts +9 -0
  218. package/src/handler-binding.ts +17 -2
  219. package/src/index.ts +47 -1
  220. package/src/judged-dispatcher.ts +100 -9
  221. package/src/judgment-binding.ts +62 -0
  222. package/src/live-version-binding.ts +237 -0
  223. package/src/middleware/authorize.ts +18 -2
  224. package/src/middleware/project-ref.ts +88 -0
  225. package/src/openapi/generate.ts +5 -0
  226. package/src/openapi/operations.ts +664 -27
  227. package/src/openapi/schemas.ts +1175 -33
  228. package/src/publish-refused.ts +31 -0
  229. package/src/registry-read-only.ts +43 -0
  230. package/src/retention-binding.ts +30 -0
  231. package/src/routes/agent-releases.ts +581 -0
  232. package/src/routes/agents.ts +53 -3
  233. package/src/routes/approvals.ts +36 -8
  234. package/src/routes/audit.ts +10 -0
  235. package/src/routes/auth.ts +1 -1
  236. package/src/routes/blocks.ts +39 -14
  237. package/src/routes/conversations.ts +31 -3
  238. package/src/routes/deployments.ts +75 -5
  239. package/src/routes/env.ts +1 -0
  240. package/src/routes/eval-comparison.ts +59 -11
  241. package/src/routes/eval-runs.ts +46 -3
  242. package/src/routes/eval-versions.ts +110 -0
  243. package/src/routes/flows.ts +34 -11
  244. package/src/routes/gate-policies.ts +226 -0
  245. package/src/routes/gate-policy-spec.ts +229 -0
  246. package/src/routes/gate-policy-wire.ts +20 -0
  247. package/src/routes/guardrails.ts +7 -0
  248. package/src/routes/hierarchy-errors.ts +23 -0
  249. package/src/routes/judged-suites.ts +5 -0
  250. package/src/routes/judgments.ts +107 -23
  251. package/src/routes/live-scope-wire.ts +90 -0
  252. package/src/routes/policies.ts +61 -3
  253. package/src/routes/projects.ts +72 -9
  254. package/src/routes/retention.ts +21 -2
  255. package/src/routes/reviewers.ts +23 -4
  256. package/src/routes/runs.ts +34 -20
  257. package/src/routes/secrets.ts +2 -0
  258. package/src/routes/segments.ts +75 -0
  259. package/src/routes/teams.ts +66 -8
  260. package/src/routes/tools.ts +7 -0
  261. package/src/routes/uuid-param.ts +28 -0
  262. package/src/tool-binding.ts +9 -0
@@ -2,7 +2,8 @@
2
2
  // Copyright (C) 2026 Kindgi Inc.
3
3
 
4
4
  import type { AgentId } from '@kindgi/agents';
5
- import type { FlowId, ProjectId, RunId, Semver, TenantId } from '@kindgi/types';
5
+ import type { FlowVersionOverrides } from '@kindgi/flow';
6
+ import type { FlowId, ProjectId, RunId, ScopeSegment, Semver, TenantId } from '@kindgi/types';
6
7
 
7
8
  /**
8
9
  * Callback the platform HTTP surface hands off to when it receives
@@ -72,8 +73,18 @@ export interface InvokeAgentBindingInput {
72
73
  */
73
74
  readonly projectId?: ProjectId;
74
75
  readonly agentId: AgentId;
75
- /** Omit → binding chooses the latest registered version. */
76
+ /**
77
+ * Omit → the binding chooses: the conversation's own version for a
78
+ * follow-up turn, else the version live for the run's scope, else the
79
+ * latest registered.
80
+ */
76
81
  readonly agentVersion?: Semver;
82
+ /**
83
+ * The run's segment path below its project, coarse to fine (an
84
+ * app-defined finer scope, e.g. company then role): a live version
85
+ * pinned on a prefix of it serves the run.
86
+ */
87
+ readonly segments?: readonly ScopeSegment[];
77
88
  /**
78
89
  * Opaque payload forwarded from the request body's `input` field.
79
90
  * The binding is responsible for coercing this into whatever shape
@@ -97,6 +108,8 @@ export interface InvokeFlowBindingInput {
97
108
  readonly projectId?: ProjectId;
98
109
  readonly flowId: FlowId;
99
110
  readonly flowVersion?: Semver;
111
+ /** The run's segment path, as for an agent run: its agent steps resolve live versions with it. */
112
+ readonly segments?: readonly ScopeSegment[];
100
113
  readonly input: unknown;
101
114
  readonly dryRun?: boolean;
102
115
  /**
@@ -107,6 +120,8 @@ export interface InvokeFlowBindingInput {
107
120
  * can't run in the background may treat `false` like `true`.
108
121
  */
109
122
  readonly wait?: boolean;
123
+ /** Agents and tools to run at other exact versions than the flow version's pins (`RunFlowInput.versions`). */
124
+ readonly versions?: FlowVersionOverrides;
110
125
  }
111
126
 
112
127
  export type RunHandlerOutcome =
package/src/index.ts CHANGED
@@ -239,6 +239,7 @@ export type {
239
239
  AgentUnregisterInput,
240
240
  AgentUnregisterOutcome,
241
241
  } from './agent-binding.js';
242
+ export type { RegistryReadOnly } from './registry-read-only.js';
242
243
  export type {
243
244
  FlowGetInput,
244
245
  FlowGetVersionInput,
@@ -330,7 +331,12 @@ export type {
330
331
  PolicyVersionPage,
331
332
  PolicyVersionRow,
332
333
  } from '@kindgi/policy-contract';
333
- export { JUDGE_CLASS_SCOPE_KINDS, VERDICTS, judgeClassApplies } from './judgment-binding.js';
334
+ export {
335
+ JUDGE_CLASS_SCOPE_KINDS,
336
+ VERDICTS,
337
+ judgeClassApplies,
338
+ whyNotAssertable,
339
+ } from './judgment-binding.js';
334
340
  export type {
335
341
  EvalCaseListInput,
336
342
  EvalCasePage,
@@ -343,6 +349,8 @@ export { MAX_JUDGED_CASES } from './routes/judged-suites.js';
343
349
  export { MAX_JUDGED_HISTORY } from './routes/judgment-context.js';
344
350
  export type {
345
351
  JudgeClass,
352
+ JudgeClassAssertableBy,
353
+ JudgeClassAsserter,
346
354
  JudgeClassCreateInput,
347
355
  JudgeClassCreateOutcome,
348
356
  JudgeClassGetInput,
@@ -385,6 +393,42 @@ export type {
385
393
  BlockReinstateOutcome,
386
394
  BlockVersionInput,
387
395
  } from './block-binding.js';
396
+ export type {
397
+ AgentReleaseBindings,
398
+ ListPromotionsInput,
399
+ LivePin,
400
+ LiveResolution,
401
+ LiveResolveInput,
402
+ LiveVersionBinding,
403
+ PromoteInput,
404
+ Promotion,
405
+ PromotionAction,
406
+ PromotionActor,
407
+ PromotionBinding,
408
+ PromotionError,
409
+ PromotionErrorCode,
410
+ PromotionRequestInput,
411
+ PromotionStatus,
412
+ RollbackInput,
413
+ UnpinInput,
414
+ } from './live-version-binding.js';
415
+ export type {
416
+ GateMetricName,
417
+ GateMetricSpec,
418
+ GatePolicy,
419
+ GatePolicyBinding,
420
+ GatePolicyError,
421
+ GatePolicyErrorCode,
422
+ GatePolicyListInput,
423
+ GatePolicyPublishInput,
424
+ GatePolicyRef,
425
+ GatePolicySpec,
426
+ GatePolicyVersionInput,
427
+ } from './gate-policy-binding.js';
428
+ export { GATE_METRICS } from './gate-policy-binding.js';
429
+ export type { GateApproval, GateCheck, GateInput, GateResult } from './gate.js';
430
+ export { evaluateGate, gateApproval } from './gate.js';
431
+ export type { AgentReleaseGateDeps } from './routes/agent-releases.js';
388
432
  export { EVAL_KINDS } from './eval-suite-binding.js';
389
433
  export type {
390
434
  EvalKind,
@@ -406,6 +450,7 @@ export { EVAL_RUN_STATUSES } from './eval-run-binding.js';
406
450
  export type {
407
451
  AgentRef,
408
452
  EvalBaseline,
453
+ EvalClassWeights,
409
454
  EvalComparison,
410
455
  EvalReads,
411
456
  EvalRun,
@@ -684,6 +729,7 @@ export type { JsonSchema } from './openapi/schemas.js';
684
729
 
685
730
  export type {
686
731
  RetentionBinding,
732
+ RetentionPolicyConflict,
687
733
  RetentionScheduledInput,
688
734
  RetentionScheduledItem,
689
735
  RetentionScheduledPage,
@@ -16,10 +16,12 @@
16
16
  */
17
17
 
18
18
  import type { ReplayTurnReport } from '@kindgi/agents';
19
+ import type { FlowVersionOverrides } from '@kindgi/flow';
19
20
  import type { RunId } from '@kindgi/types';
20
21
 
22
+ import type { AgentRegistryBinding } from './agent-binding.js';
21
23
  import type { EvalCaseStoreBinding, JudgedEvalCase } from './eval-case-binding.js';
22
- import type { AgentRef, EvalComparison, FlowRef } from './eval-run-binding.js';
24
+ import type { AgentRef, EvalClassWeights, EvalComparison, FlowRef } from './eval-run-binding.js';
23
25
  import type {
24
26
  DispatchContext,
25
27
  DispatchResult,
@@ -80,8 +82,24 @@ export type RecordedVersion =
80
82
 
81
83
  /** What ran on the cases: an agent version, or a flow version. */
82
84
  export type ComparisonCandidate =
83
- | { readonly kind: 'agent'; readonly agentId: string; readonly version: string }
84
- | { readonly kind: 'flow'; readonly flowId: string; readonly version: string };
85
+ | {
86
+ readonly kind: 'agent';
87
+ readonly agentId: string;
88
+ readonly version: string;
89
+ /**
90
+ * The version's `pinsDigest`: what it ran, as a promotion gate checks.
91
+ * Absent for a version published before pins, and from a dispatcher
92
+ * without an agent registry.
93
+ */
94
+ readonly pinsDigest?: string;
95
+ }
96
+ | {
97
+ readonly kind: 'flow';
98
+ readonly flowId: string;
99
+ readonly version: string;
100
+ /** Agents and tools its replays ran at other versions than the flow version's pins. */
101
+ readonly versions?: FlowVersionOverrides;
102
+ };
85
103
 
86
104
  /** What a comparison eval run concluded: what a promotion gate reads. */
87
105
  export interface JudgedComparisonSummary {
@@ -106,6 +124,12 @@ export interface JudgedComparisonSummary {
106
124
  */
107
125
  readonly stopped: number;
108
126
  readonly reads: EvalComparison['reads'];
127
+ /**
128
+ * Which judgments counted: `restricted-only` weighs a judgment not
129
+ * recorded under a restricted class 0 (a gate can require it). Absent
130
+ * from a summary recorded before T200: `as-recorded`.
131
+ */
132
+ readonly classWeights?: EvalClassWeights;
109
133
  /** The models that answered the candidate's replays, and how many replays each. */
110
134
  readonly sampling: {
111
135
  readonly models: readonly {
@@ -145,11 +169,16 @@ export interface JudgedCaseResult {
145
169
 
146
170
  export interface JudgedDispatcherOptions {
147
171
  readonly cases: EvalCaseStoreBinding;
172
+ /** Where the candidate's `pinsDigest` is read, for the summary. */
173
+ readonly agents?: Pick<AgentRegistryBinding, 'getVersion'>;
148
174
  }
149
175
 
150
176
  /** Cases read per page. */
151
177
  const CASE_PAGE = 100;
152
178
 
179
+ export const VERSIONS_NEED_A_FLOW =
180
+ '`versions` runs a flow with some of its agents or tools at other versions: it needs `flowRef`.';
181
+
153
182
  function validateComparison(
154
183
  suite: { readonly spec: Readonly<Record<string, unknown>> },
155
184
  target: AgentRef | FlowRef,
@@ -168,6 +197,9 @@ function validateComparison(
168
197
  return { kind: 'err', message: 'The test set has no cases.' };
169
198
  }
170
199
  const c = comparison ?? DEFAULT_COMPARISON;
200
+ if (c.versions !== undefined && 'agentId' in target) {
201
+ return { kind: 'err', message: VERSIONS_NEED_A_FLOW };
202
+ }
171
203
  if (c.baseline !== 'recorded') {
172
204
  return { kind: 'err', message: "Only `baseline: 'recorded'` runs today." };
173
205
  }
@@ -180,7 +212,9 @@ export function createJudgedDispatcher(options: JudgedDispatcherOptions): EvalRu
180
212
  validate: validateComparison,
181
213
  async dispatch(ctx): Promise<DispatchResult> {
182
214
  const comparison = ctx.comparison ?? DEFAULT_COMPARISON;
183
- const all = await allCases(options.cases, ctx);
215
+ const stored = await allCases(options.cases, ctx);
216
+ const all =
217
+ comparison.classWeights === 'restricted-only' ? stored.map(restrictedOnly) : stored;
184
218
  if (ctx.dryRun) {
185
219
  return { result: { dryRun: true, cases: all.length, comparison } };
186
220
  }
@@ -192,7 +226,8 @@ export function createJudgedDispatcher(options: JudgedDispatcherOptions): EvalRu
192
226
  results.push(result);
193
227
  ctx.onProgress(result as unknown as Readonly<Record<string, unknown>>);
194
228
  }
195
- const summary = summarize(ctx, comparison, all, results, [...models.values()]);
229
+ const pinsDigest = await candidatePins(options.agents, ctx);
230
+ const summary = summarize(ctx, comparison, all, results, [...models.values()], pinsDigest);
196
231
  return {
197
232
  result: { summary, perCase: results },
198
233
  ...(ctx.abortSignal.aborted && { error: 'cancelled' }),
@@ -337,6 +372,8 @@ async function invokeCase(
337
372
  evalRunId: ctx.runId as unknown as string,
338
373
  },
339
374
  ...(judgedCase.subject.kind === 'agent' && { history: judgedCase.context?.history ?? [] }),
375
+ ...(!('agentId' in ctx.target) &&
376
+ ctx.comparison?.versions !== undefined && { versions: ctx.comparison.versions }),
340
377
  });
341
378
  } catch (cause) {
342
379
  return { error: cause instanceof Error ? cause.message : String(cause) };
@@ -436,6 +473,7 @@ function summarize(
436
473
  cases: readonly JudgedEvalCase[],
437
474
  results: readonly JudgedCaseResult[],
438
475
  models: readonly { providerId: string; model: string; runs: number }[],
476
+ pinsDigest: string | undefined,
439
477
  ): JudgedComparisonSummary {
440
478
  const errors = results.filter((r) => r.error !== undefined).length;
441
479
  const stopped = results.filter((r) => r.stopped !== undefined).length;
@@ -467,7 +505,7 @@ function summarize(
467
505
  : 'failed',
468
506
  completedAt: new Date().toISOString(),
469
507
  suite: { id: ctx.suite.id, version: ctx.suite.version },
470
- candidate: candidateOf(ctx.target),
508
+ candidate: candidateOf(ctx.target, ctx.comparison?.versions, pinsDigest),
471
509
  baseline: {
472
510
  kind: 'recorded',
473
511
  versions: [...versions.values()].map(
@@ -484,6 +522,7 @@ function summarize(
484
522
  errors,
485
523
  stopped,
486
524
  reads: comparison.reads,
525
+ classWeights: comparison.classWeights ?? 'as-recorded',
487
526
  sampling: { models },
488
527
  repetitions: comparison.repetitions,
489
528
  metrics: {
@@ -500,8 +539,60 @@ function summarize(
500
539
  };
501
540
  }
502
541
 
503
- function candidateOf(target: AgentRef | FlowRef): ComparisonCandidate {
542
+ /**
543
+ * A case as a `restricted-only` comparison counts it: each item at its
544
+ * restricted judgments' weights; an item with none is left out, so it
545
+ * counts as unjudged.
546
+ */
547
+ function restrictedOnly(c: JudgedEvalCase): JudgedEvalCase {
548
+ return {
549
+ ...c,
550
+ items: c.items.flatMap((item) =>
551
+ item.restricted !== undefined && item.restricted.totalWeight > 0
552
+ ? [
553
+ {
554
+ ...item,
555
+ yesWeight: item.restricted.yesWeight,
556
+ totalWeight: item.restricted.totalWeight,
557
+ },
558
+ ]
559
+ : [],
560
+ ),
561
+ };
562
+ }
563
+
564
+ /** The agent candidate's `pinsDigest`, from the registry. */
565
+ async function candidatePins(
566
+ agents: JudgedDispatcherOptions['agents'],
567
+ ctx: DispatchContext,
568
+ ): Promise<string | undefined> {
569
+ if (agents === undefined || !('agentId' in ctx.target) || ctx.target.version === undefined) {
570
+ return undefined;
571
+ }
572
+ const found = await agents.getVersion({
573
+ tenantId: ctx.tenantId,
574
+ agentId: ctx.target.agentId,
575
+ version: ctx.target.version,
576
+ });
577
+ return found?.pinsDigest;
578
+ }
579
+
580
+ function candidateOf(
581
+ target: AgentRef | FlowRef,
582
+ versions: FlowVersionOverrides | undefined,
583
+ pinsDigest?: string,
584
+ ): ComparisonCandidate {
504
585
  return 'agentId' in target
505
- ? { kind: 'agent', agentId: target.agentId as unknown as string, version: target.version ?? '' }
506
- : { kind: 'flow', flowId: target.flowId as unknown as string, version: target.version ?? '' };
586
+ ? {
587
+ kind: 'agent',
588
+ agentId: target.agentId as unknown as string,
589
+ version: target.version ?? '',
590
+ ...(pinsDigest !== undefined && { pinsDigest }),
591
+ }
592
+ : {
593
+ kind: 'flow',
594
+ flowId: target.flowId as unknown as string,
595
+ version: target.version ?? '',
596
+ ...(versions !== undefined && { versions }),
597
+ };
507
598
  }
@@ -1,6 +1,7 @@
1
1
  // SPDX-License-Identifier: Apache-2.0
2
2
  // Copyright (C) 2026 Kindgi Inc.
3
3
 
4
+ import { REVIEWER_ROLE_RANK, type ReviewerRole } from '@kindgi/authz';
4
5
  import type { Cursor, ListScope, ProjectId, TenantId } from '@kindgi/types';
5
6
 
6
7
  /**
@@ -77,6 +78,54 @@ export type JudgeClassScope =
77
78
  export const JUDGE_CLASS_SCOPE_KINDS = ['tenant', 'project', 'agent'] as const;
78
79
  export type JudgeClassScopeKind = (typeof JUDGE_CLASS_SCOPE_KINDS)[number];
79
80
 
81
+ /**
82
+ * Who may assert a judge class (T200): every part that's set must hold.
83
+ * Absent: anyone who may judge the run may assert the class, as before.
84
+ * A class with one is *restricted*; a judgment recorded while it is
85
+ * carries `restricted`, and a comparison weighted `restricted-only`
86
+ * counts only those.
87
+ */
88
+ export interface JudgeClassAssertableBy {
89
+ /** The caller's reviewer role is at least this (its token's, or the roster's). */
90
+ readonly minReviewerRole?: ReviewerRole;
91
+ /** Users, service tokens, or both. */
92
+ readonly principalKinds?: readonly ('user' | 'service')[];
93
+ /** Only these principals: user ids, or service token ids. */
94
+ readonly principalIds?: readonly string[];
95
+ }
96
+
97
+ /** Who asserts a judgment, as `JudgeClassAssertableBy` checks it. */
98
+ export interface JudgeClassAsserter {
99
+ readonly kind: 'user' | 'service';
100
+ readonly id: string;
101
+ /** The caller's reviewer role; absent for a caller who isn't a reviewer. */
102
+ readonly reviewerRole?: ReviewerRole;
103
+ }
104
+
105
+ /** Why `asserter` may not assert a class with `assertableBy`, or `undefined` when they may. */
106
+ export function whyNotAssertable(
107
+ assertableBy: JudgeClassAssertableBy,
108
+ asserter: JudgeClassAsserter,
109
+ ): string | undefined {
110
+ if (
111
+ assertableBy.principalKinds !== undefined &&
112
+ !assertableBy.principalKinds.includes(asserter.kind)
113
+ ) {
114
+ return `only ${assertableBy.principalKinds.map((k) => (k === 'user' ? 'users' : 'service tokens')).join(' and ')} may assert it`;
115
+ }
116
+ if (assertableBy.principalIds !== undefined && !assertableBy.principalIds.includes(asserter.id)) {
117
+ return "it's restricted to named people or tokens, and you aren't one of them";
118
+ }
119
+ if (assertableBy.minReviewerRole !== undefined) {
120
+ const need = assertableBy.minReviewerRole;
121
+ const role = asserter.reviewerRole;
122
+ if (role === undefined || REVIEWER_ROLE_RANK[role] < REVIEWER_ROLE_RANK[need]) {
123
+ return `it needs a ${need} reviewer or above${role === undefined ? ", and you aren't a reviewer" : `; you're a ${role} reviewer`}`;
124
+ }
125
+ }
126
+ return undefined;
127
+ }
128
+
80
129
  export interface JudgeClass {
81
130
  readonly id: string;
82
131
  readonly tenantId: TenantId;
@@ -86,6 +135,8 @@ export interface JudgeClass {
86
135
  /** How much a judgment of this class counts (≥ 0); an unclassified judgment counts 1. */
87
136
  readonly weight: number;
88
137
  readonly description?: string;
138
+ /** Who may assert it; absent: anyone who may judge the run (T200). */
139
+ readonly assertableBy?: JudgeClassAssertableBy;
89
140
  readonly createdAt: string;
90
141
  readonly updatedAt: string;
91
142
  /** Set when the class was retired. */
@@ -98,6 +149,7 @@ export interface JudgeClassCreateInput {
98
149
  readonly name: string;
99
150
  readonly weight: number;
100
151
  readonly description?: string;
152
+ readonly assertableBy?: JudgeClassAssertableBy;
101
153
  }
102
154
 
103
155
  export type JudgeClassCreateOutcome =
@@ -128,6 +180,8 @@ export interface JudgeClassUpdateInput {
128
180
  readonly judgeClassId: string;
129
181
  readonly weight?: number;
130
182
  readonly description?: string;
183
+ /** Who may assert it; `null` lifts the restriction. Absent: unchanged. */
184
+ readonly assertableBy?: JudgeClassAssertableBy | null;
131
185
  }
132
186
 
133
187
  /** Whether a class's scope covers a run in `projectId` whose subject is `subject`. */
@@ -189,6 +243,12 @@ export interface Judgment {
189
243
  readonly reason?: string;
190
244
  /** The judge's class; absent for an unclassified judgment (weight 1). */
191
245
  readonly judgeClassId?: string;
246
+ /**
247
+ * Set when the class was restricted (`assertableBy`) when the judgment
248
+ * was recorded, so the judge was checked against it. A restriction added
249
+ * or lifted later doesn't change it (T200).
250
+ */
251
+ readonly restricted?: true;
192
252
  readonly assertedBy: JudgmentAssertedBy;
193
253
  /** The app's opaque id for its end user who judged, when an app judged on their behalf. */
194
254
  readonly participantId?: string;
@@ -292,6 +352,8 @@ export interface JudgmentRecordInput {
292
352
  readonly verdict: Verdict;
293
353
  readonly reason?: string;
294
354
  readonly judgeClassId?: string;
355
+ /** The class was restricted, and the judge met it (see `Judgment.restricted`). */
356
+ readonly restricted?: true;
295
357
  readonly assertedBy: JudgmentAssertedBy;
296
358
  readonly participantId?: string;
297
359
  }
@@ -0,0 +1,237 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright (C) 2026 Kindgi Inc.
3
+
4
+ import type {
5
+ Cursor,
6
+ LiveScope,
7
+ OrgId,
8
+ ProjectId,
9
+ Result,
10
+ ScopeSegment,
11
+ Semver,
12
+ TenantId,
13
+ Timestamp,
14
+ } from '@kindgi/types';
15
+
16
+ import type { GatePolicyBinding, GatePolicyRef } from './gate-policy-binding.js';
17
+ import type { GateApproval, GateCheck } from './gate.js';
18
+
19
+ /**
20
+ * Live versions of agents: which version serves a scope, and the
21
+ * promotions that put it there (evals step 4). A run that names no
22
+ * version takes the most specific live version that covers it (a
23
+ * segment path, then its project, its org, the tenant), else the latest
24
+ * registered one, as before anything is pinned.
25
+ */
26
+
27
+ /** What a run's version is resolved for: its project (and org) and its segment path. */
28
+ export interface LiveResolveInput {
29
+ readonly tenantId: TenantId;
30
+ readonly agentId: string;
31
+ /** Absent → only the org's pin (with `orgId`) and the tenant-wide pin apply. */
32
+ readonly projectId?: ProjectId;
33
+ /**
34
+ * The project's org, when it has one (the resolver may look it up from
35
+ * `projectId` itself). Without `projectId`: an org scope's own
36
+ * coordinates, as the promotion gate resolves what serves an org.
37
+ */
38
+ readonly orgId?: OrgId;
39
+ /** Coarse to fine (needs `projectId`); a segment pin covers every path that starts with its own. */
40
+ readonly segments?: readonly ScopeSegment[];
41
+ }
42
+
43
+ export interface LiveResolution {
44
+ readonly version: Semver;
45
+ /** The pin that matched. */
46
+ readonly scope: LiveScope;
47
+ }
48
+
49
+ export interface LivePin {
50
+ readonly agentId: string;
51
+ readonly scope: LiveScope;
52
+ readonly version: Semver;
53
+ /** The promotion that set it. */
54
+ readonly promotionId: string;
55
+ readonly setAt: Timestamp;
56
+ }
57
+
58
+ export interface LiveVersionBinding {
59
+ /** The most specific live version covering the input; `null` when none is pinned on the way up. */
60
+ resolve(input: LiveResolveInput): Promise<LiveResolution | null>;
61
+ /** Every pin of an agent. */
62
+ list(input: { readonly tenantId: TenantId; readonly agentId: string }): Promise<
63
+ readonly LivePin[]
64
+ >;
65
+ }
66
+
67
+ /** Who asked: the request's principal (a user, or a service token). */
68
+ export interface PromotionActor {
69
+ readonly kind: 'user' | 'service';
70
+ readonly id: string;
71
+ }
72
+
73
+ export type PromotionAction = 'promote' | 'rollback' | 'unpin';
74
+
75
+ /**
76
+ * Where a promotion request stands. Only `promote` rows have one; rollback
77
+ * and unpin are immediate. A request's status changes once, from
78
+ * `pending-approval` to its final state.
79
+ */
80
+ export type PromotionStatus =
81
+ /** The version is live for the scope. */
82
+ | 'promoted'
83
+ /** The gate passed; a reviewer's approval is open. The live version is unchanged. */
84
+ | 'pending-approval'
85
+ /** The gate failed (`checks` say why). The live version is unchanged. */
86
+ | 'refused'
87
+ /** Approved, but the scope's live version or policy changed meanwhile: check again. */
88
+ | 'superseded'
89
+ /** The reviewer rejected it. */
90
+ | 'rejected'
91
+ /** The approval expired undecided. */
92
+ | 'expired';
93
+
94
+ /**
95
+ * One change of a scope's live version, kept for good (the audit trail):
96
+ * what was live before, what is after, who asked and why.
97
+ */
98
+ export interface Promotion {
99
+ readonly id: string;
100
+ readonly agentId: string;
101
+ readonly scope: LiveScope;
102
+ readonly action: PromotionAction;
103
+ /** The scope's own pin before; `null` when it had none. */
104
+ readonly fromVersion: Semver | null;
105
+ /** The scope's own pin after; `null` after an unpin. */
106
+ readonly toVersion: Semver | null;
107
+ readonly requestedBy: PromotionActor;
108
+ readonly reason?: string;
109
+ /** The comparison the change was judged on, when there was one. */
110
+ readonly evalRunId?: string;
111
+ readonly createdAt: Timestamp;
112
+ /** A `promote` row's state; absent on a promotion made before gates (`promoted`). */
113
+ readonly status?: PromotionStatus;
114
+ /** The gate policy that applied; `null` when none did. Absent before gates. */
115
+ readonly policy?: GatePolicyRef | null;
116
+ /** The gate's checks, as they ran. */
117
+ readonly checks?: readonly GateCheck[];
118
+ /** The approval a `pending-approval` promotion waits on (kept once decided). */
119
+ readonly approvalId?: string;
120
+ /** When a `pending-approval` promotion reached its final state. */
121
+ readonly resolvedAt?: Timestamp;
122
+ }
123
+
124
+ export type PromotionErrorCode =
125
+ /** The version isn't registered for this agent, or was unregistered. */
126
+ | 'agent-version-not-found'
127
+ /** The scope names an org or project the tenant doesn't have. */
128
+ | 'scope-invalid'
129
+ /** Rollback: the scope has no earlier live version to go back to. */
130
+ | 'nothing-to-roll-back'
131
+ /** Unpin or rollback: the scope has no pin of its own. */
132
+ | 'not-pinned'
133
+ /** The scope's live version changed while the gate ran: check again. */
134
+ | 'promotion-superseded'
135
+ /**
136
+ * Unpin or rollback: it would leave a scope a gate policy applies to
137
+ * resolving to the latest version, where publishing goes live ungated.
138
+ */
139
+ | 'gate-policy-needs-pin'
140
+ | 'persistence-error';
141
+
142
+ export interface PromotionError {
143
+ readonly code: PromotionErrorCode;
144
+ readonly message: string;
145
+ }
146
+
147
+ export interface PromoteInput {
148
+ readonly tenantId: TenantId;
149
+ readonly agentId: string;
150
+ readonly version: Semver;
151
+ readonly scope: LiveScope;
152
+ readonly requestedBy: PromotionActor;
153
+ readonly reason?: string;
154
+ readonly evalRunId?: string;
155
+ }
156
+
157
+ /**
158
+ * A promotion with the gate's verdict, recorded as one request: refused,
159
+ * waiting for approval, or promoted. The route runs the gate; the
160
+ * binding records the outcome atomically.
161
+ */
162
+ export interface PromotionRequestInput extends PromoteInput {
163
+ readonly gate: {
164
+ readonly policy: GatePolicyRef | null;
165
+ readonly checks: readonly GateCheck[];
166
+ readonly passed: boolean;
167
+ /** A passing promotion waits for this approval instead of going live. */
168
+ readonly approval?: GateApproval;
169
+ /**
170
+ * What served the scope when the gate ran. A binding refuses with
171
+ * `promotion-superseded` when it no longer does, and an approved
172
+ * promotion whose scope moved on becomes `superseded`.
173
+ */
174
+ readonly servingVersion: Semver;
175
+ };
176
+ }
177
+
178
+ export interface RollbackInput {
179
+ readonly tenantId: TenantId;
180
+ readonly agentId: string;
181
+ readonly scope: LiveScope;
182
+ /** Back to this earlier version; absent → the scope's previous live version. */
183
+ readonly toVersion?: Semver;
184
+ readonly requestedBy: PromotionActor;
185
+ readonly reason?: string;
186
+ }
187
+
188
+ export interface UnpinInput {
189
+ readonly tenantId: TenantId;
190
+ readonly agentId: string;
191
+ readonly scope: LiveScope;
192
+ readonly requestedBy: PromotionActor;
193
+ readonly reason?: string;
194
+ }
195
+
196
+ export interface ListPromotionsInput {
197
+ readonly tenantId: TenantId;
198
+ readonly agentId: string;
199
+ /** Only this scope's history. */
200
+ readonly scope?: LiveScope;
201
+ readonly limit: number;
202
+ readonly cursor?: Cursor;
203
+ }
204
+
205
+ export interface PromotionBinding {
206
+ /** Make `version` live for `scope`. The version must be registered and active. */
207
+ promote(input: PromoteInput): Promise<Result<Promotion, PromotionError>>;
208
+ /**
209
+ * Record a gated promotion request (evals step 4b): `refused` when the
210
+ * gate failed, `pending-approval` (opening a HITL approval, subject
211
+ * `agent-promotion`) when it passed and the policy wants an approval,
212
+ * else `promoted`. Optional: without it, the route refuses a promotion
213
+ * a gate policy applies to (`501`), rather than promoting ungated.
214
+ */
215
+ request?(input: PromotionRequestInput): Promise<Result<Promotion, PromotionError>>;
216
+ /** Back to the scope's previous live version, or a named earlier one. */
217
+ rollback(input: RollbackInput): Promise<Result<Promotion, PromotionError>>;
218
+ /**
219
+ * Remove the scope's own pin: it falls back to the next scope up. With
220
+ * gate policies, `gate-policy-needs-pin` when that would leave a gated
221
+ * scope resolving to the latest version.
222
+ */
223
+ unpin(input: UnpinInput): Promise<Result<Promotion, PromotionError>>;
224
+ list(input: ListPromotionsInput): Promise<{
225
+ readonly data: readonly Promotion[];
226
+ readonly nextCursor?: Cursor;
227
+ }>;
228
+ get(tenantId: TenantId, promotionId: string): Promise<Promotion | null>;
229
+ }
230
+
231
+ /** The bindings behind an agent's live versions and promotions (`createApp({ agentReleases })`). */
232
+ export interface AgentReleaseBindings {
233
+ readonly live: LiveVersionBinding;
234
+ readonly promotions: PromotionBinding;
235
+ /** Gate policies (evals step 4b). Absent: no gates, every promotion goes through as before. */
236
+ readonly gatePolicies?: GatePolicyBinding;
237
+ }