@kindgi/api 0.1.3 → 0.1.4-rc.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/dist/agent-binding.d.ts +18 -4
  2. package/dist/agent-binding.d.ts.map +1 -1
  3. package/dist/agent-pins.d.ts +48 -0
  4. package/dist/agent-pins.d.ts.map +1 -0
  5. package/dist/agent-pins.js +102 -0
  6. package/dist/agent-pins.js.map +1 -0
  7. package/dist/app.d.ts +22 -0
  8. package/dist/app.d.ts.map +1 -1
  9. package/dist/app.js +20 -2
  10. package/dist/app.js.map +1 -1
  11. package/dist/block-binding.d.ts +132 -0
  12. package/dist/block-binding.d.ts.map +1 -0
  13. package/dist/block-binding.js +4 -0
  14. package/dist/block-binding.js.map +1 -0
  15. package/dist/block-pins.d.ts +22 -0
  16. package/dist/block-pins.d.ts.map +1 -0
  17. package/dist/block-pins.js +112 -0
  18. package/dist/block-pins.js.map +1 -0
  19. package/dist/cost-binding.d.ts +20 -1
  20. package/dist/cost-binding.d.ts.map +1 -1
  21. package/dist/cost-binding.js +4 -0
  22. package/dist/cost-binding.js.map +1 -1
  23. package/dist/deploy-versions.d.ts +58 -0
  24. package/dist/deploy-versions.d.ts.map +1 -0
  25. package/dist/deploy-versions.js +91 -0
  26. package/dist/deploy-versions.js.map +1 -0
  27. package/dist/deployment-binding.d.ts +24 -3
  28. package/dist/deployment-binding.d.ts.map +1 -1
  29. package/dist/derive-agent-version.d.ts +69 -0
  30. package/dist/derive-agent-version.d.ts.map +1 -0
  31. package/dist/derive-agent-version.js +139 -0
  32. package/dist/derive-agent-version.js.map +1 -0
  33. package/dist/errors.d.ts.map +1 -1
  34. package/dist/errors.js +17 -0
  35. package/dist/errors.js.map +1 -1
  36. package/dist/eval-case-binding.d.ts +65 -0
  37. package/dist/eval-case-binding.d.ts.map +1 -0
  38. package/dist/eval-case-binding.js +4 -0
  39. package/dist/eval-case-binding.js.map +1 -0
  40. package/dist/eval-run-binding.d.ts +30 -0
  41. package/dist/eval-run-binding.d.ts.map +1 -1
  42. package/dist/eval-run-dispatcher.d.ts +38 -3
  43. package/dist/eval-run-dispatcher.d.ts.map +1 -1
  44. package/dist/eval-run-dispatcher.js +21 -15
  45. package/dist/eval-run-dispatcher.js.map +1 -1
  46. package/dist/eval-suite-binding.d.ts +1 -1
  47. package/dist/eval-suite-binding.d.ts.map +1 -1
  48. package/dist/eval-suite-binding.js +2 -0
  49. package/dist/eval-suite-binding.js.map +1 -1
  50. package/dist/flow-binding.d.ts +10 -4
  51. package/dist/flow-binding.d.ts.map +1 -1
  52. package/dist/flow-pins.d.ts +36 -0
  53. package/dist/flow-pins.d.ts.map +1 -0
  54. package/dist/flow-pins.js +81 -0
  55. package/dist/flow-pins.js.map +1 -0
  56. package/dist/index.d.ts +17 -6
  57. package/dist/index.d.ts.map +1 -1
  58. package/dist/index.js +8 -2
  59. package/dist/index.js.map +1 -1
  60. package/dist/judged-dispatcher.d.ts +134 -0
  61. package/dist/judged-dispatcher.d.ts.map +1 -0
  62. package/dist/judged-dispatcher.js +297 -0
  63. package/dist/judged-dispatcher.js.map +1 -0
  64. package/dist/judged-items.d.ts +86 -0
  65. package/dist/judged-items.d.ts.map +1 -0
  66. package/dist/judged-items.js +184 -0
  67. package/dist/judged-items.js.map +1 -0
  68. package/dist/judgment-binding.d.ts +316 -0
  69. package/dist/judgment-binding.d.ts.map +1 -0
  70. package/dist/judgment-binding.js +19 -0
  71. package/dist/judgment-binding.js.map +1 -0
  72. package/dist/openapi/generate.d.ts.map +1 -1
  73. package/dist/openapi/generate.js +4 -1
  74. package/dist/openapi/generate.js.map +1 -1
  75. package/dist/openapi/operations.d.ts.map +1 -1
  76. package/dist/openapi/operations.js +461 -7
  77. package/dist/openapi/operations.js.map +1 -1
  78. package/dist/openapi/schemas.d.ts +40 -0
  79. package/dist/openapi/schemas.d.ts.map +1 -1
  80. package/dist/openapi/schemas.js +843 -7
  81. package/dist/openapi/schemas.js.map +1 -1
  82. package/dist/provider-binding.d.ts +12 -7
  83. package/dist/provider-binding.d.ts.map +1 -1
  84. package/dist/routes/agents.d.ts +9 -1
  85. package/dist/routes/agents.d.ts.map +1 -1
  86. package/dist/routes/agents.js +175 -11
  87. package/dist/routes/agents.js.map +1 -1
  88. package/dist/routes/blocks.d.ts +19 -0
  89. package/dist/routes/blocks.d.ts.map +1 -0
  90. package/dist/routes/blocks.js +281 -0
  91. package/dist/routes/blocks.js.map +1 -0
  92. package/dist/routes/cost.d.ts.map +1 -1
  93. package/dist/routes/cost.js +47 -2
  94. package/dist/routes/cost.js.map +1 -1
  95. package/dist/routes/deployments.d.ts +3 -0
  96. package/dist/routes/deployments.d.ts.map +1 -1
  97. package/dist/routes/deployments.js +208 -55
  98. package/dist/routes/deployments.js.map +1 -1
  99. package/dist/routes/eval-comparison.d.ts +14 -0
  100. package/dist/routes/eval-comparison.d.ts.map +1 -0
  101. package/dist/routes/eval-comparison.js +87 -0
  102. package/dist/routes/eval-comparison.js.map +1 -0
  103. package/dist/routes/eval-runs.d.ts.map +1 -1
  104. package/dist/routes/eval-runs.js +8 -0
  105. package/dist/routes/eval-runs.js.map +1 -1
  106. package/dist/routes/flows.d.ts +13 -1
  107. package/dist/routes/flows.d.ts.map +1 -1
  108. package/dist/routes/flows.js +44 -3
  109. package/dist/routes/flows.js.map +1 -1
  110. package/dist/routes/hierarchy-errors.d.ts +35 -0
  111. package/dist/routes/hierarchy-errors.d.ts.map +1 -0
  112. package/dist/routes/hierarchy-errors.js +39 -0
  113. package/dist/routes/hierarchy-errors.js.map +1 -0
  114. package/dist/routes/judged-suites.d.ts +20 -0
  115. package/dist/routes/judged-suites.d.ts.map +1 -0
  116. package/dist/routes/judged-suites.js +272 -0
  117. package/dist/routes/judged-suites.js.map +1 -0
  118. package/dist/routes/judgment-context.d.ts +22 -0
  119. package/dist/routes/judgment-context.d.ts.map +1 -0
  120. package/dist/routes/judgment-context.js +88 -0
  121. package/dist/routes/judgment-context.js.map +1 -0
  122. package/dist/routes/judgment-flow-context.d.ts +32 -0
  123. package/dist/routes/judgment-flow-context.d.ts.map +1 -0
  124. package/dist/routes/judgment-flow-context.js +195 -0
  125. package/dist/routes/judgment-flow-context.js.map +1 -0
  126. package/dist/routes/judgments.d.ts +41 -0
  127. package/dist/routes/judgments.d.ts.map +1 -0
  128. package/dist/routes/judgments.js +566 -0
  129. package/dist/routes/judgments.js.map +1 -0
  130. package/dist/routes/orgs.d.ts +5 -2
  131. package/dist/routes/orgs.d.ts.map +1 -1
  132. package/dist/routes/orgs.js +38 -22
  133. package/dist/routes/orgs.js.map +1 -1
  134. package/dist/routes/policies.d.ts.map +1 -1
  135. package/dist/routes/policies.js +12 -1
  136. package/dist/routes/policies.js.map +1 -1
  137. package/dist/routes/projects.d.ts +10 -2
  138. package/dist/routes/projects.d.ts.map +1 -1
  139. package/dist/routes/projects.js +87 -79
  140. package/dist/routes/projects.js.map +1 -1
  141. package/dist/routes/providers.d.ts.map +1 -1
  142. package/dist/routes/providers.js +6 -1
  143. package/dist/routes/providers.js.map +1 -1
  144. package/dist/routes/runs.js +24 -3
  145. package/dist/routes/runs.js.map +1 -1
  146. package/dist/routes/teams.d.ts +6 -2
  147. package/dist/routes/teams.d.ts.map +1 -1
  148. package/dist/routes/teams.js +77 -73
  149. package/dist/routes/teams.js.map +1 -1
  150. package/openapi.json +13545 -10103
  151. package/package.json +21 -21
  152. package/src/agent-binding.ts +19 -4
  153. package/src/agent-pins.ts +147 -0
  154. package/src/app.ts +71 -2
  155. package/src/block-binding.ts +137 -0
  156. package/src/block-pins.ts +148 -0
  157. package/src/cost-binding.ts +21 -1
  158. package/src/deploy-versions.ts +157 -0
  159. package/src/deployment-binding.ts +27 -3
  160. package/src/derive-agent-version.ts +206 -0
  161. package/src/errors.ts +17 -0
  162. package/src/eval-case-binding.ts +71 -0
  163. package/src/eval-run-binding.ts +33 -0
  164. package/src/eval-run-dispatcher.ts +57 -16
  165. package/src/eval-suite-binding.ts +2 -0
  166. package/src/flow-binding.ts +11 -4
  167. package/src/flow-pins.ts +113 -0
  168. package/src/index.ts +85 -2
  169. package/src/judged-dispatcher.ts +507 -0
  170. package/src/judged-items.ts +263 -0
  171. package/src/judgment-binding.ts +349 -0
  172. package/src/openapi/generate.ts +7 -1
  173. package/src/openapi/operations.ts +523 -7
  174. package/src/openapi/schemas.ts +1009 -88
  175. package/src/provider-binding.ts +12 -7
  176. package/src/routes/agents.ts +243 -19
  177. package/src/routes/blocks.ts +362 -0
  178. package/src/routes/cost.ts +55 -1
  179. package/src/routes/deployments.ts +266 -56
  180. package/src/routes/eval-comparison.ts +101 -0
  181. package/src/routes/eval-runs.ts +11 -0
  182. package/src/routes/flows.ts +63 -5
  183. package/src/routes/hierarchy-errors.ts +51 -0
  184. package/src/routes/judged-suites.ts +363 -0
  185. package/src/routes/judgment-context.ts +128 -0
  186. package/src/routes/judgment-flow-context.ts +245 -0
  187. package/src/routes/judgments.ts +743 -0
  188. package/src/routes/orgs.ts +44 -27
  189. package/src/routes/policies.ts +19 -0
  190. package/src/routes/projects.ts +106 -95
  191. package/src/routes/providers.ts +5 -0
  192. package/src/routes/runs.ts +28 -3
  193. package/src/routes/teams.ts +96 -90
@@ -84,6 +84,20 @@ const RunAgentIdQueryParam = {
84
84
  description: "Only this agent's turns, at any version. Turns that ran before Kindgi 0.1.3 don't name their agent and aren't matched.",
85
85
  schema: { type: 'string', minLength: 1 },
86
86
  };
87
+ const RunReplaysQueryParam = {
88
+ name: 'replays',
89
+ in: 'query',
90
+ required: false,
91
+ description: 'Replay runs (an eval run re-running a past run). `exclude` (default) leaves them out; `include` lists them with the other runs; `only` lists just them.',
92
+ schema: { type: 'string', enum: ['exclude', 'include', 'only'], default: 'exclude' },
93
+ };
94
+ const RunEvalRunIdQueryParam = {
95
+ name: 'evalRunId',
96
+ in: 'query',
97
+ required: false,
98
+ description: 'Only the replay runs of this eval run. Implies replays are included; cannot be combined with `replays=exclude`.',
99
+ schema: { type: 'string', minLength: 1 },
100
+ };
87
101
  const RunIncludeQueryParam = {
88
102
  name: 'include',
89
103
  in: 'query',
@@ -427,6 +441,13 @@ const CostGroupByQueryParam = {
427
441
  description: "Comma-separated list of dimensions to aggregate over. Each value must be one of `agentId | runId | category | providerId | day | month | tenant | conversationId | model | servedModel | projectId | orgId | rootRunId | flowId`. `model` is the model actually called; `servedModel` the exact version the vendor reported; `orgId` the org of the record's project. Duplicates collapse.",
428
442
  schema: { type: 'string' },
429
443
  };
444
+ const CostAggregateLimitQueryParam = {
445
+ name: 'limit',
446
+ in: 'query',
447
+ required: false,
448
+ description: 'The most groups to return: the most expensive ones (`groups` is ordered by `totalUsd`, highest first). 1 to 10000, default 1000. When there were more, `truncated` is `true` and `totalGroups` says how many; the totals still cover every record.',
449
+ schema: { type: 'integer', minimum: 1, maximum: 10000, default: 1000 },
450
+ };
430
451
  const CostModelQueryParam = {
431
452
  name: 'model',
432
453
  in: 'query',
@@ -611,7 +632,7 @@ const EvalSuiteKindFilterQueryParam = {
611
632
  name: 'kind',
612
633
  in: 'query',
613
634
  required: false,
614
- description: 'Filter to eval suites of a single kind. Values: `accuracy | pairwise | regression | human-review | benchmark | custom`.',
635
+ description: 'Filter to eval suites of a single kind. Values: `accuracy | pairwise | regression | human-review | benchmark | custom | judged`.',
615
636
  schema: { $ref: '#/components/schemas/EvalKind' },
616
637
  };
617
638
  const EvalSuiteNameFilterQueryParam = {
@@ -621,6 +642,34 @@ const EvalSuiteNameFilterQueryParam = {
621
642
  description: 'Prefix match on `EvalSuite.id`. Dotted namespaces are the natural filter shape.',
622
643
  schema: { type: 'string' },
623
644
  };
645
+ const BlockIdPathParam = {
646
+ name: 'blockId',
647
+ in: 'path',
648
+ required: true,
649
+ description: 'Block id: dotted lowercase (e.g. `acme.intake-prompt`).',
650
+ schema: { type: 'string', minLength: 1 },
651
+ };
652
+ const BlockVersionPathParam = {
653
+ name: 'version',
654
+ in: 'path',
655
+ required: true,
656
+ description: 'Exact block version (`major.minor.patch`).',
657
+ schema: { type: 'string', minLength: 1 },
658
+ };
659
+ const BlockKindFilterQueryParam = {
660
+ name: 'kind',
661
+ in: 'query',
662
+ required: false,
663
+ description: 'Only blocks of this kind: `prompt` or `settings`.',
664
+ schema: { $ref: '#/components/schemas/BlockKind' },
665
+ };
666
+ const BlockNameFilterQueryParam = {
667
+ name: 'name',
668
+ in: 'query',
669
+ required: false,
670
+ description: 'Prefix match on the block id.',
671
+ schema: { type: 'string' },
672
+ };
624
673
  // Admin plane — eval-run data plane.
625
674
  const EvalRunIdPathParam = {
626
675
  name: 'runId',
@@ -723,6 +772,55 @@ const SecretNamePathParam = {
723
772
  description: 'Secret name (opaque string within the tenant + envName).',
724
773
  schema: { type: 'string', minLength: 1 },
725
774
  };
775
+ // ---------------- judgments + judge classes: parameters ----------------
776
+ const JudgmentIdPathParam = {
777
+ name: 'judgmentId',
778
+ in: 'path',
779
+ required: true,
780
+ description: 'Judgment id.',
781
+ schema: { type: 'string', minLength: 1 },
782
+ };
783
+ const JudgeClassIdPathParam = {
784
+ name: 'judgeClassId',
785
+ in: 'path',
786
+ required: true,
787
+ description: 'Judge class id.',
788
+ schema: { type: 'string', minLength: 1 },
789
+ };
790
+ const judgmentQuery = (name, description, schema) => ({
791
+ name,
792
+ in: 'query',
793
+ required: false,
794
+ description,
795
+ schema,
796
+ });
797
+ const JudgmentListQueryParams = [
798
+ judgmentQuery('runId', 'Only judgments of this run.', { type: 'string', minLength: 1 }),
799
+ judgmentQuery('agentId', 'Only judgments of runs of this agent.', {
800
+ type: 'string',
801
+ minLength: 1,
802
+ }),
803
+ judgmentQuery('agentVersion', 'Only judgments of runs of this agent version. Needs `agentId`.', {
804
+ type: 'string',
805
+ minLength: 1,
806
+ }),
807
+ judgmentQuery('flowId', 'Only judgments of runs of this flow.', { type: 'string', minLength: 1 }),
808
+ judgmentQuery('verdict', 'Only judgments with this verdict.', {
809
+ type: 'string',
810
+ enum: ['yes', 'no'],
811
+ }),
812
+ judgmentQuery('judgeClassId', 'Only judgments recorded under this judge class.', {
813
+ type: 'string',
814
+ minLength: 1,
815
+ }),
816
+ judgmentQuery('participantId', "Only judgments made for this app end user's opaque id.", {
817
+ type: 'string',
818
+ minLength: 1,
819
+ }),
820
+ ];
821
+ const JudgeClassScopeKindQueryParam = judgmentQuery('scopeKind', 'Only classes of this scope kind. `project` and `agent` need `projectId`; `agent` also needs `agentId`.', { type: 'string', enum: ['tenant', 'project', 'agent'] });
822
+ const JudgeClassProjectIdQueryParam = judgmentQuery('projectId', 'Project of the scope, for `scopeKind=project|agent`.', { type: 'string', minLength: 1 });
823
+ const JudgeClassAgentIdQueryParam = judgmentQuery('agentId', 'Agent of the scope, for `scopeKind=agent`.', { type: 'string', minLength: 1 });
726
824
  // ---------------- shared responses ----------------
727
825
  const ErrorResponse = (description) => ({
728
826
  description,
@@ -810,6 +908,8 @@ export const OPERATIONS = [
810
908
  ParentRunIdQueryParam,
811
909
  TopLevelQueryParam,
812
910
  RunAgentIdQueryParam,
911
+ RunReplaysQueryParam,
912
+ RunEvalRunIdQueryParam,
813
913
  RunIncludeQueryParam,
814
914
  ],
815
915
  responses: {
@@ -1317,6 +1417,24 @@ export const OPERATIONS = [
1317
1417
  '404': ErrorResponse('No agent with that id under this tenant.'),
1318
1418
  },
1319
1419
  },
1420
+ {
1421
+ method: 'post',
1422
+ honoPath: '/v1/agents/:agentId/versions',
1423
+ openapiPath: '/v1/agents/{agentId}/versions',
1424
+ operationId: 'agents.deriveVersion',
1425
+ summary: 'Derive an agent version with data-block pins swapped',
1426
+ description: "An expert's edit, without a code change: a new agent version that is `from`'s bag with some prompt or settings pins swapped (`derivedFrom: { version, reason: 'edited', label, by }`), numbered the next free patch after the agent's highest version. Only blocks `from` already references swap, to a published, active version of the right kind (model settings for the model-settings block); tool pins come from code. Needs `publish` on the agent.",
1427
+ tags: ['agents'],
1428
+ security: 'bearer',
1429
+ parameters: [AgentIdPathParam, IdempotencyKeyParam],
1430
+ requestBody: { required: true, schema: ref('DeriveAgentVersionBody') },
1431
+ responses: {
1432
+ '201': { description: 'The derived agent version.', schema: ref('Agent') },
1433
+ ...CommonMutationErrors,
1434
+ '400': ErrorResponse("`validation-failed`: `from` has no pins, a swap names a block it doesn't reference, or a version that isn't published, active or the right kind (see `details.issues`)."),
1435
+ '404': ErrorResponse('`agent-not-found`: no agent at `from`.'),
1436
+ },
1437
+ },
1320
1438
  {
1321
1439
  method: 'get',
1322
1440
  honoPath: '/v1/agents/:agentId/versions/:version',
@@ -2347,7 +2465,7 @@ export const OPERATIONS = [
2347
2465
  openapiPath: '/v1/providers',
2348
2466
  operationId: 'providers.register',
2349
2467
  summary: 'Register a model provider',
2350
- description: 'Body is a full `ProviderMetadata`. Server validates shape: provider-level `id` + `region` non-empty; `models[]` non-empty with unique `name` per entry; per-model `contextWindow` positive integer; per-model `features` against the closed enum; per-model `cost` non-negative; optional per-model `p95LatencyMs` / `maxOutputTokens` well-shaped — same rules as `@kindgi/capabilities.createProviderRegistry`. Secrets (API keys, endpoints) are NOT part of the wire shape; deployments store them inside the binding.',
2468
+ description: 'Body is a full `ProviderMetadata`. Server validates shape: provider-level `id` + `region` non-empty; `models[]` non-empty with unique `name` per entry; per-model `contextWindow` positive integer; per-model `features` against the closed enum; per-model `cost` non-negative; optional per-model `p95LatencyMs` / `maxOutputTokens` well-shaped; optional `labels` within their limits — same rules as `@kindgi/capabilities.createProviderRegistry`. Secrets (API keys, endpoints) are NOT part of the wire shape; deployments store them inside the binding.',
2351
2469
  tags: ['providers'],
2352
2470
  security: 'bearer',
2353
2471
  parameters: [IdempotencyKeyParam],
@@ -2365,13 +2483,181 @@ export const OPERATIONS = [
2365
2483
  openapiPath: '/v1/providers/{providerId}/unregister',
2366
2484
  operationId: 'providers.unregister',
2367
2485
  summary: 'Unregister a model provider',
2486
+ description: 'A tombstone, not an erase: from then on the provider is gone from list, get and capabilities, and the router never picks it. Its id is free to register again. A retention policy on the `provider` domain purges the row.',
2368
2487
  tags: ['providers'],
2369
2488
  security: 'bearer',
2370
2489
  parameters: [ProviderIdPathParam, IdempotencyKeyParam],
2371
2490
  responses: {
2372
2491
  '200': { description: 'Unregistered.', schema: ref('UnregisterProviderResult') },
2373
2492
  ...CommonMutationErrors,
2374
- '404': ErrorResponse('No provider with that id under this tenant.'),
2493
+ '404': ErrorResponse('No provider with that id under this tenant, or already unregistered.'),
2494
+ },
2495
+ },
2496
+ // ---------- judgments ----------
2497
+ {
2498
+ method: 'post',
2499
+ honoPath: '/v1/judgments',
2500
+ openapiPath: '/v1/judgments',
2501
+ operationId: 'judgments.create',
2502
+ summary: "Judge an item of a run's output",
2503
+ description: "Records yes or no, with an optional reason, about one item of a finished run's output, optionally under a judge class that applies to the run's project or agent (unclassified judgments count with weight 1). `item.pointer` (a JSON Pointer) must resolve in the run's output; its value is kept as `itemValue`. The first judgment of a run also stores a copy of the run's input and output. `assertedBy` is the authenticated caller, never the body. Judging again as the same caller for the same run, item key and `participantId` supersedes the earlier judgment. Needs `judge` on the run.",
2504
+ tags: ['judgments'],
2505
+ security: 'bearer',
2506
+ parameters: [IdempotencyKeyParam],
2507
+ requestBody: { required: true, schema: ref('CreateJudgmentBody') },
2508
+ responses: {
2509
+ '201': { description: 'Judgment recorded.', schema: ref('Judgment') },
2510
+ ...CommonMutationErrors,
2511
+ '400': ErrorResponse('Malformed body, or `item-not-found` (the pointer resolves to nothing in the output), or `judge-class-not-applicable`.'),
2512
+ '403': ErrorResponse('`permission-denied`: not allowed to judge this run.'),
2513
+ '404': ErrorResponse('`run-not-found`.'),
2514
+ '409': ErrorResponse('`run-not-finished`: the run has no output to judge yet.'),
2515
+ },
2516
+ },
2517
+ {
2518
+ method: 'get',
2519
+ honoPath: '/v1/judgments',
2520
+ openapiPath: '/v1/judgments',
2521
+ operationId: 'judgments.list',
2522
+ summary: 'List judgments',
2523
+ description: 'Live judgments (not removed or superseded), newest first, cursor-paginated. Filter by run, agent (and version), flow, verdict, judge class or participant; `?scopeKind + ?scopeId` narrow to a project.',
2524
+ tags: ['judgments'],
2525
+ security: 'bearer',
2526
+ parameters: [
2527
+ LimitQueryParam,
2528
+ CursorQueryParam,
2529
+ ...JudgmentListQueryParams,
2530
+ ScopeKindQueryParam,
2531
+ ScopeIdQueryParam,
2532
+ ],
2533
+ responses: {
2534
+ '200': { description: 'Page of judgments.', schema: ref('JudgmentCollectionPage') },
2535
+ ...CommonAuthErrors,
2536
+ '400': ErrorResponse('Malformed query parameter.'),
2537
+ },
2538
+ },
2539
+ {
2540
+ method: 'get',
2541
+ honoPath: '/v1/judgments/:judgmentId',
2542
+ openapiPath: '/v1/judgments/{judgmentId}',
2543
+ operationId: 'judgments.get',
2544
+ summary: 'Fetch a judgment with its copies',
2545
+ description: "Returns the judgment (live or not) with the stored copy of the run's input and output and, when the judgment pointed at an item, its value.",
2546
+ tags: ['judgments'],
2547
+ security: 'bearer',
2548
+ parameters: [JudgmentIdPathParam],
2549
+ responses: {
2550
+ '200': { description: 'Judgment with copies.', schema: ref('JudgmentWithCopies') },
2551
+ ...CommonAuthErrors,
2552
+ '404': ErrorResponse('No judgment with that id under this tenant.'),
2553
+ },
2554
+ },
2555
+ {
2556
+ method: 'post',
2557
+ honoPath: '/v1/judgments/:judgmentId/unregister',
2558
+ openapiPath: '/v1/judgments/{judgmentId}/unregister',
2559
+ operationId: 'judgments.unregister',
2560
+ summary: 'Remove a judgment',
2561
+ description: 'Soft delete: the judgment stops listing; retention policy decides when it is purged.',
2562
+ tags: ['judgments'],
2563
+ security: 'bearer',
2564
+ parameters: [JudgmentIdPathParam, IdempotencyKeyParam],
2565
+ responses: {
2566
+ '200': { description: 'Removed.', schema: ref('UnregisterJudgmentResult') },
2567
+ ...CommonMutationErrors,
2568
+ '403': ErrorResponse('`permission-denied`.'),
2569
+ '404': ErrorResponse('No live judgment with that id under this tenant.'),
2570
+ },
2571
+ },
2572
+ // ---------- judge classes ----------
2573
+ {
2574
+ method: 'post',
2575
+ honoPath: '/v1/judge-classes',
2576
+ openapiPath: '/v1/judge-classes',
2577
+ operationId: 'judgeClasses.create',
2578
+ summary: 'Create a judge class',
2579
+ description: 'A named kind of judge with a weight, scoped to the tenant, a project, or an agent in a project. Names are unique among the live classes of a scope. Needs `admin` on the tenant (tenant scope) or the project.',
2580
+ tags: ['judge-classes'],
2581
+ security: 'bearer',
2582
+ parameters: [IdempotencyKeyParam],
2583
+ requestBody: { required: true, schema: ref('CreateJudgeClassBody') },
2584
+ responses: {
2585
+ '201': { description: 'Judge class created.', schema: ref('JudgeClass') },
2586
+ ...CommonMutationErrors,
2587
+ '403': ErrorResponse('`permission-denied`.'),
2588
+ '409': ErrorResponse('`judge-class-name-taken`, or an idempotency conflict.'),
2589
+ },
2590
+ },
2591
+ {
2592
+ method: 'get',
2593
+ honoPath: '/v1/judge-classes',
2594
+ openapiPath: '/v1/judge-classes',
2595
+ operationId: 'judgeClasses.list',
2596
+ summary: 'List judge classes',
2597
+ description: 'Live classes, newest first, cursor-paginated. `?scopeKind=tenant|project|agent` (with `projectId` / `agentId`) narrows to one scope.',
2598
+ tags: ['judge-classes'],
2599
+ security: 'bearer',
2600
+ parameters: [
2601
+ LimitQueryParam,
2602
+ CursorQueryParam,
2603
+ JudgeClassScopeKindQueryParam,
2604
+ JudgeClassProjectIdQueryParam,
2605
+ JudgeClassAgentIdQueryParam,
2606
+ ],
2607
+ responses: {
2608
+ '200': { description: 'Page of judge classes.', schema: ref('JudgeClassCollectionPage') },
2609
+ ...CommonAuthErrors,
2610
+ '400': ErrorResponse('Malformed scope parameters.'),
2611
+ },
2612
+ },
2613
+ {
2614
+ method: 'get',
2615
+ honoPath: '/v1/judge-classes/:judgeClassId',
2616
+ openapiPath: '/v1/judge-classes/{judgeClassId}',
2617
+ operationId: 'judgeClasses.get',
2618
+ summary: 'Fetch a judge class',
2619
+ description: 'Also returns a retired class (`unregisteredAt` set): judgments keep naming theirs.',
2620
+ tags: ['judge-classes'],
2621
+ security: 'bearer',
2622
+ parameters: [JudgeClassIdPathParam],
2623
+ responses: {
2624
+ '200': { description: 'Judge class.', schema: ref('JudgeClass') },
2625
+ ...CommonAuthErrors,
2626
+ '404': ErrorResponse('No judge class with that id under this tenant.'),
2627
+ },
2628
+ },
2629
+ {
2630
+ method: 'patch',
2631
+ honoPath: '/v1/judge-classes/:judgeClassId',
2632
+ openapiPath: '/v1/judge-classes/{judgeClassId}',
2633
+ operationId: 'judgeClasses.update',
2634
+ summary: "Change a judge class's weight or description",
2635
+ tags: ['judge-classes'],
2636
+ security: 'bearer',
2637
+ parameters: [JudgeClassIdPathParam, IdempotencyKeyParam],
2638
+ requestBody: { required: true, schema: ref('UpdateJudgeClassBody') },
2639
+ responses: {
2640
+ '200': { description: 'Updated judge class.', schema: ref('JudgeClass') },
2641
+ ...CommonMutationErrors,
2642
+ '403': ErrorResponse('`permission-denied`.'),
2643
+ '404': ErrorResponse('No live judge class with that id under this tenant.'),
2644
+ },
2645
+ },
2646
+ {
2647
+ method: 'post',
2648
+ honoPath: '/v1/judge-classes/:judgeClassId/unregister',
2649
+ openapiPath: '/v1/judge-classes/{judgeClassId}/unregister',
2650
+ operationId: 'judgeClasses.unregister',
2651
+ summary: 'Retire a judge class',
2652
+ description: 'No new judgments may name it; existing judgments keep it.',
2653
+ tags: ['judge-classes'],
2654
+ security: 'bearer',
2655
+ parameters: [JudgeClassIdPathParam, IdempotencyKeyParam],
2656
+ responses: {
2657
+ '200': { description: 'Retired.', schema: ref('UnregisterJudgeClassResult') },
2658
+ ...CommonMutationErrors,
2659
+ '403': ErrorResponse('`permission-denied`.'),
2660
+ '404': ErrorResponse('No live judge class with that id under this tenant.'),
2375
2661
  },
2376
2662
  },
2377
2663
  // ---------- mcp (interop plane) ----------
@@ -2578,11 +2864,12 @@ export const OPERATIONS = [
2578
2864
  openapiPath: '/v1/cost/aggregate',
2579
2865
  operationId: 'cost.aggregate',
2580
2866
  summary: 'Aggregate cost across a time window',
2581
- description: "Primary consumer path for dashboards. `groupBy` is required (comma-separated dimensions from the closed set); time range is required (both `from` and `to`, or both omitted for the default last-30-days window echoed back in `timeRange`). Filters compose on top of the time window. `?scopeKind + ?scopeId` narrow the aggregate to a scope: `org` covers every project in the org, so one call sums an org's spend. Each group, and the total, carries its cost and its token sums (`tokens`). `inherit` has no effect on cost records, which always belong to a project.",
2867
+ description: "Primary consumer path for dashboards. `groupBy` is required (comma-separated dimensions from the closed set); time range is required (both `from` and `to`, or both omitted for the default last-30-days window echoed back in `timeRange`). Filters compose on top of the time window. `?scopeKind + ?scopeId` narrow the aggregate to a scope: `org` covers every project in the org, so one call sums an org's spend. Each group, and the total, carries its cost and its token sums (`tokens`). `groups` is ordered by `totalUsd`, highest first (ties by key), and capped at `limit` (default 1000): `truncated` and `totalGroups` say when there were more, and the totals still cover every record. For every record, page through `/v1/cost/records`. `inherit` has no effect on cost records, which always belong to a project.",
2582
2868
  tags: ['cost'],
2583
2869
  security: 'bearer',
2584
2870
  parameters: [
2585
2871
  CostGroupByQueryParam,
2872
+ CostAggregateLimitQueryParam,
2586
2873
  CostFromQueryParam,
2587
2874
  CostToQueryParam,
2588
2875
  CostCategoryQueryParam,
@@ -2757,7 +3044,7 @@ export const OPERATIONS = [
2757
3044
  openapiPath: '/v1/policies',
2758
3045
  operationId: 'policies.publish',
2759
3046
  summary: 'Publish a policy',
2760
- description: "Body is a full `Policy` — the server validates top-level shape (id, tenantId, semver version, kind ∈ closed enum, spec is an object). Deeper `spec` validation is the runtime consumer's responsibility per kind. Re-publishing an existing `(policyId, version)` returns `409 policy-already-registered`. Idempotency-Key applies (retries with the same key replay the original 201).",
3047
+ description: "Body is a full `Policy` — the server validates top-level shape (id, tenantId, semver version, kind ∈ closed enum, spec is an object). Deeper `spec` validation is the runtime consumer's responsibility per kind. Re-publishing an existing `(policyId, version)` returns `409 policy-already-registered`. A known kind that no runtime consumer applies yet (`access-control`, `adapter-allowlist`, `rate-limit`, `compliance`) is refused with `400 kind-not-applied` (`details.appliedKinds` lists the ones that are): publishing it would change nothing. Idempotency-Key applies (retries with the same key replay the original 201).",
2761
3048
  tags: ['policies'],
2762
3049
  security: 'bearer',
2763
3050
  parameters: [IdempotencyKeyParam],
@@ -2765,7 +3052,7 @@ export const OPERATIONS = [
2765
3052
  responses: {
2766
3053
  '201': { description: 'Policy published.', schema: ref('PublishPolicyResult') },
2767
3054
  ...CommonMutationErrors,
2768
- '400': ErrorResponse('Validation failed (see `details.issues`).'),
3055
+ '400': ErrorResponse('Validation failed (see `details.issues`), or `kind-not-applied`: no runtime consumer applies that kind yet.'),
2769
3056
  '409': ErrorResponse('Policy already registered at that (id, version).'),
2770
3057
  },
2771
3058
  },
@@ -2892,6 +3179,47 @@ export const OPERATIONS = [
2892
3179
  '409': ErrorResponse('Eval suite already registered at that (id, version).'),
2893
3180
  },
2894
3181
  },
3182
+ {
3183
+ method: 'post',
3184
+ honoPath: '/v1/eval-suites/:suiteId/versions/from-judgments',
3185
+ openapiPath: '/v1/eval-suites/{suiteId}/versions/from-judgments',
3186
+ operationId: 'evalSuites.buildFromJudgments',
3187
+ summary: 'Build a test set from judgments',
3188
+ description: "Publishes a `judged` eval suite version whose cases are copies of judged runs of one agent (optionally one version) or flow, newest first, at most 1000. Each case holds the run's input, what the turn read (`context`), the judged output, and each item's judgments summed up: yes and no counts, the weight behind yes and behind all judgments (an unclassified judgment counts 1), and the reasons. `judgeClassIds` counts only judgments of those classes; `minJudgments` leaves out runs with fewer. Needs `admin` on the project.",
3189
+ tags: ['eval-suites'],
3190
+ security: 'bearer',
3191
+ parameters: [EvalSuiteIdPathParam, IdempotencyKeyParam],
3192
+ requestBody: { required: true, schema: ref('BuildJudgedSuiteBody') },
3193
+ responses: {
3194
+ '201': { description: 'Version published.', schema: ref('BuildJudgedSuiteResult') },
3195
+ ...CommonMutationErrors,
3196
+ '400': ErrorResponse('Malformed body.'),
3197
+ '403': ErrorResponse('`permission-denied`.'),
3198
+ '409': ErrorResponse('Eval suite already registered at that (id, version).'),
3199
+ '501': ErrorResponse('`test-sets-not-supported`: this deployment cannot build test sets.'),
3200
+ },
3201
+ },
3202
+ {
3203
+ method: 'get',
3204
+ honoPath: '/v1/eval-suites/:suiteId/versions/:version/cases',
3205
+ openapiPath: '/v1/eval-suites/{suiteId}/versions/{version}/cases',
3206
+ operationId: 'evalSuites.listCases',
3207
+ summary: 'List the cases of a judged eval suite version',
3208
+ description: 'Cursor-paginated, in the order the cases were stored (newest judged run first).',
3209
+ tags: ['eval-suites'],
3210
+ security: 'bearer',
3211
+ parameters: [
3212
+ EvalSuiteIdPathParam,
3213
+ EvalSuiteVersionPathParam,
3214
+ LimitQueryParam,
3215
+ CursorQueryParam,
3216
+ ],
3217
+ responses: {
3218
+ '200': { description: 'Page of cases.', schema: ref('JudgedEvalCaseCollectionPage') },
3219
+ ...CommonAuthErrors,
3220
+ '404': ErrorResponse('No eval suite with that id.'),
3221
+ },
3222
+ },
2895
3223
  {
2896
3224
  method: 'post',
2897
3225
  honoPath: '/v1/eval-suites/:suiteId/versions/:version/unregister',
@@ -2924,6 +3252,129 @@ export const OPERATIONS = [
2924
3252
  },
2925
3253
  },
2926
3254
  // ---------- eval runs (data plane) ----------
3255
+ {
3256
+ method: 'get',
3257
+ honoPath: '/v1/blocks',
3258
+ openapiPath: '/v1/blocks',
3259
+ operationId: 'blocks.list',
3260
+ summary: 'List data blocks (latest version of each)',
3261
+ description: 'Cursor-paginated. Only the blocks of projects the caller can read. `?kind=` narrows to prompts or settings, `?name=` is a prefix match on the id, and `?scopeKind=` + `?scopeId=` narrow to a project or an org, as the other lists do.',
3262
+ tags: ['blocks'],
3263
+ security: 'bearer',
3264
+ parameters: [
3265
+ LimitQueryParam,
3266
+ CursorQueryParam,
3267
+ BlockKindFilterQueryParam,
3268
+ BlockNameFilterQueryParam,
3269
+ ScopeKindQueryParam,
3270
+ ScopeIdQueryParam,
3271
+ ],
3272
+ responses: {
3273
+ '200': { description: 'Page of blocks.', schema: ref('BlockCollectionPage') },
3274
+ ...CommonAuthErrors,
3275
+ '400': ErrorResponse('Malformed query parameter: an unknown `kind`, or `scope-invalid` for a malformed scope.'),
3276
+ },
3277
+ },
3278
+ {
3279
+ method: 'get',
3280
+ honoPath: '/v1/blocks/:blockId',
3281
+ openapiPath: '/v1/blocks/{blockId}',
3282
+ operationId: 'blocks.get',
3283
+ summary: 'Fetch a data block (latest version)',
3284
+ tags: ['blocks'],
3285
+ security: 'bearer',
3286
+ parameters: [BlockIdPathParam],
3287
+ responses: {
3288
+ '200': { description: 'Latest active version.', schema: ref('Block') },
3289
+ ...CommonAuthErrors,
3290
+ '404': ErrorResponse("`block-not-found`: no such block, or one in a project the caller can't read."),
3291
+ },
3292
+ },
3293
+ {
3294
+ method: 'get',
3295
+ honoPath: '/v1/blocks/:blockId/versions',
3296
+ openapiPath: '/v1/blocks/{blockId}/versions',
3297
+ operationId: 'blocks.versions.list',
3298
+ summary: 'List versions of a data block',
3299
+ description: 'Newest published first. `?includeTombstoned=true` includes unregistered versions, each with `unregisteredAt`.',
3300
+ tags: ['blocks'],
3301
+ security: 'bearer',
3302
+ parameters: [BlockIdPathParam, LimitQueryParam, CursorQueryParam, IncludeTombstonedQueryParam],
3303
+ responses: {
3304
+ '200': { description: 'Page of versions.', schema: ref('BlockCollectionPage') },
3305
+ ...CommonAuthErrors,
3306
+ '404': ErrorResponse('`block-not-found`.'),
3307
+ },
3308
+ },
3309
+ {
3310
+ method: 'get',
3311
+ honoPath: '/v1/blocks/:blockId/versions/:version',
3312
+ openapiPath: '/v1/blocks/{blockId}/versions/{version}',
3313
+ operationId: 'blocks.versions.get',
3314
+ summary: 'Fetch a specific data block version',
3315
+ description: 'An unregistered version is returned too, with `unregisteredAt`.',
3316
+ tags: ['blocks'],
3317
+ security: 'bearer',
3318
+ parameters: [BlockIdPathParam, BlockVersionPathParam],
3319
+ responses: {
3320
+ '200': { description: 'The block at that version.', schema: ref('Block') },
3321
+ ...CommonAuthErrors,
3322
+ '404': ErrorResponse('`block-not-found`.'),
3323
+ },
3324
+ },
3325
+ {
3326
+ method: 'post',
3327
+ honoPath: '/v1/blocks',
3328
+ openapiPath: '/v1/blocks',
3329
+ operationId: 'blocks.publish',
3330
+ summary: 'Publish a data block version',
3331
+ description: "Needs `write` on the project. A prompt's template must parse as Liquid; a settings block's `values` must satisfy its `schema` and the latest version's. A taken version is `409 block-already-registered` (versions never change); a block keeps its kind, and its versions stay in its first version's project (`409 block-project-mismatch`). Idempotency-Key applies.",
3332
+ tags: ['blocks'],
3333
+ security: 'bearer',
3334
+ parameters: [IdempotencyKeyParam],
3335
+ requestBody: { required: true, schema: ref('PublishBlockBody') },
3336
+ responses: {
3337
+ '201': { description: 'Published.', schema: ref('PublishBlockResult') },
3338
+ ...CommonMutationErrors,
3339
+ '400': ErrorResponse('`validation-failed` (see `details.issues`), or an unknown `projectId`.'),
3340
+ '403': ErrorResponse('`permission-denied`: no `write` on the project.'),
3341
+ '409': ErrorResponse('`block-already-registered` or `block-project-mismatch`.'),
3342
+ },
3343
+ },
3344
+ {
3345
+ method: 'post',
3346
+ honoPath: '/v1/blocks/:blockId/versions/:version/unregister',
3347
+ openapiPath: '/v1/blocks/{blockId}/versions/{version}/unregister',
3348
+ operationId: 'blocks.versions.unregister',
3349
+ summary: 'Unregister a data block version',
3350
+ description: 'Soft: no range picks it any more, but the agent versions that pin it keep running it, and GET still reads it. Needs `write` on the project.',
3351
+ tags: ['blocks'],
3352
+ security: 'bearer',
3353
+ parameters: [BlockIdPathParam, BlockVersionPathParam, IdempotencyKeyParam],
3354
+ responses: {
3355
+ '200': { description: 'Unregistered.', schema: ref('UnregisterBlockResult') },
3356
+ ...CommonMutationErrors,
3357
+ '403': ErrorResponse('`permission-denied`: no `write` on the project.'),
3358
+ '404': ErrorResponse('`block-not-found`, or already unregistered.'),
3359
+ },
3360
+ },
3361
+ {
3362
+ method: 'post',
3363
+ honoPath: '/v1/blocks/:blockId/versions/:version/reinstate',
3364
+ openapiPath: '/v1/blocks/{blockId}/versions/{version}/reinstate',
3365
+ operationId: 'blocks.versions.reinstate',
3366
+ summary: 'Reinstate an unregistered data block version',
3367
+ description: 'Unchanged, as published. Idempotent. Needs `write` on the project.',
3368
+ tags: ['blocks'],
3369
+ security: 'bearer',
3370
+ parameters: [BlockIdPathParam, BlockVersionPathParam, IdempotencyKeyParam],
3371
+ responses: {
3372
+ '200': { description: 'Reinstated.', schema: ref('ReinstateBlockResult') },
3373
+ ...CommonMutationErrors,
3374
+ '403': ErrorResponse('`permission-denied`: no `write` on the project.'),
3375
+ '404': ErrorResponse('`block-not-found`.'),
3376
+ },
3377
+ },
2927
3378
  {
2928
3379
  method: 'post',
2929
3380
  honoPath: '/v1/eval-suites/:suiteId/runs',
@@ -3699,7 +4150,7 @@ export const OPERATIONS = [
3699
4150
  openapiPath: '/v1/orgs/{orgId}',
3700
4151
  operationId: 'orgs.delete',
3701
4152
  summary: 'Delete an org (idempotent)',
3702
- description: 'Idempotent — deleting an unknown or already-deleted org returns 204 per the binding contract.',
4153
+ description: "A tombstone, not an erase: from then on the org is gone from get and list, and its slug is free for a new org. Its projects and teams stay, without an org; when one of those projects has the slug of a project that has none, nothing is deleted: `409 slug-conflict` names the slugs (rename or move those projects first). In the Kindgi runtime, the org's own secrets and secret mappings are deleted with it, for good, and its own environments and MCP endpoints are unregistered. A retention policy on the `org` domain purges the org's row. Idempotent: deleting an unknown or already-deleted org returns 204.",
3703
4154
  tags: ['orgs'],
3704
4155
  security: 'bearer',
3705
4156
  parameters: [
@@ -3715,6 +4166,7 @@ export const OPERATIONS = [
3715
4166
  responses: {
3716
4167
  '204': { description: 'Deleted (or already absent). No body.' },
3717
4168
  ...CommonAuthErrors,
4169
+ '409': ErrorResponse("slug-conflict: the org's projects would leave it with slugs that projects without an org already have."),
3718
4170
  },
3719
4171
  },
3720
4172
  // ---------- teams ----------
@@ -4009,6 +4461,7 @@ export const OPERATIONS = [
4009
4461
  openapiPath: '/v1/projects',
4010
4462
  operationId: 'projects.create',
4011
4463
  summary: 'Create a project',
4464
+ description: "A project's slug is unique within its org, and a project without an org's among the tenant's projects without one: two orgs may each have a project with the same slug. A taken slug answers `409 slug-conflict`; a second Default, `409 project-default-already-exists`.",
4012
4465
  tags: ['projects'],
4013
4466
  security: 'bearer',
4014
4467
  parameters: [IdempotencyKeyParam],
@@ -4047,6 +4500,7 @@ export const OPERATIONS = [
4047
4500
  openapiPath: '/v1/projects/{projectId}',
4048
4501
  operationId: 'projects.update',
4049
4502
  summary: 'Partially update a project',
4503
+ description: 'A new `slug`, or a move to another org (`orgId`, or `null` for none), answers `409 slug-conflict` when the slug is taken where the project ends up.',
4050
4504
  tags: ['projects'],
4051
4505
  security: 'bearer',
4052
4506
  parameters: [