@kindgi/api 0.1.4-rc.0 → 0.1.4-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/agent-binding.d.ts +8 -0
  2. package/dist/agent-binding.d.ts.map +1 -1
  3. package/dist/app.d.ts.map +1 -1
  4. package/dist/app.js +7 -1
  5. package/dist/app.js.map +1 -1
  6. package/dist/deploy-versions.d.ts +2 -0
  7. package/dist/deploy-versions.d.ts.map +1 -1
  8. package/dist/deploy-versions.js +1 -1
  9. package/dist/deploy-versions.js.map +1 -1
  10. package/dist/derive-agent-version.d.ts +9 -1
  11. package/dist/derive-agent-version.d.ts.map +1 -1
  12. package/dist/derive-agent-version.js +11 -1
  13. package/dist/derive-agent-version.js.map +1 -1
  14. package/dist/errors.d.ts.map +1 -1
  15. package/dist/errors.js +1 -0
  16. package/dist/errors.js.map +1 -1
  17. package/dist/eval-run-binding.d.ts +7 -0
  18. package/dist/eval-run-binding.d.ts.map +1 -1
  19. package/dist/eval-run-binding.js.map +1 -1
  20. package/dist/eval-run-dispatcher.d.ts +3 -0
  21. package/dist/eval-run-dispatcher.d.ts.map +1 -1
  22. package/dist/eval-run-dispatcher.js.map +1 -1
  23. package/dist/flow-binding.d.ts +8 -0
  24. package/dist/flow-binding.d.ts.map +1 -1
  25. package/dist/guardrail-binding.d.ts +8 -0
  26. package/dist/guardrail-binding.d.ts.map +1 -1
  27. package/dist/handler-binding.d.ts +3 -0
  28. package/dist/handler-binding.d.ts.map +1 -1
  29. package/dist/index.d.ts +1 -0
  30. package/dist/index.d.ts.map +1 -1
  31. package/dist/index.js.map +1 -1
  32. package/dist/judged-dispatcher.d.ts +4 -0
  33. package/dist/judged-dispatcher.d.ts.map +1 -1
  34. package/dist/judged-dispatcher.js +14 -3
  35. package/dist/judged-dispatcher.js.map +1 -1
  36. package/dist/openapi/operations.d.ts.map +1 -1
  37. package/dist/openapi/operations.js +21 -6
  38. package/dist/openapi/operations.js.map +1 -1
  39. package/dist/openapi/schemas.d.ts +6 -0
  40. package/dist/openapi/schemas.d.ts.map +1 -1
  41. package/dist/openapi/schemas.js +341 -2
  42. package/dist/openapi/schemas.js.map +1 -1
  43. package/dist/registry-read-only.d.ts +32 -0
  44. package/dist/registry-read-only.d.ts.map +1 -0
  45. package/dist/registry-read-only.js +22 -0
  46. package/dist/registry-read-only.js.map +1 -0
  47. package/dist/routes/agents.d.ts.map +1 -1
  48. package/dist/routes/agents.js +7 -1
  49. package/dist/routes/agents.js.map +1 -1
  50. package/dist/routes/blocks.d.ts.map +1 -1
  51. package/dist/routes/blocks.js +37 -12
  52. package/dist/routes/blocks.js.map +1 -1
  53. package/dist/routes/deployments.d.ts.map +1 -1
  54. package/dist/routes/deployments.js +16 -0
  55. package/dist/routes/deployments.js.map +1 -1
  56. package/dist/routes/eval-comparison.d.ts +3 -2
  57. package/dist/routes/eval-comparison.d.ts.map +1 -1
  58. package/dist/routes/eval-comparison.js +39 -3
  59. package/dist/routes/eval-comparison.js.map +1 -1
  60. package/dist/routes/eval-runs.d.ts +7 -1
  61. package/dist/routes/eval-runs.d.ts.map +1 -1
  62. package/dist/routes/eval-runs.js +34 -3
  63. package/dist/routes/eval-runs.js.map +1 -1
  64. package/dist/routes/eval-versions.d.ts +25 -0
  65. package/dist/routes/eval-versions.d.ts.map +1 -0
  66. package/dist/routes/eval-versions.js +66 -0
  67. package/dist/routes/eval-versions.js.map +1 -0
  68. package/dist/routes/flows.d.ts.map +1 -1
  69. package/dist/routes/flows.js +4 -0
  70. package/dist/routes/flows.js.map +1 -1
  71. package/dist/routes/guardrails.d.ts.map +1 -1
  72. package/dist/routes/guardrails.js +4 -0
  73. package/dist/routes/guardrails.js.map +1 -1
  74. package/dist/routes/hierarchy-errors.d.ts +10 -0
  75. package/dist/routes/hierarchy-errors.d.ts.map +1 -1
  76. package/dist/routes/hierarchy-errors.js +8 -0
  77. package/dist/routes/hierarchy-errors.js.map +1 -1
  78. package/dist/routes/projects.d.ts.map +1 -1
  79. package/dist/routes/projects.js +13 -7
  80. package/dist/routes/projects.js.map +1 -1
  81. package/dist/routes/runs.js +1 -0
  82. package/dist/routes/runs.js.map +1 -1
  83. package/dist/routes/teams.d.ts.map +1 -1
  84. package/dist/routes/teams.js +12 -6
  85. package/dist/routes/teams.js.map +1 -1
  86. package/dist/routes/tools.d.ts.map +1 -1
  87. package/dist/routes/tools.js +4 -0
  88. package/dist/routes/tools.js.map +1 -1
  89. package/dist/tool-binding.d.ts +8 -0
  90. package/dist/tool-binding.d.ts.map +1 -1
  91. package/openapi.json +729 -17
  92. package/package.json +21 -21
  93. package/src/agent-binding.ts +9 -0
  94. package/src/app.ts +10 -1
  95. package/src/deploy-versions.ts +1 -1
  96. package/src/derive-agent-version.ts +12 -1
  97. package/src/errors.ts +1 -0
  98. package/src/eval-run-binding.ts +7 -0
  99. package/src/eval-run-dispatcher.ts +3 -0
  100. package/src/flow-binding.ts +9 -0
  101. package/src/guardrail-binding.ts +9 -0
  102. package/src/handler-binding.ts +3 -0
  103. package/src/index.ts +1 -0
  104. package/src/judged-dispatcher.ts +27 -4
  105. package/src/openapi/operations.ts +56 -6
  106. package/src/openapi/schemas.ts +361 -2
  107. package/src/registry-read-only.ts +43 -0
  108. package/src/routes/agents.ts +10 -1
  109. package/src/routes/blocks.ts +39 -14
  110. package/src/routes/deployments.ts +25 -0
  111. package/src/routes/eval-comparison.ts +37 -3
  112. package/src/routes/eval-runs.ts +46 -3
  113. package/src/routes/eval-versions.ts +110 -0
  114. package/src/routes/flows.ts +7 -0
  115. package/src/routes/guardrails.ts +7 -0
  116. package/src/routes/hierarchy-errors.ts +9 -0
  117. package/src/routes/projects.ts +18 -7
  118. package/src/routes/runs.ts +1 -0
  119. package/src/routes/teams.ts +14 -6
  120. package/src/routes/tools.ts +7 -0
  121. package/src/tool-binding.ts +9 -0
@@ -194,6 +194,7 @@ export const RunSchema: JsonSchema = {
194
194
  type: 'string',
195
195
  description: 'Set on a replay run: the eval run that started it.',
196
196
  },
197
+ versions: { $ref: '#/components/schemas/FlowVersionOverrides' },
197
198
  publicAccessToken: {
198
199
  type: 'string',
199
200
  description:
@@ -1287,6 +1288,21 @@ export const FlowPinsSchema: JsonSchema = {
1287
1288
  },
1288
1289
  };
1289
1290
 
1291
+ export const FlowVersionOverridesSchema: JsonSchema = {
1292
+ description:
1293
+ "Agents and tools a flow runs at other exact versions than the flow version's pins (\"this flow, with `acme.scorer` at 0.4.0\"), without publishing a new flow version: a comparison's flow candidate (`versions` on the start body and the run's `comparison`), and the flow runs that replay it (a run's `versions`). Each id must be an agent or tool the flow uses.",
1294
+ type: 'object',
1295
+ additionalProperties: false,
1296
+ properties: {
1297
+ tools: { ...PinMapSchema, description: 'Tool id → exact version.' },
1298
+ agents: {
1299
+ ...PinMapSchema,
1300
+ description:
1301
+ 'Agent id → exact version, for agent nodes with or without a version of their own.',
1302
+ },
1303
+ },
1304
+ };
1305
+
1290
1306
  export const PublishFlowBodySchema: JsonSchema = {
1291
1307
  description:
1292
1308
  'Full flow definition. Validated server-side via `@kindgi/flow.loadFlow` — validation failures return `400 validation-failed` with the issue list under `details.issues`.',
@@ -4766,6 +4782,342 @@ export const EvalComparisonSchema: JsonSchema = {
4766
4782
  },
4767
4783
  repetitions: { type: 'integer', minimum: 1, maximum: 10 },
4768
4784
  k: { type: 'integer', minimum: 1, maximum: 100 },
4785
+ versions: { $ref: '#/components/schemas/FlowVersionOverrides' },
4786
+ },
4787
+ };
4788
+
4789
+ // ---------------- comparison results (a `judged` eval run's `result`) ----------------
4790
+
4791
+ const nullableNumber = { type: ['number', 'null'] } as const;
4792
+
4793
+ export const ComparisonMetricSchema: JsonSchema = {
4794
+ description:
4795
+ "One metric, the recorded runs beside the candidate. `null` where a side had no judged evidence; `n` / `weight` are the candidate's evidence (cases with judged items, and the judgment weight behind them), `baselineN` / `baselineWeight` the recorded side's.",
4796
+ type: 'object',
4797
+ additionalProperties: false,
4798
+ required: [
4799
+ 'baseline',
4800
+ 'candidate',
4801
+ 'delta',
4802
+ 'n',
4803
+ 'weight',
4804
+ 'baselineN',
4805
+ 'baselineWeight',
4806
+ 'direction',
4807
+ ],
4808
+ properties: {
4809
+ baseline: nullableNumber,
4810
+ candidate: nullableNumber,
4811
+ delta: nullableNumber,
4812
+ n: { type: 'integer', minimum: 0 },
4813
+ weight: { type: 'number', minimum: 0 },
4814
+ baselineN: { type: 'integer', minimum: 0 },
4815
+ baselineWeight: { type: 'number', minimum: 0 },
4816
+ direction: { type: 'string', enum: ['higher'] },
4817
+ k: {
4818
+ type: 'integer',
4819
+ minimum: 1,
4820
+ description: '`weightedPrecisionAtK`: the ranked items it looked at.',
4821
+ },
4822
+ spread: {
4823
+ type: 'number',
4824
+ description: "With more than one repetition: the candidate's max − min across them.",
4825
+ },
4826
+ },
4827
+ };
4828
+
4829
+ export const ComparisonCandidateSchema: JsonSchema = {
4830
+ description:
4831
+ 'What ran on the cases: an agent version, or a flow version (with any versions it swapped in).',
4832
+ oneOf: [
4833
+ {
4834
+ type: 'object',
4835
+ additionalProperties: false,
4836
+ required: ['kind', 'agentId', 'version'],
4837
+ properties: {
4838
+ kind: { type: 'string', enum: ['agent'] },
4839
+ agentId: { type: 'string' },
4840
+ version: { type: 'string' },
4841
+ },
4842
+ },
4843
+ {
4844
+ type: 'object',
4845
+ additionalProperties: false,
4846
+ required: ['kind', 'flowId', 'version'],
4847
+ properties: {
4848
+ kind: { type: 'string', enum: ['flow'] },
4849
+ flowId: { type: 'string' },
4850
+ version: { type: 'string' },
4851
+ versions: { $ref: '#/components/schemas/FlowVersionOverrides' },
4852
+ },
4853
+ },
4854
+ ],
4855
+ };
4856
+
4857
+ export const JudgedComparisonSummarySchema: JsonSchema = {
4858
+ description:
4859
+ 'What a comparison concluded: the candidate beside the recorded runs, the case counts, and the metrics. What a promotion gate reads.',
4860
+ type: 'object',
4861
+ additionalProperties: false,
4862
+ required: [
4863
+ 'evalRunId',
4864
+ 'status',
4865
+ 'completedAt',
4866
+ 'suite',
4867
+ 'candidate',
4868
+ 'baseline',
4869
+ 'scope',
4870
+ 'cases',
4871
+ 'diverged',
4872
+ 'refusedWrites',
4873
+ 'errors',
4874
+ 'stopped',
4875
+ 'reads',
4876
+ 'sampling',
4877
+ 'repetitions',
4878
+ 'metrics',
4879
+ ],
4880
+ properties: {
4881
+ evalRunId: { type: 'string' },
4882
+ status: { type: 'string', enum: ['completed', 'partial', 'failed'] },
4883
+ completedAt: { type: 'string', format: 'date-time' },
4884
+ suite: {
4885
+ type: 'object',
4886
+ additionalProperties: false,
4887
+ required: ['id', 'version'],
4888
+ properties: { id: { type: 'string' }, version: { type: 'string' } },
4889
+ },
4890
+ candidate: { $ref: '#/components/schemas/ComparisonCandidate' },
4891
+ baseline: {
4892
+ description:
4893
+ "What the candidate was compared with: `recorded` (the test set's recorded runs, with the versions that served them), or another version.",
4894
+ oneOf: [
4895
+ {
4896
+ type: 'object',
4897
+ additionalProperties: false,
4898
+ required: ['kind', 'versions'],
4899
+ properties: {
4900
+ kind: { type: 'string', enum: ['recorded'] },
4901
+ versions: {
4902
+ type: 'array',
4903
+ items: {
4904
+ type: 'object',
4905
+ additionalProperties: false,
4906
+ required: ['version', 'cases'],
4907
+ properties: {
4908
+ agentId: { type: 'string' },
4909
+ flowId: { type: 'string' },
4910
+ version: { type: 'string' },
4911
+ cases: { type: 'integer', minimum: 0 },
4912
+ },
4913
+ },
4914
+ },
4915
+ },
4916
+ },
4917
+ {
4918
+ type: 'object',
4919
+ additionalProperties: false,
4920
+ required: ['kind', 'agentId', 'version', 'via'],
4921
+ properties: {
4922
+ kind: { type: 'string', enum: ['version'] },
4923
+ agentId: { type: 'string' },
4924
+ version: { type: 'string' },
4925
+ via: { type: 'string', enum: ['explicit', 'live'] },
4926
+ liveScope: { type: 'object', additionalProperties: true },
4927
+ },
4928
+ },
4929
+ ],
4930
+ },
4931
+ scope: {
4932
+ type: 'object',
4933
+ additionalProperties: false,
4934
+ description: "Where the test set's judgments came from.",
4935
+ properties: { projectId: { type: 'string' } },
4936
+ },
4937
+ cases: { type: 'integer', minimum: 0 },
4938
+ diverged: {
4939
+ type: 'integer',
4940
+ minimum: 0,
4941
+ description: "Cases where a read with no recording ran live under `reads: 'recorded'`.",
4942
+ },
4943
+ refusedWrites: {
4944
+ type: 'integer',
4945
+ minimum: 0,
4946
+ description: 'Tool calls refused across the cases (what the candidate would have done).',
4947
+ },
4948
+ errors: { type: 'integer', minimum: 0, description: 'Cases none of whose repetitions ran.' },
4949
+ stopped: {
4950
+ type: 'integer',
4951
+ minimum: 0,
4952
+ description:
4953
+ "Flow cases that stopped at a write the replay refused: no output to score, so they're left out of the metrics.",
4954
+ },
4955
+ reads: { type: 'string', enum: ['recorded', 'live'] },
4956
+ sampling: {
4957
+ type: 'object',
4958
+ additionalProperties: false,
4959
+ required: ['models'],
4960
+ properties: {
4961
+ models: {
4962
+ type: 'array',
4963
+ description:
4964
+ "The models that answered the candidate's replays, and how many replays each.",
4965
+ items: {
4966
+ type: 'object',
4967
+ additionalProperties: false,
4968
+ required: ['providerId', 'model', 'runs'],
4969
+ properties: {
4970
+ providerId: { type: 'string' },
4971
+ model: { type: 'string' },
4972
+ runs: { type: 'integer', minimum: 0 },
4973
+ },
4974
+ },
4975
+ },
4976
+ },
4977
+ },
4978
+ repetitions: { type: 'integer', minimum: 1 },
4979
+ metrics: {
4980
+ type: 'object',
4981
+ additionalProperties: false,
4982
+ required: ['weightedYesShare', 'judgedCoverage', 'weightedPrecisionAtK'],
4983
+ properties: {
4984
+ weightedYesShare: { $ref: '#/components/schemas/ComparisonMetric' },
4985
+ judgedCoverage: { $ref: '#/components/schemas/ComparisonMetric' },
4986
+ weightedPrecisionAtK: { $ref: '#/components/schemas/ComparisonMetric' },
4987
+ },
4988
+ },
4989
+ },
4990
+ };
4991
+
4992
+ const outputScore = {
4993
+ type: 'object',
4994
+ additionalProperties: false,
4995
+ required: ['yesWeight', 'totalWeight', 'items', 'judgedItems', 'topK'],
4996
+ description:
4997
+ "An output's score: Σ yesWeight and Σ totalWeight over its judged items, and over those among the first `k` ranked items.",
4998
+ properties: {
4999
+ yesWeight: { type: 'number' },
5000
+ totalWeight: { type: 'number' },
5001
+ items: { type: 'integer', minimum: 0 },
5002
+ judgedItems: { type: 'integer', minimum: 0 },
5003
+ topK: {
5004
+ type: 'object',
5005
+ additionalProperties: false,
5006
+ required: ['yesWeight', 'totalWeight'],
5007
+ properties: { yesWeight: { type: 'number' }, totalWeight: { type: 'number' } },
5008
+ },
5009
+ },
5010
+ } as const;
5011
+
5012
+ export const ComparisonCaseResultSchema: JsonSchema = {
5013
+ description:
5014
+ "One case of a comparison: its replay runs, the scores, the items kept, dropped and new, the tool calls, and why it didn't run when it didn't.",
5015
+ type: 'object',
5016
+ additionalProperties: false,
5017
+ required: [
5018
+ 'caseId',
5019
+ 'runIds',
5020
+ 'baseline',
5021
+ 'candidate',
5022
+ 'diverged',
5023
+ 'refusedWrites',
5024
+ 'noContext',
5025
+ 'approvalSkipped',
5026
+ ],
5027
+ properties: {
5028
+ caseId: { type: 'string' },
5029
+ runIds: {
5030
+ type: 'array',
5031
+ items: { type: 'string' },
5032
+ description: "The candidate's replay runs, one per repetition.",
5033
+ },
5034
+ baseline: outputScore,
5035
+ candidate: { type: 'array', items: outputScore, description: 'One per repetition that ran.' },
5036
+ changes: {
5037
+ type: 'object',
5038
+ additionalProperties: false,
5039
+ required: ['kept', 'dropped', 'new'],
5040
+ description: "The first repetition's items against the judged ones.",
5041
+ properties: {
5042
+ kept: {
5043
+ type: 'array',
5044
+ items: {
5045
+ type: 'object',
5046
+ additionalProperties: false,
5047
+ required: ['key'],
5048
+ properties: {
5049
+ key: { type: 'string' },
5050
+ rankBefore: { type: 'integer' },
5051
+ rank: { type: 'integer' },
5052
+ },
5053
+ },
5054
+ },
5055
+ dropped: {
5056
+ type: 'array',
5057
+ items: {
5058
+ type: 'object',
5059
+ additionalProperties: false,
5060
+ required: ['key'],
5061
+ properties: { key: { type: 'string' }, rankBefore: { type: 'integer' } },
5062
+ },
5063
+ },
5064
+ new: {
5065
+ type: 'array',
5066
+ items: {
5067
+ type: 'object',
5068
+ additionalProperties: false,
5069
+ required: ['key', 'pointer'],
5070
+ properties: {
5071
+ key: { type: 'string' },
5072
+ pointer: { type: 'string' },
5073
+ rank: { type: 'integer' },
5074
+ },
5075
+ },
5076
+ },
5077
+ },
5078
+ },
5079
+ tools: {
5080
+ type: 'array',
5081
+ description: "The first repetition's tool calls, and what happened to each.",
5082
+ items: {
5083
+ type: 'object',
5084
+ additionalProperties: false,
5085
+ required: ['step', 'callId', 'toolId', 'toolVersion', 'arguments', 'source'],
5086
+ properties: {
5087
+ step: { type: 'integer', minimum: 0 },
5088
+ callId: { type: 'string' },
5089
+ toolId: { type: 'string' },
5090
+ toolVersion: { type: 'string' },
5091
+ arguments: {},
5092
+ source: { type: 'string', enum: ['live', 'recorded', 'refused'] },
5093
+ reason: { type: 'string' },
5094
+ },
5095
+ },
5096
+ },
5097
+ diverged: { type: 'boolean' },
5098
+ refusedWrites: { type: 'integer', minimum: 0 },
5099
+ noContext: { type: 'boolean' },
5100
+ approvalSkipped: { type: 'boolean' },
5101
+ error: { type: 'string' },
5102
+ stopped: {
5103
+ type: 'object',
5104
+ additionalProperties: false,
5105
+ required: ['toolId', 'arguments'],
5106
+ description: 'Set when the replay stopped at a refused write: what it would have done.',
5107
+ properties: { toolId: { type: 'string' }, arguments: {}, reason: { type: 'string' } },
5108
+ },
5109
+ },
5110
+ };
5111
+
5112
+ export const JudgedComparisonResultSchema: JsonSchema = {
5113
+ description:
5114
+ "A comparison's `result` (a `judged` eval run's): the summary and each case. `EvalRun.result` stays an open object, since each kind has its own; the clients read it as this (TS `comparisonOf(run)`, Python `comparison_of(run)`).",
5115
+ type: 'object',
5116
+ additionalProperties: false,
5117
+ required: ['summary', 'perCase'],
5118
+ properties: {
5119
+ summary: { $ref: '#/components/schemas/JudgedComparisonSummary' },
5120
+ perCase: { type: 'array', items: { $ref: '#/components/schemas/ComparisonCaseResult' } },
4769
5121
  },
4770
5122
  };
4771
5123
 
@@ -4798,7 +5150,7 @@ export const EvalRunSchema: JsonSchema = {
4798
5150
  type: 'object',
4799
5151
  additionalProperties: true,
4800
5152
  description:
4801
- 'Kind-specific opaque JSON. For `accuracy`, contains `{ passCount, totalCount, meanScore, perCase[] }`. For `judged` (a comparison), `{ summary, perCase[] }`: the summary has the baseline (the versions behind the recorded runs) and the candidate (`{ kind: "agent", agentId, version }` or `{ kind: "flow", flowId, version }`), the case counts (`cases`, `diverged`, `refusedWrites`, `errors`, and `stopped`: flow cases that stopped at a write the replay refused, left out of the metrics), the models that answered, and `metrics` (`weightedYesShare`, `judgedCoverage`, `weightedPrecisionAtK`, each `{ baseline, candidate, delta, n, weight, baselineN, baselineWeight, direction, k?, spread? }`); each case has its replay runs, the scores, the items kept, dropped and new, the tool calls with what happened to each, and `stopped` (what it would have done) when it stopped. Other kinds define their own shapes as their dispatchers ship.',
5153
+ 'Kind-specific opaque JSON. For `accuracy`, contains `{ passCount, totalCount, meanScore, perCase[] }`. For `judged` (a comparison), `{ summary, perCase[] }`: the summary has the baseline (the versions behind the recorded runs) and the candidate (`{ kind: "agent", agentId, version }` or `{ kind: "flow", flowId, version, versions? }`), the case counts (`cases`, `diverged`, `refusedWrites`, `errors`, and `stopped`: flow cases that stopped at a write the replay refused, left out of the metrics), the models that answered, and `metrics` (`weightedYesShare`, `judgedCoverage`, `weightedPrecisionAtK`, each `{ baseline, candidate, delta, n, weight, baselineN, baselineWeight, direction, k?, spread? }`); each case has its replay runs, the scores, the items kept, dropped and new, the tool calls with what happened to each, and `stopped` (what it would have done) when it stopped. Other kinds define their own shapes as their dispatchers ship.',
4802
5154
  },
4803
5155
  error: { type: 'string' },
4804
5156
  correlationId: { type: 'string' },
@@ -4831,9 +5183,10 @@ export const StartEvalRunBodySchema: JsonSchema = {
4831
5183
  reads: { type: 'string', enum: ['recorded', 'live'] },
4832
5184
  repetitions: { type: 'integer', minimum: 1, maximum: 10 },
4833
5185
  k: { type: 'integer', minimum: 1, maximum: 100 },
5186
+ versions: { $ref: '#/components/schemas/FlowVersionOverrides' },
4834
5187
  },
4835
5188
  description:
4836
- "Exactly one of `agentRef` or `flowRef` MUST be supplied. `dryRun: true` returns a plan preview without invoking the subject. For a `judged` suite (a test set), the run is a comparison: `agentRef` or `flowRef` with its `version` is the candidate, replayed on each case without doing anything the past run didn't (a flow stops at a write the replay refuses); `baseline` (default `'recorded'`), `reads` (default `recorded`), `repetitions` (default 1) and `k` (default 10) set how.",
5189
+ "Exactly one of `agentRef` or `flowRef` MUST be supplied. `dryRun: true` returns a plan preview without invoking the subject. For a `judged` suite (a test set), the run is a comparison: `agentRef` or `flowRef` with its `version` is the candidate, replayed on each case without doing anything the past run didn't (a flow stops at a write the replay refuses); `baseline` (default `'recorded'`), `reads` (default `recorded`), `repetitions` (default 1) and `k` (default 10) set how. With `flowRef`, `versions` runs the flow with some of its agents or tools at other versions; an id the flow doesn't use, or a version that isn't published, is refused (`400 validation-failed`, each under `details.issues`).",
4837
5190
  };
4838
5191
 
4839
5192
  export const StartEvalRunResultSchema: JsonSchema = {
@@ -7189,6 +7542,7 @@ export const COMPONENT_SCHEMAS: ReadonlyArray<readonly [string, JsonSchema]> = [
7189
7542
  ['DeriveAgentVersionBody', DeriveAgentVersionBodySchema],
7190
7543
  ['AgentPinSwaps', AgentPinSwapsSchema],
7191
7544
  ['FlowPins', FlowPinsSchema],
7545
+ ['FlowVersionOverrides', FlowVersionOverridesSchema],
7192
7546
  ['PublishAgentBody', PublishAgentBodySchema],
7193
7547
  ['PublishAgentResult', PublishAgentResultSchema],
7194
7548
  ['UnregisterAgentResult', UnregisterAgentResultSchema],
@@ -7352,6 +7706,11 @@ export const COMPONENT_SCHEMAS: ReadonlyArray<readonly [string, JsonSchema]> = [
7352
7706
  ['EvalRunFlowRef', EvalRunFlowRefSchema],
7353
7707
  ['EvalBaseline', EvalBaselineSchema],
7354
7708
  ['EvalComparison', EvalComparisonSchema],
7709
+ ['ComparisonMetric', ComparisonMetricSchema],
7710
+ ['ComparisonCandidate', ComparisonCandidateSchema],
7711
+ ['JudgedComparisonSummary', JudgedComparisonSummarySchema],
7712
+ ['ComparisonCaseResult', ComparisonCaseResultSchema],
7713
+ ['JudgedComparisonResult', JudgedComparisonResultSchema],
7355
7714
  ['EvalRun', EvalRunSchema],
7356
7715
  ['EvalRunCollectionPage', EvalRunCollectionPageSchema],
7357
7716
  ['StartEvalRunBody', StartEvalRunBodySchema],
@@ -0,0 +1,43 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // Copyright (C) 2026 Kindgi Inc.
3
+
4
+ import type { Context, MiddlewareHandler } from 'hono';
5
+
6
+ import { statusFor, toWireError } from './errors.js';
7
+ import type { AppEnv } from './types.js';
8
+
9
+ /**
10
+ * A registry that takes no writes: it serves its definitions from
11
+ * somewhere else (under `kindgi dev`, the pack's files). A binding that
12
+ * sets it has every write refused before the binding is called:
13
+ * publish, unregister and reinstate (and, for agents, deriving a
14
+ * version), and a deploy that would publish into it. The refusal is
15
+ * `409 registry-read-only` with `reason` as its message.
16
+ */
17
+ export interface RegistryReadOnly {
18
+ /** Why, and what to do instead, in plain words: the refusal's message. */
19
+ readonly reason: string;
20
+ }
21
+
22
+ /** The refusal of a write to a read-only registry. */
23
+ export function refuseReadOnly(c: Context<AppEnv>, readOnly: RegistryReadOnly) {
24
+ c.status(statusFor('registry-read-only') as never);
25
+ return c.json(
26
+ toWireError({ code: 'registry-read-only', message: readOnly.reason }, c.get('requestId')),
27
+ );
28
+ }
29
+
30
+ /**
31
+ * A router's writes (every method but GET and HEAD) are refused while
32
+ * its registry is read-only. Read at request time, so a binding can
33
+ * become writable without a restart.
34
+ */
35
+ export function refuseWritesWhenReadOnly(
36
+ readOnly: () => RegistryReadOnly | undefined,
37
+ ): MiddlewareHandler<AppEnv> {
38
+ return async (c, next) => {
39
+ const marker = readOnly();
40
+ if (marker === undefined || c.req.method === 'GET' || c.req.method === 'HEAD') return next();
41
+ return refuseReadOnly(c, marker);
42
+ };
43
+ }
@@ -24,6 +24,7 @@ import {
24
24
  } from '../derive-agent-version.js';
25
25
  import { statusFor, toWireError } from '../errors.js';
26
26
  import type { Authorizer } from '../middleware/authorize.js';
27
+ import { refuseWritesWhenReadOnly } from '../registry-read-only.js';
27
28
  import type { ToolRegistryBinding } from '../tool-binding.js';
28
29
  import type { AppEnv } from '../types.js';
29
30
  import { clampLimit } from './pagination.js';
@@ -53,6 +54,12 @@ export function agentsRouter(
53
54
  blockRegistry?: BlockRegistryBinding,
54
55
  ): Hono<AppEnv> {
55
56
  const r = new Hono<AppEnv>();
57
+ // A read-only registry (under `kindgi dev`, the pack's files) refuses
58
+ // every write before anything else runs.
59
+ r.use(
60
+ '*',
61
+ refuseWritesWhenReadOnly(() => binding.readOnly),
62
+ );
56
63
 
57
64
  // Authorization — check the resource directly (ref('agent', businessId));
58
65
  // permissions cascade from the parent project because the
@@ -323,7 +330,7 @@ export function agentsRouter(
323
330
  toWireError(
324
331
  {
325
332
  code: 'validation-failed',
326
- message: `Agent "${defined.value.id as unknown as string}" uses tool versions that aren't published (${resolved.issues.length} issue${resolved.issues.length === 1 ? '' : 's'})`,
333
+ message: `Agent "${defined.value.id as unknown as string}" uses tool or data-block versions it can't pin (${resolved.issues.length} issue${resolved.issues.length === 1 ? '' : 's'})`,
327
334
  issues: resolved.issues as unknown as Record<string, unknown>[],
328
335
  },
329
336
  requestId,
@@ -592,6 +599,8 @@ function derived(
592
599
  case 'ok':
593
600
  c.status(201);
594
601
  return c.json(serializeAgent(outcome.agent));
602
+ case 'reused':
603
+ return c.json(serializeAgent(outcome.agent));
595
604
  case 'not-found':
596
605
  return fail('agent-not-found', {
597
606
  code: 'agent-not-found',
@@ -184,14 +184,20 @@ export function blocksRouter(binding: BlockRegistryBinding, authorizer?: Authori
184
184
  return validationFailed(c, validated.error.message, validated.error.issues);
185
185
  const block = validated.value;
186
186
 
187
- // A block keeps its kind, and a settings block's latest schema holds.
187
+ // A block keeps its kind. A settings version without a schema keeps
188
+ // the latest version's (stored on it, so the check carries forward to
189
+ // every later version); one that gives a schema replaces it.
188
190
  const latest = await binding.get({ tenantId, blockId: block.id });
189
191
  const continuity = continuityIssues(latest, block);
190
192
  if (continuity.length > 0) {
191
193
  return validationFailed(c, `Block "${block.id}" can't take this version`, continuity);
192
194
  }
193
195
 
194
- const outcome = await binding.publish({ tenantId, projectId: projectId as ProjectId, block });
196
+ const outcome = await binding.publish({
197
+ tenantId,
198
+ projectId: projectId as ProjectId,
199
+ block: withCarriedSchema(latest, block),
200
+ });
195
201
  return published(c, outcome);
196
202
  });
197
203
 
@@ -243,7 +249,11 @@ function isBlockKind(value: string): value is BlockKind {
243
249
  return (BLOCK_KINDS as readonly string[]).includes(value);
244
250
  }
245
251
 
246
- /** A new version against the block's latest: same kind, and a settings block's schema holds. */
252
+ /**
253
+ * A new version against the block's latest: the same kind, and a
254
+ * settings version without a schema of its own satisfies the schema it
255
+ * keeps (`carriedSchema`).
256
+ */
247
257
  function continuityIssues(
248
258
  latest: BlockRecord | null,
249
259
  block: BlockDefinition,
@@ -257,17 +267,32 @@ function continuityIssues(
257
267
  },
258
268
  ];
259
269
  }
260
- if (
261
- latest.kind === 'settings' &&
262
- block.kind === 'settings' &&
263
- latest.content.schema !== undefined
264
- ) {
265
- return settingsSchemaIssues(block.content.values, latest.content.schema).map((i) => ({
266
- ...i,
267
- message: `${i.message} (the schema of version ${latest.version})`,
268
- }));
269
- }
270
- return [];
270
+ const carried = carriedSchema(latest, block);
271
+ if (block.kind !== 'settings' || carried === undefined) return [];
272
+ return settingsSchemaIssues(block.content.values, carried).map((i) => ({
273
+ ...i,
274
+ message: `${i.message} (the schema of version ${latest.version})`,
275
+ }));
276
+ }
277
+
278
+ /**
279
+ * The schema a settings version that gives none keeps: the latest
280
+ * version's. A version that gives a schema replaces it (`{}` drops the
281
+ * check on purpose).
282
+ */
283
+ function carriedSchema(
284
+ latest: BlockRecord | null,
285
+ block: BlockDefinition,
286
+ ): Readonly<Record<string, unknown>> | undefined {
287
+ if (latest?.kind !== 'settings' || block.kind !== 'settings') return undefined;
288
+ return block.content.schema === undefined ? latest.content.schema : undefined;
289
+ }
290
+
291
+ /** The version as stored: with the schema it keeps, if any. */
292
+ function withCarriedSchema(latest: BlockRecord | null, block: BlockDefinition): BlockDefinition {
293
+ const schema = carriedSchema(latest, block);
294
+ if (schema === undefined || block.kind !== 'settings') return block;
295
+ return { ...block, content: { ...block.content, schema } };
271
296
  }
272
297
 
273
298
  /** The response to a publish outcome. */
@@ -41,6 +41,7 @@ import type { FlowRegistryBinding } from '../flow-binding.js';
41
41
  import { publishDeployedFlow, resolveFlowPins } from '../flow-pins.js';
42
42
  import type { GuardrailRegistryBinding } from '../guardrail-binding.js';
43
43
  import type { ImageRegistryBinding } from '../image-registry-binding.js';
44
+ import { type RegistryReadOnly, refuseReadOnly } from '../registry-read-only.js';
44
45
  import type { SecretBinding } from '../secrets-binding.js';
45
46
  import type { SigningKeyBinding as SigningKeyRegistryBinding } from '../signing-key-binding.js';
46
47
  import type { ToolRegistryBinding } from '../tool-binding.js';
@@ -487,6 +488,11 @@ export function deploymentsRouter(bindings: DeploymentsRouterBindings): Hono<App
487
488
  }
488
489
  const validated = validation.value;
489
490
 
491
+ // A read-only registry (under `kindgi dev`, the pack's files) takes
492
+ // nothing a deployment brings: refuse before any write.
493
+ const readOnly = readOnlyTarget(validated, bindings);
494
+ if (readOnly !== undefined) return refuseReadOnly(c, readOnly);
495
+
490
496
  // ---- Registry upserts (with rollback tracking) ----
491
497
  //
492
498
  // Agent-registry, flow-registry, tool-registry, and
@@ -1701,3 +1707,22 @@ export function sha256HexPrefixed(bytes: Uint8Array): string {
1701
1707
  const hex = createHash('sha256').update(bytes).digest('hex');
1702
1708
  return `sha256:${hex}`;
1703
1709
  }
1710
+
1711
+ /** The first read-only registry a deployment would publish into, if any. */
1712
+ function readOnlyTarget(
1713
+ validated: {
1714
+ readonly tools: readonly unknown[];
1715
+ readonly agents: readonly unknown[];
1716
+ readonly flows: readonly unknown[];
1717
+ readonly guardrails: readonly unknown[];
1718
+ },
1719
+ bindings: DeploymentsRouterBindings,
1720
+ ): RegistryReadOnly | undefined {
1721
+ const targets = [
1722
+ [validated.tools, bindings.toolRegistry?.readOnly],
1723
+ [validated.agents, bindings.agentRegistry?.readOnly],
1724
+ [validated.flows, bindings.flowRegistry?.readOnly],
1725
+ [validated.guardrails, bindings.guardrailRegistry?.readOnly],
1726
+ ] as const;
1727
+ return targets.find(([brought, readOnly]) => brought.length > 0 && readOnly !== undefined)?.[1];
1728
+ }