@kindgi/api 0.1.4-rc.0 → 0.1.4-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/agent-binding.d.ts +8 -0
  2. package/dist/agent-binding.d.ts.map +1 -1
  3. package/dist/app.d.ts.map +1 -1
  4. package/dist/app.js +7 -1
  5. package/dist/app.js.map +1 -1
  6. package/dist/deploy-versions.d.ts +2 -0
  7. package/dist/deploy-versions.d.ts.map +1 -1
  8. package/dist/deploy-versions.js +1 -1
  9. package/dist/deploy-versions.js.map +1 -1
  10. package/dist/derive-agent-version.d.ts +9 -1
  11. package/dist/derive-agent-version.d.ts.map +1 -1
  12. package/dist/derive-agent-version.js +11 -1
  13. package/dist/derive-agent-version.js.map +1 -1
  14. package/dist/errors.d.ts.map +1 -1
  15. package/dist/errors.js +1 -0
  16. package/dist/errors.js.map +1 -1
  17. package/dist/eval-run-binding.d.ts +7 -0
  18. package/dist/eval-run-binding.d.ts.map +1 -1
  19. package/dist/eval-run-binding.js.map +1 -1
  20. package/dist/eval-run-dispatcher.d.ts +3 -0
  21. package/dist/eval-run-dispatcher.d.ts.map +1 -1
  22. package/dist/eval-run-dispatcher.js.map +1 -1
  23. package/dist/flow-binding.d.ts +8 -0
  24. package/dist/flow-binding.d.ts.map +1 -1
  25. package/dist/guardrail-binding.d.ts +8 -0
  26. package/dist/guardrail-binding.d.ts.map +1 -1
  27. package/dist/handler-binding.d.ts +3 -0
  28. package/dist/handler-binding.d.ts.map +1 -1
  29. package/dist/index.d.ts +1 -0
  30. package/dist/index.d.ts.map +1 -1
  31. package/dist/index.js.map +1 -1
  32. package/dist/judged-dispatcher.d.ts +4 -0
  33. package/dist/judged-dispatcher.d.ts.map +1 -1
  34. package/dist/judged-dispatcher.js +14 -3
  35. package/dist/judged-dispatcher.js.map +1 -1
  36. package/dist/openapi/operations.d.ts.map +1 -1
  37. package/dist/openapi/operations.js +21 -6
  38. package/dist/openapi/operations.js.map +1 -1
  39. package/dist/openapi/schemas.d.ts +6 -0
  40. package/dist/openapi/schemas.d.ts.map +1 -1
  41. package/dist/openapi/schemas.js +341 -2
  42. package/dist/openapi/schemas.js.map +1 -1
  43. package/dist/registry-read-only.d.ts +32 -0
  44. package/dist/registry-read-only.d.ts.map +1 -0
  45. package/dist/registry-read-only.js +22 -0
  46. package/dist/registry-read-only.js.map +1 -0
  47. package/dist/routes/agents.d.ts.map +1 -1
  48. package/dist/routes/agents.js +7 -1
  49. package/dist/routes/agents.js.map +1 -1
  50. package/dist/routes/blocks.d.ts.map +1 -1
  51. package/dist/routes/blocks.js +37 -12
  52. package/dist/routes/blocks.js.map +1 -1
  53. package/dist/routes/deployments.d.ts.map +1 -1
  54. package/dist/routes/deployments.js +16 -0
  55. package/dist/routes/deployments.js.map +1 -1
  56. package/dist/routes/eval-comparison.d.ts +3 -2
  57. package/dist/routes/eval-comparison.d.ts.map +1 -1
  58. package/dist/routes/eval-comparison.js +39 -3
  59. package/dist/routes/eval-comparison.js.map +1 -1
  60. package/dist/routes/eval-runs.d.ts +7 -1
  61. package/dist/routes/eval-runs.d.ts.map +1 -1
  62. package/dist/routes/eval-runs.js +34 -3
  63. package/dist/routes/eval-runs.js.map +1 -1
  64. package/dist/routes/eval-versions.d.ts +25 -0
  65. package/dist/routes/eval-versions.d.ts.map +1 -0
  66. package/dist/routes/eval-versions.js +66 -0
  67. package/dist/routes/eval-versions.js.map +1 -0
  68. package/dist/routes/flows.d.ts.map +1 -1
  69. package/dist/routes/flows.js +4 -0
  70. package/dist/routes/flows.js.map +1 -1
  71. package/dist/routes/guardrails.d.ts.map +1 -1
  72. package/dist/routes/guardrails.js +4 -0
  73. package/dist/routes/guardrails.js.map +1 -1
  74. package/dist/routes/hierarchy-errors.d.ts +10 -0
  75. package/dist/routes/hierarchy-errors.d.ts.map +1 -1
  76. package/dist/routes/hierarchy-errors.js +8 -0
  77. package/dist/routes/hierarchy-errors.js.map +1 -1
  78. package/dist/routes/projects.d.ts.map +1 -1
  79. package/dist/routes/projects.js +13 -7
  80. package/dist/routes/projects.js.map +1 -1
  81. package/dist/routes/runs.js +1 -0
  82. package/dist/routes/runs.js.map +1 -1
  83. package/dist/routes/teams.d.ts.map +1 -1
  84. package/dist/routes/teams.js +12 -6
  85. package/dist/routes/teams.js.map +1 -1
  86. package/dist/routes/tools.d.ts.map +1 -1
  87. package/dist/routes/tools.js +4 -0
  88. package/dist/routes/tools.js.map +1 -1
  89. package/dist/tool-binding.d.ts +8 -0
  90. package/dist/tool-binding.d.ts.map +1 -1
  91. package/openapi.json +729 -17
  92. package/package.json +21 -21
  93. package/src/agent-binding.ts +9 -0
  94. package/src/app.ts +10 -1
  95. package/src/deploy-versions.ts +1 -1
  96. package/src/derive-agent-version.ts +12 -1
  97. package/src/errors.ts +1 -0
  98. package/src/eval-run-binding.ts +7 -0
  99. package/src/eval-run-dispatcher.ts +3 -0
  100. package/src/flow-binding.ts +9 -0
  101. package/src/guardrail-binding.ts +9 -0
  102. package/src/handler-binding.ts +3 -0
  103. package/src/index.ts +1 -0
  104. package/src/judged-dispatcher.ts +27 -4
  105. package/src/openapi/operations.ts +56 -6
  106. package/src/openapi/schemas.ts +361 -2
  107. package/src/registry-read-only.ts +43 -0
  108. package/src/routes/agents.ts +10 -1
  109. package/src/routes/blocks.ts +39 -14
  110. package/src/routes/deployments.ts +25 -0
  111. package/src/routes/eval-comparison.ts +37 -3
  112. package/src/routes/eval-runs.ts +46 -3
  113. package/src/routes/eval-versions.ts +110 -0
  114. package/src/routes/flows.ts +7 -0
  115. package/src/routes/guardrails.ts +7 -0
  116. package/src/routes/hierarchy-errors.ts +9 -0
  117. package/src/routes/projects.ts +18 -7
  118. package/src/routes/runs.ts +1 -0
  119. package/src/routes/teams.ts +14 -6
  120. package/src/routes/tools.ts +7 -0
  121. package/src/tool-binding.ts +9 -0
package/openapi.json CHANGED
@@ -347,6 +347,9 @@
347
347
  "type": "string",
348
348
  "description": "Set on a replay run: the eval run that started it."
349
349
  },
350
+ "versions": {
351
+ "$ref": "#/components/schemas/FlowVersionOverrides"
352
+ },
350
353
  "publicAccessToken": {
351
354
  "type": "string",
352
355
  "description": "Only in the response to `POST /v1/runs`, when the deployment issues public run tokens: a read-only token for this run (and its descendants) to hand to a browser, for `GET /v1/runs/{runId}/progress` and its stream."
@@ -571,6 +574,9 @@
571
574
  "type": "string",
572
575
  "description": "Set on a replay run: the eval run that started it."
573
576
  },
577
+ "versions": {
578
+ "$ref": "#/components/schemas/FlowVersionOverrides"
579
+ },
574
580
  "publicAccessToken": {
575
581
  "type": "string",
576
582
  "description": "Only in the response to `POST /v1/runs`, when the deployment issues public run tokens: a read-only token for this run (and its descendants) to hand to a browser, for `GET /v1/runs/{runId}/progress` and its stream."
@@ -3235,6 +3241,27 @@
3235
3241
  }
3236
3242
  }
3237
3243
  },
3244
+ "FlowVersionOverrides": {
3245
+ "description": "Agents and tools a flow runs at other exact versions than the flow version's pins (\"this flow, with `acme.scorer` at 0.4.0\"), without publishing a new flow version: a comparison's flow candidate (`versions` on the start body and the run's `comparison`), and the flow runs that replay it (a run's `versions`). Each id must be an agent or tool the flow uses.",
3246
+ "type": "object",
3247
+ "additionalProperties": false,
3248
+ "properties": {
3249
+ "tools": {
3250
+ "type": "object",
3251
+ "additionalProperties": {
3252
+ "type": "string"
3253
+ },
3254
+ "description": "Tool id → exact version."
3255
+ },
3256
+ "agents": {
3257
+ "type": "object",
3258
+ "additionalProperties": {
3259
+ "type": "string"
3260
+ },
3261
+ "description": "Agent id → exact version, for agent nodes with or without a version of their own."
3262
+ }
3263
+ }
3264
+ },
3238
3265
  "PublishAgentBody": {
3239
3266
  "description": "Full `defineAgent` spec. Validated server-side via `@kindgi/agents.defineAgent` — validation failures return `400 validation-failed` with the issue list under `details.issues`.",
3240
3267
  "type": "object",
@@ -8635,6 +8662,658 @@
8635
8662
  "type": "integer",
8636
8663
  "minimum": 1,
8637
8664
  "maximum": 100
8665
+ },
8666
+ "versions": {
8667
+ "$ref": "#/components/schemas/FlowVersionOverrides"
8668
+ }
8669
+ }
8670
+ },
8671
+ "ComparisonMetric": {
8672
+ "description": "One metric, the recorded runs beside the candidate. `null` where a side had no judged evidence; `n` / `weight` are the candidate's evidence (cases with judged items, and the judgment weight behind them), `baselineN` / `baselineWeight` the recorded side's.",
8673
+ "type": "object",
8674
+ "additionalProperties": false,
8675
+ "required": [
8676
+ "baseline",
8677
+ "candidate",
8678
+ "delta",
8679
+ "n",
8680
+ "weight",
8681
+ "baselineN",
8682
+ "baselineWeight",
8683
+ "direction"
8684
+ ],
8685
+ "properties": {
8686
+ "baseline": {
8687
+ "type": [
8688
+ "number",
8689
+ "null"
8690
+ ]
8691
+ },
8692
+ "candidate": {
8693
+ "type": [
8694
+ "number",
8695
+ "null"
8696
+ ]
8697
+ },
8698
+ "delta": {
8699
+ "type": [
8700
+ "number",
8701
+ "null"
8702
+ ]
8703
+ },
8704
+ "n": {
8705
+ "type": "integer",
8706
+ "minimum": 0
8707
+ },
8708
+ "weight": {
8709
+ "type": "number",
8710
+ "minimum": 0
8711
+ },
8712
+ "baselineN": {
8713
+ "type": "integer",
8714
+ "minimum": 0
8715
+ },
8716
+ "baselineWeight": {
8717
+ "type": "number",
8718
+ "minimum": 0
8719
+ },
8720
+ "direction": {
8721
+ "type": "string",
8722
+ "enum": [
8723
+ "higher"
8724
+ ]
8725
+ },
8726
+ "k": {
8727
+ "type": "integer",
8728
+ "minimum": 1,
8729
+ "description": "`weightedPrecisionAtK`: the ranked items it looked at."
8730
+ },
8731
+ "spread": {
8732
+ "type": "number",
8733
+ "description": "With more than one repetition: the candidate's max − min across them."
8734
+ }
8735
+ }
8736
+ },
8737
+ "ComparisonCandidate": {
8738
+ "description": "What ran on the cases: an agent version, or a flow version (with any versions it swapped in).",
8739
+ "oneOf": [
8740
+ {
8741
+ "type": "object",
8742
+ "additionalProperties": false,
8743
+ "required": [
8744
+ "kind",
8745
+ "agentId",
8746
+ "version"
8747
+ ],
8748
+ "properties": {
8749
+ "kind": {
8750
+ "type": "string",
8751
+ "enum": [
8752
+ "agent"
8753
+ ]
8754
+ },
8755
+ "agentId": {
8756
+ "type": "string"
8757
+ },
8758
+ "version": {
8759
+ "type": "string"
8760
+ }
8761
+ }
8762
+ },
8763
+ {
8764
+ "type": "object",
8765
+ "additionalProperties": false,
8766
+ "required": [
8767
+ "kind",
8768
+ "flowId",
8769
+ "version"
8770
+ ],
8771
+ "properties": {
8772
+ "kind": {
8773
+ "type": "string",
8774
+ "enum": [
8775
+ "flow"
8776
+ ]
8777
+ },
8778
+ "flowId": {
8779
+ "type": "string"
8780
+ },
8781
+ "version": {
8782
+ "type": "string"
8783
+ },
8784
+ "versions": {
8785
+ "$ref": "#/components/schemas/FlowVersionOverrides"
8786
+ }
8787
+ }
8788
+ }
8789
+ ]
8790
+ },
8791
+ "JudgedComparisonSummary": {
8792
+ "description": "What a comparison concluded: the candidate beside the recorded runs, the case counts, and the metrics. What a promotion gate reads.",
8793
+ "type": "object",
8794
+ "additionalProperties": false,
8795
+ "required": [
8796
+ "evalRunId",
8797
+ "status",
8798
+ "completedAt",
8799
+ "suite",
8800
+ "candidate",
8801
+ "baseline",
8802
+ "scope",
8803
+ "cases",
8804
+ "diverged",
8805
+ "refusedWrites",
8806
+ "errors",
8807
+ "stopped",
8808
+ "reads",
8809
+ "sampling",
8810
+ "repetitions",
8811
+ "metrics"
8812
+ ],
8813
+ "properties": {
8814
+ "evalRunId": {
8815
+ "type": "string"
8816
+ },
8817
+ "status": {
8818
+ "type": "string",
8819
+ "enum": [
8820
+ "completed",
8821
+ "partial",
8822
+ "failed"
8823
+ ]
8824
+ },
8825
+ "completedAt": {
8826
+ "type": "string",
8827
+ "format": "date-time"
8828
+ },
8829
+ "suite": {
8830
+ "type": "object",
8831
+ "additionalProperties": false,
8832
+ "required": [
8833
+ "id",
8834
+ "version"
8835
+ ],
8836
+ "properties": {
8837
+ "id": {
8838
+ "type": "string"
8839
+ },
8840
+ "version": {
8841
+ "type": "string"
8842
+ }
8843
+ }
8844
+ },
8845
+ "candidate": {
8846
+ "$ref": "#/components/schemas/ComparisonCandidate"
8847
+ },
8848
+ "baseline": {
8849
+ "description": "What the candidate was compared with: `recorded` (the test set's recorded runs, with the versions that served them), or another version.",
8850
+ "oneOf": [
8851
+ {
8852
+ "type": "object",
8853
+ "additionalProperties": false,
8854
+ "required": [
8855
+ "kind",
8856
+ "versions"
8857
+ ],
8858
+ "properties": {
8859
+ "kind": {
8860
+ "type": "string",
8861
+ "enum": [
8862
+ "recorded"
8863
+ ]
8864
+ },
8865
+ "versions": {
8866
+ "type": "array",
8867
+ "items": {
8868
+ "type": "object",
8869
+ "additionalProperties": false,
8870
+ "required": [
8871
+ "version",
8872
+ "cases"
8873
+ ],
8874
+ "properties": {
8875
+ "agentId": {
8876
+ "type": "string"
8877
+ },
8878
+ "flowId": {
8879
+ "type": "string"
8880
+ },
8881
+ "version": {
8882
+ "type": "string"
8883
+ },
8884
+ "cases": {
8885
+ "type": "integer",
8886
+ "minimum": 0
8887
+ }
8888
+ }
8889
+ }
8890
+ }
8891
+ }
8892
+ },
8893
+ {
8894
+ "type": "object",
8895
+ "additionalProperties": false,
8896
+ "required": [
8897
+ "kind",
8898
+ "agentId",
8899
+ "version",
8900
+ "via"
8901
+ ],
8902
+ "properties": {
8903
+ "kind": {
8904
+ "type": "string",
8905
+ "enum": [
8906
+ "version"
8907
+ ]
8908
+ },
8909
+ "agentId": {
8910
+ "type": "string"
8911
+ },
8912
+ "version": {
8913
+ "type": "string"
8914
+ },
8915
+ "via": {
8916
+ "type": "string",
8917
+ "enum": [
8918
+ "explicit",
8919
+ "live"
8920
+ ]
8921
+ },
8922
+ "liveScope": {
8923
+ "type": "object",
8924
+ "additionalProperties": true
8925
+ }
8926
+ }
8927
+ }
8928
+ ]
8929
+ },
8930
+ "scope": {
8931
+ "type": "object",
8932
+ "additionalProperties": false,
8933
+ "description": "Where the test set's judgments came from.",
8934
+ "properties": {
8935
+ "projectId": {
8936
+ "type": "string"
8937
+ }
8938
+ }
8939
+ },
8940
+ "cases": {
8941
+ "type": "integer",
8942
+ "minimum": 0
8943
+ },
8944
+ "diverged": {
8945
+ "type": "integer",
8946
+ "minimum": 0,
8947
+ "description": "Cases where a read with no recording ran live under `reads: 'recorded'`."
8948
+ },
8949
+ "refusedWrites": {
8950
+ "type": "integer",
8951
+ "minimum": 0,
8952
+ "description": "Tool calls refused across the cases (what the candidate would have done)."
8953
+ },
8954
+ "errors": {
8955
+ "type": "integer",
8956
+ "minimum": 0,
8957
+ "description": "Cases none of whose repetitions ran."
8958
+ },
8959
+ "stopped": {
8960
+ "type": "integer",
8961
+ "minimum": 0,
8962
+ "description": "Flow cases that stopped at a write the replay refused: no output to score, so they're left out of the metrics."
8963
+ },
8964
+ "reads": {
8965
+ "type": "string",
8966
+ "enum": [
8967
+ "recorded",
8968
+ "live"
8969
+ ]
8970
+ },
8971
+ "sampling": {
8972
+ "type": "object",
8973
+ "additionalProperties": false,
8974
+ "required": [
8975
+ "models"
8976
+ ],
8977
+ "properties": {
8978
+ "models": {
8979
+ "type": "array",
8980
+ "description": "The models that answered the candidate's replays, and how many replays each.",
8981
+ "items": {
8982
+ "type": "object",
8983
+ "additionalProperties": false,
8984
+ "required": [
8985
+ "providerId",
8986
+ "model",
8987
+ "runs"
8988
+ ],
8989
+ "properties": {
8990
+ "providerId": {
8991
+ "type": "string"
8992
+ },
8993
+ "model": {
8994
+ "type": "string"
8995
+ },
8996
+ "runs": {
8997
+ "type": "integer",
8998
+ "minimum": 0
8999
+ }
9000
+ }
9001
+ }
9002
+ }
9003
+ }
9004
+ },
9005
+ "repetitions": {
9006
+ "type": "integer",
9007
+ "minimum": 1
9008
+ },
9009
+ "metrics": {
9010
+ "type": "object",
9011
+ "additionalProperties": false,
9012
+ "required": [
9013
+ "weightedYesShare",
9014
+ "judgedCoverage",
9015
+ "weightedPrecisionAtK"
9016
+ ],
9017
+ "properties": {
9018
+ "weightedYesShare": {
9019
+ "$ref": "#/components/schemas/ComparisonMetric"
9020
+ },
9021
+ "judgedCoverage": {
9022
+ "$ref": "#/components/schemas/ComparisonMetric"
9023
+ },
9024
+ "weightedPrecisionAtK": {
9025
+ "$ref": "#/components/schemas/ComparisonMetric"
9026
+ }
9027
+ }
9028
+ }
9029
+ }
9030
+ },
9031
+ "ComparisonCaseResult": {
9032
+ "description": "One case of a comparison: its replay runs, the scores, the items kept, dropped and new, the tool calls, and why it didn't run when it didn't.",
9033
+ "type": "object",
9034
+ "additionalProperties": false,
9035
+ "required": [
9036
+ "caseId",
9037
+ "runIds",
9038
+ "baseline",
9039
+ "candidate",
9040
+ "diverged",
9041
+ "refusedWrites",
9042
+ "noContext",
9043
+ "approvalSkipped"
9044
+ ],
9045
+ "properties": {
9046
+ "caseId": {
9047
+ "type": "string"
9048
+ },
9049
+ "runIds": {
9050
+ "type": "array",
9051
+ "items": {
9052
+ "type": "string"
9053
+ },
9054
+ "description": "The candidate's replay runs, one per repetition."
9055
+ },
9056
+ "baseline": {
9057
+ "type": "object",
9058
+ "additionalProperties": false,
9059
+ "required": [
9060
+ "yesWeight",
9061
+ "totalWeight",
9062
+ "items",
9063
+ "judgedItems",
9064
+ "topK"
9065
+ ],
9066
+ "description": "An output's score: Σ yesWeight and Σ totalWeight over its judged items, and over those among the first `k` ranked items.",
9067
+ "properties": {
9068
+ "yesWeight": {
9069
+ "type": "number"
9070
+ },
9071
+ "totalWeight": {
9072
+ "type": "number"
9073
+ },
9074
+ "items": {
9075
+ "type": "integer",
9076
+ "minimum": 0
9077
+ },
9078
+ "judgedItems": {
9079
+ "type": "integer",
9080
+ "minimum": 0
9081
+ },
9082
+ "topK": {
9083
+ "type": "object",
9084
+ "additionalProperties": false,
9085
+ "required": [
9086
+ "yesWeight",
9087
+ "totalWeight"
9088
+ ],
9089
+ "properties": {
9090
+ "yesWeight": {
9091
+ "type": "number"
9092
+ },
9093
+ "totalWeight": {
9094
+ "type": "number"
9095
+ }
9096
+ }
9097
+ }
9098
+ }
9099
+ },
9100
+ "candidate": {
9101
+ "type": "array",
9102
+ "items": {
9103
+ "type": "object",
9104
+ "additionalProperties": false,
9105
+ "required": [
9106
+ "yesWeight",
9107
+ "totalWeight",
9108
+ "items",
9109
+ "judgedItems",
9110
+ "topK"
9111
+ ],
9112
+ "description": "An output's score: Σ yesWeight and Σ totalWeight over its judged items, and over those among the first `k` ranked items.",
9113
+ "properties": {
9114
+ "yesWeight": {
9115
+ "type": "number"
9116
+ },
9117
+ "totalWeight": {
9118
+ "type": "number"
9119
+ },
9120
+ "items": {
9121
+ "type": "integer",
9122
+ "minimum": 0
9123
+ },
9124
+ "judgedItems": {
9125
+ "type": "integer",
9126
+ "minimum": 0
9127
+ },
9128
+ "topK": {
9129
+ "type": "object",
9130
+ "additionalProperties": false,
9131
+ "required": [
9132
+ "yesWeight",
9133
+ "totalWeight"
9134
+ ],
9135
+ "properties": {
9136
+ "yesWeight": {
9137
+ "type": "number"
9138
+ },
9139
+ "totalWeight": {
9140
+ "type": "number"
9141
+ }
9142
+ }
9143
+ }
9144
+ }
9145
+ },
9146
+ "description": "One per repetition that ran."
9147
+ },
9148
+ "changes": {
9149
+ "type": "object",
9150
+ "additionalProperties": false,
9151
+ "required": [
9152
+ "kept",
9153
+ "dropped",
9154
+ "new"
9155
+ ],
9156
+ "description": "The first repetition's items against the judged ones.",
9157
+ "properties": {
9158
+ "kept": {
9159
+ "type": "array",
9160
+ "items": {
9161
+ "type": "object",
9162
+ "additionalProperties": false,
9163
+ "required": [
9164
+ "key"
9165
+ ],
9166
+ "properties": {
9167
+ "key": {
9168
+ "type": "string"
9169
+ },
9170
+ "rankBefore": {
9171
+ "type": "integer"
9172
+ },
9173
+ "rank": {
9174
+ "type": "integer"
9175
+ }
9176
+ }
9177
+ }
9178
+ },
9179
+ "dropped": {
9180
+ "type": "array",
9181
+ "items": {
9182
+ "type": "object",
9183
+ "additionalProperties": false,
9184
+ "required": [
9185
+ "key"
9186
+ ],
9187
+ "properties": {
9188
+ "key": {
9189
+ "type": "string"
9190
+ },
9191
+ "rankBefore": {
9192
+ "type": "integer"
9193
+ }
9194
+ }
9195
+ }
9196
+ },
9197
+ "new": {
9198
+ "type": "array",
9199
+ "items": {
9200
+ "type": "object",
9201
+ "additionalProperties": false,
9202
+ "required": [
9203
+ "key",
9204
+ "pointer"
9205
+ ],
9206
+ "properties": {
9207
+ "key": {
9208
+ "type": "string"
9209
+ },
9210
+ "pointer": {
9211
+ "type": "string"
9212
+ },
9213
+ "rank": {
9214
+ "type": "integer"
9215
+ }
9216
+ }
9217
+ }
9218
+ }
9219
+ }
9220
+ },
9221
+ "tools": {
9222
+ "type": "array",
9223
+ "description": "The first repetition's tool calls, and what happened to each.",
9224
+ "items": {
9225
+ "type": "object",
9226
+ "additionalProperties": false,
9227
+ "required": [
9228
+ "step",
9229
+ "callId",
9230
+ "toolId",
9231
+ "toolVersion",
9232
+ "arguments",
9233
+ "source"
9234
+ ],
9235
+ "properties": {
9236
+ "step": {
9237
+ "type": "integer",
9238
+ "minimum": 0
9239
+ },
9240
+ "callId": {
9241
+ "type": "string"
9242
+ },
9243
+ "toolId": {
9244
+ "type": "string"
9245
+ },
9246
+ "toolVersion": {
9247
+ "type": "string"
9248
+ },
9249
+ "arguments": {},
9250
+ "source": {
9251
+ "type": "string",
9252
+ "enum": [
9253
+ "live",
9254
+ "recorded",
9255
+ "refused"
9256
+ ]
9257
+ },
9258
+ "reason": {
9259
+ "type": "string"
9260
+ }
9261
+ }
9262
+ }
9263
+ },
9264
+ "diverged": {
9265
+ "type": "boolean"
9266
+ },
9267
+ "refusedWrites": {
9268
+ "type": "integer",
9269
+ "minimum": 0
9270
+ },
9271
+ "noContext": {
9272
+ "type": "boolean"
9273
+ },
9274
+ "approvalSkipped": {
9275
+ "type": "boolean"
9276
+ },
9277
+ "error": {
9278
+ "type": "string"
9279
+ },
9280
+ "stopped": {
9281
+ "type": "object",
9282
+ "additionalProperties": false,
9283
+ "required": [
9284
+ "toolId",
9285
+ "arguments"
9286
+ ],
9287
+ "description": "Set when the replay stopped at a refused write: what it would have done.",
9288
+ "properties": {
9289
+ "toolId": {
9290
+ "type": "string"
9291
+ },
9292
+ "arguments": {},
9293
+ "reason": {
9294
+ "type": "string"
9295
+ }
9296
+ }
9297
+ }
9298
+ }
9299
+ },
9300
+ "JudgedComparisonResult": {
9301
+ "description": "A comparison's `result` (a `judged` eval run's): the summary and each case. `EvalRun.result` stays an open object, since each kind has its own; the clients read it as this (TS `comparisonOf(run)`, Python `comparison_of(run)`).",
9302
+ "type": "object",
9303
+ "additionalProperties": false,
9304
+ "required": [
9305
+ "summary",
9306
+ "perCase"
9307
+ ],
9308
+ "properties": {
9309
+ "summary": {
9310
+ "$ref": "#/components/schemas/JudgedComparisonSummary"
9311
+ },
9312
+ "perCase": {
9313
+ "type": "array",
9314
+ "items": {
9315
+ "$ref": "#/components/schemas/ComparisonCaseResult"
9316
+ }
8638
9317
  }
8639
9318
  }
8640
9319
  },
@@ -8700,7 +9379,7 @@
8700
9379
  "result": {
8701
9380
  "type": "object",
8702
9381
  "additionalProperties": true,
8703
- "description": "Kind-specific opaque JSON. For `accuracy`, contains `{ passCount, totalCount, meanScore, perCase[] }`. For `judged` (a comparison), `{ summary, perCase[] }`: the summary has the baseline (the versions behind the recorded runs) and the candidate (`{ kind: \"agent\", agentId, version }` or `{ kind: \"flow\", flowId, version }`), the case counts (`cases`, `diverged`, `refusedWrites`, `errors`, and `stopped`: flow cases that stopped at a write the replay refused, left out of the metrics), the models that answered, and `metrics` (`weightedYesShare`, `judgedCoverage`, `weightedPrecisionAtK`, each `{ baseline, candidate, delta, n, weight, baselineN, baselineWeight, direction, k?, spread? }`); each case has its replay runs, the scores, the items kept, dropped and new, the tool calls with what happened to each, and `stopped` (what it would have done) when it stopped. Other kinds define their own shapes as their dispatchers ship."
9382
+ "description": "Kind-specific opaque JSON. For `accuracy`, contains `{ passCount, totalCount, meanScore, perCase[] }`. For `judged` (a comparison), `{ summary, perCase[] }`: the summary has the baseline (the versions behind the recorded runs) and the candidate (`{ kind: \"agent\", agentId, version }` or `{ kind: \"flow\", flowId, version, versions? }`), the case counts (`cases`, `diverged`, `refusedWrites`, `errors`, and `stopped`: flow cases that stopped at a write the replay refused, left out of the metrics), the models that answered, and `metrics` (`weightedYesShare`, `judgedCoverage`, `weightedPrecisionAtK`, each `{ baseline, candidate, delta, n, weight, baselineN, baselineWeight, direction, k?, spread? }`); each case has its replay runs, the scores, the items kept, dropped and new, the tool calls with what happened to each, and `stopped` (what it would have done) when it stopped. Other kinds define their own shapes as their dispatchers ship."
8704
9383
  },
8705
9384
  "error": {
8706
9385
  "type": "string"
@@ -8778,9 +9457,12 @@
8778
9457
  "type": "integer",
8779
9458
  "minimum": 1,
8780
9459
  "maximum": 100
9460
+ },
9461
+ "versions": {
9462
+ "$ref": "#/components/schemas/FlowVersionOverrides"
8781
9463
  }
8782
9464
  },
8783
- "description": "Exactly one of `agentRef` or `flowRef` MUST be supplied. `dryRun: true` returns a plan preview without invoking the subject. For a `judged` suite (a test set), the run is a comparison: `agentRef` or `flowRef` with its `version` is the candidate, replayed on each case without doing anything the past run didn't (a flow stops at a write the replay refuses); `baseline` (default `'recorded'`), `reads` (default `recorded`), `repetitions` (default 1) and `k` (default 10) set how."
9465
+ "description": "Exactly one of `agentRef` or `flowRef` MUST be supplied. `dryRun: true` returns a plan preview without invoking the subject. For a `judged` suite (a test set), the run is a comparison: `agentRef` or `flowRef` with its `version` is the candidate, replayed on each case without doing anything the past run didn't (a flow stops at a write the replay refuses); `baseline` (default `'recorded'`), `reads` (default `recorded`), `repetitions` (default 1) and `k` (default 10) set how. With `flowRef`, `versions` runs the flow with some of its agents or tools at other versions; an id the flow doesn't use, or a version that isn't published, is refused (`400 validation-failed`, each under `details.issues`)."
8784
9466
  },
8785
9467
  "StartEvalRunResult": {
8786
9468
  "type": "object",
@@ -15397,7 +16079,7 @@
15397
16079
  }
15398
16080
  },
15399
16081
  "409": {
15400
- "description": "Agent already registered at that (id, version).",
16082
+ "description": "Agent already registered at that (id, version). Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
15401
16083
  "content": {
15402
16084
  "application/json": {
15403
16085
  "schema": {
@@ -15589,6 +16271,16 @@
15589
16271
  "agents"
15590
16272
  ],
15591
16273
  "responses": {
16274
+ "200": {
16275
+ "description": "An active version already holds this definition and these pins (the same swap derived before, or a deploy that registered it): that version, unchanged.",
16276
+ "content": {
16277
+ "application/json": {
16278
+ "schema": {
16279
+ "$ref": "#/components/schemas/Agent"
16280
+ }
16281
+ }
16282
+ }
16283
+ },
15592
16284
  "201": {
15593
16285
  "description": "The derived agent version.",
15594
16286
  "content": {
@@ -15630,7 +16322,7 @@
15630
16322
  }
15631
16323
  },
15632
16324
  "409": {
15633
- "description": "Idempotency-Key was reused with a different body, or resource-state conflict.",
16325
+ "description": "Idempotency-Key was reused with a different body, or resource-state conflict. Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
15634
16326
  "content": {
15635
16327
  "application/json": {
15636
16328
  "schema": {
@@ -15804,7 +16496,7 @@
15804
16496
  }
15805
16497
  },
15806
16498
  "409": {
15807
- "description": "Idempotency-Key was reused with a different body, or resource-state conflict.",
16499
+ "description": "Idempotency-Key was reused with a different body, or resource-state conflict. Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
15808
16500
  "content": {
15809
16501
  "application/json": {
15810
16502
  "schema": {
@@ -15910,7 +16602,7 @@
15910
16602
  }
15911
16603
  },
15912
16604
  "409": {
15913
- "description": "Idempotency-Key was reused with a different body, or resource-state conflict.",
16605
+ "description": "Idempotency-Key was reused with a different body, or resource-state conflict. Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
15914
16606
  "content": {
15915
16607
  "application/json": {
15916
16608
  "schema": {
@@ -16113,7 +16805,7 @@
16113
16805
  }
16114
16806
  },
16115
16807
  "409": {
16116
- "description": "Flow already registered at that (id, version).",
16808
+ "description": "Flow already registered at that (id, version). Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
16117
16809
  "content": {
16118
16810
  "application/json": {
16119
16811
  "schema": {
@@ -16414,7 +17106,7 @@
16414
17106
  }
16415
17107
  },
16416
17108
  "409": {
16417
- "description": "Idempotency-Key was reused with a different body, or resource-state conflict.",
17109
+ "description": "Idempotency-Key was reused with a different body, or resource-state conflict. Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
16418
17110
  "content": {
16419
17111
  "application/json": {
16420
17112
  "schema": {
@@ -16520,7 +17212,7 @@
16520
17212
  }
16521
17213
  },
16522
17214
  "409": {
16523
- "description": "Idempotency-Key was reused with a different body, or resource-state conflict.",
17215
+ "description": "Idempotency-Key was reused with a different body, or resource-state conflict. Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
16524
17216
  "content": {
16525
17217
  "application/json": {
16526
17218
  "schema": {
@@ -16723,7 +17415,7 @@
16723
17415
  }
16724
17416
  },
16725
17417
  "409": {
16726
- "description": "Tool already registered at that id.",
17418
+ "description": "Tool already registered at that id. Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
16727
17419
  "content": {
16728
17420
  "application/json": {
16729
17421
  "schema": {
@@ -17045,7 +17737,7 @@
17045
17737
  }
17046
17738
  },
17047
17739
  "409": {
17048
- "description": "Idempotency-Key was reused with a different body, or resource-state conflict.",
17740
+ "description": "Idempotency-Key was reused with a different body, or resource-state conflict. Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
17049
17741
  "content": {
17050
17742
  "application/json": {
17051
17743
  "schema": {
@@ -17151,7 +17843,7 @@
17151
17843
  }
17152
17844
  },
17153
17845
  "409": {
17154
- "description": "Idempotency-Key was reused with a different body, or resource-state conflict.",
17846
+ "description": "Idempotency-Key was reused with a different body, or resource-state conflict. Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
17155
17847
  "content": {
17156
17848
  "application/json": {
17157
17849
  "schema": {
@@ -17354,7 +18046,7 @@
17354
18046
  }
17355
18047
  },
17356
18048
  "409": {
17357
- "description": "Guardrail already registered at that id.",
18049
+ "description": "Guardrail already registered at that id. Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
17358
18050
  "content": {
17359
18051
  "application/json": {
17360
18052
  "schema": {
@@ -17510,7 +18202,7 @@
17510
18202
  }
17511
18203
  },
17512
18204
  "409": {
17513
- "description": "Idempotency-Key was reused with a different body, or resource-state conflict.",
18205
+ "description": "Idempotency-Key was reused with a different body, or resource-state conflict. Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
17514
18206
  "content": {
17515
18207
  "application/json": {
17516
18208
  "schema": {
@@ -26576,7 +27268,7 @@
26576
27268
  }
26577
27269
  },
26578
27270
  "409": {
26579
- "description": "Idempotency-Key was reused with a different body, or resource-state conflict.",
27271
+ "description": "Idempotency-Key was reused with a different body, or resource-state conflict. Or `registry-read-only`: this registry takes no writes (under `kindgi dev`, the pack's files are the source); the message says what to do instead.",
26580
27272
  "content": {
26581
27273
  "application/json": {
26582
27274
  "schema": {
@@ -27902,6 +28594,16 @@
27902
28594
  }
27903
28595
  }
27904
28596
  },
28597
+ "404": {
28598
+ "description": "org-not-found: `orgId` names no org of the tenant (it never existed, or it was deleted).",
28599
+ "content": {
28600
+ "application/json": {
28601
+ "schema": {
28602
+ "$ref": "#/components/schemas/WireError"
28603
+ }
28604
+ }
28605
+ }
28606
+ },
27905
28607
  "409": {
27906
28608
  "description": "Idempotency-Key was reused with a different body, or resource-state conflict.",
27907
28609
  "content": {
@@ -28039,7 +28741,7 @@
28039
28741
  }
28040
28742
  },
28041
28743
  "404": {
28042
- "description": "No team with that id under this tenant.",
28744
+ "description": "team-not-found: no team with that id under this tenant; or org-not-found: `orgId` names no org of the tenant (it never existed, or it was deleted).",
28043
28745
  "content": {
28044
28746
  "application/json": {
28045
28747
  "schema": {
@@ -28677,6 +29379,16 @@
28677
29379
  }
28678
29380
  }
28679
29381
  },
29382
+ "404": {
29383
+ "description": "org-not-found: `orgId` names no org of the tenant (it never existed, or it was deleted).",
29384
+ "content": {
29385
+ "application/json": {
29386
+ "schema": {
29387
+ "$ref": "#/components/schemas/WireError"
29388
+ }
29389
+ }
29390
+ }
29391
+ },
28680
29392
  "409": {
28681
29393
  "description": "Idempotency-Key was reused with a different body, or resource-state conflict.",
28682
29394
  "content": {
@@ -28815,7 +29527,7 @@
28815
29527
  }
28816
29528
  },
28817
29529
  "404": {
28818
- "description": "No project with that id under this tenant.",
29530
+ "description": "project-not-found: no project with that id under this tenant; or org-not-found: `orgId` names no org of the tenant (it never existed, or it was deleted).",
28819
29531
  "content": {
28820
29532
  "application/json": {
28821
29533
  "schema": {