@kindgi/api 0.1.4-rc.0 → 0.1.4-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-binding.d.ts +8 -0
- package/dist/agent-binding.d.ts.map +1 -1
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +7 -1
- package/dist/app.js.map +1 -1
- package/dist/deploy-versions.d.ts +2 -0
- package/dist/deploy-versions.d.ts.map +1 -1
- package/dist/deploy-versions.js +1 -1
- package/dist/deploy-versions.js.map +1 -1
- package/dist/derive-agent-version.d.ts +9 -1
- package/dist/derive-agent-version.d.ts.map +1 -1
- package/dist/derive-agent-version.js +11 -1
- package/dist/derive-agent-version.js.map +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +1 -0
- package/dist/errors.js.map +1 -1
- package/dist/eval-run-binding.d.ts +7 -0
- package/dist/eval-run-binding.d.ts.map +1 -1
- package/dist/eval-run-binding.js.map +1 -1
- package/dist/eval-run-dispatcher.d.ts +3 -0
- package/dist/eval-run-dispatcher.d.ts.map +1 -1
- package/dist/eval-run-dispatcher.js.map +1 -1
- package/dist/flow-binding.d.ts +8 -0
- package/dist/flow-binding.d.ts.map +1 -1
- package/dist/guardrail-binding.d.ts +8 -0
- package/dist/guardrail-binding.d.ts.map +1 -1
- package/dist/handler-binding.d.ts +3 -0
- package/dist/handler-binding.d.ts.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/judged-dispatcher.d.ts +4 -0
- package/dist/judged-dispatcher.d.ts.map +1 -1
- package/dist/judged-dispatcher.js +14 -3
- package/dist/judged-dispatcher.js.map +1 -1
- package/dist/openapi/operations.d.ts.map +1 -1
- package/dist/openapi/operations.js +21 -6
- package/dist/openapi/operations.js.map +1 -1
- package/dist/openapi/schemas.d.ts +6 -0
- package/dist/openapi/schemas.d.ts.map +1 -1
- package/dist/openapi/schemas.js +341 -2
- package/dist/openapi/schemas.js.map +1 -1
- package/dist/registry-read-only.d.ts +32 -0
- package/dist/registry-read-only.d.ts.map +1 -0
- package/dist/registry-read-only.js +22 -0
- package/dist/registry-read-only.js.map +1 -0
- package/dist/routes/agents.d.ts.map +1 -1
- package/dist/routes/agents.js +7 -1
- package/dist/routes/agents.js.map +1 -1
- package/dist/routes/blocks.d.ts.map +1 -1
- package/dist/routes/blocks.js +37 -12
- package/dist/routes/blocks.js.map +1 -1
- package/dist/routes/deployments.d.ts.map +1 -1
- package/dist/routes/deployments.js +16 -0
- package/dist/routes/deployments.js.map +1 -1
- package/dist/routes/eval-comparison.d.ts +3 -2
- package/dist/routes/eval-comparison.d.ts.map +1 -1
- package/dist/routes/eval-comparison.js +39 -3
- package/dist/routes/eval-comparison.js.map +1 -1
- package/dist/routes/eval-runs.d.ts +7 -1
- package/dist/routes/eval-runs.d.ts.map +1 -1
- package/dist/routes/eval-runs.js +34 -3
- package/dist/routes/eval-runs.js.map +1 -1
- package/dist/routes/eval-versions.d.ts +25 -0
- package/dist/routes/eval-versions.d.ts.map +1 -0
- package/dist/routes/eval-versions.js +66 -0
- package/dist/routes/eval-versions.js.map +1 -0
- package/dist/routes/flows.d.ts.map +1 -1
- package/dist/routes/flows.js +4 -0
- package/dist/routes/flows.js.map +1 -1
- package/dist/routes/guardrails.d.ts.map +1 -1
- package/dist/routes/guardrails.js +4 -0
- package/dist/routes/guardrails.js.map +1 -1
- package/dist/routes/hierarchy-errors.d.ts +10 -0
- package/dist/routes/hierarchy-errors.d.ts.map +1 -1
- package/dist/routes/hierarchy-errors.js +8 -0
- package/dist/routes/hierarchy-errors.js.map +1 -1
- package/dist/routes/projects.d.ts.map +1 -1
- package/dist/routes/projects.js +13 -7
- package/dist/routes/projects.js.map +1 -1
- package/dist/routes/runs.js +1 -0
- package/dist/routes/runs.js.map +1 -1
- package/dist/routes/teams.d.ts.map +1 -1
- package/dist/routes/teams.js +12 -6
- package/dist/routes/teams.js.map +1 -1
- package/dist/routes/tools.d.ts.map +1 -1
- package/dist/routes/tools.js +4 -0
- package/dist/routes/tools.js.map +1 -1
- package/dist/tool-binding.d.ts +8 -0
- package/dist/tool-binding.d.ts.map +1 -1
- package/openapi.json +729 -17
- package/package.json +21 -21
- package/src/agent-binding.ts +9 -0
- package/src/app.ts +10 -1
- package/src/deploy-versions.ts +1 -1
- package/src/derive-agent-version.ts +12 -1
- package/src/errors.ts +1 -0
- package/src/eval-run-binding.ts +7 -0
- package/src/eval-run-dispatcher.ts +3 -0
- package/src/flow-binding.ts +9 -0
- package/src/guardrail-binding.ts +9 -0
- package/src/handler-binding.ts +3 -0
- package/src/index.ts +1 -0
- package/src/judged-dispatcher.ts +27 -4
- package/src/openapi/operations.ts +56 -6
- package/src/openapi/schemas.ts +361 -2
- package/src/registry-read-only.ts +43 -0
- package/src/routes/agents.ts +10 -1
- package/src/routes/blocks.ts +39 -14
- package/src/routes/deployments.ts +25 -0
- package/src/routes/eval-comparison.ts +37 -3
- package/src/routes/eval-runs.ts +46 -3
- package/src/routes/eval-versions.ts +110 -0
- package/src/routes/flows.ts +7 -0
- package/src/routes/guardrails.ts +7 -0
- package/src/routes/hierarchy-errors.ts +9 -0
- package/src/routes/projects.ts +18 -7
- package/src/routes/runs.ts +1 -0
- package/src/routes/teams.ts +14 -6
- package/src/routes/tools.ts +7 -0
- package/src/tool-binding.ts +9 -0
package/src/openapi/schemas.ts
CHANGED
|
@@ -194,6 +194,7 @@ export const RunSchema: JsonSchema = {
|
|
|
194
194
|
type: 'string',
|
|
195
195
|
description: 'Set on a replay run: the eval run that started it.',
|
|
196
196
|
},
|
|
197
|
+
versions: { $ref: '#/components/schemas/FlowVersionOverrides' },
|
|
197
198
|
publicAccessToken: {
|
|
198
199
|
type: 'string',
|
|
199
200
|
description:
|
|
@@ -1287,6 +1288,21 @@ export const FlowPinsSchema: JsonSchema = {
|
|
|
1287
1288
|
},
|
|
1288
1289
|
};
|
|
1289
1290
|
|
|
1291
|
+
export const FlowVersionOverridesSchema: JsonSchema = {
|
|
1292
|
+
description:
|
|
1293
|
+
"Agents and tools a flow runs at other exact versions than the flow version's pins (\"this flow, with `acme.scorer` at 0.4.0\"), without publishing a new flow version: a comparison's flow candidate (`versions` on the start body and the run's `comparison`), and the flow runs that replay it (a run's `versions`). Each id must be an agent or tool the flow uses.",
|
|
1294
|
+
type: 'object',
|
|
1295
|
+
additionalProperties: false,
|
|
1296
|
+
properties: {
|
|
1297
|
+
tools: { ...PinMapSchema, description: 'Tool id → exact version.' },
|
|
1298
|
+
agents: {
|
|
1299
|
+
...PinMapSchema,
|
|
1300
|
+
description:
|
|
1301
|
+
'Agent id → exact version, for agent nodes with or without a version of their own.',
|
|
1302
|
+
},
|
|
1303
|
+
},
|
|
1304
|
+
};
|
|
1305
|
+
|
|
1290
1306
|
export const PublishFlowBodySchema: JsonSchema = {
|
|
1291
1307
|
description:
|
|
1292
1308
|
'Full flow definition. Validated server-side via `@kindgi/flow.loadFlow` — validation failures return `400 validation-failed` with the issue list under `details.issues`.',
|
|
@@ -4766,6 +4782,342 @@ export const EvalComparisonSchema: JsonSchema = {
|
|
|
4766
4782
|
},
|
|
4767
4783
|
repetitions: { type: 'integer', minimum: 1, maximum: 10 },
|
|
4768
4784
|
k: { type: 'integer', minimum: 1, maximum: 100 },
|
|
4785
|
+
versions: { $ref: '#/components/schemas/FlowVersionOverrides' },
|
|
4786
|
+
},
|
|
4787
|
+
};
|
|
4788
|
+
|
|
4789
|
+
// ---------------- comparison results (a `judged` eval run's `result`) ----------------
|
|
4790
|
+
|
|
4791
|
+
const nullableNumber = { type: ['number', 'null'] } as const;
|
|
4792
|
+
|
|
4793
|
+
export const ComparisonMetricSchema: JsonSchema = {
|
|
4794
|
+
description:
|
|
4795
|
+
"One metric, the recorded runs beside the candidate. `null` where a side had no judged evidence; `n` / `weight` are the candidate's evidence (cases with judged items, and the judgment weight behind them), `baselineN` / `baselineWeight` the recorded side's.",
|
|
4796
|
+
type: 'object',
|
|
4797
|
+
additionalProperties: false,
|
|
4798
|
+
required: [
|
|
4799
|
+
'baseline',
|
|
4800
|
+
'candidate',
|
|
4801
|
+
'delta',
|
|
4802
|
+
'n',
|
|
4803
|
+
'weight',
|
|
4804
|
+
'baselineN',
|
|
4805
|
+
'baselineWeight',
|
|
4806
|
+
'direction',
|
|
4807
|
+
],
|
|
4808
|
+
properties: {
|
|
4809
|
+
baseline: nullableNumber,
|
|
4810
|
+
candidate: nullableNumber,
|
|
4811
|
+
delta: nullableNumber,
|
|
4812
|
+
n: { type: 'integer', minimum: 0 },
|
|
4813
|
+
weight: { type: 'number', minimum: 0 },
|
|
4814
|
+
baselineN: { type: 'integer', minimum: 0 },
|
|
4815
|
+
baselineWeight: { type: 'number', minimum: 0 },
|
|
4816
|
+
direction: { type: 'string', enum: ['higher'] },
|
|
4817
|
+
k: {
|
|
4818
|
+
type: 'integer',
|
|
4819
|
+
minimum: 1,
|
|
4820
|
+
description: '`weightedPrecisionAtK`: the ranked items it looked at.',
|
|
4821
|
+
},
|
|
4822
|
+
spread: {
|
|
4823
|
+
type: 'number',
|
|
4824
|
+
description: "With more than one repetition: the candidate's max − min across them.",
|
|
4825
|
+
},
|
|
4826
|
+
},
|
|
4827
|
+
};
|
|
4828
|
+
|
|
4829
|
+
export const ComparisonCandidateSchema: JsonSchema = {
|
|
4830
|
+
description:
|
|
4831
|
+
'What ran on the cases: an agent version, or a flow version (with any versions it swapped in).',
|
|
4832
|
+
oneOf: [
|
|
4833
|
+
{
|
|
4834
|
+
type: 'object',
|
|
4835
|
+
additionalProperties: false,
|
|
4836
|
+
required: ['kind', 'agentId', 'version'],
|
|
4837
|
+
properties: {
|
|
4838
|
+
kind: { type: 'string', enum: ['agent'] },
|
|
4839
|
+
agentId: { type: 'string' },
|
|
4840
|
+
version: { type: 'string' },
|
|
4841
|
+
},
|
|
4842
|
+
},
|
|
4843
|
+
{
|
|
4844
|
+
type: 'object',
|
|
4845
|
+
additionalProperties: false,
|
|
4846
|
+
required: ['kind', 'flowId', 'version'],
|
|
4847
|
+
properties: {
|
|
4848
|
+
kind: { type: 'string', enum: ['flow'] },
|
|
4849
|
+
flowId: { type: 'string' },
|
|
4850
|
+
version: { type: 'string' },
|
|
4851
|
+
versions: { $ref: '#/components/schemas/FlowVersionOverrides' },
|
|
4852
|
+
},
|
|
4853
|
+
},
|
|
4854
|
+
],
|
|
4855
|
+
};
|
|
4856
|
+
|
|
4857
|
+
export const JudgedComparisonSummarySchema: JsonSchema = {
|
|
4858
|
+
description:
|
|
4859
|
+
'What a comparison concluded: the candidate beside the recorded runs, the case counts, and the metrics. What a promotion gate reads.',
|
|
4860
|
+
type: 'object',
|
|
4861
|
+
additionalProperties: false,
|
|
4862
|
+
required: [
|
|
4863
|
+
'evalRunId',
|
|
4864
|
+
'status',
|
|
4865
|
+
'completedAt',
|
|
4866
|
+
'suite',
|
|
4867
|
+
'candidate',
|
|
4868
|
+
'baseline',
|
|
4869
|
+
'scope',
|
|
4870
|
+
'cases',
|
|
4871
|
+
'diverged',
|
|
4872
|
+
'refusedWrites',
|
|
4873
|
+
'errors',
|
|
4874
|
+
'stopped',
|
|
4875
|
+
'reads',
|
|
4876
|
+
'sampling',
|
|
4877
|
+
'repetitions',
|
|
4878
|
+
'metrics',
|
|
4879
|
+
],
|
|
4880
|
+
properties: {
|
|
4881
|
+
evalRunId: { type: 'string' },
|
|
4882
|
+
status: { type: 'string', enum: ['completed', 'partial', 'failed'] },
|
|
4883
|
+
completedAt: { type: 'string', format: 'date-time' },
|
|
4884
|
+
suite: {
|
|
4885
|
+
type: 'object',
|
|
4886
|
+
additionalProperties: false,
|
|
4887
|
+
required: ['id', 'version'],
|
|
4888
|
+
properties: { id: { type: 'string' }, version: { type: 'string' } },
|
|
4889
|
+
},
|
|
4890
|
+
candidate: { $ref: '#/components/schemas/ComparisonCandidate' },
|
|
4891
|
+
baseline: {
|
|
4892
|
+
description:
|
|
4893
|
+
"What the candidate was compared with: `recorded` (the test set's recorded runs, with the versions that served them), or another version.",
|
|
4894
|
+
oneOf: [
|
|
4895
|
+
{
|
|
4896
|
+
type: 'object',
|
|
4897
|
+
additionalProperties: false,
|
|
4898
|
+
required: ['kind', 'versions'],
|
|
4899
|
+
properties: {
|
|
4900
|
+
kind: { type: 'string', enum: ['recorded'] },
|
|
4901
|
+
versions: {
|
|
4902
|
+
type: 'array',
|
|
4903
|
+
items: {
|
|
4904
|
+
type: 'object',
|
|
4905
|
+
additionalProperties: false,
|
|
4906
|
+
required: ['version', 'cases'],
|
|
4907
|
+
properties: {
|
|
4908
|
+
agentId: { type: 'string' },
|
|
4909
|
+
flowId: { type: 'string' },
|
|
4910
|
+
version: { type: 'string' },
|
|
4911
|
+
cases: { type: 'integer', minimum: 0 },
|
|
4912
|
+
},
|
|
4913
|
+
},
|
|
4914
|
+
},
|
|
4915
|
+
},
|
|
4916
|
+
},
|
|
4917
|
+
{
|
|
4918
|
+
type: 'object',
|
|
4919
|
+
additionalProperties: false,
|
|
4920
|
+
required: ['kind', 'agentId', 'version', 'via'],
|
|
4921
|
+
properties: {
|
|
4922
|
+
kind: { type: 'string', enum: ['version'] },
|
|
4923
|
+
agentId: { type: 'string' },
|
|
4924
|
+
version: { type: 'string' },
|
|
4925
|
+
via: { type: 'string', enum: ['explicit', 'live'] },
|
|
4926
|
+
liveScope: { type: 'object', additionalProperties: true },
|
|
4927
|
+
},
|
|
4928
|
+
},
|
|
4929
|
+
],
|
|
4930
|
+
},
|
|
4931
|
+
scope: {
|
|
4932
|
+
type: 'object',
|
|
4933
|
+
additionalProperties: false,
|
|
4934
|
+
description: "Where the test set's judgments came from.",
|
|
4935
|
+
properties: { projectId: { type: 'string' } },
|
|
4936
|
+
},
|
|
4937
|
+
cases: { type: 'integer', minimum: 0 },
|
|
4938
|
+
diverged: {
|
|
4939
|
+
type: 'integer',
|
|
4940
|
+
minimum: 0,
|
|
4941
|
+
description: "Cases where a read with no recording ran live under `reads: 'recorded'`.",
|
|
4942
|
+
},
|
|
4943
|
+
refusedWrites: {
|
|
4944
|
+
type: 'integer',
|
|
4945
|
+
minimum: 0,
|
|
4946
|
+
description: 'Tool calls refused across the cases (what the candidate would have done).',
|
|
4947
|
+
},
|
|
4948
|
+
errors: { type: 'integer', minimum: 0, description: 'Cases none of whose repetitions ran.' },
|
|
4949
|
+
stopped: {
|
|
4950
|
+
type: 'integer',
|
|
4951
|
+
minimum: 0,
|
|
4952
|
+
description:
|
|
4953
|
+
"Flow cases that stopped at a write the replay refused: no output to score, so they're left out of the metrics.",
|
|
4954
|
+
},
|
|
4955
|
+
reads: { type: 'string', enum: ['recorded', 'live'] },
|
|
4956
|
+
sampling: {
|
|
4957
|
+
type: 'object',
|
|
4958
|
+
additionalProperties: false,
|
|
4959
|
+
required: ['models'],
|
|
4960
|
+
properties: {
|
|
4961
|
+
models: {
|
|
4962
|
+
type: 'array',
|
|
4963
|
+
description:
|
|
4964
|
+
"The models that answered the candidate's replays, and how many replays each.",
|
|
4965
|
+
items: {
|
|
4966
|
+
type: 'object',
|
|
4967
|
+
additionalProperties: false,
|
|
4968
|
+
required: ['providerId', 'model', 'runs'],
|
|
4969
|
+
properties: {
|
|
4970
|
+
providerId: { type: 'string' },
|
|
4971
|
+
model: { type: 'string' },
|
|
4972
|
+
runs: { type: 'integer', minimum: 0 },
|
|
4973
|
+
},
|
|
4974
|
+
},
|
|
4975
|
+
},
|
|
4976
|
+
},
|
|
4977
|
+
},
|
|
4978
|
+
repetitions: { type: 'integer', minimum: 1 },
|
|
4979
|
+
metrics: {
|
|
4980
|
+
type: 'object',
|
|
4981
|
+
additionalProperties: false,
|
|
4982
|
+
required: ['weightedYesShare', 'judgedCoverage', 'weightedPrecisionAtK'],
|
|
4983
|
+
properties: {
|
|
4984
|
+
weightedYesShare: { $ref: '#/components/schemas/ComparisonMetric' },
|
|
4985
|
+
judgedCoverage: { $ref: '#/components/schemas/ComparisonMetric' },
|
|
4986
|
+
weightedPrecisionAtK: { $ref: '#/components/schemas/ComparisonMetric' },
|
|
4987
|
+
},
|
|
4988
|
+
},
|
|
4989
|
+
},
|
|
4990
|
+
};
|
|
4991
|
+
|
|
4992
|
+
const outputScore = {
|
|
4993
|
+
type: 'object',
|
|
4994
|
+
additionalProperties: false,
|
|
4995
|
+
required: ['yesWeight', 'totalWeight', 'items', 'judgedItems', 'topK'],
|
|
4996
|
+
description:
|
|
4997
|
+
"An output's score: Σ yesWeight and Σ totalWeight over its judged items, and over those among the first `k` ranked items.",
|
|
4998
|
+
properties: {
|
|
4999
|
+
yesWeight: { type: 'number' },
|
|
5000
|
+
totalWeight: { type: 'number' },
|
|
5001
|
+
items: { type: 'integer', minimum: 0 },
|
|
5002
|
+
judgedItems: { type: 'integer', minimum: 0 },
|
|
5003
|
+
topK: {
|
|
5004
|
+
type: 'object',
|
|
5005
|
+
additionalProperties: false,
|
|
5006
|
+
required: ['yesWeight', 'totalWeight'],
|
|
5007
|
+
properties: { yesWeight: { type: 'number' }, totalWeight: { type: 'number' } },
|
|
5008
|
+
},
|
|
5009
|
+
},
|
|
5010
|
+
} as const;
|
|
5011
|
+
|
|
5012
|
+
export const ComparisonCaseResultSchema: JsonSchema = {
|
|
5013
|
+
description:
|
|
5014
|
+
"One case of a comparison: its replay runs, the scores, the items kept, dropped and new, the tool calls, and why it didn't run when it didn't.",
|
|
5015
|
+
type: 'object',
|
|
5016
|
+
additionalProperties: false,
|
|
5017
|
+
required: [
|
|
5018
|
+
'caseId',
|
|
5019
|
+
'runIds',
|
|
5020
|
+
'baseline',
|
|
5021
|
+
'candidate',
|
|
5022
|
+
'diverged',
|
|
5023
|
+
'refusedWrites',
|
|
5024
|
+
'noContext',
|
|
5025
|
+
'approvalSkipped',
|
|
5026
|
+
],
|
|
5027
|
+
properties: {
|
|
5028
|
+
caseId: { type: 'string' },
|
|
5029
|
+
runIds: {
|
|
5030
|
+
type: 'array',
|
|
5031
|
+
items: { type: 'string' },
|
|
5032
|
+
description: "The candidate's replay runs, one per repetition.",
|
|
5033
|
+
},
|
|
5034
|
+
baseline: outputScore,
|
|
5035
|
+
candidate: { type: 'array', items: outputScore, description: 'One per repetition that ran.' },
|
|
5036
|
+
changes: {
|
|
5037
|
+
type: 'object',
|
|
5038
|
+
additionalProperties: false,
|
|
5039
|
+
required: ['kept', 'dropped', 'new'],
|
|
5040
|
+
description: "The first repetition's items against the judged ones.",
|
|
5041
|
+
properties: {
|
|
5042
|
+
kept: {
|
|
5043
|
+
type: 'array',
|
|
5044
|
+
items: {
|
|
5045
|
+
type: 'object',
|
|
5046
|
+
additionalProperties: false,
|
|
5047
|
+
required: ['key'],
|
|
5048
|
+
properties: {
|
|
5049
|
+
key: { type: 'string' },
|
|
5050
|
+
rankBefore: { type: 'integer' },
|
|
5051
|
+
rank: { type: 'integer' },
|
|
5052
|
+
},
|
|
5053
|
+
},
|
|
5054
|
+
},
|
|
5055
|
+
dropped: {
|
|
5056
|
+
type: 'array',
|
|
5057
|
+
items: {
|
|
5058
|
+
type: 'object',
|
|
5059
|
+
additionalProperties: false,
|
|
5060
|
+
required: ['key'],
|
|
5061
|
+
properties: { key: { type: 'string' }, rankBefore: { type: 'integer' } },
|
|
5062
|
+
},
|
|
5063
|
+
},
|
|
5064
|
+
new: {
|
|
5065
|
+
type: 'array',
|
|
5066
|
+
items: {
|
|
5067
|
+
type: 'object',
|
|
5068
|
+
additionalProperties: false,
|
|
5069
|
+
required: ['key', 'pointer'],
|
|
5070
|
+
properties: {
|
|
5071
|
+
key: { type: 'string' },
|
|
5072
|
+
pointer: { type: 'string' },
|
|
5073
|
+
rank: { type: 'integer' },
|
|
5074
|
+
},
|
|
5075
|
+
},
|
|
5076
|
+
},
|
|
5077
|
+
},
|
|
5078
|
+
},
|
|
5079
|
+
tools: {
|
|
5080
|
+
type: 'array',
|
|
5081
|
+
description: "The first repetition's tool calls, and what happened to each.",
|
|
5082
|
+
items: {
|
|
5083
|
+
type: 'object',
|
|
5084
|
+
additionalProperties: false,
|
|
5085
|
+
required: ['step', 'callId', 'toolId', 'toolVersion', 'arguments', 'source'],
|
|
5086
|
+
properties: {
|
|
5087
|
+
step: { type: 'integer', minimum: 0 },
|
|
5088
|
+
callId: { type: 'string' },
|
|
5089
|
+
toolId: { type: 'string' },
|
|
5090
|
+
toolVersion: { type: 'string' },
|
|
5091
|
+
arguments: {},
|
|
5092
|
+
source: { type: 'string', enum: ['live', 'recorded', 'refused'] },
|
|
5093
|
+
reason: { type: 'string' },
|
|
5094
|
+
},
|
|
5095
|
+
},
|
|
5096
|
+
},
|
|
5097
|
+
diverged: { type: 'boolean' },
|
|
5098
|
+
refusedWrites: { type: 'integer', minimum: 0 },
|
|
5099
|
+
noContext: { type: 'boolean' },
|
|
5100
|
+
approvalSkipped: { type: 'boolean' },
|
|
5101
|
+
error: { type: 'string' },
|
|
5102
|
+
stopped: {
|
|
5103
|
+
type: 'object',
|
|
5104
|
+
additionalProperties: false,
|
|
5105
|
+
required: ['toolId', 'arguments'],
|
|
5106
|
+
description: 'Set when the replay stopped at a refused write: what it would have done.',
|
|
5107
|
+
properties: { toolId: { type: 'string' }, arguments: {}, reason: { type: 'string' } },
|
|
5108
|
+
},
|
|
5109
|
+
},
|
|
5110
|
+
};
|
|
5111
|
+
|
|
5112
|
+
export const JudgedComparisonResultSchema: JsonSchema = {
|
|
5113
|
+
description:
|
|
5114
|
+
"A comparison's `result` (a `judged` eval run's): the summary and each case. `EvalRun.result` stays an open object, since each kind has its own; the clients read it as this (TS `comparisonOf(run)`, Python `comparison_of(run)`).",
|
|
5115
|
+
type: 'object',
|
|
5116
|
+
additionalProperties: false,
|
|
5117
|
+
required: ['summary', 'perCase'],
|
|
5118
|
+
properties: {
|
|
5119
|
+
summary: { $ref: '#/components/schemas/JudgedComparisonSummary' },
|
|
5120
|
+
perCase: { type: 'array', items: { $ref: '#/components/schemas/ComparisonCaseResult' } },
|
|
4769
5121
|
},
|
|
4770
5122
|
};
|
|
4771
5123
|
|
|
@@ -4798,7 +5150,7 @@ export const EvalRunSchema: JsonSchema = {
|
|
|
4798
5150
|
type: 'object',
|
|
4799
5151
|
additionalProperties: true,
|
|
4800
5152
|
description:
|
|
4801
|
-
'Kind-specific opaque JSON. For `accuracy`, contains `{ passCount, totalCount, meanScore, perCase[] }`. For `judged` (a comparison), `{ summary, perCase[] }`: the summary has the baseline (the versions behind the recorded runs) and the candidate (`{ kind: "agent", agentId, version }` or `{ kind: "flow", flowId, version }`), the case counts (`cases`, `diverged`, `refusedWrites`, `errors`, and `stopped`: flow cases that stopped at a write the replay refused, left out of the metrics), the models that answered, and `metrics` (`weightedYesShare`, `judgedCoverage`, `weightedPrecisionAtK`, each `{ baseline, candidate, delta, n, weight, baselineN, baselineWeight, direction, k?, spread? }`); each case has its replay runs, the scores, the items kept, dropped and new, the tool calls with what happened to each, and `stopped` (what it would have done) when it stopped. Other kinds define their own shapes as their dispatchers ship.',
|
|
5153
|
+
'Kind-specific opaque JSON. For `accuracy`, contains `{ passCount, totalCount, meanScore, perCase[] }`. For `judged` (a comparison), `{ summary, perCase[] }`: the summary has the baseline (the versions behind the recorded runs) and the candidate (`{ kind: "agent", agentId, version }` or `{ kind: "flow", flowId, version, versions? }`), the case counts (`cases`, `diverged`, `refusedWrites`, `errors`, and `stopped`: flow cases that stopped at a write the replay refused, left out of the metrics), the models that answered, and `metrics` (`weightedYesShare`, `judgedCoverage`, `weightedPrecisionAtK`, each `{ baseline, candidate, delta, n, weight, baselineN, baselineWeight, direction, k?, spread? }`); each case has its replay runs, the scores, the items kept, dropped and new, the tool calls with what happened to each, and `stopped` (what it would have done) when it stopped. Other kinds define their own shapes as their dispatchers ship.',
|
|
4802
5154
|
},
|
|
4803
5155
|
error: { type: 'string' },
|
|
4804
5156
|
correlationId: { type: 'string' },
|
|
@@ -4831,9 +5183,10 @@ export const StartEvalRunBodySchema: JsonSchema = {
|
|
|
4831
5183
|
reads: { type: 'string', enum: ['recorded', 'live'] },
|
|
4832
5184
|
repetitions: { type: 'integer', minimum: 1, maximum: 10 },
|
|
4833
5185
|
k: { type: 'integer', minimum: 1, maximum: 100 },
|
|
5186
|
+
versions: { $ref: '#/components/schemas/FlowVersionOverrides' },
|
|
4834
5187
|
},
|
|
4835
5188
|
description:
|
|
4836
|
-
"Exactly one of `agentRef` or `flowRef` MUST be supplied. `dryRun: true` returns a plan preview without invoking the subject. For a `judged` suite (a test set), the run is a comparison: `agentRef` or `flowRef` with its `version` is the candidate, replayed on each case without doing anything the past run didn't (a flow stops at a write the replay refuses); `baseline` (default `'recorded'`), `reads` (default `recorded`), `repetitions` (default 1) and `k` (default 10) set how.",
|
|
5189
|
+
"Exactly one of `agentRef` or `flowRef` MUST be supplied. `dryRun: true` returns a plan preview without invoking the subject. For a `judged` suite (a test set), the run is a comparison: `agentRef` or `flowRef` with its `version` is the candidate, replayed on each case without doing anything the past run didn't (a flow stops at a write the replay refuses); `baseline` (default `'recorded'`), `reads` (default `recorded`), `repetitions` (default 1) and `k` (default 10) set how. With `flowRef`, `versions` runs the flow with some of its agents or tools at other versions; an id the flow doesn't use, or a version that isn't published, is refused (`400 validation-failed`, each under `details.issues`).",
|
|
4837
5190
|
};
|
|
4838
5191
|
|
|
4839
5192
|
export const StartEvalRunResultSchema: JsonSchema = {
|
|
@@ -7189,6 +7542,7 @@ export const COMPONENT_SCHEMAS: ReadonlyArray<readonly [string, JsonSchema]> = [
|
|
|
7189
7542
|
['DeriveAgentVersionBody', DeriveAgentVersionBodySchema],
|
|
7190
7543
|
['AgentPinSwaps', AgentPinSwapsSchema],
|
|
7191
7544
|
['FlowPins', FlowPinsSchema],
|
|
7545
|
+
['FlowVersionOverrides', FlowVersionOverridesSchema],
|
|
7192
7546
|
['PublishAgentBody', PublishAgentBodySchema],
|
|
7193
7547
|
['PublishAgentResult', PublishAgentResultSchema],
|
|
7194
7548
|
['UnregisterAgentResult', UnregisterAgentResultSchema],
|
|
@@ -7352,6 +7706,11 @@ export const COMPONENT_SCHEMAS: ReadonlyArray<readonly [string, JsonSchema]> = [
|
|
|
7352
7706
|
['EvalRunFlowRef', EvalRunFlowRefSchema],
|
|
7353
7707
|
['EvalBaseline', EvalBaselineSchema],
|
|
7354
7708
|
['EvalComparison', EvalComparisonSchema],
|
|
7709
|
+
['ComparisonMetric', ComparisonMetricSchema],
|
|
7710
|
+
['ComparisonCandidate', ComparisonCandidateSchema],
|
|
7711
|
+
['JudgedComparisonSummary', JudgedComparisonSummarySchema],
|
|
7712
|
+
['ComparisonCaseResult', ComparisonCaseResultSchema],
|
|
7713
|
+
['JudgedComparisonResult', JudgedComparisonResultSchema],
|
|
7355
7714
|
['EvalRun', EvalRunSchema],
|
|
7356
7715
|
['EvalRunCollectionPage', EvalRunCollectionPageSchema],
|
|
7357
7716
|
['StartEvalRunBody', StartEvalRunBodySchema],
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Copyright (C) 2026 Kindgi Inc.
|
|
3
|
+
|
|
4
|
+
import type { Context, MiddlewareHandler } from 'hono';
|
|
5
|
+
|
|
6
|
+
import { statusFor, toWireError } from './errors.js';
|
|
7
|
+
import type { AppEnv } from './types.js';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* A registry that takes no writes: it serves its definitions from
|
|
11
|
+
* somewhere else (under `kindgi dev`, the pack's files). A binding that
|
|
12
|
+
* sets it has every write refused before the binding is called:
|
|
13
|
+
* publish, unregister and reinstate (and, for agents, deriving a
|
|
14
|
+
* version), and a deploy that would publish into it. The refusal is
|
|
15
|
+
* `409 registry-read-only` with `reason` as its message.
|
|
16
|
+
*/
|
|
17
|
+
export interface RegistryReadOnly {
|
|
18
|
+
/** Why, and what to do instead, in plain words: the refusal's message. */
|
|
19
|
+
readonly reason: string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** The refusal of a write to a read-only registry. */
|
|
23
|
+
export function refuseReadOnly(c: Context<AppEnv>, readOnly: RegistryReadOnly) {
|
|
24
|
+
c.status(statusFor('registry-read-only') as never);
|
|
25
|
+
return c.json(
|
|
26
|
+
toWireError({ code: 'registry-read-only', message: readOnly.reason }, c.get('requestId')),
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* A router's writes (every method but GET and HEAD) are refused while
|
|
32
|
+
* its registry is read-only. Read at request time, so a binding can
|
|
33
|
+
* become writable without a restart.
|
|
34
|
+
*/
|
|
35
|
+
export function refuseWritesWhenReadOnly(
|
|
36
|
+
readOnly: () => RegistryReadOnly | undefined,
|
|
37
|
+
): MiddlewareHandler<AppEnv> {
|
|
38
|
+
return async (c, next) => {
|
|
39
|
+
const marker = readOnly();
|
|
40
|
+
if (marker === undefined || c.req.method === 'GET' || c.req.method === 'HEAD') return next();
|
|
41
|
+
return refuseReadOnly(c, marker);
|
|
42
|
+
};
|
|
43
|
+
}
|
package/src/routes/agents.ts
CHANGED
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
} from '../derive-agent-version.js';
|
|
25
25
|
import { statusFor, toWireError } from '../errors.js';
|
|
26
26
|
import type { Authorizer } from '../middleware/authorize.js';
|
|
27
|
+
import { refuseWritesWhenReadOnly } from '../registry-read-only.js';
|
|
27
28
|
import type { ToolRegistryBinding } from '../tool-binding.js';
|
|
28
29
|
import type { AppEnv } from '../types.js';
|
|
29
30
|
import { clampLimit } from './pagination.js';
|
|
@@ -53,6 +54,12 @@ export function agentsRouter(
|
|
|
53
54
|
blockRegistry?: BlockRegistryBinding,
|
|
54
55
|
): Hono<AppEnv> {
|
|
55
56
|
const r = new Hono<AppEnv>();
|
|
57
|
+
// A read-only registry (under `kindgi dev`, the pack's files) refuses
|
|
58
|
+
// every write before anything else runs.
|
|
59
|
+
r.use(
|
|
60
|
+
'*',
|
|
61
|
+
refuseWritesWhenReadOnly(() => binding.readOnly),
|
|
62
|
+
);
|
|
56
63
|
|
|
57
64
|
// Authorization — check the resource directly (ref('agent', businessId));
|
|
58
65
|
// permissions cascade from the parent project because the
|
|
@@ -323,7 +330,7 @@ export function agentsRouter(
|
|
|
323
330
|
toWireError(
|
|
324
331
|
{
|
|
325
332
|
code: 'validation-failed',
|
|
326
|
-
message: `Agent "${defined.value.id as unknown as string}" uses tool versions
|
|
333
|
+
message: `Agent "${defined.value.id as unknown as string}" uses tool or data-block versions it can't pin (${resolved.issues.length} issue${resolved.issues.length === 1 ? '' : 's'})`,
|
|
327
334
|
issues: resolved.issues as unknown as Record<string, unknown>[],
|
|
328
335
|
},
|
|
329
336
|
requestId,
|
|
@@ -592,6 +599,8 @@ function derived(
|
|
|
592
599
|
case 'ok':
|
|
593
600
|
c.status(201);
|
|
594
601
|
return c.json(serializeAgent(outcome.agent));
|
|
602
|
+
case 'reused':
|
|
603
|
+
return c.json(serializeAgent(outcome.agent));
|
|
595
604
|
case 'not-found':
|
|
596
605
|
return fail('agent-not-found', {
|
|
597
606
|
code: 'agent-not-found',
|
package/src/routes/blocks.ts
CHANGED
|
@@ -184,14 +184,20 @@ export function blocksRouter(binding: BlockRegistryBinding, authorizer?: Authori
|
|
|
184
184
|
return validationFailed(c, validated.error.message, validated.error.issues);
|
|
185
185
|
const block = validated.value;
|
|
186
186
|
|
|
187
|
-
// A block keeps its kind
|
|
187
|
+
// A block keeps its kind. A settings version without a schema keeps
|
|
188
|
+
// the latest version's (stored on it, so the check carries forward to
|
|
189
|
+
// every later version); one that gives a schema replaces it.
|
|
188
190
|
const latest = await binding.get({ tenantId, blockId: block.id });
|
|
189
191
|
const continuity = continuityIssues(latest, block);
|
|
190
192
|
if (continuity.length > 0) {
|
|
191
193
|
return validationFailed(c, `Block "${block.id}" can't take this version`, continuity);
|
|
192
194
|
}
|
|
193
195
|
|
|
194
|
-
const outcome = await binding.publish({
|
|
196
|
+
const outcome = await binding.publish({
|
|
197
|
+
tenantId,
|
|
198
|
+
projectId: projectId as ProjectId,
|
|
199
|
+
block: withCarriedSchema(latest, block),
|
|
200
|
+
});
|
|
195
201
|
return published(c, outcome);
|
|
196
202
|
});
|
|
197
203
|
|
|
@@ -243,7 +249,11 @@ function isBlockKind(value: string): value is BlockKind {
|
|
|
243
249
|
return (BLOCK_KINDS as readonly string[]).includes(value);
|
|
244
250
|
}
|
|
245
251
|
|
|
246
|
-
/**
|
|
252
|
+
/**
|
|
253
|
+
* A new version against the block's latest: the same kind, and a
|
|
254
|
+
* settings version without a schema of its own satisfies the schema it
|
|
255
|
+
* keeps (`carriedSchema`).
|
|
256
|
+
*/
|
|
247
257
|
function continuityIssues(
|
|
248
258
|
latest: BlockRecord | null,
|
|
249
259
|
block: BlockDefinition,
|
|
@@ -257,17 +267,32 @@ function continuityIssues(
|
|
|
257
267
|
},
|
|
258
268
|
];
|
|
259
269
|
}
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
270
|
+
const carried = carriedSchema(latest, block);
|
|
271
|
+
if (block.kind !== 'settings' || carried === undefined) return [];
|
|
272
|
+
return settingsSchemaIssues(block.content.values, carried).map((i) => ({
|
|
273
|
+
...i,
|
|
274
|
+
message: `${i.message} (the schema of version ${latest.version})`,
|
|
275
|
+
}));
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* The schema a settings version that gives none keeps: the latest
|
|
280
|
+
* version's. A version that gives a schema replaces it (`{}` drops the
|
|
281
|
+
* check on purpose).
|
|
282
|
+
*/
|
|
283
|
+
function carriedSchema(
|
|
284
|
+
latest: BlockRecord | null,
|
|
285
|
+
block: BlockDefinition,
|
|
286
|
+
): Readonly<Record<string, unknown>> | undefined {
|
|
287
|
+
if (latest?.kind !== 'settings' || block.kind !== 'settings') return undefined;
|
|
288
|
+
return block.content.schema === undefined ? latest.content.schema : undefined;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
/** The version as stored: with the schema it keeps, if any. */
|
|
292
|
+
function withCarriedSchema(latest: BlockRecord | null, block: BlockDefinition): BlockDefinition {
|
|
293
|
+
const schema = carriedSchema(latest, block);
|
|
294
|
+
if (schema === undefined || block.kind !== 'settings') return block;
|
|
295
|
+
return { ...block, content: { ...block.content, schema } };
|
|
271
296
|
}
|
|
272
297
|
|
|
273
298
|
/** The response to a publish outcome. */
|
|
@@ -41,6 +41,7 @@ import type { FlowRegistryBinding } from '../flow-binding.js';
|
|
|
41
41
|
import { publishDeployedFlow, resolveFlowPins } from '../flow-pins.js';
|
|
42
42
|
import type { GuardrailRegistryBinding } from '../guardrail-binding.js';
|
|
43
43
|
import type { ImageRegistryBinding } from '../image-registry-binding.js';
|
|
44
|
+
import { type RegistryReadOnly, refuseReadOnly } from '../registry-read-only.js';
|
|
44
45
|
import type { SecretBinding } from '../secrets-binding.js';
|
|
45
46
|
import type { SigningKeyBinding as SigningKeyRegistryBinding } from '../signing-key-binding.js';
|
|
46
47
|
import type { ToolRegistryBinding } from '../tool-binding.js';
|
|
@@ -487,6 +488,11 @@ export function deploymentsRouter(bindings: DeploymentsRouterBindings): Hono<App
|
|
|
487
488
|
}
|
|
488
489
|
const validated = validation.value;
|
|
489
490
|
|
|
491
|
+
// A read-only registry (under `kindgi dev`, the pack's files) takes
|
|
492
|
+
// nothing a deployment brings: refuse before any write.
|
|
493
|
+
const readOnly = readOnlyTarget(validated, bindings);
|
|
494
|
+
if (readOnly !== undefined) return refuseReadOnly(c, readOnly);
|
|
495
|
+
|
|
490
496
|
// ---- Registry upserts (with rollback tracking) ----
|
|
491
497
|
//
|
|
492
498
|
// Agent-registry, flow-registry, tool-registry, and
|
|
@@ -1701,3 +1707,22 @@ export function sha256HexPrefixed(bytes: Uint8Array): string {
|
|
|
1701
1707
|
const hex = createHash('sha256').update(bytes).digest('hex');
|
|
1702
1708
|
return `sha256:${hex}`;
|
|
1703
1709
|
}
|
|
1710
|
+
|
|
1711
|
+
/** The first read-only registry a deployment would publish into, if any. */
|
|
1712
|
+
function readOnlyTarget(
|
|
1713
|
+
validated: {
|
|
1714
|
+
readonly tools: readonly unknown[];
|
|
1715
|
+
readonly agents: readonly unknown[];
|
|
1716
|
+
readonly flows: readonly unknown[];
|
|
1717
|
+
readonly guardrails: readonly unknown[];
|
|
1718
|
+
},
|
|
1719
|
+
bindings: DeploymentsRouterBindings,
|
|
1720
|
+
): RegistryReadOnly | undefined {
|
|
1721
|
+
const targets = [
|
|
1722
|
+
[validated.tools, bindings.toolRegistry?.readOnly],
|
|
1723
|
+
[validated.agents, bindings.agentRegistry?.readOnly],
|
|
1724
|
+
[validated.flows, bindings.flowRegistry?.readOnly],
|
|
1725
|
+
[validated.guardrails, bindings.guardrailRegistry?.readOnly],
|
|
1726
|
+
] as const;
|
|
1727
|
+
return targets.find(([brought, readOnly]) => brought.length > 0 && readOnly !== undefined)?.[1];
|
|
1728
|
+
}
|