@kindgi/api 0.1.2 → 0.1.4-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-binding.d.ts +18 -4
- package/dist/agent-binding.d.ts.map +1 -1
- package/dist/agent-pins.d.ts +48 -0
- package/dist/agent-pins.d.ts.map +1 -0
- package/dist/agent-pins.js +102 -0
- package/dist/agent-pins.js.map +1 -0
- package/dist/app.d.ts +22 -0
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +22 -3
- package/dist/app.js.map +1 -1
- package/dist/block-binding.d.ts +132 -0
- package/dist/block-binding.d.ts.map +1 -0
- package/dist/block-binding.js +4 -0
- package/dist/block-binding.js.map +1 -0
- package/dist/block-pins.d.ts +22 -0
- package/dist/block-pins.d.ts.map +1 -0
- package/dist/block-pins.js +112 -0
- package/dist/block-pins.js.map +1 -0
- package/dist/cost-binding.d.ts +100 -5
- package/dist/cost-binding.d.ts.map +1 -1
- package/dist/cost-binding.js +10 -0
- package/dist/cost-binding.js.map +1 -1
- package/dist/deploy-versions.d.ts +58 -0
- package/dist/deploy-versions.d.ts.map +1 -0
- package/dist/deploy-versions.js +91 -0
- package/dist/deploy-versions.js.map +1 -0
- package/dist/deployment-binding.d.ts +24 -3
- package/dist/deployment-binding.d.ts.map +1 -1
- package/dist/derive-agent-version.d.ts +69 -0
- package/dist/derive-agent-version.d.ts.map +1 -0
- package/dist/derive-agent-version.js +139 -0
- package/dist/derive-agent-version.js.map +1 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +17 -0
- package/dist/errors.js.map +1 -1
- package/dist/eval-case-binding.d.ts +65 -0
- package/dist/eval-case-binding.d.ts.map +1 -0
- package/dist/eval-case-binding.js +4 -0
- package/dist/eval-case-binding.js.map +1 -0
- package/dist/eval-run-binding.d.ts +30 -0
- package/dist/eval-run-binding.d.ts.map +1 -1
- package/dist/eval-run-dispatcher.d.ts +38 -3
- package/dist/eval-run-dispatcher.d.ts.map +1 -1
- package/dist/eval-run-dispatcher.js +21 -15
- package/dist/eval-run-dispatcher.js.map +1 -1
- package/dist/eval-suite-binding.d.ts +1 -1
- package/dist/eval-suite-binding.d.ts.map +1 -1
- package/dist/eval-suite-binding.js +2 -0
- package/dist/eval-suite-binding.js.map +1 -1
- package/dist/flow-binding.d.ts +10 -4
- package/dist/flow-binding.d.ts.map +1 -1
- package/dist/flow-pins.d.ts +36 -0
- package/dist/flow-pins.d.ts.map +1 -0
- package/dist/flow-pins.js +81 -0
- package/dist/flow-pins.js.map +1 -0
- package/dist/hitl-binding.d.ts +20 -5
- package/dist/hitl-binding.d.ts.map +1 -1
- package/dist/index.d.ts +19 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -2
- package/dist/index.js.map +1 -1
- package/dist/judged-dispatcher.d.ts +134 -0
- package/dist/judged-dispatcher.d.ts.map +1 -0
- package/dist/judged-dispatcher.js +297 -0
- package/dist/judged-dispatcher.js.map +1 -0
- package/dist/judged-items.d.ts +86 -0
- package/dist/judged-items.d.ts.map +1 -0
- package/dist/judged-items.js +184 -0
- package/dist/judged-items.js.map +1 -0
- package/dist/judgment-binding.d.ts +316 -0
- package/dist/judgment-binding.d.ts.map +1 -0
- package/dist/judgment-binding.js +19 -0
- package/dist/judgment-binding.js.map +1 -0
- package/dist/middleware/auth.d.ts +3 -2
- package/dist/middleware/auth.d.ts.map +1 -1
- package/dist/middleware/auth.js.map +1 -1
- package/dist/middleware/idempotency.d.ts +5 -1
- package/dist/middleware/idempotency.d.ts.map +1 -1
- package/dist/middleware/idempotency.js +8 -1
- package/dist/middleware/idempotency.js.map +1 -1
- package/dist/openapi/generate.d.ts.map +1 -1
- package/dist/openapi/generate.js +4 -1
- package/dist/openapi/generate.js.map +1 -1
- package/dist/openapi/operations.d.ts.map +1 -1
- package/dist/openapi/operations.js +543 -24
- package/dist/openapi/operations.js.map +1 -1
- package/dist/openapi/schemas.d.ts +58 -0
- package/dist/openapi/schemas.d.ts.map +1 -1
- package/dist/openapi/schemas.js +1058 -27
- package/dist/openapi/schemas.js.map +1 -1
- package/dist/provenance-binding.d.ts +27 -1
- package/dist/provenance-binding.d.ts.map +1 -1
- package/dist/provenance-binding.js.map +1 -1
- package/dist/provider-binding.d.ts +12 -7
- package/dist/provider-binding.d.ts.map +1 -1
- package/dist/reviewer-binding.d.ts +10 -2
- package/dist/reviewer-binding.d.ts.map +1 -1
- package/dist/reviewer-role.d.ts +13 -0
- package/dist/reviewer-role.d.ts.map +1 -0
- package/dist/reviewer-role.js +27 -0
- package/dist/reviewer-role.js.map +1 -0
- package/dist/routes/agents.d.ts +9 -1
- package/dist/routes/agents.d.ts.map +1 -1
- package/dist/routes/agents.js +175 -11
- package/dist/routes/agents.js.map +1 -1
- package/dist/routes/approvals.d.ts +5 -4
- package/dist/routes/approvals.d.ts.map +1 -1
- package/dist/routes/approvals.js +52 -16
- package/dist/routes/approvals.js.map +1 -1
- package/dist/routes/blocks.d.ts +19 -0
- package/dist/routes/blocks.d.ts.map +1 -0
- package/dist/routes/blocks.js +281 -0
- package/dist/routes/blocks.js.map +1 -0
- package/dist/routes/conversations.d.ts +7 -1
- package/dist/routes/conversations.d.ts.map +1 -1
- package/dist/routes/conversations.js +41 -1
- package/dist/routes/conversations.js.map +1 -1
- package/dist/routes/cost.d.ts.map +1 -1
- package/dist/routes/cost.js +142 -11
- package/dist/routes/cost.js.map +1 -1
- package/dist/routes/deployments.d.ts +3 -0
- package/dist/routes/deployments.d.ts.map +1 -1
- package/dist/routes/deployments.js +208 -55
- package/dist/routes/deployments.js.map +1 -1
- package/dist/routes/eval-comparison.d.ts +14 -0
- package/dist/routes/eval-comparison.d.ts.map +1 -0
- package/dist/routes/eval-comparison.js +87 -0
- package/dist/routes/eval-comparison.js.map +1 -0
- package/dist/routes/eval-runs.d.ts.map +1 -1
- package/dist/routes/eval-runs.js +8 -0
- package/dist/routes/eval-runs.js.map +1 -1
- package/dist/routes/flows.d.ts +13 -1
- package/dist/routes/flows.d.ts.map +1 -1
- package/dist/routes/flows.js +44 -3
- package/dist/routes/flows.js.map +1 -1
- package/dist/routes/hierarchy-errors.d.ts +35 -0
- package/dist/routes/hierarchy-errors.d.ts.map +1 -0
- package/dist/routes/hierarchy-errors.js +39 -0
- package/dist/routes/hierarchy-errors.js.map +1 -0
- package/dist/routes/identity.d.ts +7 -0
- package/dist/routes/identity.d.ts.map +1 -1
- package/dist/routes/identity.js +3 -2
- package/dist/routes/identity.js.map +1 -1
- package/dist/routes/judged-suites.d.ts +20 -0
- package/dist/routes/judged-suites.d.ts.map +1 -0
- package/dist/routes/judged-suites.js +272 -0
- package/dist/routes/judged-suites.js.map +1 -0
- package/dist/routes/judgment-context.d.ts +22 -0
- package/dist/routes/judgment-context.d.ts.map +1 -0
- package/dist/routes/judgment-context.js +88 -0
- package/dist/routes/judgment-context.js.map +1 -0
- package/dist/routes/judgment-flow-context.d.ts +32 -0
- package/dist/routes/judgment-flow-context.d.ts.map +1 -0
- package/dist/routes/judgment-flow-context.js +195 -0
- package/dist/routes/judgment-flow-context.js.map +1 -0
- package/dist/routes/judgments.d.ts +41 -0
- package/dist/routes/judgments.d.ts.map +1 -0
- package/dist/routes/judgments.js +566 -0
- package/dist/routes/judgments.js.map +1 -0
- package/dist/routes/orgs.d.ts +5 -2
- package/dist/routes/orgs.d.ts.map +1 -1
- package/dist/routes/orgs.js +38 -22
- package/dist/routes/orgs.js.map +1 -1
- package/dist/routes/policies.d.ts.map +1 -1
- package/dist/routes/policies.js +12 -1
- package/dist/routes/policies.js.map +1 -1
- package/dist/routes/projects.d.ts +10 -2
- package/dist/routes/projects.d.ts.map +1 -1
- package/dist/routes/projects.js +87 -79
- package/dist/routes/projects.js.map +1 -1
- package/dist/routes/provenance.d.ts.map +1 -1
- package/dist/routes/provenance.js +32 -1
- package/dist/routes/provenance.js.map +1 -1
- package/dist/routes/providers.d.ts.map +1 -1
- package/dist/routes/providers.js +6 -1
- package/dist/routes/providers.js.map +1 -1
- package/dist/routes/runs.d.ts +0 -7
- package/dist/routes/runs.d.ts.map +1 -1
- package/dist/routes/runs.js +65 -20
- package/dist/routes/runs.js.map +1 -1
- package/dist/routes/scope-params.d.ts +16 -1
- package/dist/routes/scope-params.d.ts.map +1 -1
- package/dist/routes/scope-params.js +28 -0
- package/dist/routes/scope-params.js.map +1 -1
- package/dist/routes/teams.d.ts +6 -2
- package/dist/routes/teams.d.ts.map +1 -1
- package/dist/routes/teams.js +77 -73
- package/dist/routes/teams.js.map +1 -1
- package/dist/types.d.ts +4 -3
- package/dist/types.d.ts.map +1 -1
- package/dist/webhook-endpoint-binding.d.ts +11 -0
- package/dist/webhook-endpoint-binding.d.ts.map +1 -1
- package/dist/webhook-endpoint-binding.js.map +1 -1
- package/openapi.json +13316 -9299
- package/package.json +21 -21
- package/src/agent-binding.ts +19 -4
- package/src/agent-pins.ts +147 -0
- package/src/app.ts +76 -3
- package/src/block-binding.ts +137 -0
- package/src/block-pins.ts +148 -0
- package/src/cost-binding.ts +116 -5
- package/src/deploy-versions.ts +157 -0
- package/src/deployment-binding.ts +27 -3
- package/src/derive-agent-version.ts +206 -0
- package/src/errors.ts +17 -0
- package/src/eval-case-binding.ts +71 -0
- package/src/eval-run-binding.ts +33 -0
- package/src/eval-run-dispatcher.ts +57 -16
- package/src/eval-suite-binding.ts +2 -0
- package/src/flow-binding.ts +11 -4
- package/src/flow-pins.ts +113 -0
- package/src/hitl-binding.ts +20 -4
- package/src/index.ts +88 -2
- package/src/judged-dispatcher.ts +507 -0
- package/src/judged-items.ts +263 -0
- package/src/judgment-binding.ts +349 -0
- package/src/middleware/auth.ts +3 -2
- package/src/middleware/idempotency.ts +7 -1
- package/src/openapi/generate.ts +7 -1
- package/src/openapi/operations.ts +615 -24
- package/src/openapi/schemas.ts +1157 -22
- package/src/provenance-binding.ts +42 -1
- package/src/provider-binding.ts +12 -7
- package/src/reviewer-binding.ts +10 -2
- package/src/reviewer-role.ts +35 -0
- package/src/routes/agents.ts +243 -19
- package/src/routes/approvals.ts +70 -19
- package/src/routes/blocks.ts +362 -0
- package/src/routes/conversations.ts +57 -1
- package/src/routes/cost.ts +159 -21
- package/src/routes/deployments.ts +266 -56
- package/src/routes/eval-comparison.ts +101 -0
- package/src/routes/eval-runs.ts +11 -0
- package/src/routes/flows.ts +63 -5
- package/src/routes/hierarchy-errors.ts +51 -0
- package/src/routes/identity.ts +10 -2
- package/src/routes/judged-suites.ts +363 -0
- package/src/routes/judgment-context.ts +128 -0
- package/src/routes/judgment-flow-context.ts +245 -0
- package/src/routes/judgments.ts +743 -0
- package/src/routes/orgs.ts +44 -27
- package/src/routes/policies.ts +19 -0
- package/src/routes/projects.ts +106 -95
- package/src/routes/provenance.ts +48 -1
- package/src/routes/providers.ts +5 -0
- package/src/routes/runs.ts +79 -22
- package/src/routes/scope-params.ts +35 -1
- package/src/routes/teams.ts +96 -90
- package/src/types.ts +4 -3
- package/src/webhook-endpoint-binding.ts +11 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judgment-flow-context.js","sourceRoot":"","sources":["../../src/routes/judgment-flow-context.ts"],"names":[],"mappings":"AAAA,sCAAsC;AACtC,iCAAiC;AAYjC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAYxD,mDAAmD;AACnD,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AACzC,gDAAgD;AAChD,MAAM,cAAc,GAAG,CAAC,CAAC;AACzB,wCAAwC;AACxC,MAAM,YAAY,GAAG,GAAG,CAAC;AAEzB,SAAS,GAAG,CAAC,KAAc;IACzB,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACzE,CAAC,CAAE,KAAiC;QACpC,CAAC,CAAC,SAAS,CAAC;AAChB,CAAC;AAED,wFAAwF;AACxF,SAAS,SAAS,CAAC,IAAiB;IAClC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,MAAM,IAAI,GAAG,CAAC,KAA0B,EAAQ,EAAE;QAChD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;gBAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAuB,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;iBACvE,IAAI,UAAU,CAAC,IAAI,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;iBAC5C,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ;oBAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,OAA4B,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,IAAI,IAAI,KAAK,IAAI;QAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,MAAe;IAEf,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC;IACvC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAmB,CAAC;IACxC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,SAAS,CAAC;QAC9C,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,SAAS;QAChE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACzB,IAAI,OAAO,EAAE,KAAK,QAAQ;gBAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,SAAS,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;QAC5B,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,QAAQ,EAAE,MAAM,CAAC;QAChC,MAAM,YAAY,GAAG,QAAQ,EAAE,YAAY,CAAC;QAC5C,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;YAC9F,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC;YAC3B,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;YAC1E,CAAC,CAAC,EAAE,CAAC;IACT,CAAC,CAAC,CAAC;AACL,CAAC;AAED,uEAAuE;AACvE,SAAS,OAAO,CAAC,WAAoB;IACnC,OAAO,WAAW,KAAK,SAAS,IAAI,WAAW,KAAK,IAAI;QACtD,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,SAAS;IACJ,KAAK,GAAqB,EAAE,CAAC;IAC7B,KAAK,GAAqB,EAAE,CAAC;IACtC,SAAS,GAAG,KAAK,CAAC;IAElB,GAAG,CAAC,IAAoB;QACtB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,qBAAqB,EAAE,CAAC;YAC/C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC;CACF;AAQD,2GAA2G;AAC3G,KAAK,UAAU,SAAS,CACtB,CAAS,EACT,CAAY,EACZ,GAAsF;IAEtF,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACxC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC;YAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,MAAM,EAAE,GAAG,CAAC,MAAgB;YAC5B,OAAO,EAAE,GAAG,CAAC,WAAW;SACzB,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;QAC3B,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAc,CAAC;KACzD,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO;IACvD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC1C,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC9B,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;YAAE,SAAS;QACnE,MAAM,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC3C,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,MAAM,SAAS,KAAK,IAAI,EAAE,EAAE,CAAC;QAC9C,IAAI,CAAC,CAAC,IAAI,KAAK,cAAc;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;QAC9D,IAAI,CAAC,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YAChC,CAAC,CAAC,GAAG,CAAC;gBACJ,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,CAAC;gBACrC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAW;gBACrC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;gBAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC;AAED,8DAA8D;AAC9D,KAAK,UAAU,QAAQ,CAAC,CAAS,EAAE,CAAY,EAAE,KAAsB;IACrE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IAC1B,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO;IAChC,MAAM,MAAM,GACV,KAAK,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,IAAI,SAAS,CAAC;IAC5F,MAAM,MAAM,GAAI,KAAK,CAAC,YAAqD,IAAI,SAAS,CAAC;IACzF,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,IAAI,SAAS,CAAC;IAC7C,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC;IACzC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QACX,KAAK,EAAE,KAAK,CAAC,KAA0B;QACvC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,CAAC;QACvC,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACrD,OAAO,EAAE,KAAK,CAAC,EAAE;QACjB,YAAY,EAAE,KAAK,CAAC,OAAO;QAC3B,GAAG,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;KAC9C,CAAC,CAAC;IACH,KAAK,MAAM,IAAI,IAAI,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;QACzC,CAAC,CAAC,GAAG,CAAC;YACJ,KAAK,EAAE,KAAK,CAAC,KAA0B;YACvC,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,CAAC;YACvC,GAAG,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;YACrD,GAAG,IAAI;SACR,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,wFAAwF;AACxF,KAAK,UAAU,UAAU,CAAC,CAAS,EAAE,KAAa;IAChD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC;YACvC,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,MAAM,EAAE,EAAE,KAAK,EAAE,KAAc,EAAE;YACjC,KAAK,EAAE,YAAY;SACpB,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,CAAS,EACT,CAAY,EACZ,GAAsF,EACtF,KAAa;IAEb,MAAM,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAClD,KAAK,MAAM,KAAK,IAAI,MAAM,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACrD,CAAC;aAAM,IAAI,KAAK,GAAG,cAAc,EAAE,CAAC;YAClC,MAAM,UAAU,CACd,CAAC,EACD,CAAC,EACD;gBACE,KAAK,EAAE,KAAK,CAAC,KAA0B;gBACvC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,WAAW,EAAE,KAAK,CAAC,WAAW;aAC/B,EACD,KAAK,GAAG,CAAC,CACV,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAKxC;IACC,MAAM,CAAC,GAAG,IAAI,SAAS,EAAE,CAAC;IAC1B,IAAI,CAAC;QACH,MAAM,UAAU,CACd,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,EAC9E,CAAC,EACD,KAAK,CAAC,GAAG,EACT,CAAC,CACF,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,mCAAmC;IACrC,CAAC;IACD,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACnE,MAAM,IAAI,GAAsB;QAC9B,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KACxC,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { Hono } from 'hono';
|
|
2
|
+
import type { ConversationBinding } from '@kindgi/agents';
|
|
3
|
+
import type { RunBinding } from '@kindgi/runtime';
|
|
4
|
+
import type { FlowRegistryBinding } from '../flow-binding.js';
|
|
5
|
+
import { type JudgmentRegistryBinding } from '../judgment-binding.js';
|
|
6
|
+
import type { Authorizer } from '../middleware/authorize.js';
|
|
7
|
+
import type { AppEnv } from '../types.js';
|
|
8
|
+
/**
|
|
9
|
+
* Judgments: yes or no, with an optional reason, about one item of a
|
|
10
|
+
* run's output, recorded under a judge class.
|
|
11
|
+
*
|
|
12
|
+
* - `POST /v1/judgments`: judge an item of a finished run.
|
|
13
|
+
* - `GET /v1/judgments`: live judgments, filtered by run, agent version,
|
|
14
|
+
* flow, verdict, class, participant or project scope.
|
|
15
|
+
* - `GET /v1/judgments/:id`: one judgment with the stored copies.
|
|
16
|
+
* - `POST /v1/judgments/:id/unregister`: remove a judgment (soft).
|
|
17
|
+
*
|
|
18
|
+
* Authorization: judging needs `write` on the run's project (a run
|
|
19
|
+
* inherits its permissions from its project). Reading and removing go by
|
|
20
|
+
* the judgment's project (`read` / `write`), since a judgment outlives
|
|
21
|
+
* its run.
|
|
22
|
+
*/
|
|
23
|
+
export declare function judgmentsRouter(binding: JudgmentRegistryBinding, runBinding: RunBinding, authorizer?: Authorizer, conversations?: ConversationBinding, flows?: FlowRegistryBinding): Hono<AppEnv>;
|
|
24
|
+
/**
|
|
25
|
+
* Judge classes: the deployment's named classes of judges ("expert",
|
|
26
|
+
* "user", …), each with a weight, scoped to the tenant, a project, or an
|
|
27
|
+
* agent in a project.
|
|
28
|
+
*
|
|
29
|
+
* Managing a tenant-scoped class needs `admin` on the tenant; a project-
|
|
30
|
+
* or agent-scoped class needs `admin` on its project. Listing returns the
|
|
31
|
+
* classes the caller may read.
|
|
32
|
+
*/
|
|
33
|
+
export declare function judgeClassesRouter(binding: JudgmentRegistryBinding, authorizer?: Authorizer): Hono<AppEnv>;
|
|
34
|
+
/** RFC 6901: the value at `pointer` in `doc`. "" is the whole document. */
|
|
35
|
+
export declare function resolvePointer(doc: unknown, pointer: string): {
|
|
36
|
+
readonly found: true;
|
|
37
|
+
readonly value: unknown;
|
|
38
|
+
} | {
|
|
39
|
+
readonly found: false;
|
|
40
|
+
};
|
|
41
|
+
//# sourceMappingURL=judgments.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judgments.d.ts","sourceRoot":"","sources":["../../src/routes/judgments.ts"],"names":[],"mappings":"AAGA,OAAO,EAAgB,IAAI,EAAE,MAAM,MAAM,CAAC;AAE1C,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAE1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAIlD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAQL,KAAK,uBAAuB,EAK7B,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC7D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAM1C;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,uBAAuB,EAChC,UAAU,EAAE,UAAU,EACtB,UAAU,CAAC,EAAE,UAAU,EACvB,aAAa,CAAC,EAAE,mBAAmB,EACnC,KAAK,CAAC,EAAE,mBAAmB,GAC1B,IAAI,CAAC,MAAM,CAAC,CA8Jd;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,uBAAuB,EAChC,UAAU,CAAC,EAAE,UAAU,GACtB,IAAI,CAAC,MAAM,CAAC,CAoJd;AAkTD,2EAA2E;AAC3E,wBAAgB,cAAc,CAC5B,GAAG,EAAE,OAAO,EACZ,OAAO,EAAE,MAAM,GACd;IAAE,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;CAAE,CAe/E"}
|
|
@@ -0,0 +1,566 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Copyright (C) 2026 Kindgi Inc.
|
|
3
|
+
import { Hono } from 'hono';
|
|
4
|
+
import { ref } from '@kindgi/authz';
|
|
5
|
+
import { statusFor, toWireError } from '../errors.js';
|
|
6
|
+
import { VERDICTS, judgeClassApplies, } from '../judgment-binding.js';
|
|
7
|
+
import { captureTurnContext } from './judgment-context.js';
|
|
8
|
+
import { captureFlowContext } from './judgment-flow-context.js';
|
|
9
|
+
import { clampLimit } from './pagination.js';
|
|
10
|
+
import { parseListScope } from './scope-params.js';
|
|
11
|
+
/**
|
|
12
|
+
* Judgments: yes or no, with an optional reason, about one item of a
|
|
13
|
+
* run's output, recorded under a judge class.
|
|
14
|
+
*
|
|
15
|
+
* - `POST /v1/judgments`: judge an item of a finished run.
|
|
16
|
+
* - `GET /v1/judgments`: live judgments, filtered by run, agent version,
|
|
17
|
+
* flow, verdict, class, participant or project scope.
|
|
18
|
+
* - `GET /v1/judgments/:id`: one judgment with the stored copies.
|
|
19
|
+
* - `POST /v1/judgments/:id/unregister`: remove a judgment (soft).
|
|
20
|
+
*
|
|
21
|
+
* Authorization: judging needs `write` on the run's project (a run
|
|
22
|
+
* inherits its permissions from its project). Reading and removing go by
|
|
23
|
+
* the judgment's project (`read` / `write`), since a judgment outlives
|
|
24
|
+
* its run.
|
|
25
|
+
*/
|
|
26
|
+
export function judgmentsRouter(binding, runBinding, authorizer, conversations, flows) {
|
|
27
|
+
const r = new Hono();
|
|
28
|
+
r.post('/', async (c) => {
|
|
29
|
+
const requestId = c.get('requestId');
|
|
30
|
+
const tenantId = c.get('tenantId');
|
|
31
|
+
const fail = (code, message) => {
|
|
32
|
+
c.status(statusFor(code));
|
|
33
|
+
return c.json(toWireError({ code, message }, requestId));
|
|
34
|
+
};
|
|
35
|
+
const parsed = parseJudgmentBody(await c.req.json().catch(() => null));
|
|
36
|
+
if (parsed.kind === 'err')
|
|
37
|
+
return fail('bad-input', parsed.message);
|
|
38
|
+
const body = parsed.body;
|
|
39
|
+
const asserted = assertedByOf(c.get('principal'));
|
|
40
|
+
if (asserted === undefined) {
|
|
41
|
+
return fail('permission-denied', 'Judging needs a user or a service token.');
|
|
42
|
+
}
|
|
43
|
+
const prepared = await prepareJudgment(c, body, runBinding, binding, authorizer);
|
|
44
|
+
if (prepared.kind === 'err')
|
|
45
|
+
return fail(prepared.code, prepared.message);
|
|
46
|
+
const { run, subject, projectId, itemValue, conversationId } = prepared;
|
|
47
|
+
const context = (await isFirstJudgment(binding, tenantId, body.runId))
|
|
48
|
+
? await captureContext({
|
|
49
|
+
tenantId,
|
|
50
|
+
runId: body.runId,
|
|
51
|
+
subject,
|
|
52
|
+
output: run.output,
|
|
53
|
+
conversationId,
|
|
54
|
+
runBinding,
|
|
55
|
+
conversations,
|
|
56
|
+
flows,
|
|
57
|
+
})
|
|
58
|
+
: undefined;
|
|
59
|
+
const judgment = await binding.record({
|
|
60
|
+
tenantId,
|
|
61
|
+
projectId,
|
|
62
|
+
runId: body.runId,
|
|
63
|
+
run: {
|
|
64
|
+
subject,
|
|
65
|
+
input: run.input,
|
|
66
|
+
output: run.output,
|
|
67
|
+
...(context !== undefined && { context }),
|
|
68
|
+
},
|
|
69
|
+
item: body.item,
|
|
70
|
+
...(itemValue !== undefined && { itemValue }),
|
|
71
|
+
verdict: body.verdict,
|
|
72
|
+
...(body.reason !== undefined && { reason: body.reason }),
|
|
73
|
+
...(body.judgeClassId !== undefined && { judgeClassId: body.judgeClassId }),
|
|
74
|
+
assertedBy: asserted,
|
|
75
|
+
...(body.participantId !== undefined && { participantId: body.participantId }),
|
|
76
|
+
});
|
|
77
|
+
c.status(201);
|
|
78
|
+
return c.json(serializeJudgment(judgment));
|
|
79
|
+
});
|
|
80
|
+
r.get('/', async (c) => {
|
|
81
|
+
const requestId = c.get('requestId');
|
|
82
|
+
const tenantId = c.get('tenantId');
|
|
83
|
+
const fail = (code, message) => {
|
|
84
|
+
c.status(statusFor(code));
|
|
85
|
+
return c.json(toWireError({ code, message }, requestId));
|
|
86
|
+
};
|
|
87
|
+
const q = (name) => {
|
|
88
|
+
const v = c.req.query(name);
|
|
89
|
+
return v !== undefined && v.length > 0 ? v : undefined;
|
|
90
|
+
};
|
|
91
|
+
const scope = parseListScope(c.req.query(), { tenantId });
|
|
92
|
+
if (scope.kind === 'err')
|
|
93
|
+
return fail('scope-invalid', scope.message);
|
|
94
|
+
const verdict = q('verdict');
|
|
95
|
+
if (verdict !== undefined && !isVerdict(verdict)) {
|
|
96
|
+
return fail('bad-input', `verdict must be one of: ${VERDICTS.join(', ')}.`);
|
|
97
|
+
}
|
|
98
|
+
if (q('agentVersion') !== undefined && q('agentId') === undefined) {
|
|
99
|
+
return fail('bad-input', 'agentVersion needs agentId.');
|
|
100
|
+
}
|
|
101
|
+
const page = await binding.list({
|
|
102
|
+
tenantId,
|
|
103
|
+
limit: clampLimit(c.req.query('limit')),
|
|
104
|
+
...(scope.scope !== undefined && { scope: scope.scope }),
|
|
105
|
+
...optional('runId', q('runId')),
|
|
106
|
+
...optional('agentId', q('agentId')),
|
|
107
|
+
...optional('agentVersion', q('agentVersion')),
|
|
108
|
+
...optional('flowId', q('flowId')),
|
|
109
|
+
...(verdict !== undefined && { verdict: verdict }),
|
|
110
|
+
...optional('judgeClassId', q('judgeClassId')),
|
|
111
|
+
...optional('participantId', q('participantId')),
|
|
112
|
+
...(q('cursor') !== undefined && { cursor: q('cursor') }),
|
|
113
|
+
});
|
|
114
|
+
const visible = authorizer === undefined
|
|
115
|
+
? page.data
|
|
116
|
+
: await authorizer.filterByCan(c, 'read', page.data, (j) => ref('project', j.projectId));
|
|
117
|
+
return c.json({
|
|
118
|
+
data: visible.map(serializeJudgment),
|
|
119
|
+
hasMore: page.hasMore,
|
|
120
|
+
...(page.nextCursor !== undefined && { nextCursor: page.nextCursor }),
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
r.get('/:judgmentId', async (c) => {
|
|
124
|
+
const requestId = c.get('requestId');
|
|
125
|
+
const tenantId = c.get('tenantId');
|
|
126
|
+
const judgmentId = c.req.param('judgmentId');
|
|
127
|
+
const judgment = await binding.get({ tenantId, judgmentId });
|
|
128
|
+
if (judgment === null ||
|
|
129
|
+
(authorizer !== undefined &&
|
|
130
|
+
!(await authorizer.can(c, 'read', ref('project', judgment.projectId))))) {
|
|
131
|
+
c.status(statusFor('judgment-not-found'));
|
|
132
|
+
return c.json(toWireError({ code: 'judgment-not-found', message: `No judgment "${judgmentId}".` }, requestId));
|
|
133
|
+
}
|
|
134
|
+
return c.json(serializeJudgmentWithCopies(judgment));
|
|
135
|
+
});
|
|
136
|
+
r.post('/:judgmentId/unregister', async (c) => {
|
|
137
|
+
const requestId = c.get('requestId');
|
|
138
|
+
const tenantId = c.get('tenantId');
|
|
139
|
+
const judgmentId = c.req.param('judgmentId');
|
|
140
|
+
const notFound = () => {
|
|
141
|
+
c.status(statusFor('judgment-not-found'));
|
|
142
|
+
return c.json(toWireError({ code: 'judgment-not-found', message: `No live judgment "${judgmentId}".` }, requestId));
|
|
143
|
+
};
|
|
144
|
+
const judgment = await binding.get({ tenantId, judgmentId });
|
|
145
|
+
if (judgment === null || judgment.unregisteredAt !== undefined)
|
|
146
|
+
return notFound();
|
|
147
|
+
if (authorizer !== undefined &&
|
|
148
|
+
!(await authorizer.can(c, 'write', ref('project', judgment.projectId)))) {
|
|
149
|
+
c.status(statusFor('permission-denied'));
|
|
150
|
+
return c.json(toWireError({ code: 'permission-denied', message: `Not allowed to remove judgment "${judgmentId}".` }, requestId));
|
|
151
|
+
}
|
|
152
|
+
const outcome = await binding.unregister({ tenantId, judgmentId });
|
|
153
|
+
if (!outcome.unregistered)
|
|
154
|
+
return notFound();
|
|
155
|
+
return c.json({ judgmentId, unregistered: true });
|
|
156
|
+
});
|
|
157
|
+
return r;
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Judge classes: the deployment's named classes of judges ("expert",
|
|
161
|
+
* "user", …), each with a weight, scoped to the tenant, a project, or an
|
|
162
|
+
* agent in a project.
|
|
163
|
+
*
|
|
164
|
+
* Managing a tenant-scoped class needs `admin` on the tenant; a project-
|
|
165
|
+
* or agent-scoped class needs `admin` on its project. Listing returns the
|
|
166
|
+
* classes the caller may read.
|
|
167
|
+
*/
|
|
168
|
+
export function judgeClassesRouter(binding, authorizer) {
|
|
169
|
+
const r = new Hono();
|
|
170
|
+
const scopeRef = (tenantId, scope) => scope.kind === 'tenant' ? ref('tenant', tenantId) : ref('project', scope.projectId);
|
|
171
|
+
r.post('/', async (c) => {
|
|
172
|
+
const requestId = c.get('requestId');
|
|
173
|
+
const tenantId = c.get('tenantId');
|
|
174
|
+
const fail = (code, message) => {
|
|
175
|
+
c.status(statusFor(code));
|
|
176
|
+
return c.json(toWireError({ code, message }, requestId));
|
|
177
|
+
};
|
|
178
|
+
const parsed = parseClassBody(await c.req.json().catch(() => null));
|
|
179
|
+
if (parsed.kind === 'err')
|
|
180
|
+
return fail('bad-input', parsed.message);
|
|
181
|
+
const { scope, name, weight, description } = parsed.body;
|
|
182
|
+
if (authorizer !== undefined &&
|
|
183
|
+
!(await authorizer.can(c, 'admin', scopeRef(tenantId, scope)))) {
|
|
184
|
+
return fail('permission-denied', 'Not allowed to manage judge classes in this scope.');
|
|
185
|
+
}
|
|
186
|
+
const outcome = await binding.createClass({
|
|
187
|
+
tenantId,
|
|
188
|
+
scope,
|
|
189
|
+
name,
|
|
190
|
+
weight,
|
|
191
|
+
...(description !== undefined && { description }),
|
|
192
|
+
});
|
|
193
|
+
if (outcome.kind === 'name-taken') {
|
|
194
|
+
return fail('judge-class-name-taken', `A judge class named "${name}" already exists here.`);
|
|
195
|
+
}
|
|
196
|
+
c.status(201);
|
|
197
|
+
return c.json(serializeJudgeClass(outcome.judgeClass));
|
|
198
|
+
});
|
|
199
|
+
r.get('/', async (c) => {
|
|
200
|
+
const requestId = c.get('requestId');
|
|
201
|
+
const tenantId = c.get('tenantId');
|
|
202
|
+
const scope = parseClassScopeQuery(c.req.query());
|
|
203
|
+
if (scope.kind === 'err') {
|
|
204
|
+
c.status(statusFor('bad-input'));
|
|
205
|
+
return c.json(toWireError({ code: 'bad-input', message: scope.message }, requestId));
|
|
206
|
+
}
|
|
207
|
+
const cursor = c.req.query('cursor');
|
|
208
|
+
const page = await binding.listClasses({
|
|
209
|
+
tenantId,
|
|
210
|
+
limit: clampLimit(c.req.query('limit')),
|
|
211
|
+
...(scope.scope !== undefined && { scope: scope.scope }),
|
|
212
|
+
...(cursor !== undefined && cursor.length > 0 && { cursor: cursor }),
|
|
213
|
+
});
|
|
214
|
+
const visible = authorizer === undefined
|
|
215
|
+
? page.data
|
|
216
|
+
: await authorizer.filterByCan(c, 'read', page.data, (k) => scopeRef(tenantId, k.scope));
|
|
217
|
+
return c.json({
|
|
218
|
+
data: visible.map(serializeJudgeClass),
|
|
219
|
+
hasMore: page.hasMore,
|
|
220
|
+
...(page.nextCursor !== undefined && { nextCursor: page.nextCursor }),
|
|
221
|
+
});
|
|
222
|
+
});
|
|
223
|
+
const loadClass = async (c, action) => {
|
|
224
|
+
const requestId = c.get('requestId');
|
|
225
|
+
const tenantId = c.get('tenantId');
|
|
226
|
+
const judgeClassId = c.req.param('judgeClassId') ?? '';
|
|
227
|
+
const found = await binding.getClass({
|
|
228
|
+
tenantId,
|
|
229
|
+
judgeClassId,
|
|
230
|
+
includeUnregistered: action === 'read',
|
|
231
|
+
});
|
|
232
|
+
if (found !== null && authorizer !== undefined) {
|
|
233
|
+
const allowed = await authorizer.can(c, action, scopeRef(tenantId, found.scope));
|
|
234
|
+
if (!allowed &&
|
|
235
|
+
action === 'admin' &&
|
|
236
|
+
(await authorizer.can(c, 'read', scopeRef(tenantId, found.scope)))) {
|
|
237
|
+
c.status(statusFor('permission-denied'));
|
|
238
|
+
return c.json(toWireError({
|
|
239
|
+
code: 'permission-denied',
|
|
240
|
+
message: 'Not allowed to manage judge classes in this scope.',
|
|
241
|
+
}, requestId));
|
|
242
|
+
}
|
|
243
|
+
if (!allowed)
|
|
244
|
+
return classNotFound(c, judgeClassId);
|
|
245
|
+
}
|
|
246
|
+
return found ?? classNotFound(c, judgeClassId);
|
|
247
|
+
};
|
|
248
|
+
r.get('/:judgeClassId', async (c) => {
|
|
249
|
+
const found = await loadClass(c, 'read');
|
|
250
|
+
return found instanceof Response ? found : c.json(serializeJudgeClass(found));
|
|
251
|
+
});
|
|
252
|
+
r.patch('/:judgeClassId', async (c) => {
|
|
253
|
+
const requestId = c.get('requestId');
|
|
254
|
+
const tenantId = c.get('tenantId');
|
|
255
|
+
const raw = (await c.req.json().catch(() => null));
|
|
256
|
+
const weight = raw?.weight;
|
|
257
|
+
const description = raw?.description;
|
|
258
|
+
if (raw === null ||
|
|
259
|
+
typeof raw !== 'object' ||
|
|
260
|
+
(weight === undefined && description === undefined) ||
|
|
261
|
+
(weight !== undefined && !isWeight(weight)) ||
|
|
262
|
+
(description !== undefined && typeof description !== 'string')) {
|
|
263
|
+
c.status(statusFor('bad-input'));
|
|
264
|
+
return c.json(toWireError({
|
|
265
|
+
code: 'bad-input',
|
|
266
|
+
message: 'Send weight (a number ≥ 0) and/or description (a string).',
|
|
267
|
+
}, requestId));
|
|
268
|
+
}
|
|
269
|
+
const found = await loadClass(c, 'admin');
|
|
270
|
+
if (found instanceof Response)
|
|
271
|
+
return found;
|
|
272
|
+
if (found.unregisteredAt !== undefined)
|
|
273
|
+
return classNotFound(c, found.id);
|
|
274
|
+
const updated = await binding.updateClass({
|
|
275
|
+
tenantId,
|
|
276
|
+
judgeClassId: found.id,
|
|
277
|
+
...(weight !== undefined && { weight: weight }),
|
|
278
|
+
...(description !== undefined && { description: description }),
|
|
279
|
+
});
|
|
280
|
+
return updated === null ? classNotFound(c, found.id) : c.json(serializeJudgeClass(updated));
|
|
281
|
+
});
|
|
282
|
+
r.post('/:judgeClassId/unregister', async (c) => {
|
|
283
|
+
const tenantId = c.get('tenantId');
|
|
284
|
+
const found = await loadClass(c, 'admin');
|
|
285
|
+
if (found instanceof Response)
|
|
286
|
+
return found;
|
|
287
|
+
const outcome = await binding.unregisterClass({ tenantId, judgeClassId: found.id });
|
|
288
|
+
if (!outcome.unregistered)
|
|
289
|
+
return classNotFound(c, found.id);
|
|
290
|
+
return c.json({ judgeClassId: found.id, unregistered: true });
|
|
291
|
+
});
|
|
292
|
+
return r;
|
|
293
|
+
}
|
|
294
|
+
/**
|
|
295
|
+
* Everything a judgment needs from its run, or why it can't be recorded:
|
|
296
|
+
* the run exists, the caller may judge it, it has finished with an output,
|
|
297
|
+
* the class (if any) applies to it, and the pointer (if any) resolves.
|
|
298
|
+
*/
|
|
299
|
+
async function prepareJudgment(c, body, runBinding, binding, authorizer) {
|
|
300
|
+
const err = (code, message) => ({ kind: 'err', code, message });
|
|
301
|
+
const tenantId = c.get('tenantId');
|
|
302
|
+
const run = await runBinding.getRun(tenantId, body.runId);
|
|
303
|
+
if (run === null)
|
|
304
|
+
return err('run-not-found', `No run "${body.runId}".`);
|
|
305
|
+
// A run inherits its permissions from its project (as cancelling one
|
|
306
|
+
// does): judging it needs `write` there.
|
|
307
|
+
if (authorizer !== undefined &&
|
|
308
|
+
!(await authorizer.can(c, 'write', ref('project', run.projectId)))) {
|
|
309
|
+
return err('permission-denied', `Not allowed to judge run "${body.runId}".`);
|
|
310
|
+
}
|
|
311
|
+
if (run.status !== 'completed' || run.output === undefined || run.output === null) {
|
|
312
|
+
return err('run-not-finished', `Run "${body.runId}" has no output to judge yet (status ${run.status}).`);
|
|
313
|
+
}
|
|
314
|
+
const subject = subjectOf(run);
|
|
315
|
+
const projectId = run.projectId;
|
|
316
|
+
if (body.judgeClassId !== undefined &&
|
|
317
|
+
!(await classApplies(binding, tenantId, body.judgeClassId, { projectId, subject }))) {
|
|
318
|
+
return err('judge-class-not-applicable', `Judge class "${body.judgeClassId}" doesn't exist or doesn't apply to this run's project or agent.`);
|
|
319
|
+
}
|
|
320
|
+
const copy = { input: run.input, output: run.output };
|
|
321
|
+
const turn = run.agent !== undefined ? { conversationId: run.agent.conversationId } : {};
|
|
322
|
+
if (body.item.pointer === undefined)
|
|
323
|
+
return { kind: 'ok', run: copy, subject, projectId, ...turn };
|
|
324
|
+
const found = resolvePointer(run.output, body.item.pointer);
|
|
325
|
+
if (!found.found) {
|
|
326
|
+
return err('item-not-found', `Nothing at "${body.item.pointer}" in run "${body.runId}"'s output.`);
|
|
327
|
+
}
|
|
328
|
+
return { kind: 'ok', run: copy, subject, projectId, itemValue: found.value, ...turn };
|
|
329
|
+
}
|
|
330
|
+
/** Whether the run has no live judgment yet (its copy is stored with the first). */
|
|
331
|
+
/**
|
|
332
|
+
* What a run's first judgment keeps so the run can be replayed later: a
|
|
333
|
+
* turn's history and retrieved context; a flow run's tool calls with
|
|
334
|
+
* their results.
|
|
335
|
+
*/
|
|
336
|
+
function captureContext(input) {
|
|
337
|
+
const { tenantId, runId, subject, runBinding } = input;
|
|
338
|
+
return subject.kind === 'agent'
|
|
339
|
+
? captureTurnContext({
|
|
340
|
+
tenantId,
|
|
341
|
+
runId,
|
|
342
|
+
output: input.output,
|
|
343
|
+
conversationId: input.conversationId,
|
|
344
|
+
runBinding,
|
|
345
|
+
conversations: input.conversations,
|
|
346
|
+
})
|
|
347
|
+
: captureFlowContext({
|
|
348
|
+
tenantId,
|
|
349
|
+
run: { runId, flowId: subject.id, flowVersion: subject.version },
|
|
350
|
+
runBinding,
|
|
351
|
+
flows: input.flows,
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
async function isFirstJudgment(binding, tenantId, runId) {
|
|
355
|
+
const page = await binding.list({ tenantId, runId, limit: 1 });
|
|
356
|
+
return page.data.length === 0;
|
|
357
|
+
}
|
|
358
|
+
/** Whether a live class exists and its scope covers the run. */
|
|
359
|
+
async function classApplies(binding, tenantId, judgeClassId, run) {
|
|
360
|
+
const judgeClass = await binding.getClass({ tenantId, judgeClassId });
|
|
361
|
+
return judgeClass !== null && judgeClassApplies(judgeClass.scope, run);
|
|
362
|
+
}
|
|
363
|
+
const MAX_REASON = 4000;
|
|
364
|
+
function parseJudgmentBody(raw) {
|
|
365
|
+
const err = (message) => ({ kind: 'err', message });
|
|
366
|
+
if (raw === null || typeof raw !== 'object')
|
|
367
|
+
return err('Send a JSON object.');
|
|
368
|
+
const b = raw;
|
|
369
|
+
if (!nonEmpty(b.runId))
|
|
370
|
+
return err('runId is required.');
|
|
371
|
+
if (typeof b.verdict !== 'string' || !isVerdict(b.verdict)) {
|
|
372
|
+
return err(`verdict must be one of: ${VERDICTS.join(', ')}.`);
|
|
373
|
+
}
|
|
374
|
+
const item = parseItem(b.item);
|
|
375
|
+
if (typeof item === 'string')
|
|
376
|
+
return err(item);
|
|
377
|
+
const optional = optionalFieldError(b);
|
|
378
|
+
if (optional !== undefined)
|
|
379
|
+
return err(optional);
|
|
380
|
+
return {
|
|
381
|
+
kind: 'ok',
|
|
382
|
+
body: {
|
|
383
|
+
runId: b.runId,
|
|
384
|
+
item,
|
|
385
|
+
verdict: b.verdict,
|
|
386
|
+
...(typeof b.judgeClassId === 'string' && { judgeClassId: b.judgeClassId }),
|
|
387
|
+
...(typeof b.reason === 'string' && b.reason.length > 0 && { reason: b.reason }),
|
|
388
|
+
...(typeof b.participantId === 'string' && { participantId: b.participantId }),
|
|
389
|
+
},
|
|
390
|
+
};
|
|
391
|
+
}
|
|
392
|
+
/** What's wrong with the optional fields (class, reason, participant), if anything. */
|
|
393
|
+
function optionalFieldError(b) {
|
|
394
|
+
if (b.judgeClassId !== undefined && !nonEmpty(b.judgeClassId)) {
|
|
395
|
+
return 'judgeClassId must be a judge class id, or left out for an unclassified judgment.';
|
|
396
|
+
}
|
|
397
|
+
if (b.reason !== undefined && (typeof b.reason !== 'string' || b.reason.length > MAX_REASON)) {
|
|
398
|
+
return `reason must be a string of at most ${MAX_REASON} characters.`;
|
|
399
|
+
}
|
|
400
|
+
if (b.participantId !== undefined && !nonEmpty(b.participantId)) {
|
|
401
|
+
return 'participantId must be a non-empty string.';
|
|
402
|
+
}
|
|
403
|
+
return undefined;
|
|
404
|
+
}
|
|
405
|
+
function parseItem(raw) {
|
|
406
|
+
if (raw === null || typeof raw !== 'object')
|
|
407
|
+
return 'item is required: { key, pointer?, rank? }.';
|
|
408
|
+
const i = raw;
|
|
409
|
+
if (!nonEmpty(i.key))
|
|
410
|
+
return 'item.key is required.';
|
|
411
|
+
if (i.pointer !== undefined && (typeof i.pointer !== 'string' || !isPointer(i.pointer))) {
|
|
412
|
+
return 'item.pointer must be a JSON Pointer such as "/matches/2" (or "" for the whole output).';
|
|
413
|
+
}
|
|
414
|
+
if (i.rank !== undefined && !(Number.isInteger(i.rank) && i.rank >= 0)) {
|
|
415
|
+
return 'item.rank must be a whole number ≥ 0.';
|
|
416
|
+
}
|
|
417
|
+
return {
|
|
418
|
+
key: i.key,
|
|
419
|
+
...(typeof i.pointer === 'string' && { pointer: i.pointer }),
|
|
420
|
+
...(typeof i.rank === 'number' && { rank: i.rank }),
|
|
421
|
+
};
|
|
422
|
+
}
|
|
423
|
+
function parseClassBody(raw) {
|
|
424
|
+
const err = (message) => ({ kind: 'err', message });
|
|
425
|
+
if (raw === null || typeof raw !== 'object')
|
|
426
|
+
return err('Send a JSON object.');
|
|
427
|
+
const b = raw;
|
|
428
|
+
if (!nonEmpty(b.name) || b.name.length > 100)
|
|
429
|
+
return err('name is required (at most 100 characters).');
|
|
430
|
+
if (!isWeight(b.weight))
|
|
431
|
+
return err('weight is required: a number ≥ 0.');
|
|
432
|
+
if (b.description !== undefined && typeof b.description !== 'string') {
|
|
433
|
+
return err('description must be a string.');
|
|
434
|
+
}
|
|
435
|
+
const scope = parseClassScope(b.scope);
|
|
436
|
+
if (typeof scope === 'string')
|
|
437
|
+
return err(scope);
|
|
438
|
+
return {
|
|
439
|
+
kind: 'ok',
|
|
440
|
+
body: {
|
|
441
|
+
scope,
|
|
442
|
+
name: b.name,
|
|
443
|
+
weight: b.weight,
|
|
444
|
+
...(typeof b.description === 'string' && { description: b.description }),
|
|
445
|
+
},
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
function parseClassScope(raw) {
|
|
449
|
+
const usage = 'scope is required: { kind: "tenant" }, { kind: "project", projectId }, or { kind: "agent", projectId, agentId }.';
|
|
450
|
+
if (raw === null || typeof raw !== 'object')
|
|
451
|
+
return usage;
|
|
452
|
+
const s = raw;
|
|
453
|
+
if (s.kind === 'tenant')
|
|
454
|
+
return { kind: 'tenant' };
|
|
455
|
+
if (!nonEmpty(s.projectId))
|
|
456
|
+
return usage;
|
|
457
|
+
const projectId = s.projectId;
|
|
458
|
+
if (s.kind === 'project')
|
|
459
|
+
return { kind: 'project', projectId };
|
|
460
|
+
if (s.kind === 'agent' && nonEmpty(s.agentId))
|
|
461
|
+
return { kind: 'agent', projectId, agentId: s.agentId };
|
|
462
|
+
return usage;
|
|
463
|
+
}
|
|
464
|
+
/** `?scopeKind=tenant|project|agent&projectId=&agentId=` on the class list. */
|
|
465
|
+
function parseClassScopeQuery(query) {
|
|
466
|
+
const kind = query.scopeKind;
|
|
467
|
+
if (kind === undefined || kind.length === 0)
|
|
468
|
+
return { kind: 'ok' };
|
|
469
|
+
const scope = parseClassScope({ kind, projectId: query.projectId, agentId: query.agentId });
|
|
470
|
+
return typeof scope === 'string' ? { kind: 'err', message: scope } : { kind: 'ok', scope };
|
|
471
|
+
}
|
|
472
|
+
// ---------- helpers ----------
|
|
473
|
+
function nonEmpty(v) {
|
|
474
|
+
return typeof v === 'string' && v.trim().length > 0;
|
|
475
|
+
}
|
|
476
|
+
function isVerdict(v) {
|
|
477
|
+
return VERDICTS.includes(v);
|
|
478
|
+
}
|
|
479
|
+
function isWeight(v) {
|
|
480
|
+
return typeof v === 'number' && Number.isFinite(v) && v >= 0;
|
|
481
|
+
}
|
|
482
|
+
function optional(key, value) {
|
|
483
|
+
return (value === undefined ? {} : { [key]: value });
|
|
484
|
+
}
|
|
485
|
+
function classNotFound(c, id) {
|
|
486
|
+
c.status(statusFor('judge-class-not-found'));
|
|
487
|
+
return c.json(toWireError({ code: 'judge-class-not-found', message: `No judge class "${id}".` }, c.get('requestId')));
|
|
488
|
+
}
|
|
489
|
+
/** The caller as a judgment records it, from the request's principal. */
|
|
490
|
+
function assertedByOf(principal) {
|
|
491
|
+
if (principal === undefined)
|
|
492
|
+
return undefined;
|
|
493
|
+
const actor = principal.actor;
|
|
494
|
+
return { kind: actor.kind === 'user' ? 'user' : 'service', id: actor.id };
|
|
495
|
+
}
|
|
496
|
+
/** What a run ran: its agent at a version (agent turns), else its flow. */
|
|
497
|
+
function subjectOf(run) {
|
|
498
|
+
return run.agent !== undefined
|
|
499
|
+
? { kind: 'agent', id: run.agent.id, version: run.agent.version }
|
|
500
|
+
: { kind: 'flow', id: run.flowId, version: run.flowVersion };
|
|
501
|
+
}
|
|
502
|
+
const POINTER_RE = /^(\/([^~/]|~[01])*)*$/;
|
|
503
|
+
function isPointer(p) {
|
|
504
|
+
return POINTER_RE.test(p);
|
|
505
|
+
}
|
|
506
|
+
/** RFC 6901: the value at `pointer` in `doc`. "" is the whole document. */
|
|
507
|
+
export function resolvePointer(doc, pointer) {
|
|
508
|
+
if (pointer === '')
|
|
509
|
+
return { found: true, value: doc };
|
|
510
|
+
let current = doc;
|
|
511
|
+
for (const raw of pointer.slice(1).split('/')) {
|
|
512
|
+
const token = raw.replace(/~1/g, '/').replace(/~0/g, '~');
|
|
513
|
+
if (Array.isArray(current)) {
|
|
514
|
+
if (!/^(0|[1-9]\d*)$/.test(token) || Number(token) >= current.length)
|
|
515
|
+
return { found: false };
|
|
516
|
+
current = current[Number(token)];
|
|
517
|
+
}
|
|
518
|
+
else if (current !== null && typeof current === 'object' && Object.hasOwn(current, token)) {
|
|
519
|
+
current = current[token];
|
|
520
|
+
}
|
|
521
|
+
else {
|
|
522
|
+
return { found: false };
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
return { found: true, value: current };
|
|
526
|
+
}
|
|
527
|
+
// ---------- serialization ----------
|
|
528
|
+
function serializeJudgment(j) {
|
|
529
|
+
return {
|
|
530
|
+
id: j.id,
|
|
531
|
+
tenantId: j.tenantId,
|
|
532
|
+
projectId: j.projectId,
|
|
533
|
+
runId: j.runId,
|
|
534
|
+
subject: j.subject,
|
|
535
|
+
item: j.item,
|
|
536
|
+
verdict: j.verdict,
|
|
537
|
+
...(j.reason !== undefined && { reason: j.reason }),
|
|
538
|
+
...(j.judgeClassId !== undefined && { judgeClassId: j.judgeClassId }),
|
|
539
|
+
assertedBy: j.assertedBy,
|
|
540
|
+
...(j.participantId !== undefined && { participantId: j.participantId }),
|
|
541
|
+
createdAt: j.createdAt,
|
|
542
|
+
...(j.unregisteredAt !== undefined && { unregisteredAt: j.unregisteredAt }),
|
|
543
|
+
...(j.supersededBy !== undefined && { supersededBy: j.supersededBy }),
|
|
544
|
+
};
|
|
545
|
+
}
|
|
546
|
+
function serializeJudgmentWithCopies(j) {
|
|
547
|
+
return {
|
|
548
|
+
...serializeJudgment(j),
|
|
549
|
+
run: j.run,
|
|
550
|
+
...(j.itemValue !== undefined && { itemValue: j.itemValue }),
|
|
551
|
+
};
|
|
552
|
+
}
|
|
553
|
+
function serializeJudgeClass(k) {
|
|
554
|
+
return {
|
|
555
|
+
id: k.id,
|
|
556
|
+
tenantId: k.tenantId,
|
|
557
|
+
scope: k.scope,
|
|
558
|
+
name: k.name,
|
|
559
|
+
weight: k.weight,
|
|
560
|
+
...(k.description !== undefined && { description: k.description }),
|
|
561
|
+
createdAt: k.createdAt,
|
|
562
|
+
updatedAt: k.updatedAt,
|
|
563
|
+
...(k.unregisteredAt !== undefined && { unregisteredAt: k.unregisteredAt }),
|
|
564
|
+
};
|
|
565
|
+
}
|
|
566
|
+
//# sourceMappingURL=judgments.js.map
|