@plumloom/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +2 -0
- package/README.md +71 -0
- package/dist/actions/configured-runs.d.ts +25 -0
- package/dist/actions/configured-runs.d.ts.map +1 -0
- package/dist/actions/configured-runs.js +35 -0
- package/dist/actions/configured-runs.js.map +1 -0
- package/dist/actions/context.d.ts +5 -0
- package/dist/actions/context.d.ts.map +1 -0
- package/dist/actions/context.js +2 -0
- package/dist/actions/context.js.map +1 -0
- package/dist/actions/doctor.d.ts +46 -0
- package/dist/actions/doctor.d.ts.map +1 -0
- package/dist/actions/doctor.js +130 -0
- package/dist/actions/doctor.js.map +1 -0
- package/dist/actions/evaluations.d.ts +87 -0
- package/dist/actions/evaluations.d.ts.map +1 -0
- package/dist/actions/evaluations.js +307 -0
- package/dist/actions/evaluations.js.map +1 -0
- package/dist/actions/gate.d.ts +63 -0
- package/dist/actions/gate.d.ts.map +1 -0
- package/dist/actions/gate.js +272 -0
- package/dist/actions/gate.js.map +1 -0
- package/dist/actions/identity.d.ts +4 -0
- package/dist/actions/identity.d.ts.map +1 -0
- package/dist/actions/identity.js +4 -0
- package/dist/actions/identity.js.map +1 -0
- package/dist/actions/models.d.ts +4 -0
- package/dist/actions/models.d.ts.map +1 -0
- package/dist/actions/models.js +4 -0
- package/dist/actions/models.js.map +1 -0
- package/dist/actions/quality-standards.d.ts +30 -0
- package/dist/actions/quality-standards.d.ts.map +1 -0
- package/dist/actions/quality-standards.js +86 -0
- package/dist/actions/quality-standards.js.map +1 -0
- package/dist/actions/results.d.ts +12 -0
- package/dist/actions/results.d.ts.map +1 -0
- package/dist/actions/results.js +65 -0
- package/dist/actions/results.js.map +1 -0
- package/dist/actions/runs.d.ts +22 -0
- package/dist/actions/runs.d.ts.map +1 -0
- package/dist/actions/runs.js +59 -0
- package/dist/actions/runs.js.map +1 -0
- package/dist/actions/suite-gate.d.ts +24 -0
- package/dist/actions/suite-gate.d.ts.map +1 -0
- package/dist/actions/suite-gate.js +40 -0
- package/dist/actions/suite-gate.js.map +1 -0
- package/dist/actions/suite.d.ts +49 -0
- package/dist/actions/suite.d.ts.map +1 -0
- package/dist/actions/suite.js +68 -0
- package/dist/actions/suite.js.map +1 -0
- package/dist/actions/validation.d.ts +3 -0
- package/dist/actions/validation.d.ts.map +1 -0
- package/dist/actions/validation.js +26 -0
- package/dist/actions/validation.js.map +1 -0
- package/dist/actions/workspaces.d.ts +13 -0
- package/dist/actions/workspaces.d.ts.map +1 -0
- package/dist/actions/workspaces.js +18 -0
- package/dist/actions/workspaces.js.map +1 -0
- package/dist/api/api.d.ts +237 -0
- package/dist/api/api.d.ts.map +1 -0
- package/dist/api/api.js +396 -0
- package/dist/api/api.js.map +1 -0
- package/dist/api/client.d.ts +38 -0
- package/dist/api/client.d.ts.map +1 -0
- package/dist/api/client.js +295 -0
- package/dist/api/client.js.map +1 -0
- package/dist/api/endpoints.d.ts +31 -0
- package/dist/api/endpoints.d.ts.map +1 -0
- package/dist/api/endpoints.js +32 -0
- package/dist/api/endpoints.js.map +1 -0
- package/dist/api/errors.d.ts +22 -0
- package/dist/api/errors.d.ts.map +1 -0
- package/dist/api/errors.js +84 -0
- package/dist/api/errors.js.map +1 -0
- package/dist/api/schemas.d.ts +169 -0
- package/dist/api/schemas.d.ts.map +1 -0
- package/dist/api/schemas.js +194 -0
- package/dist/api/schemas.js.map +1 -0
- package/dist/auth/credential-store.d.ts +9 -0
- package/dist/auth/credential-store.d.ts.map +1 -0
- package/dist/auth/credential-store.js +2 -0
- package/dist/auth/credential-store.js.map +1 -0
- package/dist/auth/credentials.d.ts +27 -0
- package/dist/auth/credentials.d.ts.map +1 -0
- package/dist/auth/credentials.js +81 -0
- package/dist/auth/credentials.js.map +1 -0
- package/dist/auth/keyring-store.d.ts +9 -0
- package/dist/auth/keyring-store.d.ts.map +1 -0
- package/dist/auth/keyring-store.js +57 -0
- package/dist/auth/keyring-store.js.map +1 -0
- package/dist/auth/login.d.ts +28 -0
- package/dist/auth/login.d.ts.map +1 -0
- package/dist/auth/login.js +27 -0
- package/dist/auth/login.js.map +1 -0
- package/dist/auth/password-prompt.d.ts +5 -0
- package/dist/auth/password-prompt.d.ts.map +1 -0
- package/dist/auth/password-prompt.js +10 -0
- package/dist/auth/password-prompt.js.map +1 -0
- package/dist/auth/redact.d.ts +3 -0
- package/dist/auth/redact.d.ts.map +1 -0
- package/dist/auth/redact.js +69 -0
- package/dist/auth/redact.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +4 -0
- package/dist/cli.js.map +1 -0
- package/dist/commands/executor.d.ts +36 -0
- package/dist/commands/executor.d.ts.map +1 -0
- package/dist/commands/executor.js +1218 -0
- package/dist/commands/executor.js.map +1 -0
- package/dist/commands/program.d.ts +21 -0
- package/dist/commands/program.d.ts.map +1 -0
- package/dist/commands/program.js +588 -0
- package/dist/commands/program.js.map +1 -0
- package/dist/commands/types.d.ts +139 -0
- package/dist/commands/types.d.ts.map +1 -0
- package/dist/commands/types.js +2 -0
- package/dist/commands/types.js.map +1 -0
- package/dist/config.d.ts +12 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +57 -0
- package/dist/config.js.map +1 -0
- package/dist/configured-run/file-inputs.d.ts +8 -0
- package/dist/configured-run/file-inputs.d.ts.map +1 -0
- package/dist/configured-run/file-inputs.js +160 -0
- package/dist/configured-run/file-inputs.js.map +1 -0
- package/dist/configured-run/validation.d.ts +103 -0
- package/dist/configured-run/validation.d.ts.map +1 -0
- package/dist/configured-run/validation.js +327 -0
- package/dist/configured-run/validation.js.map +1 -0
- package/dist/domain/common.d.ts +6 -0
- package/dist/domain/common.d.ts.map +1 -0
- package/dist/domain/common.js +10 -0
- package/dist/domain/common.js.map +1 -0
- package/dist/domain/types.d.ts +117 -0
- package/dist/domain/types.d.ts.map +1 -0
- package/dist/domain/types.js +2 -0
- package/dist/domain/types.js.map +1 -0
- package/dist/errors/autoeval-error.d.ts +13 -0
- package/dist/errors/autoeval-error.d.ts.map +1 -0
- package/dist/errors/autoeval-error.js +36 -0
- package/dist/errors/autoeval-error.js.map +1 -0
- package/dist/errors/hints.d.ts +3 -0
- package/dist/errors/hints.d.ts.map +1 -0
- package/dist/errors/hints.js +43 -0
- package/dist/errors/hints.js.map +1 -0
- package/dist/gate/decision.d.ts +97 -0
- package/dist/gate/decision.d.ts.map +1 -0
- package/dist/gate/decision.js +436 -0
- package/dist/gate/decision.js.map +1 -0
- package/dist/gate/policy.d.ts +31 -0
- package/dist/gate/policy.d.ts.map +1 -0
- package/dist/gate/policy.js +26 -0
- package/dist/gate/policy.js.map +1 -0
- package/dist/guide/eval-creation-backend.d.ts +20 -0
- package/dist/guide/eval-creation-backend.d.ts.map +1 -0
- package/dist/guide/eval-creation-backend.js +16 -0
- package/dist/guide/eval-creation-backend.js.map +1 -0
- package/dist/guide/eval-guide.d.ts +24 -0
- package/dist/guide/eval-guide.d.ts.map +1 -0
- package/dist/guide/eval-guide.js +219 -0
- package/dist/guide/eval-guide.js.map +1 -0
- package/dist/mcp/context.d.ts +21 -0
- package/dist/mcp/context.d.ts.map +1 -0
- package/dist/mcp/context.js +19 -0
- package/dist/mcp/context.js.map +1 -0
- package/dist/mcp/schemas.d.ts +206 -0
- package/dist/mcp/schemas.d.ts.map +1 -0
- package/dist/mcp/schemas.js +80 -0
- package/dist/mcp/schemas.js.map +1 -0
- package/dist/mcp/server.d.ts +5 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/mcp/server.js +10 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/tools.d.ts +42 -0
- package/dist/mcp/tools.d.ts.map +1 -0
- package/dist/mcp/tools.js +287 -0
- package/dist/mcp/tools.js.map +1 -0
- package/dist/mcp.d.ts +3 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +32 -0
- package/dist/mcp.js.map +1 -0
- package/dist/output/big-digits.d.ts +7 -0
- package/dist/output/big-digits.d.ts.map +1 -0
- package/dist/output/big-digits.js +43 -0
- package/dist/output/big-digits.js.map +1 -0
- package/dist/output/clipboard.d.ts +29 -0
- package/dist/output/clipboard.d.ts.map +1 -0
- package/dist/output/clipboard.js +82 -0
- package/dist/output/clipboard.js.map +1 -0
- package/dist/output/colors.d.ts +21 -0
- package/dist/output/colors.d.ts.map +1 -0
- package/dist/output/colors.js +21 -0
- package/dist/output/colors.js.map +1 -0
- package/dist/output/digest.d.ts +165 -0
- package/dist/output/digest.d.ts.map +1 -0
- package/dist/output/digest.js +771 -0
- package/dist/output/digest.js.map +1 -0
- package/dist/output/human.d.ts +95 -0
- package/dist/output/human.d.ts.map +1 -0
- package/dist/output/human.js +643 -0
- package/dist/output/human.js.map +1 -0
- package/dist/output/json.d.ts +2 -0
- package/dist/output/json.d.ts.map +1 -0
- package/dist/output/json.js +5 -0
- package/dist/output/json.js.map +1 -0
- package/dist/output/layout.d.ts +18 -0
- package/dist/output/layout.d.ts.map +1 -0
- package/dist/output/layout.js +64 -0
- package/dist/output/layout.js.map +1 -0
- package/dist/output/markdown.d.ts +16 -0
- package/dist/output/markdown.d.ts.map +1 -0
- package/dist/output/markdown.js +38 -0
- package/dist/output/markdown.js.map +1 -0
- package/dist/output/progress.d.ts +29 -0
- package/dist/output/progress.d.ts.map +1 -0
- package/dist/output/progress.js +136 -0
- package/dist/output/progress.js.map +1 -0
- package/dist/output/safe-text.d.ts +11 -0
- package/dist/output/safe-text.d.ts.map +1 -0
- package/dist/output/safe-text.js +44 -0
- package/dist/output/safe-text.js.map +1 -0
- package/dist/output/scorecard.d.ts +23 -0
- package/dist/output/scorecard.d.ts.map +1 -0
- package/dist/output/scorecard.js +506 -0
- package/dist/output/scorecard.js.map +1 -0
- package/dist/output/section.d.ts +15 -0
- package/dist/output/section.d.ts.map +1 -0
- package/dist/output/section.js +44 -0
- package/dist/output/section.js.map +1 -0
- package/dist/output/spinner.d.ts +24 -0
- package/dist/output/spinner.d.ts.map +1 -0
- package/dist/output/spinner.js +34 -0
- package/dist/output/spinner.js.map +1 -0
- package/dist/output/splash.d.ts +37 -0
- package/dist/output/splash.d.ts.map +1 -0
- package/dist/output/splash.js +169 -0
- package/dist/output/splash.js.map +1 -0
- package/dist/output/table.d.ts +33 -0
- package/dist/output/table.d.ts.map +1 -0
- package/dist/output/table.js +105 -0
- package/dist/output/table.js.map +1 -0
- package/dist/output/trend.d.ts +19 -0
- package/dist/output/trend.d.ts.map +1 -0
- package/dist/output/trend.js +38 -0
- package/dist/output/trend.js.map +1 -0
- package/dist/output/writer.d.ts +14 -0
- package/dist/output/writer.d.ts.map +1 -0
- package/dist/output/writer.js +33 -0
- package/dist/output/writer.js.map +1 -0
- package/dist/polling/clock.d.ts +9 -0
- package/dist/polling/clock.d.ts.map +1 -0
- package/dist/polling/clock.js +23 -0
- package/dist/polling/clock.js.map +1 -0
- package/dist/polling/wait-for-result.d.ts +16 -0
- package/dist/polling/wait-for-result.d.ts.map +1 -0
- package/dist/polling/wait-for-result.js +29 -0
- package/dist/polling/wait-for-result.js.map +1 -0
- package/dist/polling/wait-for-run.d.ts +14 -0
- package/dist/polling/wait-for-run.d.ts.map +1 -0
- package/dist/polling/wait-for-run.js +36 -0
- package/dist/polling/wait-for-run.js.map +1 -0
- package/dist/quickstart/prompt.d.ts +18 -0
- package/dist/quickstart/prompt.d.ts.map +1 -0
- package/dist/quickstart/prompt.js +37 -0
- package/dist/quickstart/prompt.js.map +1 -0
- package/dist/quickstart/sample.d.ts +28 -0
- package/dist/quickstart/sample.d.ts.map +1 -0
- package/dist/quickstart/sample.js +235 -0
- package/dist/quickstart/sample.js.map +1 -0
- package/dist/quickstart/selection.d.ts +53 -0
- package/dist/quickstart/selection.d.ts.map +1 -0
- package/dist/quickstart/selection.js +97 -0
- package/dist/quickstart/selection.js.map +1 -0
- package/dist/run.d.ts +28 -0
- package/dist/run.d.ts.map +1 -0
- package/dist/run.js +134 -0
- package/dist/run.js.map +1 -0
- package/dist/runtime/action-context.d.ts +19 -0
- package/dist/runtime/action-context.d.ts.map +1 -0
- package/dist/runtime/action-context.js +40 -0
- package/dist/runtime/action-context.js.map +1 -0
- package/dist/suite/execution.d.ts +46 -0
- package/dist/suite/execution.d.ts.map +1 -0
- package/dist/suite/execution.js +97 -0
- package/dist/suite/execution.js.map +1 -0
- package/dist/suite/manifest.d.ts +16 -0
- package/dist/suite/manifest.d.ts.map +1 -0
- package/dist/suite/manifest.js +92 -0
- package/dist/suite/manifest.js.map +1 -0
- package/dist/suite/reporting.d.ts +35 -0
- package/dist/suite/reporting.d.ts.map +1 -0
- package/dist/suite/reporting.js +131 -0
- package/dist/suite/reporting.js.map +1 -0
- package/dist/trace/deepseek-harness.d.ts +31 -0
- package/dist/trace/deepseek-harness.d.ts.map +1 -0
- package/dist/trace/deepseek-harness.js +412 -0
- package/dist/trace/deepseek-harness.js.map +1 -0
- package/dist/trace/import.d.ts +9 -0
- package/dist/trace/import.d.ts.map +1 -0
- package/dist/trace/import.js +45 -0
- package/dist/trace/import.js.map +1 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +3 -0
- package/dist/version.js.map +1 -0
- package/package.json +82 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluations.d.ts","sourceRoot":"","sources":["../../src/actions/evaluations.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EACV,UAAU,EAEV,cAAc,EACd,wBAAwB,EAExB,YAAY,EACZ,SAAS,EACT,SAAS,EACT,iBAAiB,EAClB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAUrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAYlD,MAAM,MAAM,iBAAiB,GAAG;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,IAAI,EAAE,cAAc,CAAC;CACtB,CAAC;AAKF,MAAM,MAAM,uBAAuB,GAAG;IACpC,qFAAqF;IACrF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,sFAAsF;IACtF,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,KAAK,sBAAsB,GAAG;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,UAAU,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,kBAAkB,CAAC,EAAE,SAAS,iBAAiB,EAAE,CAAC;IAClD,eAAe,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,sBAAsB,GAAG;IACzD,WAAW,EAAE,UAAU,CAAC;IACxB,kBAAkB,CAAC,EAAE,UAAU,CAAC;CACjC,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG,sBAAsB,GAAG;IAC7D,WAAW,EAAE,cAAc,CAAC;IAC5B,kBAAkB,CAAC,EAAE,KAAK,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG,sBAAsB,GAAG;IAC3D,WAAW,EAAE,aAAa,CAAC;IAC3B,kBAAkB,CAAC,EAAE,KAAK,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAC/B,mBAAmB,GAAG,uBAAuB,GAAG,qBAAqB,CAAC;AAExE,MAAM,MAAM,wBAAwB,GAAG;IACrC,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,uBAAuB,CAAC;IACrC,aAAa,EAAE,qBAAqB,CAAC;IACrC,KAAK,EAAE,SAAS,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;IACvC,qBAAqB,CAAC,EAAE,MAAM,MAAM,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,kCAAkC,GAAG,IAAI,CACnD,wBAAwB,EACxB,OAAO,GAAG,gBAAgB,GAAG,eAAe,GAAG,UAAU,CAC1D,CAAC;AAqGF,wBAAsB,wBAAwB,CAC5C,OAAO,EAAE,aAAa,EACtB,iBAAiB,EAAE,MAAM,EACzB,gBAAgB,CAAC,EAAE,MAAM,EACzB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,iBAAiB,CAAC,CAgC5B;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,aAAa,EACtB,iBAAiB,EAAE,MAAM,EACzB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,wBAAwB,CAAC,CAkBnC;AAED,wBAAgB,eAAe,CAC7B,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,EAC5D,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,cAAc,CAAC,CAIzB;AAED,wBAAsB,aAAa,CACjC,OAAO,EAAE,aAAa,EACtB,YAAY,EAAE,MAAM,EACpB,gBAAgB,CAAC,EAAE,MAAM,EACzB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,UAAU,CAAC,CAwBrB;AAED,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE;IAAE,WAAW,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,EACvD,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,iBAAiB,CAAC,CA2B5B;AAED,wBAAsB,6BAA6B,CACjD,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,kCAAkC,GACxC,OAAO,CAAC,SAAS,CAAC,CAsHpB;AAED,wBAAsB,mBAAmB,CACvC,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,wBAAwB,GAC9B,OAAO,CAAC,YAAY,CAAC,CA6BvB;AAED,wBAAsB,mCAAmC,CACvD,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE;IACL,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,EACD,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,sBAAsB,CAAC,CAiCjC"}
|
|
@@ -0,0 +1,307 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { AutoevalError } from '../errors/autoeval-error.js';
|
|
3
|
+
import { waitForRun } from '../polling/wait-for-run.js';
|
|
4
|
+
import { jsonObjectSchema, } from '../api/schemas.js';
|
|
5
|
+
import { requirePageSize, requireUuid } from './validation.js';
|
|
6
|
+
import { z } from 'zod';
|
|
7
|
+
const evaluationNameSchema = z.string().trim().min(1).max(120);
|
|
8
|
+
function extractEvaluationId(response) {
|
|
9
|
+
return response.evaluation_id ?? response.evaluationId ?? response.eval_id ?? response.id;
|
|
10
|
+
}
|
|
11
|
+
/** Single run per scenario unless the eval file asks for multi-run evaluation. */
|
|
12
|
+
const DEFAULT_RUNS_PER_SCENARIO = 1;
|
|
13
|
+
function extractMethodologyVersionId(response) {
|
|
14
|
+
return (response.methodology_version_id ??
|
|
15
|
+
response.methodologyVersionId ??
|
|
16
|
+
response.methodology_id ??
|
|
17
|
+
response.methodologyId ??
|
|
18
|
+
response.id);
|
|
19
|
+
}
|
|
20
|
+
function extractConfigVersionId(response) {
|
|
21
|
+
return (response.config_version_id ??
|
|
22
|
+
response.configVersionId ??
|
|
23
|
+
response.config_id ??
|
|
24
|
+
response.configId ??
|
|
25
|
+
response.id);
|
|
26
|
+
}
|
|
27
|
+
function asNonEmptyString(value) {
|
|
28
|
+
return typeof value === 'string' && value.trim().length > 0 ? value : undefined;
|
|
29
|
+
}
|
|
30
|
+
function extractCreatedAt(value) {
|
|
31
|
+
if (typeof value !== 'object' || value === null)
|
|
32
|
+
return undefined;
|
|
33
|
+
const record = value;
|
|
34
|
+
return (asNonEmptyString(record.created_at) ??
|
|
35
|
+
asNonEmptyString(record.createdAt) ??
|
|
36
|
+
asNonEmptyString(record.created));
|
|
37
|
+
}
|
|
38
|
+
function requireVersionTimeline(response, evaluationId) {
|
|
39
|
+
const timeline = response.evaluationVersions;
|
|
40
|
+
if (!timeline || timeline.items.length === 0) {
|
|
41
|
+
throw new AutoevalError(`Evaluation version timeline is unavailable for ${evaluationId}.`, {
|
|
42
|
+
kind: 'validation',
|
|
43
|
+
code: 'EVALUATION_VERSION_TIMELINE_UNAVAILABLE',
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
const currentMatches = timeline.items.filter((item) => item.version === timeline.currentVersion);
|
|
47
|
+
if (currentMatches.length !== 1) {
|
|
48
|
+
throw new AutoevalError(`Evaluation version timeline is inconsistent for ${evaluationId}.`, {
|
|
49
|
+
kind: 'validation',
|
|
50
|
+
code: 'EVALUATION_VERSION_TIMELINE_INVALID',
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
const mismatchedCurrentFlags = timeline.items.some((item) => item.isCurrent && item.version !== timeline.currentVersion);
|
|
54
|
+
if (mismatchedCurrentFlags) {
|
|
55
|
+
throw new AutoevalError(`Evaluation version timeline is inconsistent for ${evaluationId}.`, {
|
|
56
|
+
kind: 'validation',
|
|
57
|
+
code: 'EVALUATION_VERSION_TIMELINE_INVALID',
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
return timeline;
|
|
61
|
+
}
|
|
62
|
+
function buildVersionHistory(timeline) {
|
|
63
|
+
return [...timeline.items]
|
|
64
|
+
.sort((left, right) => left.version - right.version)
|
|
65
|
+
.map((item) => {
|
|
66
|
+
const createdAt = extractCreatedAt(item);
|
|
67
|
+
return {
|
|
68
|
+
version: item.version,
|
|
69
|
+
methodologyVersionId: item.methodologyId,
|
|
70
|
+
configVersionId: item.configId,
|
|
71
|
+
isCurrent: item.version === timeline.currentVersion,
|
|
72
|
+
...(createdAt ? { createdAt } : {}),
|
|
73
|
+
};
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
function selectVersion(timeline, requestedVersion) {
|
|
77
|
+
const history = buildVersionHistory(timeline);
|
|
78
|
+
if (requestedVersion !== undefined) {
|
|
79
|
+
return history.find((item) => item.version === requestedVersion);
|
|
80
|
+
}
|
|
81
|
+
return history.find((item) => item.version === timeline.currentVersion);
|
|
82
|
+
}
|
|
83
|
+
export async function resolveEvaluationVersion(context, evaluationIdInput, requestedVersion, signal) {
|
|
84
|
+
const evaluationId = requireUuid(evaluationIdInput, 'Evaluation ID');
|
|
85
|
+
const response = await context.api.getEvaluationVersions(evaluationId, signal);
|
|
86
|
+
if (response.evaluationId !== evaluationId) {
|
|
87
|
+
throw new AutoevalError('Autoeval API returned a mismatched evaluation identifier.', {
|
|
88
|
+
kind: 'upstream',
|
|
89
|
+
code: 'MISMATCHED_EVALUATION_ID',
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
const contextType = response.context_type ?? 'scenario';
|
|
93
|
+
const timeline = requireVersionTimeline(response, evaluationId);
|
|
94
|
+
const selected = selectVersion(timeline, requestedVersion);
|
|
95
|
+
if (selected) {
|
|
96
|
+
return {
|
|
97
|
+
version: selected.version,
|
|
98
|
+
evaluationId,
|
|
99
|
+
methodologyVersionId: selected.methodologyVersionId,
|
|
100
|
+
configVersionId: selected.configVersionId,
|
|
101
|
+
isCurrent: selected.isCurrent,
|
|
102
|
+
...(selected.createdAt ? { createdAt: selected.createdAt } : {}),
|
|
103
|
+
contextType,
|
|
104
|
+
raw: jsonObjectSchema.parse(response),
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
throw new AutoevalError(`Evaluation version ${requestedVersion} was not found for ${evaluationId}.`, {
|
|
108
|
+
kind: 'validation',
|
|
109
|
+
code: 'EVALUATION_VERSION_NOT_FOUND',
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
export async function listEvaluationVersions(context, evaluationIdInput, signal) {
|
|
113
|
+
const evaluationId = requireUuid(evaluationIdInput, 'Evaluation ID');
|
|
114
|
+
const response = await context.api.getEvaluationVersions(evaluationId, signal);
|
|
115
|
+
if (response.evaluationId !== evaluationId) {
|
|
116
|
+
throw new AutoevalError('Autoeval API returned a mismatched evaluation identifier.', {
|
|
117
|
+
kind: 'upstream',
|
|
118
|
+
code: 'MISMATCHED_EVALUATION_ID',
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
const timeline = requireVersionTimeline(response, evaluationId);
|
|
122
|
+
return {
|
|
123
|
+
evaluationId,
|
|
124
|
+
contextType: response.context_type ?? 'scenario',
|
|
125
|
+
currentVersion: timeline.currentVersion,
|
|
126
|
+
versions: buildVersionHistory(timeline),
|
|
127
|
+
raw: jsonObjectSchema.parse(response),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
export function listEvaluations(context, input, signal) {
|
|
131
|
+
const workspaceId = requireUuid(input.workspaceId, 'Workspace ID');
|
|
132
|
+
const size = requirePageSize(input.size ?? 100);
|
|
133
|
+
return context.api.listEvaluations(workspaceId, { page: input.page ?? 1, size }, signal);
|
|
134
|
+
}
|
|
135
|
+
export async function getEvaluation(context, evaluationId, requestedVersion, signal) {
|
|
136
|
+
const version = await resolveEvaluationVersion(context, evaluationId, requestedVersion, signal);
|
|
137
|
+
const response = await context.api.getEvaluationConfiguration(version.evaluationId, version.methodologyVersionId, version.configVersionId, signal);
|
|
138
|
+
if (response.evaluation_id !== version.evaluationId ||
|
|
139
|
+
response.methodology_version_id !== version.methodologyVersionId ||
|
|
140
|
+
response.config_version_id !== version.configVersionId) {
|
|
141
|
+
throw new AutoevalError('Autoeval API returned mismatched evaluation version identifiers.', {
|
|
142
|
+
kind: 'upstream',
|
|
143
|
+
code: 'MISMATCHED_VERSION_ID',
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
return {
|
|
147
|
+
...version,
|
|
148
|
+
configuration: jsonObjectSchema.parse(response),
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
export async function createEvaluation(context, input, signal) {
|
|
152
|
+
const workspaceId = requireUuid(input.workspaceId, 'Workspace ID');
|
|
153
|
+
const evaluationName = evaluationNameSchema.parse(input.evaluationName ?? 'Untitled');
|
|
154
|
+
const draftResponse = await context.api.createEvaluationDraft(workspaceId, signal);
|
|
155
|
+
const draftEvaluationId = extractEvaluationId(draftResponse);
|
|
156
|
+
if (!draftEvaluationId) {
|
|
157
|
+
throw new AutoevalError('Autoeval API draft response did not include an evaluation ID.', {
|
|
158
|
+
kind: 'upstream',
|
|
159
|
+
code: 'MISSING_EVALUATION_ID',
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
const evaluationId = requireUuid(draftEvaluationId, 'Evaluation ID');
|
|
163
|
+
const createResponse = await context.api.createEvaluation({ workspaceId, evaluationId, evaluationName }, signal);
|
|
164
|
+
const createEvaluationId = extractEvaluationId(createResponse);
|
|
165
|
+
if (createEvaluationId && createEvaluationId !== evaluationId) {
|
|
166
|
+
throw new AutoevalError('Autoeval API create response returned a mismatched evaluation ID.', {
|
|
167
|
+
kind: 'upstream',
|
|
168
|
+
code: 'MISMATCHED_EVALUATION_ID',
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
return { evaluationId, workspaceId, evaluationName };
|
|
172
|
+
}
|
|
173
|
+
export async function createConfiguredRunSubmission(context, input) {
|
|
174
|
+
const evaluationId = requireUuid(input.evaluationId, 'Evaluation ID');
|
|
175
|
+
const methodology = input.methodology;
|
|
176
|
+
const configuration = input.configuration;
|
|
177
|
+
// Public eval files must not embed account identity. Resolve it through the
|
|
178
|
+
// authenticated API session unless an explicit caller supplied one.
|
|
179
|
+
const userSystemId = methodology.userSystemId ?? (await context.api.getCurrentUser(input.signal)).userSystemId;
|
|
180
|
+
const methodologyResponse = await context.api.createMethodologyVersion(evaluationId, {
|
|
181
|
+
user_sys_id: userSystemId,
|
|
182
|
+
judge_model: configuration.contextType === 'scenario'
|
|
183
|
+
? requireUuid(methodology.judgeModelId, 'Judge model ID')
|
|
184
|
+
: methodology.judgeModel,
|
|
185
|
+
judge_model_id: requireUuid(methodology.judgeModelId, 'Judge model ID'),
|
|
186
|
+
evaluator_instructions: methodology.evaluatorInstructions,
|
|
187
|
+
change_log: methodology.changeLog ?? '',
|
|
188
|
+
runs_per_scenario: methodology.runsPerScenario ?? DEFAULT_RUNS_PER_SCENARIO,
|
|
189
|
+
}, input.signal);
|
|
190
|
+
const rawMethodologyVersionId = extractMethodologyVersionId(methodologyResponse);
|
|
191
|
+
if (!rawMethodologyVersionId) {
|
|
192
|
+
throw new AutoevalError('Autoeval API methodology response did not include a methodology version ID.', {
|
|
193
|
+
kind: 'upstream',
|
|
194
|
+
code: 'MISSING_METHODOLOGY_VERSION_ID',
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
const methodologyVersionId = requireUuid(rawMethodologyVersionId, 'Methodology version ID');
|
|
198
|
+
const configResponse = await context.api.createConfigVersion(evaluationId, configuration.contextType === 'scenario'
|
|
199
|
+
? {
|
|
200
|
+
// Legacy scenario fixtures may provide scenarios as [[...]].
|
|
201
|
+
// The API expects a flat scenarios list, so flatten one level when needed.
|
|
202
|
+
methodology_version_id: methodologyVersionId,
|
|
203
|
+
primary_model_id: z.uuid().parse(configuration.artifact.primaryModelId),
|
|
204
|
+
comparison_model_ids: z.array(z.uuid()).parse(configuration.artifact.comparisonModelIds),
|
|
205
|
+
scenarios: (() => {
|
|
206
|
+
const parsedScenarios = z
|
|
207
|
+
.array(z.json())
|
|
208
|
+
.min(1)
|
|
209
|
+
.parse(configuration.artifact.scenarios);
|
|
210
|
+
return Array.isArray(parsedScenarios[0])
|
|
211
|
+
? z.array(z.json()).min(1).parse(parsedScenarios[0])
|
|
212
|
+
: parsedScenarios;
|
|
213
|
+
})(),
|
|
214
|
+
prompt_text: z.string().min(1).parse(configuration.artifact.promptText),
|
|
215
|
+
selected_metrics: configuration.selectedMetrics,
|
|
216
|
+
...(configuration.referenceDocuments && configuration.referenceDocuments.length > 0
|
|
217
|
+
? { reference_documents: configuration.referenceDocuments }
|
|
218
|
+
: {}),
|
|
219
|
+
...(configuration.temperatureContext !== undefined
|
|
220
|
+
? { temperature_context: configuration.temperatureContext }
|
|
221
|
+
: {}),
|
|
222
|
+
}
|
|
223
|
+
: {
|
|
224
|
+
context_name: configuration.contextName,
|
|
225
|
+
context_type: configuration.contextType,
|
|
226
|
+
methodology_version_id: methodologyVersionId,
|
|
227
|
+
artifact: configuration.artifact,
|
|
228
|
+
expected: configuration.expected,
|
|
229
|
+
selected_metrics: configuration.selectedMetrics,
|
|
230
|
+
...(configuration.referenceDocuments && configuration.referenceDocuments.length > 0
|
|
231
|
+
? { reference_documents: configuration.referenceDocuments }
|
|
232
|
+
: {}),
|
|
233
|
+
...(configuration.temperatureContext !== undefined
|
|
234
|
+
? { temperature_context: configuration.temperatureContext }
|
|
235
|
+
: {}),
|
|
236
|
+
}, input.signal);
|
|
237
|
+
const rawConfigVersionId = extractConfigVersionId(configResponse);
|
|
238
|
+
if (!rawConfigVersionId) {
|
|
239
|
+
throw new AutoevalError('Autoeval API config response did not include a config version ID.', {
|
|
240
|
+
kind: 'upstream',
|
|
241
|
+
code: 'MISSING_CONFIG_VERSION_ID',
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
const configVersionId = requireUuid(rawConfigVersionId, 'Config version ID');
|
|
245
|
+
const idempotencyKey = (input.idempotencyKeyFactory ?? randomUUID)();
|
|
246
|
+
const runResponse = await context.api.createRun({
|
|
247
|
+
evaluationId,
|
|
248
|
+
configVersionId,
|
|
249
|
+
idempotencyKey,
|
|
250
|
+
...(input.signal ? { signal: input.signal } : {}),
|
|
251
|
+
});
|
|
252
|
+
if (runResponse.using.configVersionId !== configVersionId ||
|
|
253
|
+
runResponse.using.methodologyVersionId !== methodologyVersionId) {
|
|
254
|
+
throw new AutoevalError('Autoeval API created the run with unexpected version identifiers.', {
|
|
255
|
+
kind: 'upstream',
|
|
256
|
+
code: 'MISMATCHED_RUN_VERSION_ID',
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
const run = {
|
|
260
|
+
evaluationId,
|
|
261
|
+
runId: runResponse.runId,
|
|
262
|
+
status: runResponse.status,
|
|
263
|
+
configVersionId: runResponse.using.configVersionId,
|
|
264
|
+
methodologyVersionId: runResponse.using.methodologyVersionId,
|
|
265
|
+
};
|
|
266
|
+
return run;
|
|
267
|
+
}
|
|
268
|
+
export async function createConfiguredRun(context, input) {
|
|
269
|
+
const run = await createConfiguredRunSubmission(context, input);
|
|
270
|
+
await context.api.startRunStatusStream(run.evaluationId, run.runId, input.signal);
|
|
271
|
+
const outcome = await waitForRun({
|
|
272
|
+
api: context.api,
|
|
273
|
+
clock: input.clock,
|
|
274
|
+
evaluationId: run.evaluationId,
|
|
275
|
+
runId: run.runId,
|
|
276
|
+
intervalMs: input.pollIntervalMs,
|
|
277
|
+
timeoutMs: input.pollTimeoutMs,
|
|
278
|
+
...(input.signal ? { signal: input.signal } : {}),
|
|
279
|
+
...(input.onStatus ? { onStatus: input.onStatus } : {}),
|
|
280
|
+
});
|
|
281
|
+
if (outcome.status.state !== 'COMPLETED') {
|
|
282
|
+
const failureCode = outcome.status.failureCode ? ` (${outcome.status.failureCode})` : '';
|
|
283
|
+
throw new AutoevalError(`Run ${run.runId} ended in state ${outcome.status.state}${failureCode}.`, {
|
|
284
|
+
kind: 'run_failed',
|
|
285
|
+
code: outcome.status.failureCode ?? 'RUN_FAILED',
|
|
286
|
+
cause: outcome.status,
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
return { run, outcome };
|
|
290
|
+
}
|
|
291
|
+
export async function updateEvaluationTitleAndRefreshList(context, input, signal) {
|
|
292
|
+
const workspaceId = requireUuid(input.workspaceId, 'Workspace ID');
|
|
293
|
+
const evaluationId = requireUuid(input.evaluationId, 'Evaluation ID');
|
|
294
|
+
const evaluationName = evaluationNameSchema.parse(input.evaluationName);
|
|
295
|
+
const size = requirePageSize(input.size ?? 50);
|
|
296
|
+
const page = Math.max(1, input.page ?? 1);
|
|
297
|
+
await context.api.patchEvaluationTitle(evaluationId, {
|
|
298
|
+
name: evaluationName,
|
|
299
|
+
user_sys_id: input.userSystemId,
|
|
300
|
+
...(input.description !== undefined ? { description: input.description } : {}),
|
|
301
|
+
}, signal);
|
|
302
|
+
await context.api.putEvaltoolTitle(workspaceId, evaluationId, { eval_name: evaluationName }, signal);
|
|
303
|
+
await context.api.syncEvalsList({ workspaceId, limit: 25, offset: 0 }, signal);
|
|
304
|
+
const listPage = await context.api.listEvaluationsSorted(workspaceId, { page, size, sortBy: 'created_at', sortOrder: 'desc' }, signal);
|
|
305
|
+
return { evaluationId, workspaceId, evaluationName, page: listPage };
|
|
306
|
+
}
|
|
307
|
+
//# sourceMappingURL=evaluations.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluations.js","sourceRoot":"","sources":["../../src/actions/evaluations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAczC,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAEL,gBAAgB,GAKjB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAE/D,SAAS,mBAAmB,CAC1B,QAA4D;IAE5D,OAAO,QAAQ,CAAC,aAAa,IAAI,QAAQ,CAAC,YAAY,IAAI,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,EAAE,CAAC;AAC5F,CAAC;AAeD,kFAAkF;AAClF,MAAM,yBAAyB,GAAG,CAAC,CAAC;AA8DpC,SAAS,2BAA2B,CAClC,QAA0C;IAE1C,OAAO,CACL,QAAQ,CAAC,sBAAsB;QAC/B,QAAQ,CAAC,oBAAoB;QAC7B,QAAQ,CAAC,cAAc;QACvB,QAAQ,CAAC,aAAa;QACtB,QAAQ,CAAC,EAAE,CACZ,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,QAAqC;IACnE,OAAO,CACL,QAAQ,CAAC,iBAAiB;QAC1B,QAAQ,CAAC,eAAe;QACxB,QAAQ,CAAC,SAAS;QAClB,QAAQ,CAAC,QAAQ;QACjB,QAAQ,CAAC,EAAE,CACZ,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAClF,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,SAAS,CAAC;IAClE,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,OAAO,CACL,gBAAgB,CAAC,MAAM,CAAC,UAAU,CAAC;QACnC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC;QAClC,gBAAgB,CAAC,MAAM,CAAC,OAAO,CAAC,CACjC,CAAC;AACJ,CAAC;AAID,SAAS,sBAAsB,CAC7B,QAAoC,EACpC,YAAoB;IAEpB,MAAM,QAAQ,GAAG,QAAQ,CAAC,kBAAkB,CAAC;IAC7C,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7C,MAAM,IAAI,aAAa,CAAC,kDAAkD,YAAY,GAAG,EAAE;YACzF,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,yCAAyC;SAChD,CAAC,CAAC;IACL,CAAC;IAED,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,cAAc,CAAC,CAAC;IACjG,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,aAAa,CAAC,mDAAmD,YAAY,GAAG,EAAE;YAC1F,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,qCAAqC;SAC5C,CAAC,CAAC;IACL,CAAC;IAED,MAAM,sBAAsB,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAChD,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,cAAc,CACrE,CAAC;IACF,IAAI,sBAAsB,EAAE,CAAC;QAC3B,MAAM,IAAI,aAAa,CAAC,mDAAmD,YAAY,GAAG,EAAE;YAC1F,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,qCAAqC;SAC5C,CAAC,CAAC;IACL,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAyB;IACpD,OAAO,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC;SACvB,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;SACnD,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACzC,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,oBAAoB,EAAE,IAAI,CAAC,aAAa;YACxC,eAAe,EAAE,IAAI,CAAC,QAAQ;YAC9B,SAAS,EAAE,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,cAAc;YACnD,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpC,CAAC;IACJ,CAAC,CAAC,CAAC;AACP,CAAC;AAED,SAAS,aAAa,CACpB,QAAyB,EACzB,gBAAyB;IAEzB,MAAM,OAAO,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAE9C,IAAI,gBAAgB,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,gBAAgB,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,cAAc,CAAC,CAAC;AAC1E,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,OAAsB,EACtB,iBAAyB,EACzB,gBAAyB,EACzB,MAAoB;IAEpB,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;IACrE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC/E,IAAI,QAAQ,CAAC,YAAY,KAAK,YAAY,EAAE,CAAC;QAC3C,MAAM,IAAI,aAAa,CAAC,2DAA2D,EAAE;YACnF,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,0BAA0B;SACjC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,WAAW,GAA0B,QAAQ,CAAC,YAAY,IAAI,UAAU,CAAC;IAC/E,MAAM,QAAQ,GAAG,sBAAsB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAC3D,IAAI,QAAQ,EAAE,CAAC;QACb,OAAO;YACL,OAAO,EAAE,QAAQ,CAAC,OAAO;YACzB,YAAY;YACZ,oBAAoB,EAAE,QAAQ,CAAC,oBAAoB;YACnD,eAAe,EAAE,QAAQ,CAAC,eAAe;YACzC,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,GAAG,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChE,WAAW;YACX,GAAG,EAAE,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC;SACtC,CAAC;IACJ,CAAC;IAED,MAAM,IAAI,aAAa,CACrB,sBAAsB,gBAAgB,sBAAsB,YAAY,GAAG,EAC3E;QACE,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,8BAA8B;KACrC,CACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,OAAsB,EACtB,iBAAyB,EACzB,MAAoB;IAEpB,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;IACrE,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC/E,IAAI,QAAQ,CAAC,YAAY,KAAK,YAAY,EAAE,CAAC;QAC3C,MAAM,IAAI,aAAa,CAAC,2DAA2D,EAAE;YACnF,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,0BAA0B;SACjC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,QAAQ,GAAG,sBAAsB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAEhE,OAAO;QACL,YAAY;QACZ,WAAW,EAAE,QAAQ,CAAC,YAAY,IAAI,UAAU;QAChD,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,mBAAmB,CAAC,QAAQ,CAAC;QACvC,GAAG,EAAE,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC;KACtC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAsB,EACtB,KAA4D,EAC5D,MAAoB;IAEpB,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;IAChD,OAAO,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,WAAW,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC;AAC3F,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAsB,EACtB,YAAoB,EACpB,gBAAyB,EACzB,MAAoB;IAEpB,MAAM,OAAO,GAAG,MAAM,wBAAwB,CAAC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAChG,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAC3D,OAAO,CAAC,YAAY,EACpB,OAAO,CAAC,oBAAoB,EAC5B,OAAO,CAAC,eAAe,EACvB,MAAM,CACP,CAAC;IAEF,IACE,QAAQ,CAAC,aAAa,KAAK,OAAO,CAAC,YAAY;QAC/C,QAAQ,CAAC,sBAAsB,KAAK,OAAO,CAAC,oBAAoB;QAChE,QAAQ,CAAC,iBAAiB,KAAK,OAAO,CAAC,eAAe,EACtD,CAAC;QACD,MAAM,IAAI,aAAa,CAAC,kEAAkE,EAAE;YAC1F,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,uBAAuB;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,GAAG,OAAO;QACV,aAAa,EAAE,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC;KAChD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAsB,EACtB,KAAuD,EACvD,MAAoB;IAEpB,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IACnE,MAAM,cAAc,GAAG,oBAAoB,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,IAAI,UAAU,CAAC,CAAC;IAEtF,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACnF,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAC;IAC7D,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACvB,MAAM,IAAI,aAAa,CAAC,+DAA+D,EAAE;YACvF,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,uBAAuB;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,YAAY,GAAG,WAAW,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;IACrE,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CACvD,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,EAAE,EAC7C,MAAM,CACP,CAAC;IACF,MAAM,kBAAkB,GAAG,mBAAmB,CAAC,cAAc,CAAC,CAAC;IAC/D,IAAI,kBAAkB,IAAI,kBAAkB,KAAK,YAAY,EAAE,CAAC;QAC9D,MAAM,IAAI,aAAa,CAAC,mEAAmE,EAAE;YAC3F,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,0BAA0B;SACjC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,OAAsB,EACtB,KAAyC;IAEzC,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IACtE,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,CAAC;IACtC,MAAM,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC;IAE1C,4EAA4E;IAC5E,oEAAoE;IACpE,MAAM,YAAY,GAChB,WAAW,CAAC,YAAY,IAAI,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC;IAE5F,MAAM,mBAAmB,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,wBAAwB,CACpE,YAAY,EACZ;QACE,WAAW,EAAE,YAAY;QACzB,WAAW,EACT,aAAa,CAAC,WAAW,KAAK,UAAU;YACtC,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,YAAY,EAAE,gBAAgB,CAAC;YACzD,CAAC,CAAC,WAAW,CAAC,UAAU;QAC5B,cAAc,EAAE,WAAW,CAAC,WAAW,CAAC,YAAY,EAAE,gBAAgB,CAAC;QACvE,sBAAsB,EAAE,WAAW,CAAC,qBAAqB;QACzD,UAAU,EAAE,WAAW,CAAC,SAAS,IAAI,EAAE;QACvC,iBAAiB,EAAE,WAAW,CAAC,eAAe,IAAI,yBAAyB;KAC5E,EACD,KAAK,CAAC,MAAM,CACb,CAAC;IAEF,MAAM,uBAAuB,GAAG,2BAA2B,CAAC,mBAAmB,CAAC,CAAC;IACjF,IAAI,CAAC,uBAAuB,EAAE,CAAC;QAC7B,MAAM,IAAI,aAAa,CACrB,6EAA6E,EAC7E;YACE,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,gCAAgC;SACvC,CACF,CAAC;IACJ,CAAC;IACD,MAAM,oBAAoB,GAAG,WAAW,CAAC,uBAAuB,EAAE,wBAAwB,CAAC,CAAC;IAE5F,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAC1D,YAAY,EACZ,aAAa,CAAC,WAAW,KAAK,UAAU;QACtC,CAAC,CAAC;YACE,6DAA6D;YAC7D,2EAA2E;YAC3E,sBAAsB,EAAE,oBAAoB;YAC5C,gBAAgB,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,cAAc,CAAC;YACvE,oBAAoB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,kBAAkB,CAAC;YACxF,SAAS,EAAE,CAAC,GAAG,EAAE;gBACf,MAAM,eAAe,GAAG,CAAC;qBACtB,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;qBACf,GAAG,CAAC,CAAC,CAAC;qBACN,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;gBAC3C,OAAO,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;oBACtC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;oBACpD,CAAC,CAAC,eAAe,CAAC;YACtB,CAAC,CAAC,EAAE;YACJ,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,UAAU,CAAC;YACvE,gBAAgB,EAAE,aAAa,CAAC,eAAe;YAC/C,GAAG,CAAC,aAAa,CAAC,kBAAkB,IAAI,aAAa,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC;gBACjF,CAAC,CAAC,EAAE,mBAAmB,EAAE,aAAa,CAAC,kBAAkB,EAAE;gBAC3D,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,aAAa,CAAC,kBAAkB,KAAK,SAAS;gBAChD,CAAC,CAAC,EAAE,mBAAmB,EAAE,aAAa,CAAC,kBAAkB,EAAE;gBAC3D,CAAC,CAAC,EAAE,CAAC;SACR;QACH,CAAC,CAAC;YACE,YAAY,EAAE,aAAa,CAAC,WAAW;YACvC,YAAY,EAAE,aAAa,CAAC,WAAW;YACvC,sBAAsB,EAAE,oBAAoB;YAC5C,QAAQ,EAAE,aAAa,CAAC,QAAQ;YAChC,QAAQ,EAAE,aAAa,CAAC,QAAQ;YAChC,gBAAgB,EAAE,aAAa,CAAC,eAAe;YAC/C,GAAG,CAAC,aAAa,CAAC,kBAAkB,IAAI,aAAa,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC;gBACjF,CAAC,CAAC,EAAE,mBAAmB,EAAE,aAAa,CAAC,kBAAkB,EAAE;gBAC3D,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,aAAa,CAAC,kBAAkB,KAAK,SAAS;gBAChD,CAAC,CAAC,EAAE,mBAAmB,EAAE,aAAa,CAAC,kBAAkB,EAAE;gBAC3D,CAAC,CAAC,EAAE,CAAC;SACR,EACL,KAAK,CAAC,MAAM,CACb,CAAC;IAEF,MAAM,kBAAkB,GAAG,sBAAsB,CAAC,cAAc,CAAC,CAAC;IAClE,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACxB,MAAM,IAAI,aAAa,CAAC,mEAAmE,EAAE;YAC3F,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,2BAA2B;SAClC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,eAAe,GAAG,WAAW,CAAC,kBAAkB,EAAE,mBAAmB,CAAC,CAAC;IAE7E,MAAM,cAAc,GAAG,CAAC,KAAK,CAAC,qBAAqB,IAAI,UAAU,CAAC,EAAE,CAAC;IACrE,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;QAC9C,YAAY;QACZ,eAAe;QACf,cAAc;QACd,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD,CAAC,CAAC;IAEH,IACE,WAAW,CAAC,KAAK,CAAC,eAAe,KAAK,eAAe;QACrD,WAAW,CAAC,KAAK,CAAC,oBAAoB,KAAK,oBAAoB,EAC/D,CAAC;QACD,MAAM,IAAI,aAAa,CAAC,mEAAmE,EAAE;YAC3F,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,2BAA2B;SAClC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,GAAG,GAAG;QACV,YAAY;QACZ,KAAK,EAAE,WAAW,CAAC,KAAK;QACxB,MAAM,EAAE,WAAW,CAAC,MAAM;QAC1B,eAAe,EAAE,WAAW,CAAC,KAAK,CAAC,eAAe;QAClD,oBAAoB,EAAE,WAAW,CAAC,KAAK,CAAC,oBAAoB;KAC7D,CAAC;IAEF,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAAsB,EACtB,KAA+B;IAE/B,MAAM,GAAG,GAAG,MAAM,6BAA6B,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAEhE,MAAM,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAElF,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC;QAC/B,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,YAAY,EAAE,GAAG,CAAC,YAAY;QAC9B,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,UAAU,EAAE,KAAK,CAAC,cAAc;QAChC,SAAS,EAAE,KAAK,CAAC,aAAa;QAC9B,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxD,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACzF,MAAM,IAAI,aAAa,CACrB,OAAO,GAAG,CAAC,KAAK,mBAAmB,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,WAAW,GAAG,EACxE;YACE,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,WAAW,IAAI,YAAY;YAChD,KAAK,EAAE,OAAO,CAAC,MAAM;SACtB,CACF,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC1B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mCAAmC,CACvD,OAAsB,EACtB,KAQC,EACD,MAAoB;IAEpB,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IACnE,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IACtE,MAAM,cAAc,GAAG,oBAAoB,CAAC,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACxE,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IAE1C,MAAM,OAAO,CAAC,GAAG,CAAC,oBAAoB,CACpC,YAAY,EACZ;QACE,IAAI,EAAE,cAAc;QACpB,WAAW,EAAE,KAAK,CAAC,YAAY;QAC/B,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/E,EACD,MAAM,CACP,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAChC,WAAW,EACX,YAAY,EACZ,EAAE,SAAS,EAAE,cAAc,EAAE,EAC7B,MAAM,CACP,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAE/E,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,qBAAqB,CACtD,WAAW,EACX,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,EACvD,MAAM,CACP,CAAC;IAEF,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AACvE,CAAC"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import type { EvaluationContextType, EvaluationResults } from '../domain/types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Release-gate thresholds. Every field is optional; only configured thresholds
|
|
4
|
+
* produce checks. A gate with no configured threshold is advisory and passes.
|
|
5
|
+
*/
|
|
6
|
+
export type GateThresholds = {
|
|
7
|
+
/** Scenario only: minimum primary-model overall score. */
|
|
8
|
+
minOverall?: number;
|
|
9
|
+
/** Scenario only: minimum score for every scenario cell. */
|
|
10
|
+
minScenario?: number;
|
|
11
|
+
/** Scenario only: minimum judge agreement ratio. */
|
|
12
|
+
minJudgeAgreement?: number;
|
|
13
|
+
/** Conversation and agent trace only: minimum per-metric scores. */
|
|
14
|
+
metrics?: Readonly<Record<string, number>>;
|
|
15
|
+
};
|
|
16
|
+
export type GateVerdict = 'PASS' | 'FAIL' | 'INCONCLUSIVE';
|
|
17
|
+
export type GateCheck = {
|
|
18
|
+
/** Stable metric identifier, safe for machine consumption. */
|
|
19
|
+
metric: string;
|
|
20
|
+
label: string;
|
|
21
|
+
threshold: string;
|
|
22
|
+
actual: string;
|
|
23
|
+
decision: GateVerdict;
|
|
24
|
+
passed: boolean;
|
|
25
|
+
/** Why a check is not a pass; omitted for passing checks. */
|
|
26
|
+
reason?: string;
|
|
27
|
+
/** Extra context, such as the scenario that produced the minimum score. */
|
|
28
|
+
detail?: string;
|
|
29
|
+
};
|
|
30
|
+
export type GateMetrics = {
|
|
31
|
+
overall?: number;
|
|
32
|
+
scenarioMinimum?: {
|
|
33
|
+
score: number;
|
|
34
|
+
scenarioName?: string;
|
|
35
|
+
modelKey?: string;
|
|
36
|
+
};
|
|
37
|
+
judgeAgreement?: {
|
|
38
|
+
agreed: number;
|
|
39
|
+
total: number;
|
|
40
|
+
ratio: number;
|
|
41
|
+
};
|
|
42
|
+
trajectoryScore?: number;
|
|
43
|
+
};
|
|
44
|
+
export type GateReport = {
|
|
45
|
+
contextType: EvaluationContextType;
|
|
46
|
+
decision: GateVerdict;
|
|
47
|
+
passed: boolean;
|
|
48
|
+
/** True when no threshold was configured, so the gate could not fail. */
|
|
49
|
+
advisory: boolean;
|
|
50
|
+
thresholds: GateThresholds;
|
|
51
|
+
metrics: GateMetrics;
|
|
52
|
+
checks: GateCheck[];
|
|
53
|
+
};
|
|
54
|
+
export declare function extractGateMetrics(results: EvaluationResults): GateMetrics;
|
|
55
|
+
/**
|
|
56
|
+
* Compare evaluation results against release thresholds. Pure: it performs no
|
|
57
|
+
* I/O and never inspects credentials or raw upstream diagnostics. Scenario
|
|
58
|
+
* evaluations retain the canonical score and agreement checks. Conversation
|
|
59
|
+
* and agent-trace evaluations gate only on configured `per_metric` scores;
|
|
60
|
+
* outcome-judge fields never contribute evidence.
|
|
61
|
+
*/
|
|
62
|
+
export declare function evaluateGate(results: EvaluationResults, thresholds: GateThresholds): GateReport;
|
|
63
|
+
//# sourceMappingURL=gate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gate.d.ts","sourceRoot":"","sources":["../../src/actions/gate.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,qBAAqB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAEnF;;;GAGG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,0DAA0D;IAC1D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oDAAoD;IACpD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oEAAoE;IACpE,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,MAAM,GAAG,cAAc,CAAC;AAE3D,MAAM,MAAM,SAAS,GAAG;IACtB,8DAA8D;IAC9D,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,WAAW,CAAC;IACtB,MAAM,EAAE,OAAO,CAAC;IAChB,6DAA6D;IAC7D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2EAA2E;IAC3E,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9E,cAAc,CAAC,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAClE,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG;IACvB,WAAW,EAAE,qBAAqB,CAAC;IACnC,QAAQ,EAAE,WAAW,CAAC;IACtB,MAAM,EAAE,OAAO,CAAC;IAChB,yEAAyE;IACzE,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,cAAc,CAAC;IAC3B,OAAO,EAAE,WAAW,CAAC;IACrB,MAAM,EAAE,SAAS,EAAE,CAAC;CACrB,CAAC;AA0IF,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,iBAAiB,GAAG,WAAW,CAI1E;AA8HD;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,cAAc,GAAG,UAAU,CA+B/F"}
|