niceeval 0.15.1-canary.76 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +45 -40
- package/dist/adapter-execution-trace.cjs.map +1 -1
- package/dist/adapter-execution-trace.d.cts +26 -0
- package/dist/adapter-execution-trace.d.mts +26 -0
- package/dist/adapter-execution-trace.d.ts +26 -0
- package/dist/adapter-usage.cjs.map +1 -1
- package/dist/adapter-usage.d.cts +9 -0
- package/dist/adapter-usage.d.mts +9 -0
- package/dist/adapter-usage.d.ts +9 -0
- package/dist/adapter.cjs +19 -4
- package/dist/adapter.cjs.map +1 -1
- package/dist/adapter.d.cts +37 -12
- package/dist/adapter.d.mts +37 -12
- package/dist/adapter.d.ts +37 -12
- package/dist/agents/ai-sdk.cjs +3 -0
- package/dist/agents/ai-sdk.cjs.map +1 -1
- package/dist/agents/bub.cjs +3 -0
- package/dist/agents/bub.cjs.map +1 -1
- package/dist/agents/claude-code-native.cjs +28 -2
- package/dist/agents/claude-code-native.cjs.map +1 -1
- package/dist/agents/claude-code.cjs +3 -0
- package/dist/agents/claude-code.cjs.map +1 -1
- package/dist/agents/codex.cjs +3 -0
- package/dist/agents/codex.cjs.map +1 -1
- package/dist/agents/coding-cli-versions.cjs +9 -10
- package/dist/agents/coding-cli-versions.cjs.map +1 -1
- package/dist/agents/coding-cli-versions.d.cts +1 -1
- package/dist/agents/coding-cli-versions.d.mts +1 -1
- package/dist/agents/coding-cli-versions.d.ts +1 -1
- package/dist/agents/deepseek-harness.cjs +3 -0
- package/dist/agents/deepseek-harness.cjs.map +1 -1
- package/dist/agents/hermes.cjs +3 -0
- package/dist/agents/hermes.cjs.map +1 -1
- package/dist/agents/index.cjs.map +1 -1
- package/dist/agents/index.d.cts +3 -2
- package/dist/agents/index.d.mts +3 -2
- package/dist/agents/index.d.ts +3 -2
- package/dist/agents/omp.cjs +3 -0
- package/dist/agents/omp.cjs.map +1 -1
- package/dist/agents/openai-compat.cjs +6 -0
- package/dist/agents/openai-compat.cjs.map +1 -1
- package/dist/agents/openclaw.cjs +3 -0
- package/dist/agents/openclaw.cjs.map +1 -1
- package/dist/agents/opencode.cjs +3 -0
- package/dist/agents/opencode.cjs.map +1 -1
- package/dist/agents/sdk-streams.cjs +9 -0
- package/dist/agents/sdk-streams.cjs.map +1 -1
- package/dist/agents/ui-message-stream.cjs +3 -0
- package/dist/agents/ui-message-stream.cjs.map +1 -1
- package/dist/assertions/api.cjs.map +1 -1
- package/dist/assertions/api.d.cts +27 -5
- package/dist/assertions/api.d.mts +27 -5
- package/dist/assertions/api.d.ts +27 -5
- package/dist/assertions/collection.cjs +6 -0
- package/dist/assertions/collection.cjs.map +1 -1
- package/dist/assertions/collection.d.cts +4 -2
- package/dist/assertions/collection.d.mts +4 -2
- package/dist/assertions/collection.d.ts +4 -2
- package/dist/assertions/collection.js +12 -8
- package/dist/assertions/context-match.cjs +70 -0
- package/dist/assertions/context-match.cjs.map +1 -0
- package/dist/assertions/context-match.d.cts +107 -0
- package/dist/assertions/context-match.d.mts +107 -0
- package/dist/assertions/context-match.d.ts +107 -0
- package/dist/assertions/context-match.js +12 -0
- package/dist/assertions/context-match.js.map +1 -0
- package/dist/assertions/full-content.cjs +109 -0
- package/dist/assertions/full-content.cjs.map +1 -0
- package/dist/assertions/full-content.d.cts +6 -0
- package/dist/assertions/full-content.d.mts +6 -0
- package/dist/assertions/full-content.d.ts +6 -0
- package/dist/assertions/full-content.js +12 -0
- package/dist/assertions/full-content.js.map +1 -0
- package/dist/assertions/judge-presets.cjs +2 -34
- package/dist/assertions/judge-presets.cjs.map +1 -1
- package/dist/assertions/judge-presets.d.cts +6 -9
- package/dist/assertions/judge-presets.d.mts +6 -9
- package/dist/assertions/judge-presets.d.ts +6 -9
- package/dist/assertions/judge-presets.js +8 -8
- package/dist/assertions/judge.cjs +107 -29
- package/dist/assertions/judge.cjs.map +1 -1
- package/dist/assertions/judge.d.cts +2 -0
- package/dist/assertions/judge.d.mts +2 -0
- package/dist/assertions/judge.d.ts +2 -0
- package/dist/assertions/match.cjs +244 -5
- package/dist/assertions/match.cjs.map +1 -1
- package/dist/assertions/match.d.cts +43 -3
- package/dist/assertions/match.d.mts +43 -3
- package/dist/assertions/match.d.ts +43 -3
- package/dist/assertions/match.js +88 -78
- package/dist/assertions/material.cjs +520 -0
- package/dist/assertions/material.cjs.map +1 -0
- package/dist/assertions/material.d.cts +56 -0
- package/dist/assertions/material.d.mts +56 -0
- package/dist/assertions/material.d.ts +56 -0
- package/dist/assertions/material.js +20 -0
- package/dist/assertions/material.js.map +1 -0
- package/dist/assertions/record/attachment.cjs +5 -2
- package/dist/assertions/record/attachment.cjs.map +1 -1
- package/dist/assertions/record/codec.cjs +2 -1
- package/dist/assertions/record/codec.cjs.map +1 -1
- package/dist/assertions/record/model.cjs.map +1 -1
- package/dist/assertions/record/model.d.cts +6 -1
- package/dist/assertions/record/model.d.mts +6 -1
- package/dist/assertions/record/model.d.ts +6 -1
- package/dist/assertions/runtime.cjs +207 -14
- package/dist/assertions/runtime.cjs.map +1 -1
- package/dist/assertions/runtime.d.cts +9 -0
- package/dist/assertions/runtime.d.mts +9 -0
- package/dist/assertions/runtime.d.ts +9 -0
- package/dist/assertions/runtime.js +14 -8
- package/dist/assertions/score-match-audit.cjs +18 -8
- package/dist/assertions/score-match-audit.cjs.map +1 -1
- package/dist/assertions/score-match-gateway.cjs +83 -25
- package/dist/assertions/score-match-gateway.cjs.map +1 -1
- package/dist/assertions/score-match-gateway.d.cts +2 -0
- package/dist/assertions/score-match-gateway.d.mts +2 -0
- package/dist/assertions/score-match-gateway.d.ts +2 -0
- package/dist/assertions/value-composition.test-d.cjs +77 -0
- package/dist/assertions/value-composition.test-d.cjs.map +1 -0
- package/dist/assertions/value-composition.test-d.d.cts +1 -0
- package/dist/assertions/value-composition.test-d.d.mts +1 -0
- package/dist/assertions/value-composition.test-d.d.ts +1 -0
- package/dist/assertions/value-composition.test-d.js +4 -0
- package/dist/assertions/value-composition.test-d.js.map +1 -0
- package/dist/context/assert-first.cjs +207 -213
- package/dist/context/assert-first.cjs.map +1 -1
- package/dist/context/assert-first.d.cts +49 -23
- package/dist/context/assert-first.d.mts +49 -23
- package/dist/context/assert-first.d.ts +49 -23
- package/dist/context/session.cjs +111 -9
- package/dist/context/session.cjs.map +1 -1
- package/dist/context/session.d.cts +18 -0
- package/dist/context/session.d.mts +18 -0
- package/dist/context/session.d.ts +18 -0
- package/dist/define.cjs +17 -1
- package/dist/define.cjs.map +1 -1
- package/dist/eval/cli/contribution.cjs +2 -0
- package/dist/eval/cli/contribution.cjs.map +1 -1
- package/dist/eval/host/index.cjs +2 -0
- package/dist/eval/host/index.cjs.map +1 -1
- package/dist/eval/record/evaluation.cjs +3 -0
- package/dist/eval/record/evaluation.cjs.map +1 -1
- package/dist/eval/record/membership-provenance.d.cts +15 -15
- package/dist/eval/record/membership-provenance.d.mts +15 -15
- package/dist/eval/record/membership-provenance.d.ts +15 -15
- package/dist/eval/record/score.cjs +7 -0
- package/dist/eval/record/score.cjs.map +1 -1
- package/dist/eval/record/sealed-assertion.cjs +3 -0
- package/dist/eval/record/sealed-assertion.cjs.map +1 -1
- package/dist/eval/record/verdict.cjs +7 -0
- package/dist/eval/record/verdict.cjs.map +1 -1
- package/dist/expect/index.cjs +8 -2
- package/dist/expect/index.cjs.map +1 -1
- package/dist/expect/index.d.cts +9 -4
- package/dist/expect/index.d.mts +9 -4
- package/dist/expect/index.d.ts +9 -4
- package/dist/expect/index.js +72 -62
- package/dist/expect/index.mjs +72 -62
- package/dist/experiment/host/cli/contribution.cjs +37 -23
- package/dist/experiment/host/cli/contribution.cjs.map +1 -1
- package/dist/experiment/host/cli/contribution.d.cts +9 -1
- package/dist/experiment/host/cli/contribution.d.mts +9 -1
- package/dist/experiment/host/cli/contribution.d.ts +9 -1
- package/dist/experiment/host/current.cjs +74 -0
- package/dist/experiment/host/current.cjs.map +1 -0
- package/dist/experiment/host/current.d.cts +33 -0
- package/dist/experiment/host/current.d.mts +33 -0
- package/dist/experiment/host/current.d.ts +33 -0
- package/dist/experiment/host/current.js +10 -0
- package/dist/experiment/host/current.js.map +1 -0
- package/dist/experiment/host/index.cjs +17 -174
- package/dist/experiment/host/index.cjs.map +1 -1
- package/dist/experiment/host/index.d.cts +8 -54
- package/dist/experiment/host/index.d.mts +8 -54
- package/dist/experiment/host/index.d.ts +8 -54
- package/dist/experiment/host/operations.cjs +33 -10
- package/dist/experiment/host/operations.cjs.map +1 -1
- package/dist/experiment/host/operations.d.cts +29 -1
- package/dist/experiment/host/operations.d.mts +29 -1
- package/dist/experiment/host/operations.d.ts +29 -1
- package/dist/experiment/host/operations.js +6 -4
- package/dist/experiment/host/requirements.cjs +3 -0
- package/dist/experiment/host/requirements.cjs.map +1 -0
- package/dist/experiment/host/requirements.d.cts +3 -0
- package/dist/experiment/host/requirements.d.mts +3 -0
- package/dist/experiment/host/requirements.d.ts +3 -0
- package/dist/experiment/host/requirements.js +4 -0
- package/dist/experiment/host/requirements.js.map +1 -0
- package/dist/experiment/host/runtime.cjs +179 -0
- package/dist/experiment/host/runtime.cjs.map +1 -0
- package/dist/experiment/host/runtime.d.cts +79 -0
- package/dist/experiment/host/runtime.d.mts +79 -0
- package/dist/experiment/host/runtime.d.ts +79 -0
- package/dist/experiment/host/runtime.js +6 -0
- package/dist/experiment/host/runtime.js.map +1 -0
- package/dist/experiment/host/types.cjs.map +1 -1
- package/dist/experiment/host/types.d.cts +17 -16
- package/dist/experiment/host/types.d.mts +17 -16
- package/dist/experiment/host/types.d.ts +17 -16
- package/dist/fresh-import.cjs +2 -1
- package/dist/fresh-import.cjs.map +1 -1
- package/dist/index.cjs +4 -2
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +12 -6
- package/dist/index.d.mts +12 -6
- package/dist/index.d.ts +12 -6
- package/dist/index.js +32 -30
- package/dist/index.mjs +32 -30
- package/dist/inspection/artifacts.cjs +83 -54
- package/dist/inspection/artifacts.cjs.map +1 -1
- package/dist/inspection/artifacts.d.cts +8 -0
- package/dist/inspection/artifacts.d.mts +8 -0
- package/dist/inspection/artifacts.d.ts +8 -0
- package/dist/inspection/artifacts.js +2 -0
- package/dist/inspection/assertions.cjs +4 -1
- package/dist/inspection/assertions.cjs.map +1 -1
- package/dist/inspection/cli/contribution.cjs +27 -4
- package/dist/inspection/cli/contribution.cjs.map +1 -1
- package/dist/inspection/cli/contribution.d.cts +2 -1
- package/dist/inspection/cli/contribution.d.mts +2 -1
- package/dist/inspection/cli/contribution.d.ts +2 -1
- package/dist/inspection/execution-traces.cjs +114 -8
- package/dist/inspection/execution-traces.cjs.map +1 -1
- package/dist/inspection/execution-traces.d.cts +38 -0
- package/dist/inspection/execution-traces.d.mts +38 -0
- package/dist/inspection/execution-traces.d.ts +38 -0
- package/dist/inspection/facts.cjs +12 -0
- package/dist/inspection/facts.cjs.map +1 -1
- package/dist/inspection/limits.cjs +5 -1
- package/dist/inspection/limits.cjs.map +1 -1
- package/dist/inspection/limits.d.cts +4 -0
- package/dist/inspection/limits.d.mts +4 -0
- package/dist/inspection/limits.d.ts +4 -0
- package/dist/inspection/limits.js +6 -2
- package/dist/inspection/overview.cjs +74 -33
- package/dist/inspection/overview.cjs.map +1 -1
- package/dist/inspection/overview.d.cts +19 -1
- package/dist/inspection/overview.d.mts +19 -1
- package/dist/inspection/overview.d.ts +19 -1
- package/dist/inspection/overview.js +4 -2
- package/dist/inspection/project-input.cjs +9 -0
- package/dist/inspection/project-input.cjs.map +1 -0
- package/dist/inspection/project-input.d.cts +20 -0
- package/dist/inspection/project-input.d.mts +20 -0
- package/dist/inspection/project-input.d.ts +20 -0
- package/dist/inspection/project-input.js +8 -0
- package/dist/inspection/project-input.js.map +1 -0
- package/dist/inspection/project-result.cjs +60 -0
- package/dist/inspection/project-result.cjs.map +1 -0
- package/dist/inspection/project-result.d.cts +1264 -0
- package/dist/inspection/project-result.d.mts +1264 -0
- package/dist/inspection/project-result.d.ts +1264 -0
- package/dist/inspection/project-result.js +10 -0
- package/dist/inspection/project-result.js.map +1 -0
- package/dist/inspection/project-source.cjs +109 -0
- package/dist/inspection/project-source.cjs.map +1 -0
- package/dist/inspection/project-source.d.cts +10 -0
- package/dist/inspection/project-source.d.mts +10 -0
- package/dist/inspection/project-source.d.ts +10 -0
- package/dist/inspection/project-source.js +6 -0
- package/dist/inspection/project-source.js.map +1 -0
- package/dist/inspection/project.cjs +100 -0
- package/dist/inspection/project.cjs.map +1 -0
- package/dist/inspection/project.d.cts +3 -0
- package/dist/inspection/project.d.mts +3 -0
- package/dist/inspection/project.d.ts +3 -0
- package/dist/inspection/project.js +6 -0
- package/dist/inspection/project.js.map +1 -0
- package/dist/inspection/protocol.cjs +19 -9
- package/dist/inspection/protocol.cjs.map +1 -1
- package/dist/inspection/protocol.d.cts +5506 -38189
- package/dist/inspection/protocol.d.mts +5506 -38189
- package/dist/inspection/protocol.d.ts +5506 -38189
- package/dist/inspection/public.mjs +896 -84
- package/dist/inspection/query.cjs +2 -0
- package/dist/inspection/query.cjs.map +1 -1
- package/dist/inspection/results.cjs +234 -29
- package/dist/inspection/results.cjs.map +1 -1
- package/dist/inspection/results.d.cts +870 -13
- package/dist/inspection/results.d.mts +870 -13
- package/dist/inspection/results.d.ts +870 -13
- package/dist/inspection/results.js +36 -34
- package/dist/inspection/select.cjs +126 -14
- package/dist/inspection/select.cjs.map +1 -1
- package/dist/inspection/select.d.cts +1 -1
- package/dist/inspection/select.d.mts +1 -1
- package/dist/inspection/select.d.ts +1 -1
- package/dist/inspection/source.cjs +4 -0
- package/dist/inspection/source.cjs.map +1 -1
- package/dist/inspection/sources.cjs +5 -0
- package/dist/inspection/sources.cjs.map +1 -1
- package/dist/inspection/trace.cjs +82 -84
- package/dist/inspection/trace.cjs.map +1 -1
- package/dist/inspection/trace.d.cts +7 -2
- package/dist/inspection/trace.d.mts +7 -2
- package/dist/inspection/trace.d.ts +7 -2
- package/dist/model-slots.cjs +35 -0
- package/dist/model-slots.cjs.map +1 -0
- package/dist/model-slots.d.cts +19 -0
- package/dist/model-slots.d.mts +19 -0
- package/dist/model-slots.d.ts +19 -0
- package/dist/model-slots.js +8 -0
- package/dist/model-slots.js.map +1 -0
- package/dist/o11y/adapter-call-price.cjs +4 -0
- package/dist/o11y/adapter-call-price.cjs.map +1 -1
- package/dist/o11y/adapter-call-price.d.cts +24 -0
- package/dist/o11y/adapter-call-price.d.mts +24 -0
- package/dist/o11y/adapter-call-price.d.ts +24 -0
- package/dist/o11y/adapter-call-price.js +6 -2
- package/dist/o11y/adapter-usage-projection.cjs +158 -0
- package/dist/o11y/adapter-usage-projection.cjs.map +1 -0
- package/dist/o11y/adapter-usage-projection.d.cts +141 -0
- package/dist/o11y/adapter-usage-projection.d.mts +141 -0
- package/dist/o11y/adapter-usage-projection.d.ts +141 -0
- package/dist/o11y/adapter-usage-projection.js +20 -0
- package/dist/o11y/adapter-usage-projection.js.map +1 -0
- package/dist/o11y/eval-usage.cjs +149 -0
- package/dist/o11y/eval-usage.cjs.map +1 -0
- package/dist/o11y/eval-usage.d.cts +46 -0
- package/dist/o11y/eval-usage.d.mts +46 -0
- package/dist/o11y/eval-usage.d.ts +46 -0
- package/dist/o11y/eval-usage.js +8 -0
- package/dist/o11y/eval-usage.js.map +1 -0
- package/dist/o11y/judge-call-price.cjs +62 -0
- package/dist/o11y/judge-call-price.cjs.map +1 -0
- package/dist/o11y/judge-call-price.d.cts +3 -0
- package/dist/o11y/judge-call-price.d.mts +3 -0
- package/dist/o11y/judge-call-price.d.ts +3 -0
- package/dist/o11y/judge-call-price.js +6 -0
- package/dist/o11y/judge-call-price.js.map +1 -0
- package/dist/o11y/judge-usage-projection.cjs +134 -0
- package/dist/o11y/judge-usage-projection.cjs.map +1 -0
- package/dist/o11y/judge-usage-projection.d.cts +18 -0
- package/dist/o11y/judge-usage-projection.d.mts +18 -0
- package/dist/o11y/judge-usage-projection.d.ts +18 -0
- package/dist/o11y/judge-usage-projection.js +20 -0
- package/dist/o11y/judge-usage-projection.js.map +1 -0
- package/dist/o11y/judge-usage-receipt.cjs +135 -0
- package/dist/o11y/judge-usage-receipt.cjs.map +1 -0
- package/dist/o11y/judge-usage-receipt.d.cts +5 -0
- package/dist/o11y/judge-usage-receipt.d.mts +5 -0
- package/dist/o11y/judge-usage-receipt.d.ts +5 -0
- package/dist/o11y/judge-usage-receipt.js +6 -0
- package/dist/o11y/judge-usage-receipt.js.map +1 -0
- package/dist/o11y/judge-usage.cjs +171 -0
- package/dist/o11y/judge-usage.cjs.map +1 -0
- package/dist/o11y/judge-usage.d.cts +35 -0
- package/dist/o11y/judge-usage.d.mts +35 -0
- package/dist/o11y/judge-usage.d.ts +35 -0
- package/dist/o11y/judge-usage.js +6 -0
- package/dist/o11y/judge-usage.js.map +1 -0
- package/dist/plugin/contracts.cjs +2 -0
- package/dist/plugin/contracts.cjs.map +1 -1
- package/dist/plugin/link.cjs +7 -0
- package/dist/plugin/link.cjs.map +1 -1
- package/dist/record/attachment/protocol.cjs +8 -6
- package/dist/record/attachment/protocol.cjs.map +1 -1
- package/dist/record/attachment/protocol.d.cts +10 -0
- package/dist/record/attachment/protocol.d.mts +10 -0
- package/dist/record/attachment/protocol.d.ts +10 -0
- package/dist/record/attachment/protocol.js +38 -36
- package/dist/record/codec/seal-manifest.d.cts +3 -3
- package/dist/record/codec/seal-manifest.d.mts +3 -3
- package/dist/record/codec/seal-manifest.d.ts +3 -3
- package/dist/record/family/adapter-usage/definition.d.cts +1 -0
- package/dist/record/family/adapter-usage/definition.d.mts +1 -0
- package/dist/record/family/adapter-usage/definition.d.ts +1 -0
- package/dist/record/family/adapter-usage/persistence.cjs +1 -1
- package/dist/record/family/adapter-usage/persistence.cjs.map +1 -1
- package/dist/record/family/adapter-usage/persistence.d.cts +2 -1
- package/dist/record/family/adapter-usage/persistence.d.mts +2 -1
- package/dist/record/family/adapter-usage/persistence.d.ts +2 -1
- package/dist/record/family/adapter-usage/schema.cjs +21 -2
- package/dist/record/family/adapter-usage/schema.cjs.map +1 -1
- package/dist/record/family/adapter-usage/schema.d.cts +85 -0
- package/dist/record/family/adapter-usage/schema.d.mts +85 -0
- package/dist/record/family/adapter-usage/schema.d.ts +85 -0
- package/dist/record/family/adapter-usage/schema.js +18 -12
- package/dist/record/family/catalog.cjs +31 -27
- package/dist/record/family/catalog.cjs.map +1 -1
- package/dist/record/family/catalog.d.cts +94 -3
- package/dist/record/family/catalog.d.mts +94 -3
- package/dist/record/family/catalog.d.ts +94 -3
- package/dist/record/family/current.cjs +45 -42
- package/dist/record/family/current.cjs.map +1 -1
- package/dist/record/family/current.d.cts +240 -1
- package/dist/record/family/current.d.mts +240 -1
- package/dist/record/family/current.d.ts +240 -1
- package/dist/record/family/execution-traces/definition.d.cts +26 -0
- package/dist/record/family/execution-traces/definition.d.mts +26 -0
- package/dist/record/family/execution-traces/definition.d.ts +26 -0
- package/dist/record/family/execution-traces/definition.js +20 -14
- package/dist/record/family/execution-traces/schema.cjs +57 -1
- package/dist/record/family/execution-traces/schema.cjs.map +1 -1
- package/dist/record/family/execution-traces/schema.d.cts +114 -0
- package/dist/record/family/execution-traces/schema.d.mts +114 -0
- package/dist/record/family/execution-traces/schema.d.ts +114 -0
- package/dist/record/family/execution-traces/schema.js +18 -12
- package/dist/record/family/judge-usage/definition.cjs +12 -0
- package/dist/record/family/judge-usage/definition.cjs.map +1 -0
- package/dist/record/family/judge-usage/definition.d.cts +91 -0
- package/dist/record/family/judge-usage/definition.d.mts +91 -0
- package/dist/record/family/judge-usage/definition.d.ts +91 -0
- package/dist/record/family/judge-usage/definition.js +6 -0
- package/dist/record/family/judge-usage/definition.js.map +1 -0
- package/dist/record/family/judge-usage/persistence.cjs +10 -0
- package/dist/record/family/judge-usage/persistence.cjs.map +1 -0
- package/dist/record/family/judge-usage/persistence.d.cts +91 -0
- package/dist/record/family/judge-usage/persistence.d.mts +91 -0
- package/dist/record/family/judge-usage/persistence.d.ts +91 -0
- package/dist/record/family/judge-usage/persistence.js +6 -0
- package/dist/record/family/judge-usage/persistence.js.map +1 -0
- package/dist/record/family/judge-usage/schema.cjs +307 -0
- package/dist/record/family/judge-usage/schema.cjs.map +1 -0
- package/dist/record/family/judge-usage/schema.d.cts +351 -0
- package/dist/record/family/judge-usage/schema.d.mts +351 -0
- package/dist/record/family/judge-usage/schema.d.ts +351 -0
- package/dist/record/family/judge-usage/schema.js +18 -0
- package/dist/record/family/judge-usage/schema.js.map +1 -0
- package/dist/record/host/sqlite-host.cjs +55 -9
- package/dist/record/host/sqlite-host.cjs.map +1 -1
- package/dist/record/host/types.cjs.map +1 -1
- package/dist/record/host/types.d.cts +7 -0
- package/dist/record/host/types.d.mts +7 -0
- package/dist/record/host/types.d.ts +7 -0
- package/dist/record/model/run-context.cjs +7 -1
- package/dist/record/model/run-context.cjs.map +1 -1
- package/dist/record/model/run-context.d.cts +4 -0
- package/dist/record/model/run-context.d.mts +4 -0
- package/dist/record/model/run-context.d.ts +4 -0
- package/dist/record/model/seal-manifest.d.cts +1 -1
- package/dist/record/model/seal-manifest.d.mts +1 -1
- package/dist/record/model/seal-manifest.d.ts +1 -1
- package/dist/record/sqlite/client.cjs +18 -5
- package/dist/record/sqlite/client.cjs.map +1 -1
- package/dist/record/sqlite/client.d.cts +1 -1
- package/dist/record/sqlite/client.d.mts +1 -1
- package/dist/record/sqlite/client.d.ts +1 -1
- package/dist/record/sqlite/project-state-database.cjs +29 -8
- package/dist/record/sqlite/project-state-database.cjs.map +1 -1
- package/dist/run/host/runtime.cjs +4 -0
- package/dist/run/host/runtime.cjs.map +1 -1
- package/dist/run/storage/sqlite.cjs +20 -0
- package/dist/run/storage/sqlite.cjs.map +1 -1
- package/dist/runner/adapter-execution-trace.cjs +125 -2
- package/dist/runner/adapter-execution-trace.cjs.map +1 -1
- package/dist/runner/adapter-execution-trace.d.cts +1 -1
- package/dist/runner/adapter-execution-trace.d.mts +1 -1
- package/dist/runner/adapter-execution-trace.d.ts +1 -1
- package/dist/runner/adapter-usage.cjs +55 -7
- package/dist/runner/adapter-usage.cjs.map +1 -1
- package/dist/runner/adapter-usage.d.cts +9 -2
- package/dist/runner/adapter-usage.d.mts +9 -2
- package/dist/runner/adapter-usage.d.ts +9 -2
- package/dist/runner/adoption.cjs +9 -0
- package/dist/runner/adoption.cjs.map +1 -1
- package/dist/runner/attempt-cancellation.cjs +12 -0
- package/dist/runner/attempt-cancellation.cjs.map +1 -0
- package/dist/runner/attempt-cancellation.d.cts +6 -0
- package/dist/runner/attempt-cancellation.d.mts +6 -0
- package/dist/runner/attempt-cancellation.d.ts +6 -0
- package/dist/runner/attempt-cancellation.js +6 -0
- package/dist/runner/attempt-cancellation.js.map +1 -0
- package/dist/runner/attempt-resources.cjs +10 -4
- package/dist/runner/attempt-resources.cjs.map +1 -1
- package/dist/runner/attempt-resources.d.cts +2 -1
- package/dist/runner/attempt-resources.d.mts +2 -1
- package/dist/runner/attempt-resources.d.ts +2 -1
- package/dist/runner/attempt.cjs +1177 -1139
- package/dist/runner/attempt.cjs.map +1 -1
- package/dist/runner/config-identity.cjs +20 -1
- package/dist/runner/config-identity.cjs.map +1 -1
- package/dist/runner/config-identity.d.cts +5 -0
- package/dist/runner/config-identity.d.mts +5 -0
- package/dist/runner/config-identity.d.ts +5 -0
- package/dist/runner/discover.cjs +14 -0
- package/dist/runner/discover.cjs.map +1 -1
- package/dist/runner/discover.d.cts +3 -0
- package/dist/runner/discover.d.mts +3 -0
- package/dist/runner/discover.d.ts +3 -0
- package/dist/runner/feedback/human.cjs +9 -4
- package/dist/runner/feedback/human.cjs.map +1 -1
- package/dist/runner/fingerprint.cjs +1 -0
- package/dist/runner/fingerprint.cjs.map +1 -1
- package/dist/runner/judge-usage.cjs +18 -0
- package/dist/runner/judge-usage.cjs.map +1 -0
- package/dist/runner/judge-usage.d.cts +5 -0
- package/dist/runner/judge-usage.d.mts +5 -0
- package/dist/runner/judge-usage.d.ts +5 -0
- package/dist/runner/judge-usage.js +8 -0
- package/dist/runner/judge-usage.js.map +1 -0
- package/dist/runner/project-target.cjs.map +1 -1
- package/dist/runner/project-target.d.cts +2 -0
- package/dist/runner/project-target.d.mts +2 -0
- package/dist/runner/project-target.d.ts +2 -0
- package/dist/runner/record/planning.cjs +1 -0
- package/dist/runner/record/planning.cjs.map +1 -1
- package/dist/runner/record.cjs +4 -0
- package/dist/runner/record.cjs.map +1 -1
- package/dist/runner/reporters/braintrust.cjs +2 -0
- package/dist/runner/reporters/braintrust.cjs.map +1 -1
- package/dist/runner/reuse-diagnostics.cjs +9 -10
- package/dist/runner/reuse-diagnostics.cjs.map +1 -1
- package/dist/runner/reuse-diagnostics.d.cts +5 -6
- package/dist/runner/reuse-diagnostics.d.mts +5 -6
- package/dist/runner/reuse-diagnostics.d.ts +5 -6
- package/dist/runner/reuse-diagnostics.js +4 -4
- package/dist/runner/reuse-plan.cjs +100 -48
- package/dist/runner/reuse-plan.cjs.map +1 -1
- package/dist/runner/reuse-plan.d.cts +60 -3
- package/dist/runner/reuse-plan.d.mts +60 -3
- package/dist/runner/reuse-plan.d.ts +60 -3
- package/dist/runner/reuse-plan.js +14 -10
- package/dist/runner/run.cjs +8 -6
- package/dist/runner/run.cjs.map +1 -1
- package/dist/runner/session.cjs +1 -1
- package/dist/runner/session.cjs.map +1 -1
- package/dist/runner/session.d.cts +1 -1
- package/dist/runner/session.d.mts +1 -1
- package/dist/runner/session.d.ts +1 -1
- package/dist/runner/source-producer.cjs +3 -1
- package/dist/runner/source-producer.cjs.map +1 -1
- package/dist/runner/types.cjs.map +1 -1
- package/dist/runner/types.d.cts +26 -5
- package/dist/runner/types.d.mts +26 -5
- package/dist/runner/types.d.ts +26 -5
- package/dist/sandbox/action.cjs +3 -0
- package/dist/sandbox/action.cjs.map +1 -1
- package/dist/sandbox/commands.cjs +3 -0
- package/dist/sandbox/commands.cjs.map +1 -1
- package/dist/sandbox/compose.cjs +6 -0
- package/dist/sandbox/compose.cjs.map +1 -1
- package/dist/sandbox/docker-profile/registry.cjs +7 -0
- package/dist/sandbox/docker-profile/registry.cjs.map +1 -1
- package/dist/sandbox/docker-profile/runtime.cjs +3 -0
- package/dist/sandbox/docker-profile/runtime.cjs.map +1 -1
- package/dist/sandbox/docker-profile/schema.cjs +7 -0
- package/dist/sandbox/docker-profile/schema.cjs.map +1 -1
- package/dist/sandbox/incus.cjs +7 -0
- package/dist/sandbox/incus.cjs.map +1 -1
- package/dist/sandbox/layer.cjs +12 -0
- package/dist/sandbox/layer.cjs.map +1 -1
- package/dist/shared/adapter-cleanup.cjs +18 -0
- package/dist/shared/adapter-cleanup.cjs.map +1 -0
- package/dist/shared/adapter-cleanup.d.cts +4 -0
- package/dist/shared/adapter-cleanup.d.mts +4 -0
- package/dist/shared/adapter-cleanup.d.ts +4 -0
- package/dist/shared/adapter-cleanup.js +10 -0
- package/dist/shared/adapter-cleanup.js.map +1 -0
- package/dist/shared/attempt-lifecycle.cjs +3 -0
- package/dist/shared/attempt-lifecycle.cjs.map +1 -0
- package/dist/shared/attempt-lifecycle.d.cts +13 -0
- package/dist/shared/attempt-lifecycle.d.mts +13 -0
- package/dist/shared/attempt-lifecycle.d.ts +13 -0
- package/dist/shared/attempt-lifecycle.js +4 -0
- package/dist/shared/attempt-lifecycle.js.map +1 -0
- package/dist/show/contribution.cjs +52 -14
- package/dist/show/contribution.cjs.map +1 -1
- package/dist/show/contribution.d.cts +23 -1
- package/dist/show/contribution.d.mts +23 -1
- package/dist/show/contribution.d.ts +23 -1
- package/dist/show/model.cjs +8 -1
- package/dist/show/model.cjs.map +1 -1
- package/dist/show/model.d.cts +10 -1
- package/dist/show/model.d.mts +10 -1
- package/dist/show/model.d.ts +10 -1
- package/dist/show/render.cjs +287 -49
- package/dist/show/render.cjs.map +1 -1
- package/dist/show/render.d.cts +3 -0
- package/dist/show/render.d.mts +3 -0
- package/dist/show/render.d.ts +3 -0
- package/dist/sources/projector.cjs +3 -0
- package/dist/sources/projector.cjs.map +1 -1
- package/dist/types.cjs.map +1 -1
- package/dist/types.d.cts +1 -1
- package/dist/types.d.mts +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/view/app-dist/.vite/manifest.json +2 -2
- package/dist/view/app-dist/assets/index-CmVoXbwG.js +63 -0
- package/dist/view/app-dist/assets/{quality-cost-scatter-CFB3wnkN.js → quality-cost-scatter-BuLISaH4.js} +1 -1
- package/dist/view/app-dist/index.html +1 -1
- package/dist/view/cli/contribution.cjs +54 -21
- package/dist/view/cli/contribution.cjs.map +1 -1
- package/dist/view/cli/contribution.d.cts +2 -1
- package/dist/view/cli/contribution.d.mts +2 -1
- package/dist/view/cli/contribution.d.ts +2 -1
- package/dist/view/function-runtime.mjs +13231 -11647
- package/dist/view/http-protocol.cjs +3 -0
- package/dist/view/http-protocol.cjs.map +1 -1
- package/dist/view/http-protocol.d.cts +8 -0
- package/dist/view/http-protocol.d.mts +8 -0
- package/dist/view/http-protocol.d.ts +8 -0
- package/dist/view/inspection-host.cjs +17 -1
- package/dist/view/inspection-host.cjs.map +1 -1
- package/dist/view/render.cjs +1 -0
- package/dist/view/render.cjs.map +1 -1
- package/dist/view/render.d.cts +2 -1
- package/dist/view/render.d.mts +2 -1
- package/dist/view/render.d.ts +2 -1
- package/dist/view/revision.cjs.map +1 -1
- package/dist/view/revision.d.cts +12 -0
- package/dist/view/revision.d.mts +12 -0
- package/dist/view/revision.d.ts +12 -0
- package/dist/view/server.cjs +10 -9
- package/dist/view/server.cjs.map +1 -1
- package/docs-site/zh/examples/ai-agent-application.mdx +1 -1
- package/docs-site/zh/explanation/adapter.mdx +2 -2
- package/docs-site/zh/explanation/assert.mdx +60 -24
- package/docs-site/zh/explanation/drive.mdx +1 -1
- package/docs-site/zh/explanation/evals.mdx +31 -29
- package/docs-site/zh/explanation/experiment.mdx +15 -15
- package/docs-site/zh/explanation/judge.mdx +23 -15
- package/docs-site/zh/explanation/overview.mdx +44 -21
- package/docs-site/zh/explanation/record.mdx +28 -13
- package/docs-site/zh/explanation/results-flow.mdx +37 -0
- package/docs-site/zh/explanation/runner.mdx +27 -6
- package/docs-site/zh/index.mdx +22 -115
- package/docs-site/zh/introduction.mdx +89 -61
- package/docs-site/zh/reference/cli.mdx +50 -26
- package/docs-site/zh/reference/define-agent.mdx +37 -5
- package/docs-site/zh/reference/define-eval.mdx +130 -29
- package/docs-site/zh/reference/define-experiment.mdx +251 -0
- package/docs-site/zh/reference/events.mdx +34 -30
- package/docs-site/zh/reference/expect.mdx +77 -2
- package/docs-site/zh/reference/official-adapters.mdx +2 -2
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +43 -24
- package/docs-site/zh/troubleshooting/debugging.mdx +65 -42
- package/docs-site/zh/troubleshooting/recover-after-kill.mdx +35 -28
- package/docs-site/zh/tutorials/accept-results.mdx +34 -24
- package/docs-site/zh/tutorials/agent-feedback-loop.mdx +74 -43
- package/docs-site/zh/tutorials/agent-onboarding.mdx +27 -20
- package/docs-site/zh/tutorials/authoring.mdx +138 -121
- package/docs-site/zh/tutorials/ci-integration.mdx +46 -22
- package/docs-site/zh/tutorials/compare-runs.mdx +40 -13
- package/docs-site/zh/tutorials/concurrency.mdx +66 -62
- package/docs-site/zh/tutorials/configuration.mdx +17 -17
- package/docs-site/zh/tutorials/connect-otel.mdx +48 -37
- package/docs-site/zh/tutorials/connect-your-agent.mdx +80 -173
- package/docs-site/zh/tutorials/control-run-cost.mdx +63 -12
- package/docs-site/zh/tutorials/criteria-files.mdx +28 -21
- package/docs-site/zh/tutorials/custom-application.mdx +75 -55
- package/docs-site/zh/tutorials/dataset-fanout.mdx +32 -22
- package/docs-site/zh/tutorials/debug-lifecycle-plan.mdx +38 -31
- package/docs-site/zh/tutorials/docker-in-docker.mdx +83 -89
- package/docs-site/zh/tutorials/eval-groups.mdx +70 -89
- package/docs-site/zh/tutorials/evaluation-kinds.mdx +12 -5
- package/docs-site/zh/tutorials/experiment-services.mdx +243 -0
- package/docs-site/zh/tutorials/fixtures.mdx +42 -26
- package/docs-site/zh/tutorials/handle-execution-failures.mdx +41 -19
- package/docs-site/zh/tutorials/install-custom-sandbox-agent.mdx +21 -16
- package/docs-site/zh/tutorials/niceeval-taste-application.mdx +394 -0
- package/docs-site/zh/tutorials/niceeval-taste.mdx +190 -0
- package/docs-site/zh/tutorials/nixos-managed-dind.mdx +51 -39
- package/docs-site/zh/tutorials/plugins.mdx +55 -31
- package/docs-site/zh/tutorials/prebuilt-sandbox.mdx +204 -0
- package/docs-site/zh/tutorials/quickstart.mdx +91 -47
- package/docs-site/zh/tutorials/reporters.mdx +45 -30
- package/docs-site/zh/tutorials/rerun-and-cache.mdx +39 -40
- package/docs-site/zh/tutorials/sandbox-agent.mdx +65 -40
- package/docs-site/zh/tutorials/sandbox-providers.mdx +104 -282
- package/docs-site/zh/tutorials/sandbox-reuse.mdx +65 -97
- package/docs-site/zh/tutorials/select-and-preview.mdx +25 -20
- package/docs-site/zh/tutorials/verify-judge.mdx +49 -26
- package/docs-site/zh/tutorials/viewing-results.mdx +130 -80
- package/docs-site/zh/tutorials/write-experiment.mdx +116 -298
- package/docs-site/zh/tutorials/write-send.mdx +53 -52
- package/node_modules/undici/docs/docs/api/Agent.md +8 -8
- package/node_modules/undici/docs/docs/api/DiagnosticsChannel.md +9 -6
- package/node_modules/undici/docs/docs/api/Errors.md +25 -0
- package/node_modules/undici/docs/docs/api/Interceptors.md +4 -3
- package/node_modules/undici/docs/docs/api/Pool.md +5 -6
- package/node_modules/undici/lib/api/readable.js +1 -2
- package/node_modules/undici/lib/core/request.js +23 -2
- package/node_modules/undici/lib/core/util.js +0 -39
- package/node_modules/undici/lib/dispatcher/client-h1.js +11 -4
- package/node_modules/undici/lib/dispatcher/client-h2.js +52 -8
- package/node_modules/undici/lib/dispatcher/client.js +1 -1
- package/node_modules/undici/lib/dispatcher/dispatcher1-wrapper.js +3 -2
- package/node_modules/undici/lib/dispatcher/env-http-proxy-agent.js +21 -6
- package/node_modules/undici/lib/dispatcher/pool-base.js +63 -2
- package/node_modules/undici/lib/dispatcher/pool.js +90 -7
- package/node_modules/undici/lib/dispatcher/proxy-agent.js +16 -3
- package/node_modules/undici/lib/handler/deduplication-handler.js +24 -11
- package/node_modules/undici/lib/handler/redirect-handler.js +5 -0
- package/node_modules/undici/lib/handler/retry-handler.js +72 -13
- package/node_modules/undici/lib/interceptor/cache.js +4 -1
- package/node_modules/undici/lib/interceptor/decompress.js +236 -69
- package/node_modules/undici/lib/web/cache/cache.js +17 -8
- package/node_modules/undici/lib/web/cache/cachestorage.js +11 -5
- package/node_modules/undici/lib/web/cookies/index.js +19 -7
- package/node_modules/undici/lib/web/cookies/util.js +3 -1
- package/node_modules/undici/lib/web/eventsource/eventsource.js +25 -1
- package/node_modules/undici/lib/web/fetch/body.js +22 -15
- package/node_modules/undici/lib/web/fetch/data-url.js +6 -6
- package/node_modules/undici/lib/web/fetch/formdata.js +30 -36
- package/node_modules/undici/lib/web/fetch/headers.js +34 -31
- package/node_modules/undici/lib/web/fetch/index.js +38 -14
- package/node_modules/undici/lib/web/fetch/request.js +74 -79
- package/node_modules/undici/lib/web/fetch/response.js +43 -44
- package/node_modules/undici/lib/web/fetch/util.js +7 -7
- package/node_modules/undici/lib/web/infra/index.js +16 -1
- package/node_modules/undici/lib/web/webidl/index.js +2 -14
- package/node_modules/undici/lib/web/websocket/events.js +43 -26
- package/node_modules/undici/lib/web/websocket/receiver.js +5 -0
- package/node_modules/undici/lib/web/websocket/stream/websocketerror.js +13 -10
- package/node_modules/undici/lib/web/websocket/websocket.js +44 -41
- package/node_modules/undici/package.json +1 -1
- package/node_modules/undici/types/diagnostics-channel.d.ts +2 -1
- package/node_modules/undici/types/interceptors.d.ts +1 -1
- package/node_modules/undici/types/webidl.d.ts +13 -7
- package/package.json +4 -4
- package/dist/view/app-dist/assets/index-DFzw1L97.js +0 -63
- package/docs-site/zh/tutorials/experiments.mdx +0 -101
|
@@ -501,7 +501,8 @@ var AssertionCoverageSchema = Schema.Union([
|
|
|
501
501
|
"sampled",
|
|
502
502
|
"truncated",
|
|
503
503
|
"redacted",
|
|
504
|
-
"provider-limited"
|
|
504
|
+
"provider-limited",
|
|
505
|
+
"capacity-limited"
|
|
505
506
|
])
|
|
506
507
|
}),
|
|
507
508
|
Schema.Struct({
|
|
@@ -531,7 +532,13 @@ var AssertionLimitationSchema$1 = Schema.Union([
|
|
|
531
532
|
kind: Schema.Literal("truncated"),
|
|
532
533
|
omittedBytes: NonNegativeIntegerSchema$1
|
|
533
534
|
}),
|
|
534
|
-
Schema.Struct({ kind: Schema.Literal("provider-limited") })
|
|
535
|
+
Schema.Struct({ kind: Schema.Literal("provider-limited") }),
|
|
536
|
+
Schema.Struct({
|
|
537
|
+
kind: Schema.Literal("capacity-limited"),
|
|
538
|
+
capturedItems: NonNegativeIntegerSchema$1,
|
|
539
|
+
knownTotalItems: NonNegativeIntegerSchema$1,
|
|
540
|
+
omittedBytes: Schema.NullOr(NonNegativeIntegerSchema$1)
|
|
541
|
+
})
|
|
535
542
|
]);
|
|
536
543
|
var NoScoreContributionSchema = Schema.Struct({ state: Schema.Literal("not-scored") });
|
|
537
544
|
var EarnedScoreContributionSchema = Schema.Struct({
|
|
@@ -1282,7 +1289,7 @@ Schema.String.check(Schema.makeFilter(isRecordFormatId, {
|
|
|
1282
1289
|
var RecordFormatSchema = Schema.Literals([RECORD_FORMAT]).pipe(Schema.brand(RECORD_FORMAT_ID_BRAND));
|
|
1283
1290
|
//#endregion
|
|
1284
1291
|
//#region src/record/definition/canonical.ts
|
|
1285
|
-
var encoder$
|
|
1292
|
+
var encoder$3 = new TextEncoder();
|
|
1286
1293
|
/**
|
|
1287
1294
|
* Blob refs are intentionally opaque to the generic canonicalizer. Their
|
|
1288
1295
|
* bounded, deterministic projection makes them count toward the generic JSON
|
|
@@ -1298,8 +1305,8 @@ function failure$1(state, code, path) {
|
|
|
1298
1305
|
}
|
|
1299
1306
|
function compareUtf8(left, right) {
|
|
1300
1307
|
if (left === right) return 0;
|
|
1301
|
-
const leftBytes = encoder$
|
|
1302
|
-
const rightBytes = encoder$
|
|
1308
|
+
const leftBytes = encoder$3.encode(left);
|
|
1309
|
+
const rightBytes = encoder$3.encode(right);
|
|
1303
1310
|
const length = Math.min(leftBytes.byteLength, rightBytes.byteLength);
|
|
1304
1311
|
for (let index = 0; index < length; index += 1) {
|
|
1305
1312
|
const difference = leftBytes[index] - rightBytes[index];
|
|
@@ -1348,7 +1355,7 @@ function enter(value, depth, path, limits, state) {
|
|
|
1348
1355
|
failure$1(state, "record-json-invalid", path);
|
|
1349
1356
|
return false;
|
|
1350
1357
|
}
|
|
1351
|
-
if (encoder$
|
|
1358
|
+
if (encoder$3.encode(value).byteLength > limits.maximumStringUtf8Bytes) {
|
|
1352
1359
|
failure$1(state, "record-json-limit-exceeded", path);
|
|
1353
1360
|
return false;
|
|
1354
1361
|
}
|
|
@@ -1362,7 +1369,7 @@ function sortedObjectKeys(value, path, limits, state) {
|
|
|
1362
1369
|
const textKeys = keys;
|
|
1363
1370
|
for (const key of textKeys) {
|
|
1364
1371
|
if (!isUnicodeScalarString(key)) return failure$1(state, "record-json-invalid", [...path, key]);
|
|
1365
|
-
if (key.length === 0 || encoder$
|
|
1372
|
+
if (key.length === 0 || encoder$3.encode(key).byteLength > limits.maximumKeyUtf8Bytes) return failure$1(state, "record-json-limit-exceeded", [...path, key]);
|
|
1366
1373
|
}
|
|
1367
1374
|
return Object.freeze([...textKeys].sort(compareUtf8));
|
|
1368
1375
|
}
|
|
@@ -1450,7 +1457,7 @@ function projectBlobRefsForBudget(value, options) {
|
|
|
1450
1457
|
}
|
|
1451
1458
|
function jsonBytes(value) {
|
|
1452
1459
|
const text = JSON.stringify(value);
|
|
1453
|
-
return text === void 0 ? void 0 : encoder$
|
|
1460
|
+
return text === void 0 ? void 0 : encoder$3.encode(text).byteLength;
|
|
1454
1461
|
}
|
|
1455
1462
|
/**
|
|
1456
1463
|
* The only generic JSON normalizer. It intentionally knows no durable field
|
|
@@ -1882,6 +1889,16 @@ var persistences = /* @__PURE__ */ new WeakSet();
|
|
|
1882
1889
|
var references = /* @__PURE__ */ new WeakMap();
|
|
1883
1890
|
var referenceDeclarations = /* @__PURE__ */ new WeakMap();
|
|
1884
1891
|
var referenceDeclarationRuns = /* @__PURE__ */ new WeakMap();
|
|
1892
|
+
/** Shared admission limits for producers using the current attachment encoder. */
|
|
1893
|
+
var RecordAttachmentEncodingLimits = Object.freeze({
|
|
1894
|
+
maximumJsonBytes: 1048576,
|
|
1895
|
+
maximumDepth: 64,
|
|
1896
|
+
maximumNodes: 1e5,
|
|
1897
|
+
maximumObjectKeys: 1e4,
|
|
1898
|
+
maximumArrayItems: 1e5,
|
|
1899
|
+
maximumKeyUtf8Bytes: 16384,
|
|
1900
|
+
maximumStringUtf8Bytes: 1048576
|
|
1901
|
+
});
|
|
1885
1902
|
function mintedReferenceDeclarationRuntime(value) {
|
|
1886
1903
|
const direct = referenceDeclarations.get(value);
|
|
1887
1904
|
if (direct !== void 0 || !SchemaAST.isAST(value) || !SchemaAST.isDeclaration(value)) return direct;
|
|
@@ -1904,15 +1921,7 @@ function isRecordAttachmentSchema(schema) {
|
|
|
1904
1921
|
/** Defines only the current logical fact: no revision, migration, writer or storage selector. */
|
|
1905
1922
|
function defineRecordAttachment(input) {
|
|
1906
1923
|
if (!validOwner(input.owner) || !isRecordAttachmentName(input.family) || !isRecordAttachmentSchema(input.schema) || typeof input.validate !== "undefined" && typeof input.validate !== "function") invalid$1();
|
|
1907
|
-
const limits =
|
|
1908
|
-
maximumJsonBytes: 1048576,
|
|
1909
|
-
maximumDepth: 64,
|
|
1910
|
-
maximumNodes: 1e5,
|
|
1911
|
-
maximumObjectKeys: 1e4,
|
|
1912
|
-
maximumArrayItems: 1e5,
|
|
1913
|
-
maximumKeyUtf8Bytes: 16384,
|
|
1914
|
-
maximumStringUtf8Bytes: 1048576
|
|
1915
|
-
};
|
|
1924
|
+
const limits = RecordAttachmentEncodingLimits;
|
|
1916
1925
|
const definition = Object.freeze({
|
|
1917
1926
|
owner: input.owner,
|
|
1918
1927
|
family: input.family,
|
|
@@ -2202,6 +2211,24 @@ var RunGetDocumentSchema = Schema.Struct({
|
|
|
2202
2211
|
});
|
|
2203
2212
|
Schema.Union([RunListDocumentSchema, RunGetDocumentSchema]);
|
|
2204
2213
|
//#endregion
|
|
2214
|
+
//#region src/model-slots.ts
|
|
2215
|
+
var modelText = Schema.String.check(Schema.isPattern(/\S/u));
|
|
2216
|
+
var slotKey = Schema.String.check(Schema.isPattern(/^[A-Za-z][A-Za-z0-9_-]{0,63}$(?![\s\S])/u));
|
|
2217
|
+
Schema.Record(slotKey, Schema.Struct({
|
|
2218
|
+
model: modelText,
|
|
2219
|
+
reasoningEffort: Schema.optionalKey(modelText)
|
|
2220
|
+
})).check(Schema.makeFilter((value) => Object.keys(value).length > 0 && Object.keys(value).length <= 64));
|
|
2221
|
+
var ResolvedModelSlotsSchema = Schema.Record(slotKey, Schema.Struct({
|
|
2222
|
+
model: Schema.NullOr(modelText),
|
|
2223
|
+
reasoningEffort: Schema.NullOr(modelText)
|
|
2224
|
+
})).check(Schema.makeFilter((value) => Object.keys(value).length <= 64));
|
|
2225
|
+
//#endregion
|
|
2226
|
+
//#region src/shared/adapter-cleanup.ts
|
|
2227
|
+
var AdapterCleanupTimeoutMsSchema = Schema.Finite.check(Schema.isInt(), Schema.isBetween({
|
|
2228
|
+
minimum: 1,
|
|
2229
|
+
maximum: 3e5
|
|
2230
|
+
}));
|
|
2231
|
+
//#endregion
|
|
2205
2232
|
//#region src/record/model/run-context.ts
|
|
2206
2233
|
/** The complete budget for one immutable, self-explaining Run context. */
|
|
2207
2234
|
var RunContextLimits = Object.freeze({
|
|
@@ -2230,9 +2257,11 @@ var NonEmptyStringSchema = Schema.String.pipe(Schema.check(Schema.isMinLength(1)
|
|
|
2230
2257
|
var AdapterIdentitySchema = Schema.Struct({
|
|
2231
2258
|
name: NonEmptyStringSchema,
|
|
2232
2259
|
contract: NonEmptyStringSchema,
|
|
2233
|
-
behaviorRevision: Schema.NullOr(NonEmptyStringSchema)
|
|
2260
|
+
behaviorRevision: Schema.NullOr(NonEmptyStringSchema),
|
|
2261
|
+
cleanupTimeoutMs: Schema.optionalKey(AdapterCleanupTimeoutMsSchema)
|
|
2234
2262
|
});
|
|
2235
2263
|
var RunExecutionContextSchema = Schema.Struct({
|
|
2264
|
+
models: Schema.optionalKey(ResolvedModelSlotsSchema),
|
|
2236
2265
|
adapter: AdapterIdentitySchema,
|
|
2237
2266
|
model: Schema.NullOr(Schema.String),
|
|
2238
2267
|
reasoningEffort: Schema.NullOr(Schema.String),
|
|
@@ -2242,7 +2271,7 @@ var RunExecutionContextSchema = Schema.Struct({
|
|
|
2242
2271
|
setup: Schema.Literals(["absent", "opaque"]),
|
|
2243
2272
|
teardown: Schema.Literals(["absent", "opaque"])
|
|
2244
2273
|
}))
|
|
2245
|
-
});
|
|
2274
|
+
}).check(Schema.makeFilter((value) => value.models === void 0 || value.model === (value.models.default?.model ?? null) && value.reasoningEffort === (value.models.default?.reasoningEffort ?? null)));
|
|
2246
2275
|
var RunLabelsSchema = Schema.Record(Schema.String, Schema.String);
|
|
2247
2276
|
/** Run Context is Core, not an Attachment. */
|
|
2248
2277
|
var RunContextDefinition = defineRecordCore({
|
|
@@ -2650,7 +2679,7 @@ var PositiveSafeIntegerSchema$1 = Schema.Number.pipe(Schema.check(Schema.makeFil
|
|
|
2650
2679
|
identifier: "RecordPositiveSafeInteger",
|
|
2651
2680
|
description: "a positive JSON-safe integer"
|
|
2652
2681
|
})));
|
|
2653
|
-
Schema.Number.pipe(Schema.check(Schema.makeFilter((value) => Number.isFinite(value) && value >= 0, {
|
|
2682
|
+
var FiniteNonNegativeNumberSchema = Schema.Number.pipe(Schema.check(Schema.makeFilter((value) => Number.isFinite(value) && value >= 0, {
|
|
2654
2683
|
identifier: "RecordFiniteNonNegativeNumber",
|
|
2655
2684
|
description: "a finite non-negative JSON number"
|
|
2656
2685
|
})));
|
|
@@ -4185,40 +4214,97 @@ var AdapterCallPriceBuckets = Object.freeze([
|
|
|
4185
4214
|
"cache-read",
|
|
4186
4215
|
"cache-write"
|
|
4187
4216
|
]);
|
|
4217
|
+
function decimalParts(value) {
|
|
4218
|
+
const [integer, fraction = ""] = value.split(".");
|
|
4219
|
+
return Object.freeze({
|
|
4220
|
+
digits: BigInt(`${integer}${fraction}`),
|
|
4221
|
+
scale: fraction.length
|
|
4222
|
+
});
|
|
4223
|
+
}
|
|
4224
|
+
function canonicalDecimal(parts) {
|
|
4225
|
+
if (parts.digits === 0n) return "0";
|
|
4226
|
+
if (parts.scale === 0) return parts.digits.toString();
|
|
4227
|
+
const padded = parts.digits.toString().padStart(parts.scale + 1, "0");
|
|
4228
|
+
const integer = padded.slice(0, -parts.scale);
|
|
4229
|
+
const fraction = padded.slice(-parts.scale).replace(/0+$/u, "");
|
|
4230
|
+
return fraction.length === 0 ? integer : `${integer}.${fraction}`;
|
|
4231
|
+
}
|
|
4232
|
+
/** Converts one finite non-negative JS config number to its exact canonical decimal spelling. */
|
|
4233
|
+
function canonicalDecimalFromNumber(value) {
|
|
4234
|
+
if (!Number.isFinite(value) || value < 0) return void 0;
|
|
4235
|
+
const text = value.toString().toLowerCase();
|
|
4236
|
+
if (!text.includes("e")) return text.includes(".") ? text.replace(/\.?0+$/u, "") : text;
|
|
4237
|
+
const [coefficient, exponentText] = text.split("e");
|
|
4238
|
+
const exponent = Number(exponentText);
|
|
4239
|
+
if (coefficient === void 0 || !Number.isSafeInteger(exponent)) return void 0;
|
|
4240
|
+
const [integer, fraction = ""] = coefficient.split(".");
|
|
4241
|
+
const digits = `${integer}${fraction}`;
|
|
4242
|
+
const point = integer.length + exponent;
|
|
4243
|
+
if (point <= 0) return canonicalDecimal({
|
|
4244
|
+
digits: BigInt(digits),
|
|
4245
|
+
scale: digits.length - point
|
|
4246
|
+
});
|
|
4247
|
+
if (point >= digits.length) return `${digits}${"0".repeat(point - digits.length)}`;
|
|
4248
|
+
return canonicalDecimal({
|
|
4249
|
+
digits: BigInt(digits),
|
|
4250
|
+
scale: digits.length - point
|
|
4251
|
+
});
|
|
4252
|
+
}
|
|
4253
|
+
function multiplyCanonicalDecimalPerMillion(rate, tokens) {
|
|
4254
|
+
const value = decimalParts(rate);
|
|
4255
|
+
return canonicalDecimal({
|
|
4256
|
+
digits: value.digits * BigInt(tokens),
|
|
4257
|
+
scale: value.scale + 6
|
|
4258
|
+
});
|
|
4259
|
+
}
|
|
4260
|
+
function addCanonicalDecimals(values) {
|
|
4261
|
+
if (values.length === 0) return "0";
|
|
4262
|
+
const parts = values.map(decimalParts);
|
|
4263
|
+
const scale = Math.max(...parts.map((value) => value.scale));
|
|
4264
|
+
return canonicalDecimal({
|
|
4265
|
+
digits: parts.reduce((sum, value) => sum + value.digits * 10n ** BigInt(scale - value.scale), 0n),
|
|
4266
|
+
scale
|
|
4267
|
+
});
|
|
4268
|
+
}
|
|
4269
|
+
/** Stable digest of the effective profile only; it is intentionally not the catalog snapshot digest. */
|
|
4270
|
+
function adapterCallPricingProfileDigest(value) {
|
|
4271
|
+
const bytes = new TextEncoder().encode(JSON.stringify(value));
|
|
4272
|
+
return `sha256:${new RecordSha256().update(bytes).digestHex()}`;
|
|
4273
|
+
}
|
|
4188
4274
|
//#endregion
|
|
4189
4275
|
//#region src/record/family/adapter-usage/schema.ts
|
|
4190
|
-
var Identity = Schema.String.pipe(Schema.check(Schema.makeFilter((value) => value.trim().length > 0 && new TextEncoder().encode(value).byteLength <= 256 && !/[\u0000-\u001f\u007f]/u.test(value))));
|
|
4191
|
-
var Tokens = Schema.NullOr(NonNegativeSafeIntegerSchema$2);
|
|
4276
|
+
var Identity$1 = Schema.String.pipe(Schema.check(Schema.makeFilter((value) => value.trim().length > 0 && new TextEncoder().encode(value).byteLength <= 256 && !/[\u0000-\u001f\u007f]/u.test(value))));
|
|
4277
|
+
var Tokens$1 = Schema.NullOr(NonNegativeSafeIntegerSchema$2);
|
|
4192
4278
|
var AdapterUsageCallRevision1Schema = Schema.Struct({
|
|
4193
|
-
callId: Identity,
|
|
4194
|
-
retryOf: Schema.NullOr(Identity),
|
|
4195
|
-
provider: Schema.NullOr(Identity),
|
|
4196
|
-
model: Schema.NullOr(Identity),
|
|
4279
|
+
callId: Identity$1,
|
|
4280
|
+
retryOf: Schema.NullOr(Identity$1),
|
|
4281
|
+
provider: Schema.NullOr(Identity$1),
|
|
4282
|
+
model: Schema.NullOr(Identity$1),
|
|
4197
4283
|
status: Schema.Literals([
|
|
4198
4284
|
"succeeded",
|
|
4199
4285
|
"failed",
|
|
4200
4286
|
"cancelled",
|
|
4201
4287
|
"unknown"
|
|
4202
4288
|
]),
|
|
4203
|
-
inputTokens: Tokens,
|
|
4204
|
-
inputTotalTokens: Tokens,
|
|
4205
|
-
outputTokens: Tokens,
|
|
4206
|
-
cacheReadTokens: Tokens,
|
|
4207
|
-
cacheWriteTokens: Tokens
|
|
4289
|
+
inputTokens: Tokens$1,
|
|
4290
|
+
inputTotalTokens: Tokens$1,
|
|
4291
|
+
outputTokens: Tokens$1,
|
|
4292
|
+
cacheReadTokens: Tokens$1,
|
|
4293
|
+
cacheWriteTokens: Tokens$1
|
|
4208
4294
|
});
|
|
4209
4295
|
var AdapterUsageRouteSchema = Schema.Struct({
|
|
4210
|
-
transportProvider: Schema.NullOr(Identity),
|
|
4211
|
-
endpointId: Schema.NullOr(Identity)
|
|
4296
|
+
transportProvider: Schema.NullOr(Identity$1),
|
|
4297
|
+
endpointId: Schema.NullOr(Identity$1)
|
|
4212
4298
|
});
|
|
4213
4299
|
var AdapterUsageReportedCostSchema = Schema.Struct({
|
|
4214
4300
|
amount: CanonicalDecimalSchema,
|
|
4215
4301
|
currency: CurrencyCodeSchema,
|
|
4216
4302
|
source: Schema.Struct({
|
|
4217
4303
|
kind: Schema.Literal("reported"),
|
|
4218
|
-
id: Identity
|
|
4304
|
+
id: Identity$1
|
|
4219
4305
|
})
|
|
4220
4306
|
});
|
|
4221
|
-
var Digest = Schema.String.pipe(Schema.check(Schema.makeFilter((value) => /^sha256:[0-9a-f]{64}$/u.test(value))));
|
|
4307
|
+
var Digest$1 = Schema.String.pipe(Schema.check(Schema.makeFilter((value) => /^sha256:[0-9a-f]{64}$/u.test(value))));
|
|
4222
4308
|
var NullableRateSchema = Schema.NullOr(CanonicalDecimalSchema);
|
|
4223
4309
|
var AdapterCallPriceRatesSchema = Schema.Struct({
|
|
4224
4310
|
input: NullableRateSchema,
|
|
@@ -4228,16 +4314,16 @@ var AdapterCallPriceRatesSchema = Schema.Struct({
|
|
|
4228
4314
|
});
|
|
4229
4315
|
var AdapterCallPriceSourceSchema = Schema.Struct({
|
|
4230
4316
|
kind: Schema.Literal("configured-profile"),
|
|
4231
|
-
id: Identity,
|
|
4317
|
+
id: Identity$1,
|
|
4232
4318
|
asOf: Schema.NullOr(PositiveSafeIntegerSchema$1),
|
|
4233
|
-
profileDigest: Digest
|
|
4319
|
+
profileDigest: Digest$1
|
|
4234
4320
|
});
|
|
4235
4321
|
var AdapterCallPricingSchema = Schema.Struct({
|
|
4236
4322
|
basis: Schema.Literal("catalog-reference"),
|
|
4237
4323
|
currency: Schema.Literal("USD"),
|
|
4238
4324
|
source: AdapterCallPriceSourceSchema,
|
|
4239
|
-
requestModel: Identity,
|
|
4240
|
-
selector: Identity,
|
|
4325
|
+
requestModel: Identity$1,
|
|
4326
|
+
selector: Identity$1,
|
|
4241
4327
|
match: Schema.Literals(["exact", "provider-wildcard"]),
|
|
4242
4328
|
ratesPerMTok: AdapterCallPriceRatesSchema
|
|
4243
4329
|
});
|
|
@@ -4253,28 +4339,38 @@ var AdapterCallPriceMissingSchema = Schema.Struct({
|
|
|
4253
4339
|
});
|
|
4254
4340
|
var AdapterCallPriceReceiptSchema = Schema.Struct({
|
|
4255
4341
|
kind: Schema.Literal("adapter-call-price-estimate"),
|
|
4256
|
-
callId: Identity,
|
|
4342
|
+
callId: Identity$1,
|
|
4257
4343
|
state: Schema.Literals(["complete", "partial"]),
|
|
4258
4344
|
amount: CanonicalDecimalSchema,
|
|
4259
4345
|
currency: Schema.Literal("USD"),
|
|
4260
4346
|
source: Schema.Struct({
|
|
4261
4347
|
kind: Schema.Literal("estimated"),
|
|
4262
|
-
id: Identity
|
|
4348
|
+
id: Identity$1
|
|
4263
4349
|
}),
|
|
4264
4350
|
pricing: AdapterCallPricingSchema,
|
|
4265
4351
|
charges: Schema.NonEmptyArray(AdapterCallPriceChargeSchema),
|
|
4266
4352
|
missing: Schema.Array(AdapterCallPriceMissingSchema)
|
|
4267
4353
|
});
|
|
4268
|
-
var
|
|
4354
|
+
var AdapterUsageModelSlotSchema = Schema.String.check(Schema.isPattern(/^[A-Za-z][A-Za-z0-9_-]{0,63}$(?![\s\S])/u));
|
|
4355
|
+
var AdapterUsageCallRevision2Schema = Schema.Struct({
|
|
4269
4356
|
...AdapterUsageCallRevision1Schema.fields,
|
|
4270
4357
|
route: AdapterUsageRouteSchema,
|
|
4271
4358
|
cost: Schema.NullOr(AdapterUsageReportedCostSchema)
|
|
4272
4359
|
});
|
|
4360
|
+
var AdapterUsageCallSchema = Schema.Struct({
|
|
4361
|
+
...AdapterUsageCallRevision2Schema.fields,
|
|
4362
|
+
modelSlot: Schema.NullOr(AdapterUsageModelSlotSchema)
|
|
4363
|
+
});
|
|
4273
4364
|
Object.freeze({ maximumCalls: 4e3 });
|
|
4274
4365
|
Schema.Struct({
|
|
4275
4366
|
collection: CollectionStateSchema,
|
|
4276
4367
|
calls: Schema.Array(AdapterUsageCallRevision1Schema)
|
|
4277
4368
|
});
|
|
4369
|
+
Schema.Struct({
|
|
4370
|
+
collection: CollectionStateSchema,
|
|
4371
|
+
calls: Schema.Array(AdapterUsageCallRevision2Schema),
|
|
4372
|
+
priceReceipts: Schema.Array(AdapterCallPriceReceiptSchema)
|
|
4373
|
+
});
|
|
4278
4374
|
Schema.Struct({
|
|
4279
4375
|
collection: CollectionStateSchema,
|
|
4280
4376
|
calls: Schema.Array(AdapterUsageCallSchema),
|
|
@@ -4282,6 +4378,310 @@ Schema.Struct({
|
|
|
4282
4378
|
priceReceipts: Schema.Array(AdapterCallPriceReceiptSchema)
|
|
4283
4379
|
});
|
|
4284
4380
|
//#endregion
|
|
4381
|
+
//#region src/record/family/judge-usage/schema.ts
|
|
4382
|
+
var JudgeUsageLimits = Object.freeze({
|
|
4383
|
+
maximumCalls: 4e3,
|
|
4384
|
+
maximumModelBytes: 8192,
|
|
4385
|
+
maximumReceiptBytes: 32768,
|
|
4386
|
+
maximumReceiptFields: 24
|
|
4387
|
+
});
|
|
4388
|
+
var encoder$2 = new TextEncoder();
|
|
4389
|
+
function boundedIdentity(maximumBytes) {
|
|
4390
|
+
return Schema.String.check(Schema.makeFilter((value) => value.trim().length > 0 && !/[\ud800-\udfff]/u.test(value) && encoder$2.encode(value).byteLength <= maximumBytes && !/[\u0000-\u001f\u007f]/u.test(value)));
|
|
4391
|
+
}
|
|
4392
|
+
var Identity = boundedIdentity(256);
|
|
4393
|
+
/** @internal Used to validate observations before retaining their response fields. */
|
|
4394
|
+
var JudgeUsageModelSchema = boundedIdentity(JudgeUsageLimits.maximumModelBytes);
|
|
4395
|
+
var Digest = Schema.String.check(Schema.isPattern(/^sha256:[0-9a-f]{64}$/u));
|
|
4396
|
+
var Tokens = Schema.NullOr(NonNegativeSafeIntegerSchema$2);
|
|
4397
|
+
var NullableRate = Schema.NullOr(CanonicalDecimalSchema);
|
|
4398
|
+
/** Only these protocol facts may enter durable receipts; business text is excluded. */
|
|
4399
|
+
var JudgeUsageReceiptPaths = Object.freeze([
|
|
4400
|
+
"model",
|
|
4401
|
+
"provider",
|
|
4402
|
+
"usage.completion_tokens",
|
|
4403
|
+
"usage.cost",
|
|
4404
|
+
"usage.input_tokens",
|
|
4405
|
+
"usage.output_tokens",
|
|
4406
|
+
"usage.prompt_tokens",
|
|
4407
|
+
"usage.prompt_tokens_details.cache_write_tokens",
|
|
4408
|
+
"usage.prompt_tokens_details.cached_tokens",
|
|
4409
|
+
"usage.total_tokens"
|
|
4410
|
+
]);
|
|
4411
|
+
var ReceiptFieldSchema = Schema.Struct({
|
|
4412
|
+
path: Schema.Literals(JudgeUsageReceiptPaths),
|
|
4413
|
+
value: Schema.Union([
|
|
4414
|
+
Schema.String.check(Schema.makeFilter((value) => !/[\ud800-\udfff]/u.test(value) && encoder$2.encode(value).byteLength <= JudgeUsageLimits.maximumModelBytes && !/[\u0000-\u001f\u007f]/u.test(value))),
|
|
4415
|
+
FiniteNonNegativeNumberSchema.check(Schema.makeFilter((value) => !Object.is(value, -0))),
|
|
4416
|
+
Schema.Null
|
|
4417
|
+
])
|
|
4418
|
+
});
|
|
4419
|
+
var ReceiptSchema = Schema.Struct({
|
|
4420
|
+
state: Schema.Literals(["available", "unavailable"]),
|
|
4421
|
+
reason: Schema.NullOr(Schema.Literals([
|
|
4422
|
+
"response-unavailable",
|
|
4423
|
+
"cancelled",
|
|
4424
|
+
"response-too-large",
|
|
4425
|
+
"invalid-json",
|
|
4426
|
+
"usage-not-reported",
|
|
4427
|
+
"invalid-usage",
|
|
4428
|
+
"response-model-too-large"
|
|
4429
|
+
])),
|
|
4430
|
+
responseDigest: Schema.NullOr(Digest),
|
|
4431
|
+
fields: Schema.Array(ReceiptFieldSchema)
|
|
4432
|
+
}).check(Schema.makeFilter((receipt) => {
|
|
4433
|
+
if (receipt.fields.length > JudgeUsageLimits.maximumReceiptFields || encoder$2.encode(JSON.stringify(receipt)).byteLength > JudgeUsageLimits.maximumReceiptBytes || receipt.fields.some((field, index) => index > 0 && receipt.fields[index - 1].path >= field.path)) return false;
|
|
4434
|
+
if (receipt.state === "available") return receipt.fields.some((field) => field.value !== null) && receipt.responseDigest !== null && (receipt.reason === null || receipt.reason === "invalid-usage" || receipt.reason === "usage-not-reported" || receipt.reason === "response-model-too-large");
|
|
4435
|
+
return receipt.fields.length === 0 && receipt.reason !== null && (receipt.reason === "invalid-json" || receipt.reason === "invalid-usage" || receipt.reason === "usage-not-reported" || receipt.reason === "response-model-too-large" ? receipt.responseDigest !== null : receipt.responseDigest === null);
|
|
4436
|
+
}));
|
|
4437
|
+
var JudgeUsageCallSchema = Schema.Struct({
|
|
4438
|
+
callId: Identity,
|
|
4439
|
+
entryIndex: NonNegativeSafeIntegerSchema$2,
|
|
4440
|
+
entryId: AssertionEntryIdSchema,
|
|
4441
|
+
logicalOrdinal: PositiveSafeIntegerSchema$1,
|
|
4442
|
+
transmissionOrdinal: PositiveSafeIntegerSchema$1,
|
|
4443
|
+
operation: Schema.Literals([
|
|
4444
|
+
"score",
|
|
4445
|
+
"classify",
|
|
4446
|
+
"extract",
|
|
4447
|
+
"batchClassify"
|
|
4448
|
+
]),
|
|
4449
|
+
requestModel: JudgeUsageModelSchema,
|
|
4450
|
+
transportProvider: Schema.Literals([
|
|
4451
|
+
"openai",
|
|
4452
|
+
"vercel",
|
|
4453
|
+
"openrouter",
|
|
4454
|
+
"typesafe"
|
|
4455
|
+
]),
|
|
4456
|
+
provider: Schema.NullOr(Identity),
|
|
4457
|
+
model: Schema.NullOr(JudgeUsageModelSchema),
|
|
4458
|
+
status: Schema.Literals([
|
|
4459
|
+
"succeeded",
|
|
4460
|
+
"failed",
|
|
4461
|
+
"cancelled",
|
|
4462
|
+
"unknown"
|
|
4463
|
+
]),
|
|
4464
|
+
httpStatus: Schema.NullOr(NonNegativeSafeIntegerSchema$2.check(Schema.makeFilter((value) => value >= 100 && value <= 599))),
|
|
4465
|
+
inputTokens: Tokens,
|
|
4466
|
+
inputTotalTokens: Tokens,
|
|
4467
|
+
outputTokens: Tokens,
|
|
4468
|
+
cacheReadTokens: Tokens,
|
|
4469
|
+
cacheWriteTokens: Tokens,
|
|
4470
|
+
cost: Schema.NullOr(Schema.Struct({
|
|
4471
|
+
amount: CanonicalDecimalSchema,
|
|
4472
|
+
currency: CurrencyCodeSchema,
|
|
4473
|
+
source: Schema.Struct({
|
|
4474
|
+
kind: Schema.Literal("reported"),
|
|
4475
|
+
id: Identity
|
|
4476
|
+
})
|
|
4477
|
+
})),
|
|
4478
|
+
receipt: ReceiptSchema
|
|
4479
|
+
}).check(Schema.makeFilter((call) => validCall$1(call)));
|
|
4480
|
+
var JudgePriceReceiptSchema = Schema.Struct({
|
|
4481
|
+
kind: Schema.Literal("judge-call-price-estimate"),
|
|
4482
|
+
callId: Identity,
|
|
4483
|
+
state: Schema.Literals(["complete", "partial"]),
|
|
4484
|
+
amount: CanonicalDecimalSchema,
|
|
4485
|
+
currency: Schema.Literal("USD"),
|
|
4486
|
+
source: Schema.Struct({
|
|
4487
|
+
kind: Schema.Literal("estimated"),
|
|
4488
|
+
id: Identity
|
|
4489
|
+
}),
|
|
4490
|
+
pricing: Schema.Struct({
|
|
4491
|
+
basis: Schema.Literal("catalog-reference"),
|
|
4492
|
+
currency: Schema.Literal("USD"),
|
|
4493
|
+
source: Schema.Struct({
|
|
4494
|
+
kind: Schema.Literal("configured-profile"),
|
|
4495
|
+
id: Identity,
|
|
4496
|
+
asOf: Schema.NullOr(PositiveSafeIntegerSchema$1),
|
|
4497
|
+
profileDigest: Digest
|
|
4498
|
+
}),
|
|
4499
|
+
requestModel: JudgeUsageModelSchema,
|
|
4500
|
+
selector: JudgeUsageModelSchema,
|
|
4501
|
+
match: Schema.Literals(["exact", "provider-wildcard"]),
|
|
4502
|
+
ratesPerMTok: Schema.Struct({
|
|
4503
|
+
input: NullableRate,
|
|
4504
|
+
output: NullableRate,
|
|
4505
|
+
cacheRead: NullableRate,
|
|
4506
|
+
cacheWrite: NullableRate
|
|
4507
|
+
})
|
|
4508
|
+
}),
|
|
4509
|
+
charges: Schema.NonEmptyArray(Schema.Struct({
|
|
4510
|
+
bucket: Schema.Literals(AdapterCallPriceBuckets),
|
|
4511
|
+
tokens: NonNegativeSafeIntegerSchema$2,
|
|
4512
|
+
ratePerMTok: NullableRate,
|
|
4513
|
+
amount: CanonicalDecimalSchema
|
|
4514
|
+
})),
|
|
4515
|
+
missing: Schema.Array(Schema.Struct({
|
|
4516
|
+
bucket: Schema.Literals(AdapterCallPriceBuckets),
|
|
4517
|
+
reason: Schema.Literals(["tokens-unknown", "rate-unknown"])
|
|
4518
|
+
}))
|
|
4519
|
+
}).check(Schema.makeFilter((receipt) => validPriceReceipt(receipt)));
|
|
4520
|
+
Schema.Struct({
|
|
4521
|
+
collection: CollectionStateSchema,
|
|
4522
|
+
calls: Schema.Array(JudgeUsageCallSchema),
|
|
4523
|
+
priceReceipts: Schema.Array(JudgePriceReceiptSchema)
|
|
4524
|
+
}).check(Schema.makeFilter((value) => validateJudgeUsageAttachment(value).length === 0));
|
|
4525
|
+
function validCall$1(call) {
|
|
4526
|
+
if (call.callId !== `judge:${call.entryIndex}:${call.logicalOrdinal}:${call.transmissionOrdinal}`) return false;
|
|
4527
|
+
const buckets = [
|
|
4528
|
+
call.inputTokens,
|
|
4529
|
+
call.cacheReadTokens,
|
|
4530
|
+
call.cacheWriteTokens
|
|
4531
|
+
];
|
|
4532
|
+
const known = buckets.filter((value) => value !== null);
|
|
4533
|
+
const sum = known.reduce((total, value) => total + value, 0);
|
|
4534
|
+
if (!Number.isSafeInteger(sum) || call.inputTotalTokens !== null && (sum > call.inputTotalTokens || known.length === 3 && sum !== call.inputTotalTokens)) return false;
|
|
4535
|
+
if (call.inputTotalTokens !== null && call.outputTokens !== null && !Number.isSafeInteger(call.inputTotalTokens + call.outputTokens)) return false;
|
|
4536
|
+
const fields = new Map(call.receipt.fields.map((field) => [field.path, field.value]));
|
|
4537
|
+
for (const field of call.receipt.fields) {
|
|
4538
|
+
if (field.value === null) continue;
|
|
4539
|
+
if (field.path === "model") {
|
|
4540
|
+
if (typeof field.value !== "string" || !Schema.is(JudgeUsageModelSchema)(field.value)) return false;
|
|
4541
|
+
} else if (field.path === "provider") {
|
|
4542
|
+
if (call.transportProvider !== "openrouter" || typeof field.value !== "string" || !Schema.is(Identity)(field.value)) return false;
|
|
4543
|
+
} else if (field.path === "usage.cost") {
|
|
4544
|
+
const amount = typeof field.value === "number" ? canonicalDecimalFromNumber(field.value) : field.value;
|
|
4545
|
+
if (call.transportProvider !== "openrouter" || amount === void 0 || !Schema.is(CanonicalDecimalSchema)(amount)) return false;
|
|
4546
|
+
} else if (typeof field.value !== "number" || !Schema.is(NonNegativeSafeIntegerSchema$2)(field.value) || (call.transportProvider === "typesafe" ? ![
|
|
4547
|
+
"usage.input_tokens",
|
|
4548
|
+
"usage.output_tokens",
|
|
4549
|
+
"usage.total_tokens"
|
|
4550
|
+
].includes(field.path) : ["usage.input_tokens", "usage.output_tokens"].includes(field.path)) || field.path === "usage.prompt_tokens_details.cache_write_tokens" && call.transportProvider !== "openrouter") return false;
|
|
4551
|
+
}
|
|
4552
|
+
if ((fields.get("model") ?? null) !== call.model || (fields.get("provider") ?? null) !== call.provider) return false;
|
|
4553
|
+
const inputPath = call.transportProvider === "typesafe" ? "usage.input_tokens" : "usage.prompt_tokens";
|
|
4554
|
+
const outputPath = call.transportProvider === "typesafe" ? "usage.output_tokens" : "usage.completion_tokens";
|
|
4555
|
+
if (call.inputTotalTokens !== null && fields.get(inputPath) !== call.inputTotalTokens || call.outputTokens !== null && fields.get(outputPath) !== call.outputTokens) return false;
|
|
4556
|
+
if (call.receipt.reason !== "invalid-usage" && (typeof fields.get(inputPath) === "number" && call.inputTotalTokens === null || typeof fields.get(outputPath) === "number" && call.outputTokens === null || typeof fields.get("usage.prompt_tokens_details.cached_tokens") === "number" && call.cacheReadTokens === null || typeof fields.get("usage.prompt_tokens_details.cache_write_tokens") === "number" && call.cacheWriteTokens === null)) return false;
|
|
4557
|
+
const total = fields.get("usage.total_tokens");
|
|
4558
|
+
if (typeof total === "number" && (call.inputTotalTokens !== null && call.inputTotalTokens > total || call.outputTokens !== null && call.outputTokens > total || call.inputTotalTokens !== null && call.outputTokens !== null && call.inputTotalTokens + call.outputTokens !== total)) return false;
|
|
4559
|
+
if (call.transportProvider === "typesafe" && buckets.some((value) => value !== null)) return false;
|
|
4560
|
+
if (call.transportProvider !== "typesafe" && call.inputTotalTokens === 0 && buckets.some((value) => value !== 0)) return false;
|
|
4561
|
+
if (call.cacheReadTokens !== null && fields.get("usage.prompt_tokens_details.cached_tokens") !== call.cacheReadTokens && !(call.inputTotalTokens === 0 && call.cacheReadTokens === 0 && !fields.has("usage.prompt_tokens_details.cached_tokens"))) return false;
|
|
4562
|
+
if (call.cacheWriteTokens !== null && fields.get("usage.prompt_tokens_details.cache_write_tokens") !== call.cacheWriteTokens && !(call.inputTotalTokens === 0 && call.cacheWriteTokens === 0 && !fields.has("usage.prompt_tokens_details.cache_write_tokens"))) return false;
|
|
4563
|
+
if (call.inputTokens !== null && (call.inputTotalTokens === null || call.cacheReadTokens === null || call.cacheWriteTokens === null || call.inputTokens !== call.inputTotalTokens - call.cacheReadTokens - call.cacheWriteTokens)) return false;
|
|
4564
|
+
if (call.inputTotalTokens !== null && call.cacheReadTokens !== null && call.cacheWriteTokens !== null && call.inputTokens === null) return false;
|
|
4565
|
+
const rawCost = fields.get("usage.cost");
|
|
4566
|
+
if (rawCost !== void 0 && rawCost !== null && call.cost === null) return false;
|
|
4567
|
+
if (call.cost !== null) {
|
|
4568
|
+
const raw = rawCost;
|
|
4569
|
+
const amount = typeof raw === "number" ? canonicalDecimalFromNumber(raw) : raw;
|
|
4570
|
+
if (call.transportProvider !== "openrouter" || call.cost.currency !== "USD" || call.cost.source.id !== "openrouter.usage.cost" || call.cost.amount !== amount) return false;
|
|
4571
|
+
}
|
|
4572
|
+
return true;
|
|
4573
|
+
}
|
|
4574
|
+
function validPriceReceipt(receipt) {
|
|
4575
|
+
const profile = receipt.pricing;
|
|
4576
|
+
const expectedSelector = profile.match === "exact" ? profile.requestModel : profile.requestModel.includes("/") ? `${profile.requestModel.slice(0, profile.requestModel.indexOf("/"))}/*` : void 0;
|
|
4577
|
+
if (profile.selector !== expectedSelector || receipt.source.id !== profile.source.id || profile.source.profileDigest !== adapterCallPricingProfileDigest({
|
|
4578
|
+
basis: profile.basis,
|
|
4579
|
+
currency: profile.currency,
|
|
4580
|
+
source: {
|
|
4581
|
+
kind: profile.source.kind,
|
|
4582
|
+
id: profile.source.id,
|
|
4583
|
+
asOf: profile.source.asOf
|
|
4584
|
+
},
|
|
4585
|
+
requestModel: profile.requestModel,
|
|
4586
|
+
selector: profile.selector,
|
|
4587
|
+
match: profile.match,
|
|
4588
|
+
ratesPerMTok: {
|
|
4589
|
+
input: profile.ratesPerMTok.input,
|
|
4590
|
+
output: profile.ratesPerMTok.output,
|
|
4591
|
+
cacheRead: profile.ratesPerMTok.cacheRead,
|
|
4592
|
+
cacheWrite: profile.ratesPerMTok.cacheWrite
|
|
4593
|
+
}
|
|
4594
|
+
})) return false;
|
|
4595
|
+
const charges = new Map(receipt.charges.map((charge) => [charge.bucket, charge]));
|
|
4596
|
+
const missing = new Map(receipt.missing.map((entry) => [entry.bucket, entry]));
|
|
4597
|
+
if (charges.size !== receipt.charges.length || missing.size !== receipt.missing.length || AdapterCallPriceBuckets.some((bucket) => Number(charges.has(bucket)) + Number(missing.has(bucket)) !== 1)) return false;
|
|
4598
|
+
for (const charge of receipt.charges) {
|
|
4599
|
+
const rate = bucketRate(profile.ratesPerMTok, charge.bucket);
|
|
4600
|
+
if (charge.ratePerMTok !== rate || charge.tokens > 0 && rate === null || charge.amount !== (rate === null ? "0" : multiplyCanonicalDecimalPerMillion(rate, charge.tokens))) return false;
|
|
4601
|
+
}
|
|
4602
|
+
return receipt.state === (missing.size === 0 ? "complete" : "partial") && receipt.amount === addCanonicalDecimals(receipt.charges.map((charge) => charge.amount));
|
|
4603
|
+
}
|
|
4604
|
+
function bucketRate(rates, bucket) {
|
|
4605
|
+
switch (bucket) {
|
|
4606
|
+
case "input": return rates.input;
|
|
4607
|
+
case "output": return rates.output;
|
|
4608
|
+
case "cache-read": return rates.cacheRead;
|
|
4609
|
+
case "cache-write": return rates.cacheWrite;
|
|
4610
|
+
}
|
|
4611
|
+
}
|
|
4612
|
+
function bucketTokens(call, bucket) {
|
|
4613
|
+
switch (bucket) {
|
|
4614
|
+
case "input": return call.inputTokens;
|
|
4615
|
+
case "output": return call.outputTokens;
|
|
4616
|
+
case "cache-read": return call.cacheReadTokens;
|
|
4617
|
+
case "cache-write": return call.cacheWriteTokens;
|
|
4618
|
+
}
|
|
4619
|
+
}
|
|
4620
|
+
function validateJudgeUsageAttachment(value) {
|
|
4621
|
+
const issue = (path) => [recordAttachmentIssue("record-attachment-schema-invalid", path)];
|
|
4622
|
+
if (value.calls.length > JudgeUsageLimits.maximumCalls) return issue(["calls"]);
|
|
4623
|
+
const calls = /* @__PURE__ */ new Map();
|
|
4624
|
+
const entries = /* @__PURE__ */ new Map();
|
|
4625
|
+
const entryIds = /* @__PURE__ */ new Map();
|
|
4626
|
+
const transmissions = /* @__PURE__ */ new Map();
|
|
4627
|
+
for (const call of value.calls) {
|
|
4628
|
+
if (!validCall$1(call) || calls.has(call.callId) || entries.has(call.entryIndex) && entries.get(call.entryIndex) !== call.entryId || entryIds.has(call.entryId) && entryIds.get(call.entryId) !== call.entryIndex) return issue(["calls", "callId"]);
|
|
4629
|
+
calls.set(call.callId, call);
|
|
4630
|
+
entries.set(call.entryIndex, call.entryId);
|
|
4631
|
+
entryIds.set(call.entryId, call.entryIndex);
|
|
4632
|
+
const key = `${call.entryIndex}:${call.logicalOrdinal}`;
|
|
4633
|
+
let logical = transmissions.get(key);
|
|
4634
|
+
if (logical === void 0) {
|
|
4635
|
+
logical = {
|
|
4636
|
+
operation: call.operation,
|
|
4637
|
+
model: call.requestModel,
|
|
4638
|
+
provider: call.transportProvider,
|
|
4639
|
+
ordinals: /* @__PURE__ */ new Set()
|
|
4640
|
+
};
|
|
4641
|
+
transmissions.set(key, logical);
|
|
4642
|
+
}
|
|
4643
|
+
if (logical.operation !== call.operation || logical.model !== call.requestModel || logical.provider !== call.transportProvider) return issue(["calls", "logicalOrdinal"]);
|
|
4644
|
+
logical.ordinals.add(call.transmissionOrdinal);
|
|
4645
|
+
}
|
|
4646
|
+
for (const logical of transmissions.values()) for (let ordinal = 1; ordinal <= logical.ordinals.size; ordinal++) if (!logical.ordinals.has(ordinal)) return issue(["calls", "transmissionOrdinal"]);
|
|
4647
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4648
|
+
for (const receipt of value.priceReceipts) {
|
|
4649
|
+
const call = calls.get(receipt.callId);
|
|
4650
|
+
if (call === void 0 || seen.has(receipt.callId) || call.cost !== null || receipt.pricing.requestModel !== call.requestModel || !validPriceReceipt(receipt)) return issue(["priceReceipts", "callId"]);
|
|
4651
|
+
seen.add(receipt.callId);
|
|
4652
|
+
const charges = new Map(receipt.charges.map((charge) => [charge.bucket, charge]));
|
|
4653
|
+
const missing = new Map(receipt.missing.map((entry) => [entry.bucket, entry]));
|
|
4654
|
+
for (const bucket of AdapterCallPriceBuckets) {
|
|
4655
|
+
const tokens = bucketTokens(call, bucket);
|
|
4656
|
+
const rate = bucketRate(receipt.pricing.ratesPerMTok, bucket);
|
|
4657
|
+
const charge = charges.get(bucket);
|
|
4658
|
+
if (tokens === null ? charge !== void 0 || missing.get(bucket)?.reason !== "tokens-unknown" : tokens === 0 || rate !== null ? charge?.tokens !== tokens : charge !== void 0 || missing.get(bucket)?.reason !== "rate-unknown") return issue(["priceReceipts", "charges"]);
|
|
4659
|
+
}
|
|
4660
|
+
}
|
|
4661
|
+
return [];
|
|
4662
|
+
}
|
|
4663
|
+
//#endregion
|
|
4664
|
+
//#region src/o11y/adapter-usage-projection.ts
|
|
4665
|
+
/** Tuple order is stable across hosts, with missing identities before strings. */
|
|
4666
|
+
function compareAdapterModelGroups(left, right) {
|
|
4667
|
+
for (const key of [
|
|
4668
|
+
"modelSlot",
|
|
4669
|
+
"provider",
|
|
4670
|
+
"model"
|
|
4671
|
+
]) {
|
|
4672
|
+
const a = left[key];
|
|
4673
|
+
const b = right[key];
|
|
4674
|
+
if (a === b) continue;
|
|
4675
|
+
if (a === null) return -1;
|
|
4676
|
+
if (b === null) return 1;
|
|
4677
|
+
return a < b ? -1 : 1;
|
|
4678
|
+
}
|
|
4679
|
+
return 0;
|
|
4680
|
+
}
|
|
4681
|
+
function compareJudgeUsageCalls(left, right) {
|
|
4682
|
+
return left.entryIndex - right.entryIndex || left.logicalOrdinal - right.logicalOrdinal || left.transmissionOrdinal - right.transmissionOrdinal;
|
|
4683
|
+
}
|
|
4684
|
+
//#endregion
|
|
4285
4685
|
//#region src/record/family/protocol-values.ts
|
|
4286
4686
|
/** Canonical finite protocol vocabularies shared by current receipt owners and projections. */
|
|
4287
4687
|
var AGENT_TURN_OUTCOME = Object.freeze({
|
|
@@ -4800,6 +5200,83 @@ var LinkSchema = Schema.Union([Schema.Struct({
|
|
|
4800
5200
|
reason: Schema.String
|
|
4801
5201
|
})
|
|
4802
5202
|
})]);
|
|
5203
|
+
var DisplayRoleSchema = Schema.Literals([
|
|
5204
|
+
"user",
|
|
5205
|
+
"assistant",
|
|
5206
|
+
"system",
|
|
5207
|
+
"other"
|
|
5208
|
+
]);
|
|
5209
|
+
var DisplayScalarSchema = Schema.Union([
|
|
5210
|
+
Schema.String,
|
|
5211
|
+
FiniteNumberSchema,
|
|
5212
|
+
Schema.Boolean,
|
|
5213
|
+
Schema.Null
|
|
5214
|
+
]);
|
|
5215
|
+
var DisplayFieldsSchema = Schema.Array(Schema.Struct({
|
|
5216
|
+
label: Schema.String,
|
|
5217
|
+
value: DisplayScalarSchema
|
|
5218
|
+
}));
|
|
5219
|
+
var InputDisplayBlockSchema = Schema.Union([
|
|
5220
|
+
Schema.Struct({
|
|
5221
|
+
kind: Schema.Literal("text"),
|
|
5222
|
+
text: Schema.String
|
|
5223
|
+
}),
|
|
5224
|
+
Schema.Struct({
|
|
5225
|
+
kind: Schema.Literal("message"),
|
|
5226
|
+
role: DisplayRoleSchema,
|
|
5227
|
+
speaker: Schema.optional(Schema.String),
|
|
5228
|
+
text: Schema.String
|
|
5229
|
+
}),
|
|
5230
|
+
Schema.Struct({
|
|
5231
|
+
kind: Schema.Literal("fields"),
|
|
5232
|
+
fields: DisplayFieldsSchema
|
|
5233
|
+
}),
|
|
5234
|
+
Schema.Struct({
|
|
5235
|
+
kind: Schema.Literal("code"),
|
|
5236
|
+
language: Schema.optional(Schema.String),
|
|
5237
|
+
text: Schema.String
|
|
5238
|
+
}),
|
|
5239
|
+
Schema.Struct({
|
|
5240
|
+
kind: Schema.Literal("image"),
|
|
5241
|
+
artifactId: BoundedIdentifierSchema,
|
|
5242
|
+
alt: Schema.String
|
|
5243
|
+
})
|
|
5244
|
+
]);
|
|
5245
|
+
/** Persisted image blocks carry the descriptor fixed at acceptance. */
|
|
5246
|
+
var ExecutionDisplayBlockRecordSchema = Schema.Union([
|
|
5247
|
+
Schema.Struct({
|
|
5248
|
+
kind: Schema.Literal("text"),
|
|
5249
|
+
text: Schema.String
|
|
5250
|
+
}),
|
|
5251
|
+
Schema.Struct({
|
|
5252
|
+
kind: Schema.Literal("message"),
|
|
5253
|
+
role: DisplayRoleSchema,
|
|
5254
|
+
speaker: Schema.optional(Schema.String),
|
|
5255
|
+
text: Schema.String
|
|
5256
|
+
}),
|
|
5257
|
+
Schema.Struct({
|
|
5258
|
+
kind: Schema.Literal("fields"),
|
|
5259
|
+
fields: DisplayFieldsSchema
|
|
5260
|
+
}),
|
|
5261
|
+
Schema.Struct({
|
|
5262
|
+
kind: Schema.Literal("code"),
|
|
5263
|
+
language: Schema.optional(Schema.String),
|
|
5264
|
+
text: Schema.String
|
|
5265
|
+
}),
|
|
5266
|
+
Schema.Struct({
|
|
5267
|
+
kind: Schema.Literal("image"),
|
|
5268
|
+
artifactId: BoundedIdentifierSchema,
|
|
5269
|
+
alt: Schema.String,
|
|
5270
|
+
mediaType: Schema.Literals([
|
|
5271
|
+
"image/png",
|
|
5272
|
+
"image/jpeg",
|
|
5273
|
+
"image/webp",
|
|
5274
|
+
"image/gif"
|
|
5275
|
+
]),
|
|
5276
|
+
byteLength: NonNegativeSafeIntegerSchema,
|
|
5277
|
+
sha256: Sha256Schema
|
|
5278
|
+
})
|
|
5279
|
+
]);
|
|
4803
5280
|
var InputEvidenceSchema = Schema.Struct({
|
|
4804
5281
|
key: BoundedIdentifierSchema,
|
|
4805
5282
|
label: Schema.String,
|
|
@@ -4834,7 +5311,8 @@ var InputEventSchema = Schema.Struct({
|
|
|
4834
5311
|
"excluded",
|
|
4835
5312
|
"unknown"
|
|
4836
5313
|
])
|
|
4837
|
-
})))
|
|
5314
|
+
}))),
|
|
5315
|
+
display: Schema.optional(Schema.Array(InputDisplayBlockSchema))
|
|
4838
5316
|
});
|
|
4839
5317
|
Schema.Struct({
|
|
4840
5318
|
traceId: BoundedIdentifierSchema,
|
|
@@ -4895,7 +5373,8 @@ var ExecutionTraceEventRecordSchema = Schema.Struct({
|
|
|
4895
5373
|
"excluded",
|
|
4896
5374
|
"unknown"
|
|
4897
5375
|
])
|
|
4898
|
-
}))
|
|
5376
|
+
})),
|
|
5377
|
+
display: Schema.optional(Schema.NonEmptyArray(ExecutionDisplayBlockRecordSchema))
|
|
4899
5378
|
});
|
|
4900
5379
|
Schema.Union([ExecutionTraceHeaderRecordSchema, ExecutionTraceEventRecordSchema]);
|
|
4901
5380
|
Object.freeze({
|
|
@@ -4914,7 +5393,15 @@ Object.freeze({
|
|
|
4914
5393
|
maximumDetailPreviewBytes: 2048,
|
|
4915
5394
|
maximumDetailBytes: 524288,
|
|
4916
5395
|
maximumOutlineEvents: 32,
|
|
4917
|
-
maximumOutlineBytes: 65536
|
|
5396
|
+
maximumOutlineBytes: 65536,
|
|
5397
|
+
maximumDisplayBlocksPerEvent: 4,
|
|
5398
|
+
maximumDisplayBytesPerEvent: 8192,
|
|
5399
|
+
maximumDisplayFields: 16,
|
|
5400
|
+
maximumDisplayLabelBytes: 128,
|
|
5401
|
+
maximumDisplayAltBytes: 512,
|
|
5402
|
+
maximumDisplayPreviewBytes: 1024,
|
|
5403
|
+
maximumOutlineEventBytes: 8192,
|
|
5404
|
+
maximumDisplayInlineImageBytes: 16777216
|
|
4918
5405
|
});
|
|
4919
5406
|
//#endregion
|
|
4920
5407
|
//#region src/record/family/file-changes/schema.ts
|
|
@@ -5131,7 +5618,13 @@ var AssertionLimitationSchema = Schema.Union([
|
|
|
5131
5618
|
kind: Schema.Literal("truncated"),
|
|
5132
5619
|
omittedBytes: Schema.Number
|
|
5133
5620
|
}),
|
|
5134
|
-
Schema.Struct({ kind: Schema.Literal("provider-limited") })
|
|
5621
|
+
Schema.Struct({ kind: Schema.Literal("provider-limited") }),
|
|
5622
|
+
Schema.Struct({
|
|
5623
|
+
kind: Schema.Literal("capacity-limited"),
|
|
5624
|
+
capturedItems: Schema.Number,
|
|
5625
|
+
knownTotalItems: Schema.Number,
|
|
5626
|
+
omittedBytes: Schema.NullOr(Schema.Number)
|
|
5627
|
+
})
|
|
5135
5628
|
]);
|
|
5136
5629
|
var OverviewIssueSchema = Schema.Union([
|
|
5137
5630
|
Schema.Struct({
|
|
@@ -5223,6 +5716,7 @@ var OverviewCoverageSchema = Schema.Union([Schema.Struct({
|
|
|
5223
5716
|
"truncated",
|
|
5224
5717
|
"redacted",
|
|
5225
5718
|
"provider-limited",
|
|
5719
|
+
"capacity-limited",
|
|
5226
5720
|
"not-collected",
|
|
5227
5721
|
"source-unavailable",
|
|
5228
5722
|
"producer-failed",
|
|
@@ -5321,6 +5815,24 @@ var InspectionOverviewResultSchema = Schema.Struct({
|
|
|
5321
5815
|
cells: Schema.Array(OverviewCellSchema)
|
|
5322
5816
|
});
|
|
5323
5817
|
var InspectionExperimentResultSchema = Schema.Struct({
|
|
5818
|
+
costSummary: Schema.suspend(() => ExperimentCostSummarySchema),
|
|
5819
|
+
modelUsage: Schema.Struct({
|
|
5820
|
+
attempts: Schema.Array(Schema.Struct({
|
|
5821
|
+
locator: Schema.String,
|
|
5822
|
+
originRunId: Schema.String,
|
|
5823
|
+
usage: Schema.suspend(() => Schema.Struct({
|
|
5824
|
+
state: InspectionAttemptUsageResultSchema.fields.state,
|
|
5825
|
+
configuredModels: InspectionAttemptUsageResultSchema.fields.configuredModels,
|
|
5826
|
+
modelGroups: InspectionAttemptUsageResultSchema.fields.modelGroups,
|
|
5827
|
+
judgeUsage: JudgeUsageSummarySchema,
|
|
5828
|
+
totalCosts: InspectionAttemptUsageResultSchema.fields.totalCosts,
|
|
5829
|
+
totals: InspectionAttemptUsageResultSchema.fields.totals
|
|
5830
|
+
}))
|
|
5831
|
+
})),
|
|
5832
|
+
totalAttemptCount: NonNegativeSafeIntegerSchema$2,
|
|
5833
|
+
omittedAttemptCount: NonNegativeSafeIntegerSchema$2,
|
|
5834
|
+
unresolvedAttemptCount: NonNegativeSafeIntegerSchema$2
|
|
5835
|
+
}).check(Schema.makeFilter((usage) => usage.attempts.length <= 64 && usage.totalAttemptCount === usage.attempts.length + usage.omittedAttemptCount && usage.attempts.every((entry, index) => index === 0 || usage.attempts[index - 1].originRunId < entry.originRunId || usage.attempts[index - 1].originRunId === entry.originRunId && usage.attempts[index - 1].locator < entry.locator))),
|
|
5324
5836
|
experiment: OverviewExperimentSchema,
|
|
5325
5837
|
cells: Schema.Array(OverviewCellSchema)
|
|
5326
5838
|
});
|
|
@@ -5427,7 +5939,8 @@ var AttemptLimitationSchema = Schema.Union([
|
|
|
5427
5939
|
"sampled",
|
|
5428
5940
|
"truncated",
|
|
5429
5941
|
"redacted",
|
|
5430
|
-
"provider-limited"
|
|
5942
|
+
"provider-limited",
|
|
5943
|
+
"capacity-limited"
|
|
5431
5944
|
]),
|
|
5432
5945
|
limitations: Schema.Array(AssertionLimitationSchema)
|
|
5433
5946
|
}),
|
|
@@ -5725,6 +6238,56 @@ var TraceDiagnosticsSchema = Schema.Struct({
|
|
|
5725
6238
|
hasMore: Schema.Boolean,
|
|
5726
6239
|
omittedDiagnosticCount: Schema.Number
|
|
5727
6240
|
});
|
|
6241
|
+
var ExecutionDisplayPreviewTextSchema = Schema.Struct({
|
|
6242
|
+
preview: Schema.String,
|
|
6243
|
+
omittedBytes: Schema.Number
|
|
6244
|
+
});
|
|
6245
|
+
var ExecutionDisplayRoleSchema = Schema.Literals([
|
|
6246
|
+
"user",
|
|
6247
|
+
"assistant",
|
|
6248
|
+
"system",
|
|
6249
|
+
"other"
|
|
6250
|
+
]);
|
|
6251
|
+
var ExecutionDisplayPreviewSchema = Schema.Union([Schema.Struct({ state: Schema.Literal("absent") }), Schema.Struct({
|
|
6252
|
+
state: Schema.Literal("present"),
|
|
6253
|
+
blocks: Schema.Array(Schema.Union([
|
|
6254
|
+
Schema.Struct({
|
|
6255
|
+
kind: Schema.Literal("text"),
|
|
6256
|
+
text: ExecutionDisplayPreviewTextSchema
|
|
6257
|
+
}),
|
|
6258
|
+
Schema.Struct({
|
|
6259
|
+
kind: Schema.Literal("message"),
|
|
6260
|
+
role: ExecutionDisplayRoleSchema,
|
|
6261
|
+
speaker: Schema.optional(Schema.String),
|
|
6262
|
+
text: ExecutionDisplayPreviewTextSchema
|
|
6263
|
+
}),
|
|
6264
|
+
Schema.Struct({
|
|
6265
|
+
kind: Schema.Literal("fields"),
|
|
6266
|
+
fields: Schema.Array(Schema.Struct({
|
|
6267
|
+
label: Schema.String,
|
|
6268
|
+
value: Schema.Union([
|
|
6269
|
+
ExecutionDisplayPreviewTextSchema,
|
|
6270
|
+
Schema.Number,
|
|
6271
|
+
Schema.Boolean,
|
|
6272
|
+
Schema.Null
|
|
6273
|
+
])
|
|
6274
|
+
}))
|
|
6275
|
+
}),
|
|
6276
|
+
Schema.Struct({
|
|
6277
|
+
kind: Schema.Literal("code"),
|
|
6278
|
+
language: Schema.optional(Schema.String),
|
|
6279
|
+
text: ExecutionDisplayPreviewTextSchema
|
|
6280
|
+
}),
|
|
6281
|
+
Schema.Struct({
|
|
6282
|
+
kind: Schema.Literal("image"),
|
|
6283
|
+
artifactId: Schema.String,
|
|
6284
|
+
alt: Schema.String,
|
|
6285
|
+
mediaType: Schema.String,
|
|
6286
|
+
byteLength: Schema.Number,
|
|
6287
|
+
sha256: Schema.String
|
|
6288
|
+
})
|
|
6289
|
+
]))
|
|
6290
|
+
})]);
|
|
5728
6291
|
var TraceItemBase = {
|
|
5729
6292
|
itemId: ItemIdSchema$1,
|
|
5730
6293
|
turnId: Schema.String,
|
|
@@ -5861,7 +6424,8 @@ var InspectionTraceResultSchema = Schema.Struct({
|
|
|
5861
6424
|
key: Schema.String,
|
|
5862
6425
|
label: Schema.String
|
|
5863
6426
|
})),
|
|
5864
|
-
scopeMemberships: ExecutionTraceEventRecordSchema.fields.scopeMemberships
|
|
6427
|
+
scopeMemberships: ExecutionTraceEventRecordSchema.fields.scopeMemberships,
|
|
6428
|
+
display: ExecutionDisplayPreviewSchema
|
|
5865
6429
|
})),
|
|
5866
6430
|
identityIndex: Schema.Struct({
|
|
5867
6431
|
traceIds: Schema.Array(Schema.String),
|
|
@@ -6134,9 +6698,12 @@ var UsageCostTotalSchema = Schema.Struct({
|
|
|
6134
6698
|
value: Schema.String,
|
|
6135
6699
|
observationCount: Schema.Number
|
|
6136
6700
|
});
|
|
6701
|
+
var validUsageCurrency = Schema.is(CurrencyCodeSchema);
|
|
6702
|
+
var UsageDecimalSchema = Schema.String.check(Schema.isPattern(/^(?:0|[1-9][0-9]*)(?:\.[0-9]*[1-9])?$(?![\s\S])/u));
|
|
6703
|
+
var UsageCurrencySchema = Schema.String.check(Schema.makeFilter((value) => validUsageCurrency(value)));
|
|
6137
6704
|
var EffectiveCostSchema = Schema.Struct({
|
|
6138
|
-
amount:
|
|
6139
|
-
currency:
|
|
6705
|
+
amount: UsageDecimalSchema,
|
|
6706
|
+
currency: UsageCurrencySchema,
|
|
6140
6707
|
source: Schema.Struct({
|
|
6141
6708
|
kind: Schema.Literals(["reported", "estimated"]),
|
|
6142
6709
|
id: Schema.String
|
|
@@ -6159,19 +6726,156 @@ var EffectiveCostTotalSchema = Schema.Struct({
|
|
|
6159
6726
|
"mixed"
|
|
6160
6727
|
])),
|
|
6161
6728
|
values: Schema.Array(Schema.Struct({
|
|
6162
|
-
currency:
|
|
6163
|
-
value:
|
|
6729
|
+
currency: UsageCurrencySchema,
|
|
6730
|
+
value: UsageDecimalSchema,
|
|
6164
6731
|
source: Schema.Literals([
|
|
6165
6732
|
"reported",
|
|
6166
6733
|
"estimated",
|
|
6167
6734
|
"mixed"
|
|
6168
6735
|
]),
|
|
6169
|
-
coveredCalls:
|
|
6170
|
-
reportedCalls:
|
|
6171
|
-
estimatedCalls:
|
|
6736
|
+
coveredCalls: NonNegativeSafeIntegerSchema$2,
|
|
6737
|
+
reportedCalls: NonNegativeSafeIntegerSchema$2,
|
|
6738
|
+
estimatedCalls: NonNegativeSafeIntegerSchema$2
|
|
6172
6739
|
})),
|
|
6173
|
-
totalCalls:
|
|
6740
|
+
totalCalls: NonNegativeSafeIntegerSchema$2
|
|
6174
6741
|
});
|
|
6742
|
+
var RecordedCallCountSchema = NonNegativeSafeIntegerSchema$2.check(Schema.isLessThanOrEqualTo(4e3));
|
|
6743
|
+
var ModelGroupTokenTotalSchema = Schema.Struct({
|
|
6744
|
+
state: Schema.Literals([
|
|
6745
|
+
"available",
|
|
6746
|
+
"partial",
|
|
6747
|
+
"unavailable"
|
|
6748
|
+
]),
|
|
6749
|
+
value: Schema.NullOr(NonNegativeSafeIntegerSchema$2),
|
|
6750
|
+
observationCount: RecordedCallCountSchema
|
|
6751
|
+
}).check(Schema.makeFilter((value) => value.state === "unavailable" === (value.value === null)));
|
|
6752
|
+
var ConfiguredModelsSchema = Schema.Union([Schema.Struct({ state: Schema.Literal("not-recorded") }), Schema.Struct({
|
|
6753
|
+
state: Schema.Literal("available"),
|
|
6754
|
+
bindings: Schema.Array(Schema.Struct({
|
|
6755
|
+
modelSlot: AdapterUsageModelSlotSchema,
|
|
6756
|
+
model: Schema.NullOr(Schema.String.check(Schema.isPattern(/\S/u))),
|
|
6757
|
+
reasoningEffort: Schema.NullOr(Schema.String.check(Schema.isPattern(/\S/u))),
|
|
6758
|
+
recordedCalls: Schema.NullOr(RecordedCallCountSchema)
|
|
6759
|
+
})).check(Schema.makeFilter((bindings) => bindings.length <= 64 && bindings.every((entry, index) => index === 0 || bindings[index - 1].modelSlot < entry.modelSlot)))
|
|
6760
|
+
})]);
|
|
6761
|
+
var ModelGroupSchema = Schema.Struct({
|
|
6762
|
+
modelSlot: Schema.NullOr(AdapterUsageModelSlotSchema),
|
|
6763
|
+
provider: AdapterUsageCallSchema.fields.provider,
|
|
6764
|
+
model: AdapterUsageCallSchema.fields.model,
|
|
6765
|
+
recordedCalls: RecordedCallCountSchema.check(Schema.isGreaterThan(0)),
|
|
6766
|
+
tokens: Schema.Struct({
|
|
6767
|
+
inputTotalTokens: ModelGroupTokenTotalSchema,
|
|
6768
|
+
outputTokens: ModelGroupTokenTotalSchema,
|
|
6769
|
+
totalTokens: ModelGroupTokenTotalSchema
|
|
6770
|
+
}),
|
|
6771
|
+
costs: EffectiveCostTotalSchema
|
|
6772
|
+
}).check(Schema.makeFilter((group) => group.costs.totalCalls === group.recordedCalls && Object.values(group.tokens).every((total) => total.observationCount <= group.recordedCalls)));
|
|
6773
|
+
var ModelGroupsSchema = Schema.Union([Schema.Struct({
|
|
6774
|
+
state: Schema.Literal("available"),
|
|
6775
|
+
basis: Schema.Literal("recorded-calls"),
|
|
6776
|
+
groups: Schema.Array(ModelGroupSchema).check(Schema.makeFilter((groups) => groups.length <= 64 && groups.every((entry, index) => index === 0 || compareAdapterModelGroups(groups[index - 1], entry) < 0))),
|
|
6777
|
+
totalGroupCount: RecordedCallCountSchema,
|
|
6778
|
+
groupsTruncated: Schema.Boolean,
|
|
6779
|
+
omittedGroupCount: RecordedCallCountSchema
|
|
6780
|
+
}).check(Schema.makeFilter((value) => value.totalGroupCount === value.groups.length + value.omittedGroupCount && value.groups.length === Math.min(value.totalGroupCount, 64) && value.groupsTruncated === value.omittedGroupCount > 0)), Schema.Struct({
|
|
6781
|
+
state: Schema.Literal("unavailable"),
|
|
6782
|
+
basis: Schema.Literals([
|
|
6783
|
+
"recorded-calls",
|
|
6784
|
+
"reported-sends",
|
|
6785
|
+
"unavailable"
|
|
6786
|
+
]),
|
|
6787
|
+
reason: Schema.Literals([
|
|
6788
|
+
"source-invalid",
|
|
6789
|
+
"usage-not-recorded",
|
|
6790
|
+
"physical-call-identity-not-recorded"
|
|
6791
|
+
]),
|
|
6792
|
+
groups: Schema.Tuple([]),
|
|
6793
|
+
totalGroupCount: Schema.Null,
|
|
6794
|
+
groupsTruncated: Schema.Literal(false),
|
|
6795
|
+
omittedGroupCount: Schema.Literal(0)
|
|
6796
|
+
})]);
|
|
6797
|
+
var TotalUsageCostsSchema = Schema.Struct({
|
|
6798
|
+
state: Schema.Literals([
|
|
6799
|
+
"complete",
|
|
6800
|
+
"partial",
|
|
6801
|
+
"unavailable"
|
|
6802
|
+
]),
|
|
6803
|
+
values: Schema.Array(Schema.Struct({
|
|
6804
|
+
currency: UsageCurrencySchema,
|
|
6805
|
+
value: UsageDecimalSchema,
|
|
6806
|
+
source: Schema.Literals([
|
|
6807
|
+
"reported",
|
|
6808
|
+
"estimated",
|
|
6809
|
+
"mixed"
|
|
6810
|
+
])
|
|
6811
|
+
})),
|
|
6812
|
+
missingSources: Schema.Array(Schema.Literals(["application", "judge"])).check(Schema.makeFilter((sources) => sources.length <= 2 && sources.every((source, index) => index === 0 || sources[index - 1] === "application" && source === "judge")))
|
|
6813
|
+
}).check(Schema.makeFilter((value) => value.state === (value.missingSources.length === 0 ? "complete" : value.values.length === 0 ? "unavailable" : "partial") && value.values.every((entry, index) => index === 0 || value.values[index - 1].currency < entry.currency)));
|
|
6814
|
+
var ExperimentCostSummarySchema = Schema.Struct({
|
|
6815
|
+
scope: Schema.Literal("latest-recorded-slots"),
|
|
6816
|
+
totalCosts: TotalUsageCostsSchema,
|
|
6817
|
+
coverage: Schema.Struct({
|
|
6818
|
+
selectedSlotCount: NonNegativeSafeIntegerSchema$2,
|
|
6819
|
+
resolvedSlotCount: NonNegativeSafeIntegerSchema$2,
|
|
6820
|
+
originAttemptCount: NonNegativeSafeIntegerSchema$2,
|
|
6821
|
+
completeAttemptCount: NonNegativeSafeIntegerSchema$2,
|
|
6822
|
+
partialAttemptCount: NonNegativeSafeIntegerSchema$2,
|
|
6823
|
+
unavailableAttemptCount: NonNegativeSafeIntegerSchema$2,
|
|
6824
|
+
unresolvedSlotCount: NonNegativeSafeIntegerSchema$2
|
|
6825
|
+
}).check(Schema.makeFilter((value) => value.selectedSlotCount === value.resolvedSlotCount + value.unresolvedSlotCount && value.originAttemptCount <= value.resolvedSlotCount && value.originAttemptCount === value.completeAttemptCount + value.partialAttemptCount + value.unavailableAttemptCount))
|
|
6826
|
+
});
|
|
6827
|
+
var JudgeCostTotalsSchema = EffectiveCostTotalSchema.check(Schema.makeFilter((costs) => costs.totalCalls <= 4e3 && costs.source === null === (costs.values.length === 0) && costs.values.every((entry, index) => entry.coveredCalls <= entry.reportedCalls + entry.estimatedCalls && entry.reportedCalls + entry.estimatedCalls <= costs.totalCalls && entry.source === (entry.reportedCalls === 0 ? "estimated" : entry.estimatedCalls === 0 ? "reported" : "mixed") && (index === 0 || costs.values[index - 1].currency < entry.currency)) && costs.values.reduce((count, entry) => count + entry.reportedCalls + entry.estimatedCalls, 0) <= costs.totalCalls && (costs.state !== "unavailable" || costs.values.length === 0) && (costs.state !== "complete" || costs.values.reduce((count, entry) => count + entry.coveredCalls, 0) === costs.totalCalls)));
|
|
6828
|
+
var JudgeUsageUnavailableSchema = Schema.Union([Schema.Struct({
|
|
6829
|
+
state: Schema.Literal("unavailable"),
|
|
6830
|
+
reason: Schema.Literal("judge-usage-not-recorded")
|
|
6831
|
+
}), Schema.Struct({
|
|
6832
|
+
state: Schema.Literal("invalid"),
|
|
6833
|
+
reason: Schema.Literal("judge-usage-source-invalid")
|
|
6834
|
+
})]);
|
|
6835
|
+
var JudgeUsageSummaryFields = {
|
|
6836
|
+
state: Schema.Literals(["complete", "partial"]),
|
|
6837
|
+
coverage: Schema.Literal("physical-transmissions"),
|
|
6838
|
+
collection: CollectionStateSchema,
|
|
6839
|
+
totals: Schema.Struct({
|
|
6840
|
+
requests: ModelGroupTokenTotalSchema,
|
|
6841
|
+
inputTotalTokens: ModelGroupTokenTotalSchema,
|
|
6842
|
+
outputTokens: ModelGroupTokenTotalSchema,
|
|
6843
|
+
totalTokens: ModelGroupTokenTotalSchema,
|
|
6844
|
+
costs: JudgeCostTotalsSchema
|
|
6845
|
+
})
|
|
6846
|
+
};
|
|
6847
|
+
var JudgeUsageAvailableSummarySchema = Schema.Struct(JudgeUsageSummaryFields).check(Schema.makeFilter((usage) => {
|
|
6848
|
+
const requests = usage.totals.requests;
|
|
6849
|
+
return usage.state === usage.collection.state && requests.value === requests.observationCount && requests.state === (usage.state === "complete" ? "available" : "partial") && usage.totals.costs.totalCalls === requests.value && (usage.state === "complete" || usage.totals.costs.state !== "complete") && [
|
|
6850
|
+
usage.totals.inputTotalTokens,
|
|
6851
|
+
usage.totals.outputTokens,
|
|
6852
|
+
usage.totals.totalTokens
|
|
6853
|
+
].every((total) => total.observationCount <= requests.observationCount && (total.state !== "available" || usage.state === "complete" && total.observationCount === requests.observationCount)) && (requests.value !== 0 || usage.state !== "complete" || [
|
|
6854
|
+
usage.totals.inputTotalTokens,
|
|
6855
|
+
usage.totals.outputTokens,
|
|
6856
|
+
usage.totals.totalTokens
|
|
6857
|
+
].every((total) => total.state === "available" && total.value === 0) && usage.totals.costs.state === "complete");
|
|
6858
|
+
}));
|
|
6859
|
+
var JudgeUsageSummarySchema = Schema.Union([JudgeUsageUnavailableSchema, JudgeUsageAvailableSummarySchema]);
|
|
6860
|
+
var JudgeUsageSchema = Schema.Union([JudgeUsageUnavailableSchema, Schema.Struct({
|
|
6861
|
+
...JudgeUsageSummaryFields,
|
|
6862
|
+
calls: Schema.Array(JudgeUsageCallSchema),
|
|
6863
|
+
callsTruncated: Schema.Boolean,
|
|
6864
|
+
omittedCallCount: RecordedCallCountSchema,
|
|
6865
|
+
priceReceipts: Schema.Array(JudgePriceReceiptSchema)
|
|
6866
|
+
}).check(Schema.makeFilter((usage) => Schema.is(JudgeUsageAvailableSummarySchema)({
|
|
6867
|
+
state: usage.state,
|
|
6868
|
+
coverage: usage.coverage,
|
|
6869
|
+
collection: usage.collection,
|
|
6870
|
+
totals: usage.totals
|
|
6871
|
+
}) && usage.calls.length <= 128 && usage.calls.length + usage.omittedCallCount === usage.totals.requests.value && usage.callsTruncated === usage.omittedCallCount > 0 && usage.calls.every((call, index) => index === 0 || compareJudgeUsageCalls(usage.calls[index - 1], call) < 0) && utf8ByteLength$2(JSON.stringify({
|
|
6872
|
+
calls: usage.calls,
|
|
6873
|
+
priceReceipts: usage.priceReceipts
|
|
6874
|
+
})) <= 131072 && validateJudgeUsageAttachment({
|
|
6875
|
+
collection: usage.collection,
|
|
6876
|
+
calls: usage.calls,
|
|
6877
|
+
priceReceipts: usage.priceReceipts
|
|
6878
|
+
}).length === 0))]);
|
|
6175
6879
|
var UsageTotalsSchema = Schema.Struct({
|
|
6176
6880
|
inputTotalTokens: Schema.optional(UsageNumericTotalSchema),
|
|
6177
6881
|
inputTokens: UsageNumericTotalSchema,
|
|
@@ -6189,6 +6893,10 @@ var UsageTotalsSchema = Schema.Struct({
|
|
|
6189
6893
|
costs: Schema.optional(EffectiveCostTotalSchema)
|
|
6190
6894
|
});
|
|
6191
6895
|
var InspectionAttemptUsageResultSchema = Schema.Struct({
|
|
6896
|
+
configuredModels: ConfiguredModelsSchema,
|
|
6897
|
+
modelGroups: ModelGroupsSchema,
|
|
6898
|
+
judgeUsage: JudgeUsageSchema,
|
|
6899
|
+
totalCosts: TotalUsageCostsSchema,
|
|
6192
6900
|
source: Schema.optional(Schema.Literal("adapter")),
|
|
6193
6901
|
coverage: Schema.optional(Schema.Literal("recorded-calls")),
|
|
6194
6902
|
calls: Schema.optional(Schema.Array(AdapterUsageInspectionCallSchema)),
|
|
@@ -6393,6 +7101,72 @@ var InspectionAttemptDiffResultSchema = Schema.Union([
|
|
|
6393
7101
|
})
|
|
6394
7102
|
]);
|
|
6395
7103
|
//#endregion
|
|
7104
|
+
//#region src/inspection/project-result.ts
|
|
7105
|
+
var LocatorSchema = Schema.String.check(Schema.makeFilter((value) => ATTEMPT_LOCATOR_PATTERN.test(value)));
|
|
7106
|
+
var PositionFields = {
|
|
7107
|
+
experimentId: ExperimentIdSchema,
|
|
7108
|
+
evalId: EvalIdSchema,
|
|
7109
|
+
attemptOrdinal: NonNegativeSafeIntegerSchema$2
|
|
7110
|
+
};
|
|
7111
|
+
var PreviousResultSchema = Schema.Struct({
|
|
7112
|
+
sourceRunId: RunIdSchema,
|
|
7113
|
+
locator: LocatorSchema
|
|
7114
|
+
});
|
|
7115
|
+
var ProjectIssueSchema = Schema.suspend(() => Schema.Union([
|
|
7116
|
+
Schema.Null,
|
|
7117
|
+
Schema.Boolean,
|
|
7118
|
+
Schema.Number,
|
|
7119
|
+
Schema.String,
|
|
7120
|
+
Schema.Array(ProjectIssueSchema),
|
|
7121
|
+
Schema.Record(Schema.String, ProjectIssueSchema)
|
|
7122
|
+
]));
|
|
7123
|
+
var InspectionProjectGapReasonSchema = Schema.Literals([
|
|
7124
|
+
"no-result",
|
|
7125
|
+
"pending",
|
|
7126
|
+
"not-published",
|
|
7127
|
+
"identity-mismatch",
|
|
7128
|
+
"outcome-ineligible",
|
|
7129
|
+
"score-incomplete",
|
|
7130
|
+
"evidence-unavailable",
|
|
7131
|
+
"timeout-exceeded",
|
|
7132
|
+
"adoption-unproven"
|
|
7133
|
+
]);
|
|
7134
|
+
var InspectionProjectSlotSchema = Schema.Union([Schema.Struct({
|
|
7135
|
+
...PositionFields,
|
|
7136
|
+
state: Schema.Literal("reuse"),
|
|
7137
|
+
sourceRunId: RunIdSchema,
|
|
7138
|
+
locator: LocatorSchema,
|
|
7139
|
+
relation: Schema.Literals(["origin", "reference"]),
|
|
7140
|
+
action: Schema.Literals([
|
|
7141
|
+
"executed",
|
|
7142
|
+
"carried",
|
|
7143
|
+
"accepted"
|
|
7144
|
+
])
|
|
7145
|
+
}), Schema.Struct({
|
|
7146
|
+
...PositionFields,
|
|
7147
|
+
state: Schema.Literal("gap"),
|
|
7148
|
+
reason: InspectionProjectGapReasonSchema,
|
|
7149
|
+
issues: Schema.Array(ProjectIssueSchema),
|
|
7150
|
+
sourceRunId: Schema.NullOr(RunIdSchema),
|
|
7151
|
+
previous: Schema.NullOr(PreviousResultSchema)
|
|
7152
|
+
})]);
|
|
7153
|
+
/** Current quality metrics retain the aggregate shapes owned by Inspection. */
|
|
7154
|
+
var InspectionProjectResultSchema = Schema.Struct({
|
|
7155
|
+
targetIdentity: Schema.NonEmptyString,
|
|
7156
|
+
coverage: Schema.Struct({
|
|
7157
|
+
expected: NonNegativeSafeIntegerSchema$2,
|
|
7158
|
+
covered: NonNegativeSafeIntegerSchema$2,
|
|
7159
|
+
gaps: NonNegativeSafeIntegerSchema$2
|
|
7160
|
+
}),
|
|
7161
|
+
slots: Schema.Array(InspectionProjectSlotSchema),
|
|
7162
|
+
...InspectionOverviewResultSchema.fields,
|
|
7163
|
+
history: Schema.Array(Schema.Struct({
|
|
7164
|
+
...PositionFields,
|
|
7165
|
+
sourceRunId: RunIdSchema,
|
|
7166
|
+
locator: Schema.NullOr(LocatorSchema)
|
|
7167
|
+
}))
|
|
7168
|
+
}).check(Schema.makeFilter((value) => value.coverage.covered + value.coverage.gaps === value.coverage.expected && value.slots.length === value.coverage.expected && value.slots.filter((slot) => slot.state === "reuse").length === value.coverage.covered));
|
|
7169
|
+
//#endregion
|
|
6396
7170
|
//#region src/inspection/protocol.ts
|
|
6397
7171
|
var AttemptLocatorSchema = Schema.String.pipe(Schema.refine((value) => ATTEMPT_LOCATOR_PATTERN.test(value), {
|
|
6398
7172
|
identifier: "AttemptLocator",
|
|
@@ -6402,6 +7176,7 @@ var ItemIdSchema = Schema.String.pipe(Schema.check(Schema.makeFilter(isItemId)))
|
|
|
6402
7176
|
var ToolOccurrenceIdSchema = Schema.String.pipe(Schema.check(Schema.makeFilter(isToolOccurrenceId)));
|
|
6403
7177
|
var CommandIdSchema = Schema.String.pipe(Schema.check(Schema.makeFilter(isCommandId)));
|
|
6404
7178
|
var RunIdsSchema = Schema.Array(RunIdSchema);
|
|
7179
|
+
var ProjectExperimentIdsSchema = Schema.Array(ExperimentIdSchema).check(Schema.makeFilter((ids) => ids.length > 0 && new Set(ids).size === ids.length));
|
|
6405
7180
|
var operation = (kind, fields) => Schema.Struct({
|
|
6406
7181
|
kind: Schema.Literal(kind),
|
|
6407
7182
|
...fields
|
|
@@ -6521,8 +7296,18 @@ var RunsCompareResultSchema = Schema.Struct({
|
|
|
6521
7296
|
}))
|
|
6522
7297
|
});
|
|
6523
7298
|
var spec = (fields) => Object.freeze(fields);
|
|
6524
|
-
/** The sole
|
|
7299
|
+
/** The sole protocol catalog owner. Every request, result and descriptor projection is derived from this registry. */
|
|
6525
7300
|
var inspectionProtocolRegistry = Object.freeze({
|
|
7301
|
+
"project.get": spec({
|
|
7302
|
+
request: operation("project.get", { experimentIds: Schema.optional(ProjectExperimentIdsSchema) }),
|
|
7303
|
+
result: { project: InspectionProjectResultSchema },
|
|
7304
|
+
factKinds: [
|
|
7305
|
+
"current-project",
|
|
7306
|
+
"core",
|
|
7307
|
+
"assertions",
|
|
7308
|
+
"attempt-cost"
|
|
7309
|
+
]
|
|
7310
|
+
}),
|
|
6526
7311
|
"overview.get": spec({
|
|
6527
7312
|
request: operation("overview.get", { runIds: Schema.optional(RunIdsSchema) }),
|
|
6528
7313
|
result: { overview: InspectionOverviewResultSchema },
|
|
@@ -6538,7 +7323,10 @@ var inspectionProtocolRegistry = Object.freeze({
|
|
|
6538
7323
|
factKinds: [
|
|
6539
7324
|
"core",
|
|
6540
7325
|
"assertions",
|
|
6541
|
-
"attempt-cost"
|
|
7326
|
+
"attempt-cost",
|
|
7327
|
+
"agent-turns",
|
|
7328
|
+
"adapter-usage",
|
|
7329
|
+
"judge-usage"
|
|
6542
7330
|
]
|
|
6543
7331
|
}),
|
|
6544
7332
|
"runs.list": spec({
|
|
@@ -6619,6 +7407,7 @@ var inspectionProtocolRegistry = Object.freeze({
|
|
|
6619
7407
|
traceId: Schema.optional(Schema.String),
|
|
6620
7408
|
sourceId: Schema.optional(Schema.String),
|
|
6621
7409
|
actorId: Schema.optional(Schema.String),
|
|
7410
|
+
eventType: Schema.optional(Schema.String),
|
|
6622
7411
|
continuation: Schema.optional(Schema.String)
|
|
6623
7412
|
}),
|
|
6624
7413
|
result: { trace: InspectionTraceResultSchema },
|
|
@@ -6662,7 +7451,11 @@ var inspectionProtocolRegistry = Object.freeze({
|
|
|
6662
7451
|
"attempt.usage": spec({
|
|
6663
7452
|
request: operation("attempt.usage", { locator: AttemptLocatorSchema }),
|
|
6664
7453
|
result: { usage: InspectionAttemptUsageResultSchema },
|
|
6665
|
-
factKinds: [
|
|
7454
|
+
factKinds: [
|
|
7455
|
+
"agent-turns",
|
|
7456
|
+
"adapter-usage",
|
|
7457
|
+
"judge-usage"
|
|
7458
|
+
]
|
|
6666
7459
|
}),
|
|
6667
7460
|
"attempt.diff": spec({
|
|
6668
7461
|
request: operation("attempt.diff", { locator: AttemptLocatorSchema }),
|
|
@@ -6719,9 +7512,7 @@ var successSchema = (id, entry) => {
|
|
|
6719
7512
|
operation: Schema.Literal(id),
|
|
6720
7513
|
...entry.result
|
|
6721
7514
|
});
|
|
6722
|
-
|
|
6723
|
-
if (id === "runs.compare") return CompareSuccessMetadataSchema.pipe(fields);
|
|
6724
|
-
return SuccessMetadataSchema.pipe(fields);
|
|
7515
|
+
return id === "runs.list" ? RunsListSuccessMetadataSchema.pipe(fields) : id === "runs.compare" ? CompareSuccessMetadataSchema.pipe(fields) : SuccessMetadataSchema.pipe(fields);
|
|
6725
7516
|
};
|
|
6726
7517
|
var InspectionSuccessDocumentSchema = Schema.Union(specs.map(([id, entry]) => successSchema(id, entry)));
|
|
6727
7518
|
var explanationSchema = (id, entry) => {
|
|
@@ -6753,7 +7544,8 @@ var InspectionFailureDocumentSchema = Schema.Struct({
|
|
|
6753
7544
|
"inspection-operation-failed",
|
|
6754
7545
|
"inspection-result-invalid",
|
|
6755
7546
|
"evidence-budget-exceeded",
|
|
6756
|
-
"restart-required"
|
|
7547
|
+
"restart-required",
|
|
7548
|
+
"current-target-unavailable"
|
|
6757
7549
|
]),
|
|
6758
7550
|
reason: Schema.String,
|
|
6759
7551
|
identity: Schema.optional(Schema.Struct({ runId: Schema.String })),
|
|
@@ -6994,22 +7786,26 @@ function primitiveRequest(operation, value) {
|
|
|
6994
7786
|
return;
|
|
6995
7787
|
}
|
|
6996
7788
|
if (canonicalScoreMatchAuditJson(system) !== systemMessage.content || canonicalScoreMatchAuditJson(user) !== userMessage.content) return void 0;
|
|
7789
|
+
const systemKeys = operation === "score" ? [
|
|
7790
|
+
"operation",
|
|
7791
|
+
"rubric",
|
|
7792
|
+
"anchors"
|
|
7793
|
+
] : operation === "classify" || operation === "batchClassify" ? [
|
|
7794
|
+
"operation",
|
|
7795
|
+
"rubric",
|
|
7796
|
+
"choices"
|
|
7797
|
+
] : [
|
|
7798
|
+
"operation",
|
|
7799
|
+
"rubric",
|
|
7800
|
+
"maxItems"
|
|
7801
|
+
];
|
|
7802
|
+
const evidenceIds = operation === "classify" ? record(system)?.evidenceIds : void 0;
|
|
7803
|
+
if (evidenceIds !== void 0 && !validEvidenceIds(evidenceIds)) return void 0;
|
|
6997
7804
|
const systemInput = exact(system, [
|
|
6998
|
-
...
|
|
6999
|
-
"operation",
|
|
7000
|
-
"rubric",
|
|
7001
|
-
"anchors"
|
|
7002
|
-
] : operation === "classify" || operation === "batchClassify" ? [
|
|
7003
|
-
"operation",
|
|
7004
|
-
"rubric",
|
|
7005
|
-
"choices"
|
|
7006
|
-
] : [
|
|
7007
|
-
"operation",
|
|
7008
|
-
"rubric",
|
|
7009
|
-
"maxItems"
|
|
7010
|
-
],
|
|
7805
|
+
...systemKeys,
|
|
7011
7806
|
"protocol",
|
|
7012
|
-
"instruction"
|
|
7807
|
+
"instruction",
|
|
7808
|
+
...evidenceIds === void 0 ? [] : ["evidenceIds"]
|
|
7013
7809
|
]);
|
|
7014
7810
|
const userInput = exact(user, operation === "batchClassify" ? ["material", "items"] : ["material"]);
|
|
7015
7811
|
if (systemInput === void 0 || userInput === void 0 || systemInput.protocol !== protocol || !text(systemInput.instruction) || systemInput.operation !== operation || !text(systemInput.rubric)) return void 0;
|
|
@@ -7028,7 +7824,10 @@ function primitiveRequest(operation, value) {
|
|
|
7028
7824
|
if (!Array.isArray(systemInput.choices) || systemInput.choices.length < 2 || systemInput.choices.length > 32 || systemInput.choices.some((choice) => !text(choice))) return void 0;
|
|
7029
7825
|
const choices = new Set(systemInput.choices);
|
|
7030
7826
|
if (choices.size !== systemInput.choices.length) return void 0;
|
|
7031
|
-
if (operation === "classify") return Object.freeze({
|
|
7827
|
+
if (operation === "classify") return Object.freeze({
|
|
7828
|
+
choices,
|
|
7829
|
+
...evidenceIds === void 0 ? {} : { evidenceIds: new Set(evidenceIds) }
|
|
7830
|
+
});
|
|
7032
7831
|
if (!Array.isArray(userInput.items) || userInput.items.length === 0 || userInput.items.length > 32) return void 0;
|
|
7033
7832
|
const itemIds = /* @__PURE__ */ new Set();
|
|
7034
7833
|
for (const raw of userInput.items) {
|
|
@@ -7058,8 +7857,12 @@ function primitiveOutput(operation, output, request) {
|
|
|
7058
7857
|
return item !== void 0 && unit(item.measurement) && text(item.rationale);
|
|
7059
7858
|
}
|
|
7060
7859
|
if (operation === "classify") {
|
|
7061
|
-
const item = exact(value, [
|
|
7062
|
-
|
|
7860
|
+
const item = exact(value, [
|
|
7861
|
+
"choice",
|
|
7862
|
+
"rationale",
|
|
7863
|
+
...requestInput.evidenceIds === void 0 ? [] : ["citations"]
|
|
7864
|
+
]);
|
|
7865
|
+
return item !== void 0 && text(item.choice) && text(item.rationale) && requestInput.choices?.has(item.choice) === true && (requestInput.evidenceIds === void 0 || validEvidenceIds(item.citations, true) && item.citations.every((id) => requestInput.evidenceIds.has(id)));
|
|
7063
7866
|
}
|
|
7064
7867
|
if (operation === "extract") {
|
|
7065
7868
|
const item = exact(value, [
|
|
@@ -7187,7 +7990,9 @@ function mappingRequestAndOutput(mappingValue, response, requestModel) {
|
|
|
7187
7990
|
"items",
|
|
7188
7991
|
"material"
|
|
7189
7992
|
];
|
|
7190
|
-
const
|
|
7993
|
+
const evidenceIds = operation === "classify" ? record(mapping.input)?.evidenceIds : void 0;
|
|
7994
|
+
if (evidenceIds !== void 0 && !validEvidenceIds(evidenceIds)) return void 0;
|
|
7995
|
+
const input = exactKeys(mapping.input, [...inputKeys, ...evidenceIds === void 0 ? [] : ["evidenceIds"]]);
|
|
7191
7996
|
if (input === void 0 || !text(input.rubric)) return void 0;
|
|
7192
7997
|
let parsedResponse;
|
|
7193
7998
|
try {
|
|
@@ -7580,7 +8385,7 @@ function readScoreMatchAudit(value, expectedName, expectedMeasurement, imageCont
|
|
|
7580
8385
|
"digest",
|
|
7581
8386
|
"chunkByteLengths"
|
|
7582
8387
|
]);
|
|
7583
|
-
if (outer === void 0 || manifest === void 0 || manifest.schemaVersion !== 1 && manifest.schemaVersion !== 2 && manifest.schemaVersion !== 3 || manifest.protocol !== expectedProtocol || !Array.isArray(outer.content) || outer.content.length === 0 || !Array.isArray(manifest.chunkByteLengths) || manifest.chunkByteLengths.length !== outer.content.length || !positiveInteger(manifest.byteLength,
|
|
8388
|
+
if (outer === void 0 || manifest === void 0 || manifest.schemaVersion !== 1 && manifest.schemaVersion !== 2 && manifest.schemaVersion !== 3 || manifest.protocol !== expectedProtocol || !Array.isArray(outer.content) || outer.content.length === 0 || outer.content.length > 2050 || !Array.isArray(manifest.chunkByteLengths) || manifest.chunkByteLengths.length !== outer.content.length || !positiveInteger(manifest.byteLength, 8388608) || typeof manifest.digest !== "string" || !/^[a-f0-9]{64}$/u.test(manifest.digest)) return Object.freeze({ state: "invalid" });
|
|
7584
8389
|
const chunks = [];
|
|
7585
8390
|
for (let index = 0; index < outer.content.length; index += 1) {
|
|
7586
8391
|
const chunk = outer.content[index];
|
|
@@ -7620,7 +8425,10 @@ function readScoreMatchAudit(value, expectedName, expectedMeasurement, imageCont
|
|
|
7620
8425
|
"maxMaterialBytes",
|
|
7621
8426
|
"maxAuditBytes"
|
|
7622
8427
|
]);
|
|
7623
|
-
if (audit === void 0 || audit.schemaVersion !== manifest.schemaVersion || audit.protocol !== expectedProtocol || definition === void 0 || !text(definition.name) || !text(definition.version) || expectedName !== void 0 && definition.name !== expectedName || typeof definition.config !== "string" || typeof definition.digest !== "string" || limits === void 0 || !positiveInteger(limits.maxCalls, 16) || !positiveInteger(limits.maxMaterialBytes,
|
|
8428
|
+
if (audit === void 0 || audit.schemaVersion !== manifest.schemaVersion || audit.protocol !== expectedProtocol || definition === void 0 || !text(definition.name) || !text(definition.version) || expectedName !== void 0 && definition.name !== expectedName || typeof definition.config !== "string" || typeof definition.digest !== "string" || limits === void 0 || !positiveInteger(limits.maxCalls, 16) || !positiveInteger(limits.maxMaterialBytes, 4194304) || !positiveInteger(limits.maxAuditBytes, 8388608) || manifest.byteLength > limits.maxAuditBytes || utf8ByteLength$2(canonicalScoreMatchAuditJson({
|
|
8429
|
+
manifest,
|
|
8430
|
+
content: chunks
|
|
8431
|
+
})) > limits.maxAuditBytes) return Object.freeze({ state: "invalid" });
|
|
7624
8432
|
if (canonicalScoreMatchAuditJson(JSON.parse(definition.config)) !== definition.config || definition.digest !== scoreMatchDefinitionDigest({
|
|
7625
8433
|
name: definition.name,
|
|
7626
8434
|
version: definition.version,
|
|
@@ -7628,7 +8436,8 @@ function readScoreMatchAudit(value, expectedName, expectedMeasurement, imageCont
|
|
|
7628
8436
|
limits
|
|
7629
8437
|
})) return Object.freeze({ state: "invalid" });
|
|
7630
8438
|
const images = manifest.schemaVersion === 3 ? validImages(audit.images, audit.input, imageContent) : void 0;
|
|
7631
|
-
|
|
8439
|
+
const initialMaterialRejected = record(audit.result)?.state === "unavailable" && record(audit.result)?.code === "score-match-material-budget" && Array.isArray(audit.calls) && audit.calls.length === 0;
|
|
8440
|
+
if (typeof audit.input !== "string" || utf8ByteLength$2(audit.input) > limits.maxMaterialBytes && !initialMaterialRejected || canonicalScoreMatchAuditJson(JSON.parse(audit.input)) !== audit.input || manifest.schemaVersion === 3 && images === void 0 || !Array.isArray(audit.calls) || audit.calls.length > limits.maxCalls + 1 || audit.calls.length > limits.maxCalls && record(audit.calls.at(-1))?.state !== "rejected" || audit.calls.some((call, index) => manifest.schemaVersion === 1 ? !validCall(call, index + 1) : manifest.schemaVersion === 2 ? !validTypeSafeCall(call, index + 1) : !validV3Call(call, index + 1, images))) return Object.freeze({ state: "invalid" });
|
|
7632
8441
|
const result = record(audit.result);
|
|
7633
8442
|
if (!(result?.state === "measured" ? exact(result, ["state", "value"]) !== void 0 && unit(result.value) && (expectedMeasurement === void 0 || Object.is(result.value, expectedMeasurement)) : failure(result) || result?.state === "interrupted" && exact(result, ["state"]) !== void 0) || result?.state === "measured" && audit.calls.some((call) => record(call)?.state !== "admitted" || record(record(call)?.result)?.state !== "completed")) return Object.freeze({ state: "invalid" });
|
|
7634
8443
|
return Object.freeze({
|
|
@@ -7639,5 +8448,8 @@ function readScoreMatchAudit(value, expectedName, expectedMeasurement, imageCont
|
|
|
7639
8448
|
return Object.freeze({ state: "invalid" });
|
|
7640
8449
|
}
|
|
7641
8450
|
}
|
|
8451
|
+
function validEvidenceIds(value, empty = false) {
|
|
8452
|
+
return Array.isArray(value) && value.length <= 16384 && (empty || value.length > 0) && value.every((id) => text(id) && id.trim() !== "" && new TextEncoder().encode(id).length <= 128 && !/[\u0000-\u001f\u007f]/u.test(id)) && new Set(value).size === value.length;
|
|
8453
|
+
}
|
|
7642
8454
|
//#endregion
|
|
7643
8455
|
export { INSPECTION_BEHAVIOR_VERSION, INSPECTION_OPERATION_IDS, InspectionDiscoveryDocumentSchema, InspectionDocumentSchema, InspectionExplanationDocumentSchema, InspectionFailureDocumentSchema, InspectionOperationIdSchema, InspectionRequestSchema, InspectionSuccessDocumentSchema, QUERY_PROTOCOL, canonicalInspectionJson, canonicalJsonValue, closeInspectionJson, decodeInspectionDocument, decodeInspectionOperation, decodeInspectionRequest, inspectionProtocolRegistry, narrowInspectionExplanation, narrowInspectionSuccess, readScoreMatchAudit };
|