niceeval 0.15.1-canary.76 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +45 -40
- package/dist/adapter-execution-trace.cjs.map +1 -1
- package/dist/adapter-execution-trace.d.cts +26 -0
- package/dist/adapter-execution-trace.d.mts +26 -0
- package/dist/adapter-execution-trace.d.ts +26 -0
- package/dist/adapter-usage.cjs.map +1 -1
- package/dist/adapter-usage.d.cts +9 -0
- package/dist/adapter-usage.d.mts +9 -0
- package/dist/adapter-usage.d.ts +9 -0
- package/dist/adapter.cjs +19 -4
- package/dist/adapter.cjs.map +1 -1
- package/dist/adapter.d.cts +37 -12
- package/dist/adapter.d.mts +37 -12
- package/dist/adapter.d.ts +37 -12
- package/dist/agents/ai-sdk.cjs +3 -0
- package/dist/agents/ai-sdk.cjs.map +1 -1
- package/dist/agents/bub.cjs +3 -0
- package/dist/agents/bub.cjs.map +1 -1
- package/dist/agents/claude-code-native.cjs +28 -2
- package/dist/agents/claude-code-native.cjs.map +1 -1
- package/dist/agents/claude-code.cjs +3 -0
- package/dist/agents/claude-code.cjs.map +1 -1
- package/dist/agents/codex.cjs +3 -0
- package/dist/agents/codex.cjs.map +1 -1
- package/dist/agents/coding-cli-versions.cjs +9 -10
- package/dist/agents/coding-cli-versions.cjs.map +1 -1
- package/dist/agents/coding-cli-versions.d.cts +1 -1
- package/dist/agents/coding-cli-versions.d.mts +1 -1
- package/dist/agents/coding-cli-versions.d.ts +1 -1
- package/dist/agents/deepseek-harness.cjs +3 -0
- package/dist/agents/deepseek-harness.cjs.map +1 -1
- package/dist/agents/hermes.cjs +3 -0
- package/dist/agents/hermes.cjs.map +1 -1
- package/dist/agents/index.cjs.map +1 -1
- package/dist/agents/index.d.cts +3 -2
- package/dist/agents/index.d.mts +3 -2
- package/dist/agents/index.d.ts +3 -2
- package/dist/agents/omp.cjs +3 -0
- package/dist/agents/omp.cjs.map +1 -1
- package/dist/agents/openai-compat.cjs +6 -0
- package/dist/agents/openai-compat.cjs.map +1 -1
- package/dist/agents/openclaw.cjs +3 -0
- package/dist/agents/openclaw.cjs.map +1 -1
- package/dist/agents/opencode.cjs +3 -0
- package/dist/agents/opencode.cjs.map +1 -1
- package/dist/agents/sdk-streams.cjs +9 -0
- package/dist/agents/sdk-streams.cjs.map +1 -1
- package/dist/agents/ui-message-stream.cjs +3 -0
- package/dist/agents/ui-message-stream.cjs.map +1 -1
- package/dist/assertions/api.cjs.map +1 -1
- package/dist/assertions/api.d.cts +27 -5
- package/dist/assertions/api.d.mts +27 -5
- package/dist/assertions/api.d.ts +27 -5
- package/dist/assertions/collection.cjs +6 -0
- package/dist/assertions/collection.cjs.map +1 -1
- package/dist/assertions/collection.d.cts +4 -2
- package/dist/assertions/collection.d.mts +4 -2
- package/dist/assertions/collection.d.ts +4 -2
- package/dist/assertions/collection.js +12 -8
- package/dist/assertions/context-match.cjs +70 -0
- package/dist/assertions/context-match.cjs.map +1 -0
- package/dist/assertions/context-match.d.cts +107 -0
- package/dist/assertions/context-match.d.mts +107 -0
- package/dist/assertions/context-match.d.ts +107 -0
- package/dist/assertions/context-match.js +12 -0
- package/dist/assertions/context-match.js.map +1 -0
- package/dist/assertions/full-content.cjs +109 -0
- package/dist/assertions/full-content.cjs.map +1 -0
- package/dist/assertions/full-content.d.cts +6 -0
- package/dist/assertions/full-content.d.mts +6 -0
- package/dist/assertions/full-content.d.ts +6 -0
- package/dist/assertions/full-content.js +12 -0
- package/dist/assertions/full-content.js.map +1 -0
- package/dist/assertions/judge-presets.cjs +2 -34
- package/dist/assertions/judge-presets.cjs.map +1 -1
- package/dist/assertions/judge-presets.d.cts +6 -9
- package/dist/assertions/judge-presets.d.mts +6 -9
- package/dist/assertions/judge-presets.d.ts +6 -9
- package/dist/assertions/judge-presets.js +8 -8
- package/dist/assertions/judge.cjs +107 -29
- package/dist/assertions/judge.cjs.map +1 -1
- package/dist/assertions/judge.d.cts +2 -0
- package/dist/assertions/judge.d.mts +2 -0
- package/dist/assertions/judge.d.ts +2 -0
- package/dist/assertions/match.cjs +244 -5
- package/dist/assertions/match.cjs.map +1 -1
- package/dist/assertions/match.d.cts +43 -3
- package/dist/assertions/match.d.mts +43 -3
- package/dist/assertions/match.d.ts +43 -3
- package/dist/assertions/match.js +88 -78
- package/dist/assertions/material.cjs +520 -0
- package/dist/assertions/material.cjs.map +1 -0
- package/dist/assertions/material.d.cts +56 -0
- package/dist/assertions/material.d.mts +56 -0
- package/dist/assertions/material.d.ts +56 -0
- package/dist/assertions/material.js +20 -0
- package/dist/assertions/material.js.map +1 -0
- package/dist/assertions/record/attachment.cjs +5 -2
- package/dist/assertions/record/attachment.cjs.map +1 -1
- package/dist/assertions/record/codec.cjs +2 -1
- package/dist/assertions/record/codec.cjs.map +1 -1
- package/dist/assertions/record/model.cjs.map +1 -1
- package/dist/assertions/record/model.d.cts +6 -1
- package/dist/assertions/record/model.d.mts +6 -1
- package/dist/assertions/record/model.d.ts +6 -1
- package/dist/assertions/runtime.cjs +207 -14
- package/dist/assertions/runtime.cjs.map +1 -1
- package/dist/assertions/runtime.d.cts +9 -0
- package/dist/assertions/runtime.d.mts +9 -0
- package/dist/assertions/runtime.d.ts +9 -0
- package/dist/assertions/runtime.js +14 -8
- package/dist/assertions/score-match-audit.cjs +18 -8
- package/dist/assertions/score-match-audit.cjs.map +1 -1
- package/dist/assertions/score-match-gateway.cjs +83 -25
- package/dist/assertions/score-match-gateway.cjs.map +1 -1
- package/dist/assertions/score-match-gateway.d.cts +2 -0
- package/dist/assertions/score-match-gateway.d.mts +2 -0
- package/dist/assertions/score-match-gateway.d.ts +2 -0
- package/dist/assertions/value-composition.test-d.cjs +77 -0
- package/dist/assertions/value-composition.test-d.cjs.map +1 -0
- package/dist/assertions/value-composition.test-d.d.cts +1 -0
- package/dist/assertions/value-composition.test-d.d.mts +1 -0
- package/dist/assertions/value-composition.test-d.d.ts +1 -0
- package/dist/assertions/value-composition.test-d.js +4 -0
- package/dist/assertions/value-composition.test-d.js.map +1 -0
- package/dist/context/assert-first.cjs +207 -213
- package/dist/context/assert-first.cjs.map +1 -1
- package/dist/context/assert-first.d.cts +49 -23
- package/dist/context/assert-first.d.mts +49 -23
- package/dist/context/assert-first.d.ts +49 -23
- package/dist/context/session.cjs +111 -9
- package/dist/context/session.cjs.map +1 -1
- package/dist/context/session.d.cts +18 -0
- package/dist/context/session.d.mts +18 -0
- package/dist/context/session.d.ts +18 -0
- package/dist/define.cjs +17 -1
- package/dist/define.cjs.map +1 -1
- package/dist/eval/cli/contribution.cjs +2 -0
- package/dist/eval/cli/contribution.cjs.map +1 -1
- package/dist/eval/host/index.cjs +2 -0
- package/dist/eval/host/index.cjs.map +1 -1
- package/dist/eval/record/evaluation.cjs +3 -0
- package/dist/eval/record/evaluation.cjs.map +1 -1
- package/dist/eval/record/membership-provenance.d.cts +15 -15
- package/dist/eval/record/membership-provenance.d.mts +15 -15
- package/dist/eval/record/membership-provenance.d.ts +15 -15
- package/dist/eval/record/score.cjs +7 -0
- package/dist/eval/record/score.cjs.map +1 -1
- package/dist/eval/record/sealed-assertion.cjs +3 -0
- package/dist/eval/record/sealed-assertion.cjs.map +1 -1
- package/dist/eval/record/verdict.cjs +7 -0
- package/dist/eval/record/verdict.cjs.map +1 -1
- package/dist/expect/index.cjs +8 -2
- package/dist/expect/index.cjs.map +1 -1
- package/dist/expect/index.d.cts +9 -4
- package/dist/expect/index.d.mts +9 -4
- package/dist/expect/index.d.ts +9 -4
- package/dist/expect/index.js +72 -62
- package/dist/expect/index.mjs +72 -62
- package/dist/experiment/host/cli/contribution.cjs +37 -23
- package/dist/experiment/host/cli/contribution.cjs.map +1 -1
- package/dist/experiment/host/cli/contribution.d.cts +9 -1
- package/dist/experiment/host/cli/contribution.d.mts +9 -1
- package/dist/experiment/host/cli/contribution.d.ts +9 -1
- package/dist/experiment/host/current.cjs +74 -0
- package/dist/experiment/host/current.cjs.map +1 -0
- package/dist/experiment/host/current.d.cts +33 -0
- package/dist/experiment/host/current.d.mts +33 -0
- package/dist/experiment/host/current.d.ts +33 -0
- package/dist/experiment/host/current.js +10 -0
- package/dist/experiment/host/current.js.map +1 -0
- package/dist/experiment/host/index.cjs +17 -174
- package/dist/experiment/host/index.cjs.map +1 -1
- package/dist/experiment/host/index.d.cts +8 -54
- package/dist/experiment/host/index.d.mts +8 -54
- package/dist/experiment/host/index.d.ts +8 -54
- package/dist/experiment/host/operations.cjs +33 -10
- package/dist/experiment/host/operations.cjs.map +1 -1
- package/dist/experiment/host/operations.d.cts +29 -1
- package/dist/experiment/host/operations.d.mts +29 -1
- package/dist/experiment/host/operations.d.ts +29 -1
- package/dist/experiment/host/operations.js +6 -4
- package/dist/experiment/host/requirements.cjs +3 -0
- package/dist/experiment/host/requirements.cjs.map +1 -0
- package/dist/experiment/host/requirements.d.cts +3 -0
- package/dist/experiment/host/requirements.d.mts +3 -0
- package/dist/experiment/host/requirements.d.ts +3 -0
- package/dist/experiment/host/requirements.js +4 -0
- package/dist/experiment/host/requirements.js.map +1 -0
- package/dist/experiment/host/runtime.cjs +179 -0
- package/dist/experiment/host/runtime.cjs.map +1 -0
- package/dist/experiment/host/runtime.d.cts +79 -0
- package/dist/experiment/host/runtime.d.mts +79 -0
- package/dist/experiment/host/runtime.d.ts +79 -0
- package/dist/experiment/host/runtime.js +6 -0
- package/dist/experiment/host/runtime.js.map +1 -0
- package/dist/experiment/host/types.cjs.map +1 -1
- package/dist/experiment/host/types.d.cts +17 -16
- package/dist/experiment/host/types.d.mts +17 -16
- package/dist/experiment/host/types.d.ts +17 -16
- package/dist/fresh-import.cjs +2 -1
- package/dist/fresh-import.cjs.map +1 -1
- package/dist/index.cjs +4 -2
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +12 -6
- package/dist/index.d.mts +12 -6
- package/dist/index.d.ts +12 -6
- package/dist/index.js +32 -30
- package/dist/index.mjs +32 -30
- package/dist/inspection/artifacts.cjs +83 -54
- package/dist/inspection/artifacts.cjs.map +1 -1
- package/dist/inspection/artifacts.d.cts +8 -0
- package/dist/inspection/artifacts.d.mts +8 -0
- package/dist/inspection/artifacts.d.ts +8 -0
- package/dist/inspection/artifacts.js +2 -0
- package/dist/inspection/assertions.cjs +4 -1
- package/dist/inspection/assertions.cjs.map +1 -1
- package/dist/inspection/cli/contribution.cjs +27 -4
- package/dist/inspection/cli/contribution.cjs.map +1 -1
- package/dist/inspection/cli/contribution.d.cts +2 -1
- package/dist/inspection/cli/contribution.d.mts +2 -1
- package/dist/inspection/cli/contribution.d.ts +2 -1
- package/dist/inspection/execution-traces.cjs +114 -8
- package/dist/inspection/execution-traces.cjs.map +1 -1
- package/dist/inspection/execution-traces.d.cts +38 -0
- package/dist/inspection/execution-traces.d.mts +38 -0
- package/dist/inspection/execution-traces.d.ts +38 -0
- package/dist/inspection/facts.cjs +12 -0
- package/dist/inspection/facts.cjs.map +1 -1
- package/dist/inspection/limits.cjs +5 -1
- package/dist/inspection/limits.cjs.map +1 -1
- package/dist/inspection/limits.d.cts +4 -0
- package/dist/inspection/limits.d.mts +4 -0
- package/dist/inspection/limits.d.ts +4 -0
- package/dist/inspection/limits.js +6 -2
- package/dist/inspection/overview.cjs +74 -33
- package/dist/inspection/overview.cjs.map +1 -1
- package/dist/inspection/overview.d.cts +19 -1
- package/dist/inspection/overview.d.mts +19 -1
- package/dist/inspection/overview.d.ts +19 -1
- package/dist/inspection/overview.js +4 -2
- package/dist/inspection/project-input.cjs +9 -0
- package/dist/inspection/project-input.cjs.map +1 -0
- package/dist/inspection/project-input.d.cts +20 -0
- package/dist/inspection/project-input.d.mts +20 -0
- package/dist/inspection/project-input.d.ts +20 -0
- package/dist/inspection/project-input.js +8 -0
- package/dist/inspection/project-input.js.map +1 -0
- package/dist/inspection/project-result.cjs +60 -0
- package/dist/inspection/project-result.cjs.map +1 -0
- package/dist/inspection/project-result.d.cts +1264 -0
- package/dist/inspection/project-result.d.mts +1264 -0
- package/dist/inspection/project-result.d.ts +1264 -0
- package/dist/inspection/project-result.js +10 -0
- package/dist/inspection/project-result.js.map +1 -0
- package/dist/inspection/project-source.cjs +109 -0
- package/dist/inspection/project-source.cjs.map +1 -0
- package/dist/inspection/project-source.d.cts +10 -0
- package/dist/inspection/project-source.d.mts +10 -0
- package/dist/inspection/project-source.d.ts +10 -0
- package/dist/inspection/project-source.js +6 -0
- package/dist/inspection/project-source.js.map +1 -0
- package/dist/inspection/project.cjs +100 -0
- package/dist/inspection/project.cjs.map +1 -0
- package/dist/inspection/project.d.cts +3 -0
- package/dist/inspection/project.d.mts +3 -0
- package/dist/inspection/project.d.ts +3 -0
- package/dist/inspection/project.js +6 -0
- package/dist/inspection/project.js.map +1 -0
- package/dist/inspection/protocol.cjs +19 -9
- package/dist/inspection/protocol.cjs.map +1 -1
- package/dist/inspection/protocol.d.cts +5506 -38189
- package/dist/inspection/protocol.d.mts +5506 -38189
- package/dist/inspection/protocol.d.ts +5506 -38189
- package/dist/inspection/public.mjs +896 -84
- package/dist/inspection/query.cjs +2 -0
- package/dist/inspection/query.cjs.map +1 -1
- package/dist/inspection/results.cjs +234 -29
- package/dist/inspection/results.cjs.map +1 -1
- package/dist/inspection/results.d.cts +870 -13
- package/dist/inspection/results.d.mts +870 -13
- package/dist/inspection/results.d.ts +870 -13
- package/dist/inspection/results.js +36 -34
- package/dist/inspection/select.cjs +126 -14
- package/dist/inspection/select.cjs.map +1 -1
- package/dist/inspection/select.d.cts +1 -1
- package/dist/inspection/select.d.mts +1 -1
- package/dist/inspection/select.d.ts +1 -1
- package/dist/inspection/source.cjs +4 -0
- package/dist/inspection/source.cjs.map +1 -1
- package/dist/inspection/sources.cjs +5 -0
- package/dist/inspection/sources.cjs.map +1 -1
- package/dist/inspection/trace.cjs +82 -84
- package/dist/inspection/trace.cjs.map +1 -1
- package/dist/inspection/trace.d.cts +7 -2
- package/dist/inspection/trace.d.mts +7 -2
- package/dist/inspection/trace.d.ts +7 -2
- package/dist/model-slots.cjs +35 -0
- package/dist/model-slots.cjs.map +1 -0
- package/dist/model-slots.d.cts +19 -0
- package/dist/model-slots.d.mts +19 -0
- package/dist/model-slots.d.ts +19 -0
- package/dist/model-slots.js +8 -0
- package/dist/model-slots.js.map +1 -0
- package/dist/o11y/adapter-call-price.cjs +4 -0
- package/dist/o11y/adapter-call-price.cjs.map +1 -1
- package/dist/o11y/adapter-call-price.d.cts +24 -0
- package/dist/o11y/adapter-call-price.d.mts +24 -0
- package/dist/o11y/adapter-call-price.d.ts +24 -0
- package/dist/o11y/adapter-call-price.js +6 -2
- package/dist/o11y/adapter-usage-projection.cjs +158 -0
- package/dist/o11y/adapter-usage-projection.cjs.map +1 -0
- package/dist/o11y/adapter-usage-projection.d.cts +141 -0
- package/dist/o11y/adapter-usage-projection.d.mts +141 -0
- package/dist/o11y/adapter-usage-projection.d.ts +141 -0
- package/dist/o11y/adapter-usage-projection.js +20 -0
- package/dist/o11y/adapter-usage-projection.js.map +1 -0
- package/dist/o11y/eval-usage.cjs +149 -0
- package/dist/o11y/eval-usage.cjs.map +1 -0
- package/dist/o11y/eval-usage.d.cts +46 -0
- package/dist/o11y/eval-usage.d.mts +46 -0
- package/dist/o11y/eval-usage.d.ts +46 -0
- package/dist/o11y/eval-usage.js +8 -0
- package/dist/o11y/eval-usage.js.map +1 -0
- package/dist/o11y/judge-call-price.cjs +62 -0
- package/dist/o11y/judge-call-price.cjs.map +1 -0
- package/dist/o11y/judge-call-price.d.cts +3 -0
- package/dist/o11y/judge-call-price.d.mts +3 -0
- package/dist/o11y/judge-call-price.d.ts +3 -0
- package/dist/o11y/judge-call-price.js +6 -0
- package/dist/o11y/judge-call-price.js.map +1 -0
- package/dist/o11y/judge-usage-projection.cjs +134 -0
- package/dist/o11y/judge-usage-projection.cjs.map +1 -0
- package/dist/o11y/judge-usage-projection.d.cts +18 -0
- package/dist/o11y/judge-usage-projection.d.mts +18 -0
- package/dist/o11y/judge-usage-projection.d.ts +18 -0
- package/dist/o11y/judge-usage-projection.js +20 -0
- package/dist/o11y/judge-usage-projection.js.map +1 -0
- package/dist/o11y/judge-usage-receipt.cjs +135 -0
- package/dist/o11y/judge-usage-receipt.cjs.map +1 -0
- package/dist/o11y/judge-usage-receipt.d.cts +5 -0
- package/dist/o11y/judge-usage-receipt.d.mts +5 -0
- package/dist/o11y/judge-usage-receipt.d.ts +5 -0
- package/dist/o11y/judge-usage-receipt.js +6 -0
- package/dist/o11y/judge-usage-receipt.js.map +1 -0
- package/dist/o11y/judge-usage.cjs +171 -0
- package/dist/o11y/judge-usage.cjs.map +1 -0
- package/dist/o11y/judge-usage.d.cts +35 -0
- package/dist/o11y/judge-usage.d.mts +35 -0
- package/dist/o11y/judge-usage.d.ts +35 -0
- package/dist/o11y/judge-usage.js +6 -0
- package/dist/o11y/judge-usage.js.map +1 -0
- package/dist/plugin/contracts.cjs +2 -0
- package/dist/plugin/contracts.cjs.map +1 -1
- package/dist/plugin/link.cjs +7 -0
- package/dist/plugin/link.cjs.map +1 -1
- package/dist/record/attachment/protocol.cjs +8 -6
- package/dist/record/attachment/protocol.cjs.map +1 -1
- package/dist/record/attachment/protocol.d.cts +10 -0
- package/dist/record/attachment/protocol.d.mts +10 -0
- package/dist/record/attachment/protocol.d.ts +10 -0
- package/dist/record/attachment/protocol.js +38 -36
- package/dist/record/codec/seal-manifest.d.cts +3 -3
- package/dist/record/codec/seal-manifest.d.mts +3 -3
- package/dist/record/codec/seal-manifest.d.ts +3 -3
- package/dist/record/family/adapter-usage/definition.d.cts +1 -0
- package/dist/record/family/adapter-usage/definition.d.mts +1 -0
- package/dist/record/family/adapter-usage/definition.d.ts +1 -0
- package/dist/record/family/adapter-usage/persistence.cjs +1 -1
- package/dist/record/family/adapter-usage/persistence.cjs.map +1 -1
- package/dist/record/family/adapter-usage/persistence.d.cts +2 -1
- package/dist/record/family/adapter-usage/persistence.d.mts +2 -1
- package/dist/record/family/adapter-usage/persistence.d.ts +2 -1
- package/dist/record/family/adapter-usage/schema.cjs +21 -2
- package/dist/record/family/adapter-usage/schema.cjs.map +1 -1
- package/dist/record/family/adapter-usage/schema.d.cts +85 -0
- package/dist/record/family/adapter-usage/schema.d.mts +85 -0
- package/dist/record/family/adapter-usage/schema.d.ts +85 -0
- package/dist/record/family/adapter-usage/schema.js +18 -12
- package/dist/record/family/catalog.cjs +31 -27
- package/dist/record/family/catalog.cjs.map +1 -1
- package/dist/record/family/catalog.d.cts +94 -3
- package/dist/record/family/catalog.d.mts +94 -3
- package/dist/record/family/catalog.d.ts +94 -3
- package/dist/record/family/current.cjs +45 -42
- package/dist/record/family/current.cjs.map +1 -1
- package/dist/record/family/current.d.cts +240 -1
- package/dist/record/family/current.d.mts +240 -1
- package/dist/record/family/current.d.ts +240 -1
- package/dist/record/family/execution-traces/definition.d.cts +26 -0
- package/dist/record/family/execution-traces/definition.d.mts +26 -0
- package/dist/record/family/execution-traces/definition.d.ts +26 -0
- package/dist/record/family/execution-traces/definition.js +20 -14
- package/dist/record/family/execution-traces/schema.cjs +57 -1
- package/dist/record/family/execution-traces/schema.cjs.map +1 -1
- package/dist/record/family/execution-traces/schema.d.cts +114 -0
- package/dist/record/family/execution-traces/schema.d.mts +114 -0
- package/dist/record/family/execution-traces/schema.d.ts +114 -0
- package/dist/record/family/execution-traces/schema.js +18 -12
- package/dist/record/family/judge-usage/definition.cjs +12 -0
- package/dist/record/family/judge-usage/definition.cjs.map +1 -0
- package/dist/record/family/judge-usage/definition.d.cts +91 -0
- package/dist/record/family/judge-usage/definition.d.mts +91 -0
- package/dist/record/family/judge-usage/definition.d.ts +91 -0
- package/dist/record/family/judge-usage/definition.js +6 -0
- package/dist/record/family/judge-usage/definition.js.map +1 -0
- package/dist/record/family/judge-usage/persistence.cjs +10 -0
- package/dist/record/family/judge-usage/persistence.cjs.map +1 -0
- package/dist/record/family/judge-usage/persistence.d.cts +91 -0
- package/dist/record/family/judge-usage/persistence.d.mts +91 -0
- package/dist/record/family/judge-usage/persistence.d.ts +91 -0
- package/dist/record/family/judge-usage/persistence.js +6 -0
- package/dist/record/family/judge-usage/persistence.js.map +1 -0
- package/dist/record/family/judge-usage/schema.cjs +307 -0
- package/dist/record/family/judge-usage/schema.cjs.map +1 -0
- package/dist/record/family/judge-usage/schema.d.cts +351 -0
- package/dist/record/family/judge-usage/schema.d.mts +351 -0
- package/dist/record/family/judge-usage/schema.d.ts +351 -0
- package/dist/record/family/judge-usage/schema.js +18 -0
- package/dist/record/family/judge-usage/schema.js.map +1 -0
- package/dist/record/host/sqlite-host.cjs +55 -9
- package/dist/record/host/sqlite-host.cjs.map +1 -1
- package/dist/record/host/types.cjs.map +1 -1
- package/dist/record/host/types.d.cts +7 -0
- package/dist/record/host/types.d.mts +7 -0
- package/dist/record/host/types.d.ts +7 -0
- package/dist/record/model/run-context.cjs +7 -1
- package/dist/record/model/run-context.cjs.map +1 -1
- package/dist/record/model/run-context.d.cts +4 -0
- package/dist/record/model/run-context.d.mts +4 -0
- package/dist/record/model/run-context.d.ts +4 -0
- package/dist/record/model/seal-manifest.d.cts +1 -1
- package/dist/record/model/seal-manifest.d.mts +1 -1
- package/dist/record/model/seal-manifest.d.ts +1 -1
- package/dist/record/sqlite/client.cjs +18 -5
- package/dist/record/sqlite/client.cjs.map +1 -1
- package/dist/record/sqlite/client.d.cts +1 -1
- package/dist/record/sqlite/client.d.mts +1 -1
- package/dist/record/sqlite/client.d.ts +1 -1
- package/dist/record/sqlite/project-state-database.cjs +29 -8
- package/dist/record/sqlite/project-state-database.cjs.map +1 -1
- package/dist/run/host/runtime.cjs +4 -0
- package/dist/run/host/runtime.cjs.map +1 -1
- package/dist/run/storage/sqlite.cjs +20 -0
- package/dist/run/storage/sqlite.cjs.map +1 -1
- package/dist/runner/adapter-execution-trace.cjs +125 -2
- package/dist/runner/adapter-execution-trace.cjs.map +1 -1
- package/dist/runner/adapter-execution-trace.d.cts +1 -1
- package/dist/runner/adapter-execution-trace.d.mts +1 -1
- package/dist/runner/adapter-execution-trace.d.ts +1 -1
- package/dist/runner/adapter-usage.cjs +55 -7
- package/dist/runner/adapter-usage.cjs.map +1 -1
- package/dist/runner/adapter-usage.d.cts +9 -2
- package/dist/runner/adapter-usage.d.mts +9 -2
- package/dist/runner/adapter-usage.d.ts +9 -2
- package/dist/runner/adoption.cjs +9 -0
- package/dist/runner/adoption.cjs.map +1 -1
- package/dist/runner/attempt-cancellation.cjs +12 -0
- package/dist/runner/attempt-cancellation.cjs.map +1 -0
- package/dist/runner/attempt-cancellation.d.cts +6 -0
- package/dist/runner/attempt-cancellation.d.mts +6 -0
- package/dist/runner/attempt-cancellation.d.ts +6 -0
- package/dist/runner/attempt-cancellation.js +6 -0
- package/dist/runner/attempt-cancellation.js.map +1 -0
- package/dist/runner/attempt-resources.cjs +10 -4
- package/dist/runner/attempt-resources.cjs.map +1 -1
- package/dist/runner/attempt-resources.d.cts +2 -1
- package/dist/runner/attempt-resources.d.mts +2 -1
- package/dist/runner/attempt-resources.d.ts +2 -1
- package/dist/runner/attempt.cjs +1177 -1139
- package/dist/runner/attempt.cjs.map +1 -1
- package/dist/runner/config-identity.cjs +20 -1
- package/dist/runner/config-identity.cjs.map +1 -1
- package/dist/runner/config-identity.d.cts +5 -0
- package/dist/runner/config-identity.d.mts +5 -0
- package/dist/runner/config-identity.d.ts +5 -0
- package/dist/runner/discover.cjs +14 -0
- package/dist/runner/discover.cjs.map +1 -1
- package/dist/runner/discover.d.cts +3 -0
- package/dist/runner/discover.d.mts +3 -0
- package/dist/runner/discover.d.ts +3 -0
- package/dist/runner/feedback/human.cjs +9 -4
- package/dist/runner/feedback/human.cjs.map +1 -1
- package/dist/runner/fingerprint.cjs +1 -0
- package/dist/runner/fingerprint.cjs.map +1 -1
- package/dist/runner/judge-usage.cjs +18 -0
- package/dist/runner/judge-usage.cjs.map +1 -0
- package/dist/runner/judge-usage.d.cts +5 -0
- package/dist/runner/judge-usage.d.mts +5 -0
- package/dist/runner/judge-usage.d.ts +5 -0
- package/dist/runner/judge-usage.js +8 -0
- package/dist/runner/judge-usage.js.map +1 -0
- package/dist/runner/project-target.cjs.map +1 -1
- package/dist/runner/project-target.d.cts +2 -0
- package/dist/runner/project-target.d.mts +2 -0
- package/dist/runner/project-target.d.ts +2 -0
- package/dist/runner/record/planning.cjs +1 -0
- package/dist/runner/record/planning.cjs.map +1 -1
- package/dist/runner/record.cjs +4 -0
- package/dist/runner/record.cjs.map +1 -1
- package/dist/runner/reporters/braintrust.cjs +2 -0
- package/dist/runner/reporters/braintrust.cjs.map +1 -1
- package/dist/runner/reuse-diagnostics.cjs +9 -10
- package/dist/runner/reuse-diagnostics.cjs.map +1 -1
- package/dist/runner/reuse-diagnostics.d.cts +5 -6
- package/dist/runner/reuse-diagnostics.d.mts +5 -6
- package/dist/runner/reuse-diagnostics.d.ts +5 -6
- package/dist/runner/reuse-diagnostics.js +4 -4
- package/dist/runner/reuse-plan.cjs +100 -48
- package/dist/runner/reuse-plan.cjs.map +1 -1
- package/dist/runner/reuse-plan.d.cts +60 -3
- package/dist/runner/reuse-plan.d.mts +60 -3
- package/dist/runner/reuse-plan.d.ts +60 -3
- package/dist/runner/reuse-plan.js +14 -10
- package/dist/runner/run.cjs +8 -6
- package/dist/runner/run.cjs.map +1 -1
- package/dist/runner/session.cjs +1 -1
- package/dist/runner/session.cjs.map +1 -1
- package/dist/runner/session.d.cts +1 -1
- package/dist/runner/session.d.mts +1 -1
- package/dist/runner/session.d.ts +1 -1
- package/dist/runner/source-producer.cjs +3 -1
- package/dist/runner/source-producer.cjs.map +1 -1
- package/dist/runner/types.cjs.map +1 -1
- package/dist/runner/types.d.cts +26 -5
- package/dist/runner/types.d.mts +26 -5
- package/dist/runner/types.d.ts +26 -5
- package/dist/sandbox/action.cjs +3 -0
- package/dist/sandbox/action.cjs.map +1 -1
- package/dist/sandbox/commands.cjs +3 -0
- package/dist/sandbox/commands.cjs.map +1 -1
- package/dist/sandbox/compose.cjs +6 -0
- package/dist/sandbox/compose.cjs.map +1 -1
- package/dist/sandbox/docker-profile/registry.cjs +7 -0
- package/dist/sandbox/docker-profile/registry.cjs.map +1 -1
- package/dist/sandbox/docker-profile/runtime.cjs +3 -0
- package/dist/sandbox/docker-profile/runtime.cjs.map +1 -1
- package/dist/sandbox/docker-profile/schema.cjs +7 -0
- package/dist/sandbox/docker-profile/schema.cjs.map +1 -1
- package/dist/sandbox/incus.cjs +7 -0
- package/dist/sandbox/incus.cjs.map +1 -1
- package/dist/sandbox/layer.cjs +12 -0
- package/dist/sandbox/layer.cjs.map +1 -1
- package/dist/shared/adapter-cleanup.cjs +18 -0
- package/dist/shared/adapter-cleanup.cjs.map +1 -0
- package/dist/shared/adapter-cleanup.d.cts +4 -0
- package/dist/shared/adapter-cleanup.d.mts +4 -0
- package/dist/shared/adapter-cleanup.d.ts +4 -0
- package/dist/shared/adapter-cleanup.js +10 -0
- package/dist/shared/adapter-cleanup.js.map +1 -0
- package/dist/shared/attempt-lifecycle.cjs +3 -0
- package/dist/shared/attempt-lifecycle.cjs.map +1 -0
- package/dist/shared/attempt-lifecycle.d.cts +13 -0
- package/dist/shared/attempt-lifecycle.d.mts +13 -0
- package/dist/shared/attempt-lifecycle.d.ts +13 -0
- package/dist/shared/attempt-lifecycle.js +4 -0
- package/dist/shared/attempt-lifecycle.js.map +1 -0
- package/dist/show/contribution.cjs +52 -14
- package/dist/show/contribution.cjs.map +1 -1
- package/dist/show/contribution.d.cts +23 -1
- package/dist/show/contribution.d.mts +23 -1
- package/dist/show/contribution.d.ts +23 -1
- package/dist/show/model.cjs +8 -1
- package/dist/show/model.cjs.map +1 -1
- package/dist/show/model.d.cts +10 -1
- package/dist/show/model.d.mts +10 -1
- package/dist/show/model.d.ts +10 -1
- package/dist/show/render.cjs +287 -49
- package/dist/show/render.cjs.map +1 -1
- package/dist/show/render.d.cts +3 -0
- package/dist/show/render.d.mts +3 -0
- package/dist/show/render.d.ts +3 -0
- package/dist/sources/projector.cjs +3 -0
- package/dist/sources/projector.cjs.map +1 -1
- package/dist/types.cjs.map +1 -1
- package/dist/types.d.cts +1 -1
- package/dist/types.d.mts +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/view/app-dist/.vite/manifest.json +2 -2
- package/dist/view/app-dist/assets/index-CmVoXbwG.js +63 -0
- package/dist/view/app-dist/assets/{quality-cost-scatter-CFB3wnkN.js → quality-cost-scatter-BuLISaH4.js} +1 -1
- package/dist/view/app-dist/index.html +1 -1
- package/dist/view/cli/contribution.cjs +54 -21
- package/dist/view/cli/contribution.cjs.map +1 -1
- package/dist/view/cli/contribution.d.cts +2 -1
- package/dist/view/cli/contribution.d.mts +2 -1
- package/dist/view/cli/contribution.d.ts +2 -1
- package/dist/view/function-runtime.mjs +13231 -11647
- package/dist/view/http-protocol.cjs +3 -0
- package/dist/view/http-protocol.cjs.map +1 -1
- package/dist/view/http-protocol.d.cts +8 -0
- package/dist/view/http-protocol.d.mts +8 -0
- package/dist/view/http-protocol.d.ts +8 -0
- package/dist/view/inspection-host.cjs +17 -1
- package/dist/view/inspection-host.cjs.map +1 -1
- package/dist/view/render.cjs +1 -0
- package/dist/view/render.cjs.map +1 -1
- package/dist/view/render.d.cts +2 -1
- package/dist/view/render.d.mts +2 -1
- package/dist/view/render.d.ts +2 -1
- package/dist/view/revision.cjs.map +1 -1
- package/dist/view/revision.d.cts +12 -0
- package/dist/view/revision.d.mts +12 -0
- package/dist/view/revision.d.ts +12 -0
- package/dist/view/server.cjs +10 -9
- package/dist/view/server.cjs.map +1 -1
- package/docs-site/zh/examples/ai-agent-application.mdx +1 -1
- package/docs-site/zh/explanation/adapter.mdx +2 -2
- package/docs-site/zh/explanation/assert.mdx +60 -24
- package/docs-site/zh/explanation/drive.mdx +1 -1
- package/docs-site/zh/explanation/evals.mdx +31 -29
- package/docs-site/zh/explanation/experiment.mdx +15 -15
- package/docs-site/zh/explanation/judge.mdx +23 -15
- package/docs-site/zh/explanation/overview.mdx +44 -21
- package/docs-site/zh/explanation/record.mdx +28 -13
- package/docs-site/zh/explanation/results-flow.mdx +37 -0
- package/docs-site/zh/explanation/runner.mdx +27 -6
- package/docs-site/zh/index.mdx +22 -115
- package/docs-site/zh/introduction.mdx +89 -61
- package/docs-site/zh/reference/cli.mdx +50 -26
- package/docs-site/zh/reference/define-agent.mdx +37 -5
- package/docs-site/zh/reference/define-eval.mdx +130 -29
- package/docs-site/zh/reference/define-experiment.mdx +251 -0
- package/docs-site/zh/reference/events.mdx +34 -30
- package/docs-site/zh/reference/expect.mdx +77 -2
- package/docs-site/zh/reference/official-adapters.mdx +2 -2
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +43 -24
- package/docs-site/zh/troubleshooting/debugging.mdx +65 -42
- package/docs-site/zh/troubleshooting/recover-after-kill.mdx +35 -28
- package/docs-site/zh/tutorials/accept-results.mdx +34 -24
- package/docs-site/zh/tutorials/agent-feedback-loop.mdx +74 -43
- package/docs-site/zh/tutorials/agent-onboarding.mdx +27 -20
- package/docs-site/zh/tutorials/authoring.mdx +138 -121
- package/docs-site/zh/tutorials/ci-integration.mdx +46 -22
- package/docs-site/zh/tutorials/compare-runs.mdx +40 -13
- package/docs-site/zh/tutorials/concurrency.mdx +66 -62
- package/docs-site/zh/tutorials/configuration.mdx +17 -17
- package/docs-site/zh/tutorials/connect-otel.mdx +48 -37
- package/docs-site/zh/tutorials/connect-your-agent.mdx +80 -173
- package/docs-site/zh/tutorials/control-run-cost.mdx +63 -12
- package/docs-site/zh/tutorials/criteria-files.mdx +28 -21
- package/docs-site/zh/tutorials/custom-application.mdx +75 -55
- package/docs-site/zh/tutorials/dataset-fanout.mdx +32 -22
- package/docs-site/zh/tutorials/debug-lifecycle-plan.mdx +38 -31
- package/docs-site/zh/tutorials/docker-in-docker.mdx +83 -89
- package/docs-site/zh/tutorials/eval-groups.mdx +70 -89
- package/docs-site/zh/tutorials/evaluation-kinds.mdx +12 -5
- package/docs-site/zh/tutorials/experiment-services.mdx +243 -0
- package/docs-site/zh/tutorials/fixtures.mdx +42 -26
- package/docs-site/zh/tutorials/handle-execution-failures.mdx +41 -19
- package/docs-site/zh/tutorials/install-custom-sandbox-agent.mdx +21 -16
- package/docs-site/zh/tutorials/niceeval-taste-application.mdx +394 -0
- package/docs-site/zh/tutorials/niceeval-taste.mdx +190 -0
- package/docs-site/zh/tutorials/nixos-managed-dind.mdx +51 -39
- package/docs-site/zh/tutorials/plugins.mdx +55 -31
- package/docs-site/zh/tutorials/prebuilt-sandbox.mdx +204 -0
- package/docs-site/zh/tutorials/quickstart.mdx +91 -47
- package/docs-site/zh/tutorials/reporters.mdx +45 -30
- package/docs-site/zh/tutorials/rerun-and-cache.mdx +39 -40
- package/docs-site/zh/tutorials/sandbox-agent.mdx +65 -40
- package/docs-site/zh/tutorials/sandbox-providers.mdx +104 -282
- package/docs-site/zh/tutorials/sandbox-reuse.mdx +65 -97
- package/docs-site/zh/tutorials/select-and-preview.mdx +25 -20
- package/docs-site/zh/tutorials/verify-judge.mdx +49 -26
- package/docs-site/zh/tutorials/viewing-results.mdx +130 -80
- package/docs-site/zh/tutorials/write-experiment.mdx +116 -298
- package/docs-site/zh/tutorials/write-send.mdx +53 -52
- package/node_modules/undici/docs/docs/api/Agent.md +8 -8
- package/node_modules/undici/docs/docs/api/DiagnosticsChannel.md +9 -6
- package/node_modules/undici/docs/docs/api/Errors.md +25 -0
- package/node_modules/undici/docs/docs/api/Interceptors.md +4 -3
- package/node_modules/undici/docs/docs/api/Pool.md +5 -6
- package/node_modules/undici/lib/api/readable.js +1 -2
- package/node_modules/undici/lib/core/request.js +23 -2
- package/node_modules/undici/lib/core/util.js +0 -39
- package/node_modules/undici/lib/dispatcher/client-h1.js +11 -4
- package/node_modules/undici/lib/dispatcher/client-h2.js +52 -8
- package/node_modules/undici/lib/dispatcher/client.js +1 -1
- package/node_modules/undici/lib/dispatcher/dispatcher1-wrapper.js +3 -2
- package/node_modules/undici/lib/dispatcher/env-http-proxy-agent.js +21 -6
- package/node_modules/undici/lib/dispatcher/pool-base.js +63 -2
- package/node_modules/undici/lib/dispatcher/pool.js +90 -7
- package/node_modules/undici/lib/dispatcher/proxy-agent.js +16 -3
- package/node_modules/undici/lib/handler/deduplication-handler.js +24 -11
- package/node_modules/undici/lib/handler/redirect-handler.js +5 -0
- package/node_modules/undici/lib/handler/retry-handler.js +72 -13
- package/node_modules/undici/lib/interceptor/cache.js +4 -1
- package/node_modules/undici/lib/interceptor/decompress.js +236 -69
- package/node_modules/undici/lib/web/cache/cache.js +17 -8
- package/node_modules/undici/lib/web/cache/cachestorage.js +11 -5
- package/node_modules/undici/lib/web/cookies/index.js +19 -7
- package/node_modules/undici/lib/web/cookies/util.js +3 -1
- package/node_modules/undici/lib/web/eventsource/eventsource.js +25 -1
- package/node_modules/undici/lib/web/fetch/body.js +22 -15
- package/node_modules/undici/lib/web/fetch/data-url.js +6 -6
- package/node_modules/undici/lib/web/fetch/formdata.js +30 -36
- package/node_modules/undici/lib/web/fetch/headers.js +34 -31
- package/node_modules/undici/lib/web/fetch/index.js +38 -14
- package/node_modules/undici/lib/web/fetch/request.js +74 -79
- package/node_modules/undici/lib/web/fetch/response.js +43 -44
- package/node_modules/undici/lib/web/fetch/util.js +7 -7
- package/node_modules/undici/lib/web/infra/index.js +16 -1
- package/node_modules/undici/lib/web/webidl/index.js +2 -14
- package/node_modules/undici/lib/web/websocket/events.js +43 -26
- package/node_modules/undici/lib/web/websocket/receiver.js +5 -0
- package/node_modules/undici/lib/web/websocket/stream/websocketerror.js +13 -10
- package/node_modules/undici/lib/web/websocket/websocket.js +44 -41
- package/node_modules/undici/package.json +1 -1
- package/node_modules/undici/types/diagnostics-channel.d.ts +2 -1
- package/node_modules/undici/types/interceptors.d.ts +1 -1
- package/node_modules/undici/types/webidl.d.ts +13 -7
- package/package.json +4 -4
- package/dist/view/app-dist/assets/index-DFzw1L97.js +0 -63
- package/docs-site/zh/tutorials/experiments.mdx +0 -101
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { NumericMaterial } from "../assertions/match.cjs";
|
|
2
|
+
import type { ResolvedEvidenceCoverage } from "../assertions/coverage.cjs";
|
|
3
|
+
import type { PriceOverride, Usage } from "../types.cjs";
|
|
4
|
+
import type { ReadableAdapterUsageAttachment } from "../record/family/adapter-usage/schema.cjs";
|
|
5
|
+
/** Internal source-owned shape; package-root exposure belongs to the author facade. */
|
|
6
|
+
export interface EvalUsage {
|
|
7
|
+
readonly source: "agent" | "adapter" | "unavailable";
|
|
8
|
+
readonly scope: "turn" | "session" | "attempt";
|
|
9
|
+
readonly basis: "reported-sends" | "recorded-calls" | "unbound";
|
|
10
|
+
readonly cut: "call-time";
|
|
11
|
+
readonly inputTokens: NumericMaterial;
|
|
12
|
+
readonly inputTotalTokens: NumericMaterial;
|
|
13
|
+
readonly outputTokens: NumericMaterial;
|
|
14
|
+
readonly cacheReadTokens: NumericMaterial;
|
|
15
|
+
readonly cacheWriteTokens: NumericMaterial;
|
|
16
|
+
readonly totalTokens: NumericMaterial;
|
|
17
|
+
readonly costs: {
|
|
18
|
+
readonly state: "complete" | "partial" | "unavailable";
|
|
19
|
+
readonly values: readonly {
|
|
20
|
+
readonly currency: string;
|
|
21
|
+
readonly value: string;
|
|
22
|
+
readonly source: "reported" | "estimated" | "mixed";
|
|
23
|
+
readonly coveredContributions: number;
|
|
24
|
+
readonly reportedContributions: number;
|
|
25
|
+
readonly estimatedContributions: number;
|
|
26
|
+
}[];
|
|
27
|
+
readonly totalContributions: number;
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export interface AgentUsageContribution {
|
|
31
|
+
readonly sessionScopeId: string;
|
|
32
|
+
readonly turnId: string;
|
|
33
|
+
readonly sendAttempt: number;
|
|
34
|
+
readonly state: "pending" | "terminal";
|
|
35
|
+
readonly outcome: "completed" | "failed" | "interrupted" | null;
|
|
36
|
+
readonly model: string | null;
|
|
37
|
+
readonly usage: Readonly<Usage> | null;
|
|
38
|
+
readonly coverage: ResolvedEvidenceCoverage["usage"];
|
|
39
|
+
}
|
|
40
|
+
export declare function projectAdapterEvalUsage(usage: ReadableAdapterUsageAttachment): EvalUsage;
|
|
41
|
+
/** The receiver selects scope membership before calling this pure projection. */
|
|
42
|
+
export declare function projectAgentEvalUsage(input: {
|
|
43
|
+
contributions: readonly AgentUsageContribution[];
|
|
44
|
+
scope: "turn" | "session" | "attempt";
|
|
45
|
+
pricing?: Record<string, PriceOverride>;
|
|
46
|
+
}): EvalUsage;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { NumericMaterial } from "../assertions/match.mjs";
|
|
2
|
+
import type { ResolvedEvidenceCoverage } from "../assertions/coverage.mjs";
|
|
3
|
+
import type { PriceOverride, Usage } from "../types.mjs";
|
|
4
|
+
import type { ReadableAdapterUsageAttachment } from "../record/family/adapter-usage/schema.mjs";
|
|
5
|
+
/** Internal source-owned shape; package-root exposure belongs to the author facade. */
|
|
6
|
+
export interface EvalUsage {
|
|
7
|
+
readonly source: "agent" | "adapter" | "unavailable";
|
|
8
|
+
readonly scope: "turn" | "session" | "attempt";
|
|
9
|
+
readonly basis: "reported-sends" | "recorded-calls" | "unbound";
|
|
10
|
+
readonly cut: "call-time";
|
|
11
|
+
readonly inputTokens: NumericMaterial;
|
|
12
|
+
readonly inputTotalTokens: NumericMaterial;
|
|
13
|
+
readonly outputTokens: NumericMaterial;
|
|
14
|
+
readonly cacheReadTokens: NumericMaterial;
|
|
15
|
+
readonly cacheWriteTokens: NumericMaterial;
|
|
16
|
+
readonly totalTokens: NumericMaterial;
|
|
17
|
+
readonly costs: {
|
|
18
|
+
readonly state: "complete" | "partial" | "unavailable";
|
|
19
|
+
readonly values: readonly {
|
|
20
|
+
readonly currency: string;
|
|
21
|
+
readonly value: string;
|
|
22
|
+
readonly source: "reported" | "estimated" | "mixed";
|
|
23
|
+
readonly coveredContributions: number;
|
|
24
|
+
readonly reportedContributions: number;
|
|
25
|
+
readonly estimatedContributions: number;
|
|
26
|
+
}[];
|
|
27
|
+
readonly totalContributions: number;
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export interface AgentUsageContribution {
|
|
31
|
+
readonly sessionScopeId: string;
|
|
32
|
+
readonly turnId: string;
|
|
33
|
+
readonly sendAttempt: number;
|
|
34
|
+
readonly state: "pending" | "terminal";
|
|
35
|
+
readonly outcome: "completed" | "failed" | "interrupted" | null;
|
|
36
|
+
readonly model: string | null;
|
|
37
|
+
readonly usage: Readonly<Usage> | null;
|
|
38
|
+
readonly coverage: ResolvedEvidenceCoverage["usage"];
|
|
39
|
+
}
|
|
40
|
+
export declare function projectAdapterEvalUsage(usage: ReadableAdapterUsageAttachment): EvalUsage;
|
|
41
|
+
/** The receiver selects scope membership before calling this pure projection. */
|
|
42
|
+
export declare function projectAgentEvalUsage(input: {
|
|
43
|
+
contributions: readonly AgentUsageContribution[];
|
|
44
|
+
scope: "turn" | "session" | "attempt";
|
|
45
|
+
pricing?: Record<string, PriceOverride>;
|
|
46
|
+
}): EvalUsage;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { NumericMaterial } from "../assertions/match.js";
|
|
2
|
+
import type { ResolvedEvidenceCoverage } from "../assertions/coverage.js";
|
|
3
|
+
import type { PriceOverride, Usage } from "../types.js";
|
|
4
|
+
import type { ReadableAdapterUsageAttachment } from "../record/family/adapter-usage/schema.js";
|
|
5
|
+
/** Internal source-owned shape; package-root exposure belongs to the author facade. */
|
|
6
|
+
export interface EvalUsage {
|
|
7
|
+
readonly source: "agent" | "adapter" | "unavailable";
|
|
8
|
+
readonly scope: "turn" | "session" | "attempt";
|
|
9
|
+
readonly basis: "reported-sends" | "recorded-calls" | "unbound";
|
|
10
|
+
readonly cut: "call-time";
|
|
11
|
+
readonly inputTokens: NumericMaterial;
|
|
12
|
+
readonly inputTotalTokens: NumericMaterial;
|
|
13
|
+
readonly outputTokens: NumericMaterial;
|
|
14
|
+
readonly cacheReadTokens: NumericMaterial;
|
|
15
|
+
readonly cacheWriteTokens: NumericMaterial;
|
|
16
|
+
readonly totalTokens: NumericMaterial;
|
|
17
|
+
readonly costs: {
|
|
18
|
+
readonly state: "complete" | "partial" | "unavailable";
|
|
19
|
+
readonly values: readonly {
|
|
20
|
+
readonly currency: string;
|
|
21
|
+
readonly value: string;
|
|
22
|
+
readonly source: "reported" | "estimated" | "mixed";
|
|
23
|
+
readonly coveredContributions: number;
|
|
24
|
+
readonly reportedContributions: number;
|
|
25
|
+
readonly estimatedContributions: number;
|
|
26
|
+
}[];
|
|
27
|
+
readonly totalContributions: number;
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
export interface AgentUsageContribution {
|
|
31
|
+
readonly sessionScopeId: string;
|
|
32
|
+
readonly turnId: string;
|
|
33
|
+
readonly sendAttempt: number;
|
|
34
|
+
readonly state: "pending" | "terminal";
|
|
35
|
+
readonly outcome: "completed" | "failed" | "interrupted" | null;
|
|
36
|
+
readonly model: string | null;
|
|
37
|
+
readonly usage: Readonly<Usage> | null;
|
|
38
|
+
readonly coverage: ResolvedEvidenceCoverage["usage"];
|
|
39
|
+
}
|
|
40
|
+
export declare function projectAdapterEvalUsage(usage: ReadableAdapterUsageAttachment): EvalUsage;
|
|
41
|
+
/** The receiver selects scope membership before calling this pure projection. */
|
|
42
|
+
export declare function projectAgentEvalUsage(input: {
|
|
43
|
+
contributions: readonly AgentUsageContribution[];
|
|
44
|
+
scope: "turn" | "session" | "attempt";
|
|
45
|
+
pricing?: Record<string, PriceOverride>;
|
|
46
|
+
}): EvalUsage;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { createRequire as __niceevalCreateRequire } from "node:module";
|
|
2
|
+
const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
|
|
3
|
+
const __niceevalCanonical = __niceevalRequire("./eval-usage.cjs");
|
|
4
|
+
const __niceevalExport0 = __niceevalCanonical["projectAdapterEvalUsage"];
|
|
5
|
+
export { __niceevalExport0 as projectAdapterEvalUsage };
|
|
6
|
+
const __niceevalExport1 = __niceevalCanonical["projectAgentEvalUsage"];
|
|
7
|
+
export { __niceevalExport1 as projectAgentEvalUsage };
|
|
8
|
+
//# sourceMappingURL=eval-usage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval-usage.js","sources":[],"names":[],"mappings":""}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createJudgeCallPriceReceipts = createJudgeCallPriceReceipts;
|
|
4
|
+
const effect_1 = require("effect");
|
|
5
|
+
const core_ts_1 = require("../record/codec/core.cjs");
|
|
6
|
+
const schema_ts_1 = require("../record/family/judge-usage/schema.cjs");
|
|
7
|
+
const pricing_ts_1 = require("../record/family/adapter-usage/pricing.cjs");
|
|
8
|
+
const adapter_call_price_ts_1 = require("./adapter-call-price.cjs");
|
|
9
|
+
function tokensFor(call, bucket) {
|
|
10
|
+
switch (bucket) {
|
|
11
|
+
case "input": return call.inputTokens;
|
|
12
|
+
case "output": return call.outputTokens;
|
|
13
|
+
case "cache-read": return call.cacheReadTokens;
|
|
14
|
+
case "cache-write": return call.cacheWriteTokens;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
function createJudgeCallPriceReceipts(calls, pricing) {
|
|
18
|
+
return Object.freeze(calls.flatMap((call) => {
|
|
19
|
+
// A reported zero is a complete reported observation, not a missing price.
|
|
20
|
+
if (call.cost !== null)
|
|
21
|
+
return [];
|
|
22
|
+
const selected = (0, adapter_call_price_ts_1.configuredProfile)(call.requestModel, pricing);
|
|
23
|
+
if (selected === undefined)
|
|
24
|
+
return [];
|
|
25
|
+
const digestInput = {
|
|
26
|
+
basis: selected.basis, currency: selected.currency, source: selected.source,
|
|
27
|
+
requestModel: selected.requestModel, selector: selected.selector,
|
|
28
|
+
match: selected.match, ratesPerMTok: selected.ratesPerMTok,
|
|
29
|
+
};
|
|
30
|
+
const charges = [];
|
|
31
|
+
const missing = [];
|
|
32
|
+
for (const bucket of pricing_ts_1.AdapterCallPriceBuckets) {
|
|
33
|
+
const tokens = tokensFor(call, bucket);
|
|
34
|
+
const rate = (0, adapter_call_price_ts_1.rateFor)(selected, bucket);
|
|
35
|
+
if (tokens === null)
|
|
36
|
+
missing.push({ bucket, reason: "tokens-unknown" });
|
|
37
|
+
else if (tokens === 0)
|
|
38
|
+
charges.push({ bucket, tokens, ratePerMTok: rate, amount: "0" });
|
|
39
|
+
else if (rate === null)
|
|
40
|
+
missing.push({ bucket, reason: "rate-unknown" });
|
|
41
|
+
else
|
|
42
|
+
charges.push({ bucket, tokens, ratePerMTok: rate, amount: (0, pricing_ts_1.multiplyCanonicalDecimalPerMillion)(rate, tokens) });
|
|
43
|
+
}
|
|
44
|
+
if (charges.length === 0)
|
|
45
|
+
return [];
|
|
46
|
+
const decoded = effect_1.Schema.decodeUnknownResult(schema_ts_1.JudgePriceReceiptSchema, core_ts_1.RecordExactParseOptions)({
|
|
47
|
+
kind: "judge-call-price-estimate", callId: call.callId,
|
|
48
|
+
state: missing.length === 0 ? "complete" : "partial",
|
|
49
|
+
amount: (0, pricing_ts_1.addCanonicalDecimals)(charges.map((charge) => String(charge.amount))),
|
|
50
|
+
currency: "USD", source: { kind: "estimated", id: selected.source.id },
|
|
51
|
+
pricing: {
|
|
52
|
+
...digestInput,
|
|
53
|
+
source: { ...selected.source, profileDigest: (0, pricing_ts_1.adapterCallPricingProfileDigest)(digestInput) },
|
|
54
|
+
},
|
|
55
|
+
charges, missing,
|
|
56
|
+
});
|
|
57
|
+
if (effect_1.Result.isFailure(decoded))
|
|
58
|
+
throw new Error("Generated Judge call price receipt is invalid");
|
|
59
|
+
return [Object.freeze(decoded.success)];
|
|
60
|
+
}));
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=judge-call-price.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-call-price.js","sourceRoot":"","sources":["judge-call-price.ts"],"names":[],"mappings":";;AAcA,oEA4CC;AA1DD,mCAAwC;AAExC,qDAAkE;AAClE,sEAA8H;AAC9H,0EAAgL;AAChL,mEAAqE;AACrE,SAAS,SAAS,CAAC,IAAoB,EAAE,MAAgD;IACrF,QAAQ,MAAM,EAAE,CAAC;QACb,KAAK,OAAO,CAAC,CAAC,OAAO,IAAI,CAAC,WAAW,CAAC;QACtC,KAAK,QAAQ,CAAC,CAAC,OAAO,IAAI,CAAC,YAAY,CAAC;QACxC,KAAK,YAAY,CAAC,CAAC,OAAO,IAAI,CAAC,eAAe,CAAC;QAC/C,KAAK,aAAa,CAAC,CAAC,OAAO,IAAI,CAAC,gBAAgB,CAAC;IACrD,CAAC;AACL,CAAC;AACD,SAAgB,4BAA4B,CAAC,KAAgC,EAAE,OAA4D;IACvI,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAgC,EAAE;QACtE,2EAA2E;QAC3E,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI;YAClB,OAAO,EAAE,CAAC;QACd,MAAM,QAAQ,GAAG,IAAA,yCAAiB,EAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAC/D,IAAI,QAAQ,KAAK,SAAS;YACtB,OAAO,EAAE,CAAC;QACd,MAAM,WAAW,GAAG;YAChB,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM;YAC3E,YAAY,EAAE,QAAQ,CAAC,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAChE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY,EAAE,QAAQ,CAAC,YAAY;SAC7D,CAAC;QACF,MAAM,OAAO,GAAmC,EAAE,CAAC;QACnD,MAAM,OAAO,GAAmC,EAAE,CAAC;QACnD,KAAK,MAAM,MAAM,IAAI,oCAAuB,EAAE,CAAC;YAC3C,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACvC,MAAM,IAAI,GAAG,IAAA,+BAAO,EAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YACvC,IAAI,MAAM,KAAK,IAAI;gBACf,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,CAAC,CAAC;iBAClD,IAAI,MAAM,KAAK,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;iBAChE,IAAI,IAAI,KAAK,IAAI;gBAClB,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;;gBAEjD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,IAAA,+CAAkC,EAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACtH,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YACpB,OAAO,EAAE,CAAC;QACd,MAAM,OAAO,GAAG,eAAM,CAAC,mBAAmB,CAAC,mCAAuB,EAAE,iCAAuB,CAAC,CAAC;YACzF,IAAI,EAAE,2BAA2B,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM;YACtD,KAAK,EAAE,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YACpD,MAAM,EAAE,IAAA,iCAAoB,EAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YAC5E,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE;YACtE,OAAO,EAAE;gBACL,GAAG,WAAW;gBACd,MAAM,EAAE,EAAE,GAAG,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,IAAA,4CAA+B,EAAC,WAAW,CAAC,EAAE;aAC9F;YACD,OAAO,EAAE,OAAO;SACnB,CAAC,CAAC;QACH,IAAI,eAAM,CAAC,SAAS,CAAC,OAAO,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACrE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC,CAAC;AACR,CAAC","sourcesContent":["import { Result, Schema } from \"effect\";\nimport type { PriceOverride } from \"../runner/types.ts\";\nimport { RecordExactParseOptions } from \"../record/codec/core.ts\";\nimport { JudgePriceReceiptSchema, type JudgePriceReceipt, type JudgeUsageCall } from \"../record/family/judge-usage/schema.ts\";\nimport { AdapterCallPriceBuckets, adapterCallPricingProfileDigest, addCanonicalDecimals, multiplyCanonicalDecimalPerMillion, } from \"../record/family/adapter-usage/pricing.ts\";\nimport { configuredProfile, rateFor } from \"./adapter-call-price.ts\";\nfunction tokensFor(call: JudgeUsageCall, bucket: (typeof AdapterCallPriceBuckets)[number]): number | null {\n switch (bucket) {\n case \"input\": return call.inputTokens;\n case \"output\": return call.outputTokens;\n case \"cache-read\": return call.cacheReadTokens;\n case \"cache-write\": return call.cacheWriteTokens;\n }\n}\nexport function createJudgeCallPriceReceipts(calls: readonly JudgeUsageCall[], pricing: Readonly<Record<string, PriceOverride>> | undefined): readonly JudgePriceReceipt[] {\n return Object.freeze(calls.flatMap((call): readonly JudgePriceReceipt[] => {\n // A reported zero is a complete reported observation, not a missing price.\n if (call.cost !== null)\n return [];\n const selected = configuredProfile(call.requestModel, pricing);\n if (selected === undefined)\n return [];\n const digestInput = {\n basis: selected.basis, currency: selected.currency, source: selected.source,\n requestModel: selected.requestModel, selector: selected.selector,\n match: selected.match, ratesPerMTok: selected.ratesPerMTok,\n };\n const charges: Array<Record<string, unknown>> = [];\n const missing: Array<Record<string, unknown>> = [];\n for (const bucket of AdapterCallPriceBuckets) {\n const tokens = tokensFor(call, bucket);\n const rate = rateFor(selected, bucket);\n if (tokens === null)\n missing.push({ bucket, reason: \"tokens-unknown\" });\n else if (tokens === 0)\n charges.push({ bucket, tokens, ratePerMTok: rate, amount: \"0\" });\n else if (rate === null)\n missing.push({ bucket, reason: \"rate-unknown\" });\n else\n charges.push({ bucket, tokens, ratePerMTok: rate, amount: multiplyCanonicalDecimalPerMillion(rate, tokens) });\n }\n if (charges.length === 0)\n return [];\n const decoded = Schema.decodeUnknownResult(JudgePriceReceiptSchema, RecordExactParseOptions)({\n kind: \"judge-call-price-estimate\", callId: call.callId,\n state: missing.length === 0 ? \"complete\" : \"partial\",\n amount: addCanonicalDecimals(charges.map((charge) => String(charge.amount))),\n currency: \"USD\", source: { kind: \"estimated\", id: selected.source.id },\n pricing: {\n ...digestInput,\n source: { ...selected.source, profileDigest: adapterCallPricingProfileDigest(digestInput) },\n },\n charges, missing,\n });\n if (Result.isFailure(decoded))\n throw new Error(\"Generated Judge call price receipt is invalid\");\n return [Object.freeze(decoded.success)];\n }));\n}\n"]}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { PriceOverride } from "../runner/types.cjs";
|
|
2
|
+
import { type JudgePriceReceipt, type JudgeUsageCall } from "../record/family/judge-usage/schema.cjs";
|
|
3
|
+
export declare function createJudgeCallPriceReceipts(calls: readonly JudgeUsageCall[], pricing: Readonly<Record<string, PriceOverride>> | undefined): readonly JudgePriceReceipt[];
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { PriceOverride } from "../runner/types.mjs";
|
|
2
|
+
import { type JudgePriceReceipt, type JudgeUsageCall } from "../record/family/judge-usage/schema.mjs";
|
|
3
|
+
export declare function createJudgeCallPriceReceipts(calls: readonly JudgeUsageCall[], pricing: Readonly<Record<string, PriceOverride>> | undefined): readonly JudgePriceReceipt[];
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
import type { PriceOverride } from "../runner/types.js";
|
|
2
|
+
import { type JudgePriceReceipt, type JudgeUsageCall } from "../record/family/judge-usage/schema.js";
|
|
3
|
+
export declare function createJudgeCallPriceReceipts(calls: readonly JudgeUsageCall[], pricing: Readonly<Record<string, PriceOverride>> | undefined): readonly JudgePriceReceipt[];
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { createRequire as __niceevalCreateRequire } from "node:module";
|
|
2
|
+
const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
|
|
3
|
+
const __niceevalCanonical = __niceevalRequire("./judge-call-price.cjs");
|
|
4
|
+
const __niceevalExport0 = __niceevalCanonical["createJudgeCallPriceReceipts"];
|
|
5
|
+
export { __niceevalExport0 as createJudgeCallPriceReceipts };
|
|
6
|
+
//# sourceMappingURL=judge-call-price.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-call-price.js","sources":[],"names":[],"mappings":""}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.JUDGE_USAGE_PREVIEW_BYTE_LIMIT = exports.JUDGE_USAGE_PREVIEW_CALL_LIMIT = void 0;
|
|
4
|
+
exports.compareJudgeUsageCalls = compareJudgeUsageCalls;
|
|
5
|
+
exports.projectJudgeUsage = projectJudgeUsage;
|
|
6
|
+
exports.takeJudgeUsagePreview = takeJudgeUsagePreview;
|
|
7
|
+
exports.summarizeJudgeUsage = summarizeJudgeUsage;
|
|
8
|
+
exports.projectTotalUsageCosts = projectTotalUsageCosts;
|
|
9
|
+
exports.combineTotalUsageCosts = combineTotalUsageCosts;
|
|
10
|
+
const pricing_ts_1 = require("../record/family/adapter-usage/pricing.cjs");
|
|
11
|
+
const adapter_usage_projection_ts_1 = require("./adapter-usage-projection.cjs");
|
|
12
|
+
const bytes_ts_1 = require("../inspection/bytes.cjs");
|
|
13
|
+
exports.JUDGE_USAGE_PREVIEW_CALL_LIMIT = 128;
|
|
14
|
+
exports.JUDGE_USAGE_PREVIEW_BYTE_LIMIT = 128 * 1024;
|
|
15
|
+
function compareJudgeUsageCalls(left, right) {
|
|
16
|
+
return left.entryIndex - right.entryIndex || left.logicalOrdinal - right.logicalOrdinal ||
|
|
17
|
+
left.transmissionOrdinal - right.transmissionOrdinal;
|
|
18
|
+
}
|
|
19
|
+
/** The ledger is already validated; independent metrics retain their own coverage. */
|
|
20
|
+
function projectJudgeUsage(ledger) {
|
|
21
|
+
const receipts = new Map(ledger.priceReceipts.map((receipt) => [receipt.callId, receipt]));
|
|
22
|
+
const costCalls = ledger.calls.map((call) => {
|
|
23
|
+
const receipt = receipts.get(call.callId);
|
|
24
|
+
return { effectiveCost: call.cost !== null ? { ...call.cost, state: "complete" }
|
|
25
|
+
: receipt === undefined ? null
|
|
26
|
+
: { amount: receipt.amount, currency: receipt.currency, source: receipt.source, state: receipt.state } };
|
|
27
|
+
});
|
|
28
|
+
const costs = (0, adapter_usage_projection_ts_1.effectiveCostTotals)(costCalls, ledger.collection.state);
|
|
29
|
+
const numeric = (metric) => {
|
|
30
|
+
let value = 0;
|
|
31
|
+
let observationCount = 0;
|
|
32
|
+
let complete = ledger.collection.state === "complete";
|
|
33
|
+
for (const call of ledger.calls) {
|
|
34
|
+
const inputBuckets = [call.inputTokens, call.cacheReadTokens, call.cacheWriteTokens];
|
|
35
|
+
const knownInputs = inputBuckets.filter((bucket) => bucket !== null);
|
|
36
|
+
const input = call.inputTotalTokens ?? (knownInputs.length === 0 ? null
|
|
37
|
+
: knownInputs.reduce((sum, bucket) => sum + bucket, 0));
|
|
38
|
+
const buckets = metric === "inputTotalTokens" ? [input]
|
|
39
|
+
: metric === "outputTokens" ? [call.outputTokens] : [input, call.outputTokens];
|
|
40
|
+
if (buckets.some((bucket) => bucket !== null))
|
|
41
|
+
observationCount++;
|
|
42
|
+
for (const bucket of buckets) {
|
|
43
|
+
if (bucket === null)
|
|
44
|
+
complete = false;
|
|
45
|
+
else
|
|
46
|
+
value += bucket;
|
|
47
|
+
}
|
|
48
|
+
if (metric !== "outputTokens" && call.inputTotalTokens === null && knownInputs.length !== 3)
|
|
49
|
+
complete = false;
|
|
50
|
+
}
|
|
51
|
+
if (!Number.isSafeInteger(value) || observationCount === 0 && ledger.calls.length > 0 ||
|
|
52
|
+
ledger.calls.length === 0 && !complete) {
|
|
53
|
+
return Object.freeze({ state: "unavailable", value: null, observationCount });
|
|
54
|
+
}
|
|
55
|
+
return Object.freeze({ state: complete ? "available" : "partial", value, observationCount });
|
|
56
|
+
};
|
|
57
|
+
const all = Object.freeze({
|
|
58
|
+
state: ledger.collection.state,
|
|
59
|
+
coverage: "physical-transmissions",
|
|
60
|
+
collection: ledger.collection,
|
|
61
|
+
calls: Object.freeze([...ledger.calls].sort(compareJudgeUsageCalls)),
|
|
62
|
+
callsTruncated: false,
|
|
63
|
+
omittedCallCount: 0,
|
|
64
|
+
totals: Object.freeze({
|
|
65
|
+
requests: Object.freeze({ state: ledger.collection.state === "complete" ? "available" : "partial",
|
|
66
|
+
value: ledger.calls.length, observationCount: ledger.calls.length }),
|
|
67
|
+
inputTotalTokens: numeric("inputTotalTokens"), outputTokens: numeric("outputTokens"), totalTokens: numeric("totalTokens"),
|
|
68
|
+
costs: ledger.collection.state === "complete" && ledger.calls.length === 0
|
|
69
|
+
? Object.freeze({ ...costs, state: "complete" }) : costs,
|
|
70
|
+
}),
|
|
71
|
+
priceReceipts: ledger.priceReceipts,
|
|
72
|
+
});
|
|
73
|
+
let preview = takeJudgeUsagePreview(all, exports.JUDGE_USAGE_PREVIEW_CALL_LIMIT);
|
|
74
|
+
while ((0, bytes_ts_1.utf8ByteLength)(JSON.stringify({ calls: preview.calls, priceReceipts: preview.priceReceipts })) >
|
|
75
|
+
exports.JUDGE_USAGE_PREVIEW_BYTE_LIMIT && preview.calls.length > 0) {
|
|
76
|
+
preview = takeJudgeUsagePreview(preview, preview.calls.length - 1);
|
|
77
|
+
}
|
|
78
|
+
return preview;
|
|
79
|
+
}
|
|
80
|
+
/** Shrinking details never changes totals and retains only the corresponding price proofs. */
|
|
81
|
+
function takeJudgeUsagePreview(usage, maximumCalls) {
|
|
82
|
+
const calls = Object.freeze(usage.calls.slice(0, maximumCalls));
|
|
83
|
+
const receipts = new Map(usage.priceReceipts.map((receipt) => [receipt.callId, receipt]));
|
|
84
|
+
const priceReceipts = Object.freeze(calls.flatMap((call) => {
|
|
85
|
+
const receipt = receipts.get(call.callId);
|
|
86
|
+
return receipt === undefined ? [] : [receipt];
|
|
87
|
+
}));
|
|
88
|
+
const omittedCallCount = usage.omittedCallCount + usage.calls.length - calls.length;
|
|
89
|
+
return Object.freeze({ ...usage, calls, priceReceipts, callsTruncated: omittedCallCount > 0, omittedCallCount });
|
|
90
|
+
}
|
|
91
|
+
function summarizeJudgeUsage(usage) {
|
|
92
|
+
if (usage.state === "unavailable" || usage.state === "invalid")
|
|
93
|
+
return usage;
|
|
94
|
+
return Object.freeze({ state: usage.state, coverage: usage.coverage, collection: usage.collection, totals: usage.totals });
|
|
95
|
+
}
|
|
96
|
+
/** Currency-preserving known subtotals; no exchange rate or fabricated zero currency. */
|
|
97
|
+
function projectTotalUsageCosts(application, judge, applicationEmptyComplete = false, applicationCollectionProven = true) {
|
|
98
|
+
const applicationValues = application.costs !== undefined ? application.costs.values
|
|
99
|
+
: (application.providerCosts?.values ?? []).map(({ currency, value }) => ({ currency, value, source: "reported" }));
|
|
100
|
+
const judgeCosts = judge.state === "complete" || judge.state === "partial" ? judge.totals.costs : undefined;
|
|
101
|
+
const completeApplication = applicationCollectionProven && (applicationEmptyComplete || application.costs?.state === "complete" ||
|
|
102
|
+
application.costs === undefined && application.providerCosts?.state === "available");
|
|
103
|
+
const missingSources = [];
|
|
104
|
+
if (!completeApplication)
|
|
105
|
+
missingSources.push("application");
|
|
106
|
+
if (judgeCosts?.state !== "complete")
|
|
107
|
+
missingSources.push("judge");
|
|
108
|
+
const values = sumCostValues([...applicationValues, ...(judgeCosts?.values ?? [])]);
|
|
109
|
+
return Object.freeze({ state: missingSources.length === 0 ? "complete" : values.length === 0 ? "unavailable" : "partial",
|
|
110
|
+
values, missingSources: Object.freeze(missingSources) });
|
|
111
|
+
}
|
|
112
|
+
/** Inputs are unique origin Attempts, never preview rows or verdict-filtered members. */
|
|
113
|
+
function combineTotalUsageCosts(totals, unresolved = false) {
|
|
114
|
+
const missingSources = ["application", "judge"].filter((source) => unresolved || totals.some((total) => total.missingSources.includes(source)));
|
|
115
|
+
const values = sumCostValues(totals.flatMap((total) => total.values));
|
|
116
|
+
return Object.freeze({ state: missingSources.length === 0 ? "complete" : values.length === 0 ? "unavailable" : "partial",
|
|
117
|
+
values, missingSources: Object.freeze(missingSources) });
|
|
118
|
+
}
|
|
119
|
+
function sumCostValues(entries) {
|
|
120
|
+
const grouped = new Map();
|
|
121
|
+
for (const entry of entries) {
|
|
122
|
+
const group = grouped.get(entry.currency);
|
|
123
|
+
if (group === undefined)
|
|
124
|
+
grouped.set(entry.currency, [entry]);
|
|
125
|
+
else
|
|
126
|
+
group.push(entry);
|
|
127
|
+
}
|
|
128
|
+
return Object.freeze([...grouped.entries()].sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0)
|
|
129
|
+
.map(([currency, entries]) => Object.freeze({
|
|
130
|
+
currency, value: (0, pricing_ts_1.addCanonicalDecimals)(entries.map((entry) => entry.value)),
|
|
131
|
+
source: entries.every((entry) => entry.source === entries[0].source) ? entries[0].source : "mixed",
|
|
132
|
+
})));
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=judge-usage-projection.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-usage-projection.js","sourceRoot":"","sources":["judge-usage-projection.ts"],"names":[],"mappings":";;;AAOA,wDAGC;AAED,8CA+DC;AAED,sDAaC;AACD,kDAIC;AAED,wDAcC;AAED,wDAKC;AArHD,0EAAiF;AACjF,+EAA4F;AAE5F,qDAAwD;AAC3C,QAAA,8BAA8B,GAAG,GAAG,CAAC;AACrC,QAAA,8BAA8B,GAAG,GAAG,GAAG,IAAI,CAAC;AACzD,SAAgB,sBAAsB,CAAC,IAAoB,EAAE,KAAqB;IAC9E,OAAO,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC,cAAc;QACnF,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC,mBAAmB,CAAC;AAC7D,CAAC;AACD,sFAAsF;AACtF,SAAgB,iBAAiB,CAAC,MAA4B;IAC1D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3F,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAEtC,EAAE;QACA,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1C,OAAO,EAAE,aAAa,EAAE,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE;gBACxE,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI;oBAC1B,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;IACzH,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,IAAA,iDAAmB,EAAC,SAAS,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,CAAC,MAA2D,EAAE,EAAE;QAC5E,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,KAAK,UAAU,CAAC;QACtD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC9B,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACrF,MAAM,WAAW,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,MAAM,EAAoB,EAAE,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;YACvF,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;gBACnE,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,MAAM,KAAK,kBAAkB,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;gBACnD,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;YACnF,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,IAAI,CAAC;gBACzC,gBAAgB,EAAE,CAAC;YACvB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC3B,IAAI,MAAM,KAAK,IAAI;oBACf,QAAQ,GAAG,KAAK,CAAC;;oBAEjB,KAAK,IAAI,MAAM,CAAC;YACxB,CAAC;YACD,IAAI,MAAM,KAAK,cAAc,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;gBACvF,QAAQ,GAAG,KAAK,CAAC;QACzB,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,gBAAgB,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YACjF,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACzC,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,aAAsB,EAAE,KAAK,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC;QAC3F,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC,WAAoB,CAAC,CAAC,CAAC,SAAkB,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACnH,CAAC,CAAC;IACF,MAAM,GAAG,GAEJ,MAAM,CAAC,MAAM,CAAC;QACf,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK;QAC9B,QAAQ,EAAE,wBAAwB;QAClC,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACpE,cAAc,EAAE,KAAK;QACrB,gBAAgB,EAAE,CAAC;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;YAClB,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;gBAC7F,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,gBAAgB,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACxE,gBAAgB,EAAE,OAAO,CAAC,kBAAkB,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,cAAc,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,aAAa,CAAC;YACzH,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,KAAK,UAAU,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;gBACtE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK;SAC/D,CAAC;QACF,aAAa,EAAE,MAAM,CAAC,aAAa;KACtC,CAAC,CAAC;IACH,IAAI,OAAO,GAAG,qBAAqB,CAAC,GAAG,EAAE,sCAA8B,CAAC,CAAC;IACzE,OAAO,IAAA,yBAAc,EAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;QACjG,sCAA8B,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7D,OAAO,GAAG,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC;AACD,8FAA8F;AAC9F,SAAgB,qBAAqB,CAAC,KAEpC,EAAE,YAAoB;IAGpB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;IAChE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QACvD,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1C,OAAO,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC,CAAC;IACJ,MAAM,gBAAgB,GAAG,KAAK,CAAC,gBAAgB,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IACpF,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,gBAAgB,GAAG,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC;AACrH,CAAC;AACD,SAAgB,mBAAmB,CAAC,KAAiB;IACjD,IAAI,KAAK,CAAC,KAAK,KAAK,aAAa,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;QAC1D,OAAO,KAAK,CAAC;IACjB,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;AAC/H,CAAC;AACD,yFAAyF;AACzF,SAAgB,sBAAsB,CAAC,WAAmD,EAAE,KAAwB,EAAE,wBAAwB,GAAG,KAAK,EAAE,2BAA2B,GAAG,IAAI;IACtL,MAAM,iBAAiB,GAAG,WAAW,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM;QAChF,CAAC,CAAC,CAAC,WAAW,CAAC,aAAa,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,UAAmB,EAAE,CAAC,CAAC,CAAC;IACjI,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,KAAK,UAAU,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IAC5G,MAAM,mBAAmB,GAAG,2BAA2B,IAAI,CAAC,wBAAwB,IAAI,WAAW,CAAC,KAAK,EAAE,KAAK,KAAK,UAAU;QAC3H,WAAW,CAAC,KAAK,KAAK,SAAS,IAAI,WAAW,CAAC,aAAa,EAAE,KAAK,KAAK,WAAW,CAAC,CAAC;IACzF,MAAM,cAAc,GAAgC,EAAE,CAAC;IACvD,IAAI,CAAC,mBAAmB;QACpB,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACvC,IAAI,UAAU,EAAE,KAAK,KAAK,UAAU;QAChC,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,GAAG,iBAAiB,EAAE,GAAG,CAAC,UAAU,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACpF,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,cAAc,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;QACpH,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;AACjE,CAAC;AACD,yFAAyF;AACzF,SAAgB,sBAAsB,CAAC,MAAkC,EAAE,UAAU,GAAG,KAAK;IACzF,MAAM,cAAc,GAAI,CAAC,aAAa,EAAE,OAAO,CAAW,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC3J,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACtE,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,cAAc,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS;QACpH,MAAM,EAAE,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;AACjE,CAAC;AACD,SAAS,aAAa,CAAC,OAAkC;IACrD,MAAM,OAAO,GAAG,IAAI,GAAG,EAGjB,CAAC;IACP,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,KAAK,KAAK,SAAS;YACnB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;;YAErC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SAC1G,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;QAC5C,QAAQ,EAAE,KAAK,EAAE,IAAA,iCAAoB,EAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC1E,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,CAAC,OAAgB;KAChH,CAAC,CAAC,CAAC,CAAC;AACT,CAAC","sourcesContent":["import type { JudgeUsageAttachment, JudgeUsageCall } from \"../record/family/judge-usage/schema.ts\";\nimport { addCanonicalDecimals } from \"../record/family/adapter-usage/pricing.ts\";\nimport { effectiveCostTotals, type EffectiveCallCost } from \"./adapter-usage-projection.ts\";\nimport type { InspectionAttemptUsageResult, JudgeUsage, JudgeUsageSummary, TotalUsageCosts, } from \"../inspection/results.ts\";\nimport { utf8ByteLength } from \"../inspection/bytes.ts\";\nexport const JUDGE_USAGE_PREVIEW_CALL_LIMIT = 128;\nexport const JUDGE_USAGE_PREVIEW_BYTE_LIMIT = 128 * 1024;\nexport function compareJudgeUsageCalls(left: JudgeUsageCall, right: JudgeUsageCall): number {\n return left.entryIndex - right.entryIndex || left.logicalOrdinal - right.logicalOrdinal ||\n left.transmissionOrdinal - right.transmissionOrdinal;\n}\n/** The ledger is already validated; independent metrics retain their own coverage. */\nexport function projectJudgeUsage(ledger: JudgeUsageAttachment): JudgeUsage {\n const receipts = new Map(ledger.priceReceipts.map((receipt) => [receipt.callId, receipt]));\n const costCalls = ledger.calls.map((call): {\n readonly effectiveCost: EffectiveCallCost | null;\n } => {\n const receipt = receipts.get(call.callId);\n return { effectiveCost: call.cost !== null ? { ...call.cost, state: \"complete\" }\n : receipt === undefined ? null\n : { amount: receipt.amount, currency: receipt.currency, source: receipt.source, state: receipt.state } };\n });\n const costs = effectiveCostTotals(costCalls, ledger.collection.state);\n const numeric = (metric: \"inputTotalTokens\" | \"outputTokens\" | \"totalTokens\") => {\n let value = 0;\n let observationCount = 0;\n let complete = ledger.collection.state === \"complete\";\n for (const call of ledger.calls) {\n const inputBuckets = [call.inputTokens, call.cacheReadTokens, call.cacheWriteTokens];\n const knownInputs = inputBuckets.filter((bucket): bucket is number => bucket !== null);\n const input = call.inputTotalTokens ?? (knownInputs.length === 0 ? null\n : knownInputs.reduce((sum, bucket) => sum + bucket, 0));\n const buckets = metric === \"inputTotalTokens\" ? [input]\n : metric === \"outputTokens\" ? [call.outputTokens] : [input, call.outputTokens];\n if (buckets.some((bucket) => bucket !== null))\n observationCount++;\n for (const bucket of buckets) {\n if (bucket === null)\n complete = false;\n else\n value += bucket;\n }\n if (metric !== \"outputTokens\" && call.inputTotalTokens === null && knownInputs.length !== 3)\n complete = false;\n }\n if (!Number.isSafeInteger(value) || observationCount === 0 && ledger.calls.length > 0 ||\n ledger.calls.length === 0 && !complete) {\n return Object.freeze({ state: \"unavailable\" as const, value: null, observationCount });\n }\n return Object.freeze({ state: complete ? \"available\" as const : \"partial\" as const, value, observationCount });\n };\n const all: Extract<JudgeUsage, {\n readonly state: \"complete\" | \"partial\";\n }> = Object.freeze({\n state: ledger.collection.state,\n coverage: \"physical-transmissions\",\n collection: ledger.collection,\n calls: Object.freeze([...ledger.calls].sort(compareJudgeUsageCalls)),\n callsTruncated: false,\n omittedCallCount: 0,\n totals: Object.freeze({\n requests: Object.freeze({ state: ledger.collection.state === \"complete\" ? \"available\" : \"partial\",\n value: ledger.calls.length, observationCount: ledger.calls.length }),\n inputTotalTokens: numeric(\"inputTotalTokens\"), outputTokens: numeric(\"outputTokens\"), totalTokens: numeric(\"totalTokens\"),\n costs: ledger.collection.state === \"complete\" && ledger.calls.length === 0\n ? Object.freeze({ ...costs, state: \"complete\" }) : costs,\n }),\n priceReceipts: ledger.priceReceipts,\n });\n let preview = takeJudgeUsagePreview(all, JUDGE_USAGE_PREVIEW_CALL_LIMIT);\n while (utf8ByteLength(JSON.stringify({ calls: preview.calls, priceReceipts: preview.priceReceipts })) >\n JUDGE_USAGE_PREVIEW_BYTE_LIMIT && preview.calls.length > 0) {\n preview = takeJudgeUsagePreview(preview, preview.calls.length - 1);\n }\n return preview;\n}\n/** Shrinking details never changes totals and retains only the corresponding price proofs. */\nexport function takeJudgeUsagePreview(usage: Extract<JudgeUsage, {\n readonly state: \"complete\" | \"partial\";\n}>, maximumCalls: number): Extract<JudgeUsage, {\n readonly state: \"complete\" | \"partial\";\n}> {\n const calls = Object.freeze(usage.calls.slice(0, maximumCalls));\n const receipts = new Map(usage.priceReceipts.map((receipt) => [receipt.callId, receipt]));\n const priceReceipts = Object.freeze(calls.flatMap((call) => {\n const receipt = receipts.get(call.callId);\n return receipt === undefined ? [] : [receipt];\n }));\n const omittedCallCount = usage.omittedCallCount + usage.calls.length - calls.length;\n return Object.freeze({ ...usage, calls, priceReceipts, callsTruncated: omittedCallCount > 0, omittedCallCount });\n}\nexport function summarizeJudgeUsage(usage: JudgeUsage): JudgeUsageSummary {\n if (usage.state === \"unavailable\" || usage.state === \"invalid\")\n return usage;\n return Object.freeze({ state: usage.state, coverage: usage.coverage, collection: usage.collection, totals: usage.totals });\n}\n/** Currency-preserving known subtotals; no exchange rate or fabricated zero currency. */\nexport function projectTotalUsageCosts(application: InspectionAttemptUsageResult[\"totals\"], judge: JudgeUsageSummary, applicationEmptyComplete = false, applicationCollectionProven = true): TotalUsageCosts {\n const applicationValues = application.costs !== undefined ? application.costs.values\n : (application.providerCosts?.values ?? []).map(({ currency, value }) => ({ currency, value, source: \"reported\" as const }));\n const judgeCosts = judge.state === \"complete\" || judge.state === \"partial\" ? judge.totals.costs : undefined;\n const completeApplication = applicationCollectionProven && (applicationEmptyComplete || application.costs?.state === \"complete\" ||\n application.costs === undefined && application.providerCosts?.state === \"available\");\n const missingSources: (\"application\" | \"judge\")[] = [];\n if (!completeApplication)\n missingSources.push(\"application\");\n if (judgeCosts?.state !== \"complete\")\n missingSources.push(\"judge\");\n const values = sumCostValues([...applicationValues, ...(judgeCosts?.values ?? [])]);\n return Object.freeze({ state: missingSources.length === 0 ? \"complete\" : values.length === 0 ? \"unavailable\" : \"partial\",\n values, missingSources: Object.freeze(missingSources) });\n}\n/** Inputs are unique origin Attempts, never preview rows or verdict-filtered members. */\nexport function combineTotalUsageCosts(totals: readonly TotalUsageCosts[], unresolved = false): TotalUsageCosts {\n const missingSources = ([\"application\", \"judge\"] as const).filter((source) => unresolved || totals.some((total) => total.missingSources.includes(source)));\n const values = sumCostValues(totals.flatMap((total) => total.values));\n return Object.freeze({ state: missingSources.length === 0 ? \"complete\" : values.length === 0 ? \"unavailable\" : \"partial\",\n values, missingSources: Object.freeze(missingSources) });\n}\nfunction sumCostValues(entries: TotalUsageCosts[\"values\"]): TotalUsageCosts[\"values\"] {\n const grouped = new Map<string, {\n readonly value: string;\n readonly source: \"reported\" | \"estimated\" | \"mixed\";\n }[]>();\n for (const entry of entries) {\n const group = grouped.get(entry.currency);\n if (group === undefined)\n grouped.set(entry.currency, [entry]);\n else\n group.push(entry);\n }\n return Object.freeze([...grouped.entries()].sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0)\n .map(([currency, entries]) => Object.freeze({\n currency, value: addCanonicalDecimals(entries.map((entry) => entry.value)),\n source: entries.every((entry) => entry.source === entries[0]!.source) ? entries[0]!.source : \"mixed\" as const,\n })));\n}\n"]}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { JudgeUsageAttachment, JudgeUsageCall } from "../record/family/judge-usage/schema.cjs";
|
|
2
|
+
import type { InspectionAttemptUsageResult, JudgeUsage, JudgeUsageSummary, TotalUsageCosts } from "../inspection/results.cjs";
|
|
3
|
+
export declare const JUDGE_USAGE_PREVIEW_CALL_LIMIT = 128;
|
|
4
|
+
export declare const JUDGE_USAGE_PREVIEW_BYTE_LIMIT: number;
|
|
5
|
+
export declare function compareJudgeUsageCalls(left: JudgeUsageCall, right: JudgeUsageCall): number;
|
|
6
|
+
/** The ledger is already validated; independent metrics retain their own coverage. */
|
|
7
|
+
export declare function projectJudgeUsage(ledger: JudgeUsageAttachment): JudgeUsage;
|
|
8
|
+
/** Shrinking details never changes totals and retains only the corresponding price proofs. */
|
|
9
|
+
export declare function takeJudgeUsagePreview(usage: Extract<JudgeUsage, {
|
|
10
|
+
readonly state: "complete" | "partial";
|
|
11
|
+
}>, maximumCalls: number): Extract<JudgeUsage, {
|
|
12
|
+
readonly state: "complete" | "partial";
|
|
13
|
+
}>;
|
|
14
|
+
export declare function summarizeJudgeUsage(usage: JudgeUsage): JudgeUsageSummary;
|
|
15
|
+
/** Currency-preserving known subtotals; no exchange rate or fabricated zero currency. */
|
|
16
|
+
export declare function projectTotalUsageCosts(application: InspectionAttemptUsageResult["totals"], judge: JudgeUsageSummary, applicationEmptyComplete?: boolean, applicationCollectionProven?: boolean): TotalUsageCosts;
|
|
17
|
+
/** Inputs are unique origin Attempts, never preview rows or verdict-filtered members. */
|
|
18
|
+
export declare function combineTotalUsageCosts(totals: readonly TotalUsageCosts[], unresolved?: boolean): TotalUsageCosts;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { JudgeUsageAttachment, JudgeUsageCall } from "../record/family/judge-usage/schema.mjs";
|
|
2
|
+
import type { InspectionAttemptUsageResult, JudgeUsage, JudgeUsageSummary, TotalUsageCosts } from "../inspection/results.mjs";
|
|
3
|
+
export declare const JUDGE_USAGE_PREVIEW_CALL_LIMIT = 128;
|
|
4
|
+
export declare const JUDGE_USAGE_PREVIEW_BYTE_LIMIT: number;
|
|
5
|
+
export declare function compareJudgeUsageCalls(left: JudgeUsageCall, right: JudgeUsageCall): number;
|
|
6
|
+
/** The ledger is already validated; independent metrics retain their own coverage. */
|
|
7
|
+
export declare function projectJudgeUsage(ledger: JudgeUsageAttachment): JudgeUsage;
|
|
8
|
+
/** Shrinking details never changes totals and retains only the corresponding price proofs. */
|
|
9
|
+
export declare function takeJudgeUsagePreview(usage: Extract<JudgeUsage, {
|
|
10
|
+
readonly state: "complete" | "partial";
|
|
11
|
+
}>, maximumCalls: number): Extract<JudgeUsage, {
|
|
12
|
+
readonly state: "complete" | "partial";
|
|
13
|
+
}>;
|
|
14
|
+
export declare function summarizeJudgeUsage(usage: JudgeUsage): JudgeUsageSummary;
|
|
15
|
+
/** Currency-preserving known subtotals; no exchange rate or fabricated zero currency. */
|
|
16
|
+
export declare function projectTotalUsageCosts(application: InspectionAttemptUsageResult["totals"], judge: JudgeUsageSummary, applicationEmptyComplete?: boolean, applicationCollectionProven?: boolean): TotalUsageCosts;
|
|
17
|
+
/** Inputs are unique origin Attempts, never preview rows or verdict-filtered members. */
|
|
18
|
+
export declare function combineTotalUsageCosts(totals: readonly TotalUsageCosts[], unresolved?: boolean): TotalUsageCosts;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { JudgeUsageAttachment, JudgeUsageCall } from "../record/family/judge-usage/schema.js";
|
|
2
|
+
import type { InspectionAttemptUsageResult, JudgeUsage, JudgeUsageSummary, TotalUsageCosts } from "../inspection/results.js";
|
|
3
|
+
export declare const JUDGE_USAGE_PREVIEW_CALL_LIMIT = 128;
|
|
4
|
+
export declare const JUDGE_USAGE_PREVIEW_BYTE_LIMIT: number;
|
|
5
|
+
export declare function compareJudgeUsageCalls(left: JudgeUsageCall, right: JudgeUsageCall): number;
|
|
6
|
+
/** The ledger is already validated; independent metrics retain their own coverage. */
|
|
7
|
+
export declare function projectJudgeUsage(ledger: JudgeUsageAttachment): JudgeUsage;
|
|
8
|
+
/** Shrinking details never changes totals and retains only the corresponding price proofs. */
|
|
9
|
+
export declare function takeJudgeUsagePreview(usage: Extract<JudgeUsage, {
|
|
10
|
+
readonly state: "complete" | "partial";
|
|
11
|
+
}>, maximumCalls: number): Extract<JudgeUsage, {
|
|
12
|
+
readonly state: "complete" | "partial";
|
|
13
|
+
}>;
|
|
14
|
+
export declare function summarizeJudgeUsage(usage: JudgeUsage): JudgeUsageSummary;
|
|
15
|
+
/** Currency-preserving known subtotals; no exchange rate or fabricated zero currency. */
|
|
16
|
+
export declare function projectTotalUsageCosts(application: InspectionAttemptUsageResult["totals"], judge: JudgeUsageSummary, applicationEmptyComplete?: boolean, applicationCollectionProven?: boolean): TotalUsageCosts;
|
|
17
|
+
/** Inputs are unique origin Attempts, never preview rows or verdict-filtered members. */
|
|
18
|
+
export declare function combineTotalUsageCosts(totals: readonly TotalUsageCosts[], unresolved?: boolean): TotalUsageCosts;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { createRequire as __niceevalCreateRequire } from "node:module";
|
|
2
|
+
const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
|
|
3
|
+
const __niceevalCanonical = __niceevalRequire("./judge-usage-projection.cjs");
|
|
4
|
+
const __niceevalExport0 = __niceevalCanonical["JUDGE_USAGE_PREVIEW_BYTE_LIMIT"];
|
|
5
|
+
export { __niceevalExport0 as JUDGE_USAGE_PREVIEW_BYTE_LIMIT };
|
|
6
|
+
const __niceevalExport1 = __niceevalCanonical["JUDGE_USAGE_PREVIEW_CALL_LIMIT"];
|
|
7
|
+
export { __niceevalExport1 as JUDGE_USAGE_PREVIEW_CALL_LIMIT };
|
|
8
|
+
const __niceevalExport2 = __niceevalCanonical["combineTotalUsageCosts"];
|
|
9
|
+
export { __niceevalExport2 as combineTotalUsageCosts };
|
|
10
|
+
const __niceevalExport3 = __niceevalCanonical["compareJudgeUsageCalls"];
|
|
11
|
+
export { __niceevalExport3 as compareJudgeUsageCalls };
|
|
12
|
+
const __niceevalExport4 = __niceevalCanonical["projectJudgeUsage"];
|
|
13
|
+
export { __niceevalExport4 as projectJudgeUsage };
|
|
14
|
+
const __niceevalExport5 = __niceevalCanonical["projectTotalUsageCosts"];
|
|
15
|
+
export { __niceevalExport5 as projectTotalUsageCosts };
|
|
16
|
+
const __niceevalExport6 = __niceevalCanonical["summarizeJudgeUsage"];
|
|
17
|
+
export { __niceevalExport6 as summarizeJudgeUsage };
|
|
18
|
+
const __niceevalExport7 = __niceevalCanonical["takeJudgeUsagePreview"];
|
|
19
|
+
export { __niceevalExport7 as takeJudgeUsagePreview };
|
|
20
|
+
//# sourceMappingURL=judge-usage-projection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-usage-projection.js","sources":[],"names":[],"mappings":""}
|