niceeval 0.13.1 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +25 -2
- package/README.md +9 -10
- package/README.zh.md +9 -10
- package/dist/agents/ai-sdk.cjs.map +1 -1
- package/dist/agents/ai-sdk.d.cts +1 -1
- package/dist/agents/ai-sdk.d.mts +1 -1
- package/dist/agents/ai-sdk.d.ts +1 -1
- package/dist/agents/bub-install-spec.cjs +19 -5
- package/dist/agents/bub-install-spec.cjs.map +1 -1
- package/dist/agents/bub-install-spec.d.cts +9 -2
- package/dist/agents/bub-install-spec.d.mts +9 -2
- package/dist/agents/bub-install-spec.d.ts +9 -2
- package/dist/agents/bub-install-spec.js +14 -12
- package/dist/agents/bub.cjs +10 -6
- package/dist/agents/bub.cjs.map +1 -1
- package/dist/agents/bub.d.cts +2 -2
- package/dist/agents/bub.d.mts +2 -2
- package/dist/agents/bub.d.ts +2 -2
- package/dist/agents/claude-code.cjs.map +1 -1
- package/dist/agents/claude-code.d.cts +2 -2
- package/dist/agents/claude-code.d.mts +2 -2
- package/dist/agents/claude-code.d.ts +2 -2
- package/dist/agents/codex.cjs +4 -4
- package/dist/agents/codex.cjs.map +1 -1
- package/dist/agents/codex.d.cts +3 -3
- package/dist/agents/codex.d.mts +3 -3
- package/dist/agents/codex.d.ts +3 -3
- package/dist/agents/coding-cli-versions.cjs +10 -1
- package/dist/agents/coding-cli-versions.cjs.map +1 -1
- package/dist/agents/coding-cli-versions.d.cts +8 -3
- package/dist/agents/coding-cli-versions.d.mts +8 -3
- package/dist/agents/coding-cli-versions.d.ts +8 -3
- package/dist/agents/coding-cli-versions.js +18 -12
- package/dist/agents/deepseek-harness.cjs +151 -0
- package/dist/agents/deepseek-harness.cjs.map +1 -0
- package/dist/agents/deepseek-harness.d.cts +14 -0
- package/dist/agents/deepseek-harness.d.mts +14 -0
- package/dist/agents/deepseek-harness.d.ts +14 -0
- package/dist/agents/deepseek-harness.js +7 -0
- package/dist/agents/deepseek-harness.js.map +1 -0
- package/dist/agents/exact-npm-plugins.cjs +28 -0
- package/dist/agents/exact-npm-plugins.cjs.map +1 -0
- package/dist/agents/exact-npm-plugins.d.cts +10 -0
- package/dist/agents/exact-npm-plugins.d.mts +10 -0
- package/dist/agents/exact-npm-plugins.d.ts +10 -0
- package/dist/agents/exact-npm-plugins.js +8 -0
- package/dist/agents/exact-npm-plugins.js.map +1 -0
- package/dist/agents/index.cjs +5 -1
- package/dist/agents/index.cjs.map +1 -1
- package/dist/agents/index.d.cts +4 -0
- package/dist/agents/index.d.mts +4 -0
- package/dist/agents/index.d.ts +4 -0
- package/dist/agents/index.js +54 -50
- package/dist/agents/index.mjs +54 -50
- package/dist/agents/omp.cjs +170 -0
- package/dist/agents/omp.cjs.map +1 -0
- package/dist/agents/omp.d.cts +14 -0
- package/dist/agents/omp.d.mts +14 -0
- package/dist/agents/omp.d.ts +14 -0
- package/dist/agents/omp.js +7 -0
- package/dist/agents/omp.js.map +1 -0
- package/dist/agents/openclaw.cjs +51 -2
- package/dist/agents/openclaw.cjs.map +1 -1
- package/dist/agents/openclaw.d.cts +2 -0
- package/dist/agents/openclaw.d.mts +2 -0
- package/dist/agents/openclaw.d.ts +2 -0
- package/dist/agents/types.cjs.map +1 -1
- package/dist/agents/types.d.cts +4 -4
- package/dist/agents/types.d.mts +4 -4
- package/dist/agents/types.d.ts +4 -4
- package/dist/agents/ui-message-stream.cjs +24 -7
- package/dist/agents/ui-message-stream.cjs.map +1 -1
- package/dist/analysis/api.cjs +28 -2
- package/dist/analysis/api.cjs.map +1 -1
- package/dist/analysis/api.d.cts +22 -1
- package/dist/analysis/api.d.mts +22 -1
- package/dist/analysis/api.d.ts +22 -1
- package/dist/analysis/api.js +8 -6
- package/dist/analysis/bindings.cjs +11 -18
- package/dist/analysis/bindings.cjs.map +1 -1
- package/dist/analysis/bindings.d.cts +177 -49
- package/dist/analysis/bindings.d.mts +177 -49
- package/dist/analysis/bindings.d.ts +177 -49
- package/dist/analysis/contracts.cjs +1 -0
- package/dist/analysis/contracts.cjs.map +1 -1
- package/dist/analysis/contracts.d.cts +43 -3
- package/dist/analysis/contracts.d.mts +43 -3
- package/dist/analysis/contracts.d.ts +43 -3
- package/dist/analysis/cost-projection.cjs +2 -1
- package/dist/analysis/cost-projection.cjs.map +1 -1
- package/dist/analysis/cost-projection.d.cts +1 -1
- package/dist/analysis/cost-projection.d.mts +1 -1
- package/dist/analysis/cost-projection.d.ts +1 -1
- package/dist/analysis/cost.cjs +13 -3
- package/dist/analysis/cost.cjs.map +1 -1
- package/dist/analysis/cost.d.cts +10 -3
- package/dist/analysis/cost.d.mts +10 -3
- package/dist/analysis/cost.d.ts +10 -3
- package/dist/analysis/experiment-groups.cjs +96 -0
- package/dist/analysis/experiment-groups.cjs.map +1 -0
- package/dist/analysis/experiment-groups.d.cts +9 -0
- package/dist/analysis/experiment-groups.d.mts +9 -0
- package/dist/analysis/experiment-groups.d.ts +9 -0
- package/dist/analysis/experiment-groups.js +10 -0
- package/dist/analysis/experiment-groups.js.map +1 -0
- package/dist/analysis/index.cjs +4 -1
- package/dist/analysis/index.cjs.map +1 -1
- package/dist/analysis/index.d.cts +4 -2
- package/dist/analysis/index.d.mts +4 -2
- package/dist/analysis/index.d.ts +4 -2
- package/dist/analysis/index.js +40 -36
- package/dist/analysis/index.mjs +40 -36
- package/dist/assertions/limits.cjs +4 -0
- package/dist/assertions/limits.cjs.map +1 -1
- package/dist/assertions/limits.d.cts +4 -0
- package/dist/assertions/limits.d.mts +4 -0
- package/dist/assertions/limits.d.ts +4 -0
- package/dist/assertions/match.cjs +63 -25
- package/dist/assertions/match.cjs.map +1 -1
- package/dist/assertions/match.d.cts +7 -0
- package/dist/assertions/match.d.mts +7 -0
- package/dist/assertions/match.d.ts +7 -0
- package/dist/assertions/record/attachment.cjs +32 -4
- package/dist/assertions/record/attachment.cjs.map +1 -1
- package/dist/assertions/record/diff.cjs +10 -4
- package/dist/assertions/record/diff.cjs.map +1 -1
- package/dist/assertions/record/producer.cjs +101 -4
- package/dist/assertions/record/producer.cjs.map +1 -1
- package/dist/assertions/record/producer.d.cts +4 -1
- package/dist/assertions/record/producer.d.mts +4 -1
- package/dist/assertions/record/producer.d.ts +4 -1
- package/dist/assertions/runtime.cjs +45 -5
- package/dist/assertions/runtime.cjs.map +1 -1
- package/dist/cli/record.cjs +113 -8
- package/dist/cli/record.cjs.map +1 -1
- package/dist/cli/record.d.cts +20 -1
- package/dist/cli/record.d.mts +20 -1
- package/dist/cli/record.d.ts +20 -1
- package/dist/cli/record.js +4 -2
- package/dist/cli.cjs +386 -31
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.cts +10 -2
- package/dist/cli.d.mts +10 -2
- package/dist/cli.d.ts +10 -2
- package/dist/context/send-retry.cjs +4 -4
- package/dist/context/send-retry.cjs.map +1 -1
- package/dist/coordination/host/runtime.cjs +8 -30
- package/dist/coordination/host/runtime.cjs.map +1 -1
- package/dist/coordination/host/runtime.d.cts +3 -3
- package/dist/coordination/host/runtime.d.mts +3 -3
- package/dist/coordination/host/runtime.d.ts +3 -3
- package/dist/coordination/host/types.cjs.map +1 -1
- package/dist/coordination/host/types.d.cts +0 -5
- package/dist/coordination/host/types.d.mts +0 -5
- package/dist/coordination/host/types.d.ts +0 -5
- package/dist/coordination/platform/node.cjs +90 -9
- package/dist/coordination/platform/node.cjs.map +1 -1
- package/dist/define.cjs +25 -1
- package/dist/define.cjs.map +1 -1
- package/dist/experiment/host/index.cjs +1 -0
- package/dist/experiment/host/index.cjs.map +1 -1
- package/dist/i18n/en.cjs +26 -8
- package/dist/i18n/en.cjs.map +1 -1
- package/dist/i18n/en.d.cts +19 -6
- package/dist/i18n/en.d.mts +19 -6
- package/dist/i18n/en.d.ts +19 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/o11y/parsers/hermes.cjs +1 -1
- package/dist/o11y/parsers/hermes.cjs.map +1 -1
- package/dist/o11y/prices.json +1 -1
- package/dist/o11y/record/families.cjs +12 -3
- package/dist/o11y/record/families.cjs.map +1 -1
- package/dist/o11y/record/families.d.cts +8 -8
- package/dist/o11y/record/families.d.mts +8 -8
- package/dist/o11y/record/families.d.ts +8 -8
- package/dist/o11y/record/family-projectors.d.cts +66 -18
- package/dist/o11y/record/family-projectors.d.mts +66 -18
- package/dist/o11y/record/family-projectors.d.ts +66 -18
- package/dist/o11y/record/family-writers.d.cts +11 -3
- package/dist/o11y/record/family-writers.d.mts +11 -3
- package/dist/o11y/record/family-writers.d.ts +11 -3
- package/dist/o11y/record/runner-producer.cjs +14 -4
- package/dist/o11y/record/runner-producer.cjs.map +1 -1
- package/dist/record/codec/core.d.cts +2 -2
- package/dist/record/codec/core.d.mts +2 -2
- package/dist/record/codec/core.d.ts +2 -2
- package/dist/record/family/catalog.d.cts +44 -12
- package/dist/record/family/catalog.d.mts +44 -12
- package/dist/record/family/catalog.d.ts +44 -12
- package/dist/record/family/index.js +128 -124
- package/dist/record/family/observability-v1.cjs +35 -0
- package/dist/record/family/observability-v1.cjs.map +1 -0
- package/dist/record/family/observability-v1.d.cts +3 -0
- package/dist/record/family/observability-v1.d.mts +3 -0
- package/dist/record/family/observability-v1.d.ts +3 -0
- package/dist/record/family/observability-v1.js +6 -0
- package/dist/record/family/observability-v1.js.map +1 -0
- package/dist/record/family/observability.cjs +58 -9
- package/dist/record/family/observability.cjs.map +1 -1
- package/dist/record/family/observability.d.cts +1014 -15
- package/dist/record/family/observability.d.mts +1014 -15
- package/dist/record/family/observability.d.ts +1014 -15
- package/dist/record/family/observability.js +22 -18
- package/dist/record/host/runtime.cjs +600 -31
- package/dist/record/host/runtime.cjs.map +1 -1
- package/dist/record/host/types.cjs.map +1 -1
- package/dist/record/host/types.d.cts +33 -9
- package/dist/record/host/types.d.mts +33 -9
- package/dist/record/host/types.d.ts +33 -9
- package/dist/record/maintenance/runtime.cjs +1 -2
- package/dist/record/maintenance/runtime.cjs.map +1 -1
- package/dist/record/model/core.cjs.map +1 -1
- package/dist/record/model/core.d.cts +1 -1
- package/dist/record/model/core.d.mts +1 -1
- package/dist/record/model/core.d.ts +1 -1
- package/dist/record/model/definition.d.cts +6 -6
- package/dist/record/model/definition.d.mts +6 -6
- package/dist/record/model/definition.d.ts +6 -6
- package/dist/record/model/identifiers.cjs +1 -1
- package/dist/record/model/identifiers.cjs.map +1 -1
- package/dist/record/model/identifiers.d.cts +1 -1
- package/dist/record/model/identifiers.d.mts +1 -1
- package/dist/record/model/identifiers.d.ts +1 -1
- package/dist/record/platform/node.cjs +159 -23
- package/dist/record/platform/node.cjs.map +1 -1
- package/dist/record/platform/services.cjs.map +1 -1
- package/dist/record/platform/services.d.cts +15 -2
- package/dist/record/platform/services.d.mts +15 -2
- package/dist/record/platform/services.d.ts +15 -2
- package/dist/record/reader/errors.cjs +37 -1
- package/dist/record/reader/errors.cjs.map +1 -1
- package/dist/record/reader/errors.d.cts +49 -1
- package/dist/record/reader/errors.d.mts +49 -1
- package/dist/record/reader/errors.d.ts +49 -1
- package/dist/record/reader/errors.js +22 -12
- package/dist/record/reader/format.cjs +76 -17
- package/dist/record/reader/format.cjs.map +1 -1
- package/dist/record/reader/format.d.cts +11 -4
- package/dist/record/reader/format.d.mts +11 -4
- package/dist/record/reader/format.d.ts +11 -4
- package/dist/record/reader/format.js +2 -0
- package/dist/record/reader/index.js +32 -18
- package/dist/record/reader/runtime.cjs +64 -0
- package/dist/record/reader/runtime.cjs.map +1 -1
- package/dist/record/reader/runtime.d.cts +15 -0
- package/dist/record/reader/runtime.d.mts +15 -0
- package/dist/record/reader/runtime.d.ts +15 -0
- package/dist/record/reader/runtime.js +2 -0
- package/dist/report/assets/styles.css +53 -29
- package/dist/report/built-in/analysis-values.cjs +21 -13
- package/dist/report/built-in/analysis-values.cjs.map +1 -1
- package/dist/report/built-in/analysis-values.d.cts +11 -22
- package/dist/report/built-in/analysis-values.d.mts +11 -22
- package/dist/report/built-in/analysis-values.d.ts +11 -22
- package/dist/report/built-in/analysis-values.js +4 -6
- package/dist/report/built-in/index.cjs +1 -3
- package/dist/report/built-in/index.cjs.map +1 -1
- package/dist/report/built-in/index.d.cts +1 -1
- package/dist/report/built-in/index.d.mts +1 -1
- package/dist/report/built-in/index.d.ts +1 -1
- package/dist/report/built-in/index.js +4 -8
- package/dist/report/built-in/index.mjs +4 -8
- package/dist/report/built-in/machine.cjs +0 -2
- package/dist/report/built-in/machine.cjs.map +1 -1
- package/dist/report/built-in/machine.d.cts +0 -2
- package/dist/report/built-in/machine.d.mts +0 -2
- package/dist/report/built-in/machine.d.ts +0 -2
- package/dist/report/built-in/result-components.cjs +49 -51
- package/dist/report/built-in/result-components.cjs.map +1 -1
- package/dist/report/built-in/result-components.d.cts +16 -17
- package/dist/report/built-in/result-components.d.mts +16 -17
- package/dist/report/built-in/result-components.d.ts +16 -17
- package/dist/report/built-in/result-components.js +0 -4
- package/dist/report/built-in/run-membership-overview.cjs +126 -17
- package/dist/report/built-in/run-membership-overview.cjs.map +1 -1
- package/dist/report/built-in/run-membership-overview.d.cts +14 -0
- package/dist/report/built-in/run-membership-overview.d.mts +14 -0
- package/dist/report/built-in/run-membership-overview.d.ts +14 -0
- package/dist/report/built-in/run-membership-overview.js +4 -2
- package/dist/report/built-in/standard.cjs +80 -45
- package/dist/report/built-in/standard.cjs.map +1 -1
- package/dist/report/built-in/standard.d.cts +19 -41
- package/dist/report/built-in/standard.d.mts +19 -41
- package/dist/report/built-in/standard.d.ts +19 -41
- package/dist/report/built-in/standard.js +6 -6
- package/dist/report/components/attempt-detail/compute.cjs +78 -2
- package/dist/report/components/attempt-detail/compute.cjs.map +1 -1
- package/dist/report/components/attempt-detail/compute.d.cts +1 -1
- package/dist/report/components/attempt-detail/compute.d.mts +1 -1
- package/dist/report/components/attempt-detail/compute.d.ts +1 -1
- package/dist/report/components/attempt-detail/content.cjs +1 -1
- package/dist/report/components/attempt-detail/content.cjs.map +1 -1
- package/dist/report/components/entity-lists/compute.cjs +42 -5
- package/dist/report/components/entity-lists/compute.cjs.map +1 -1
- package/dist/report/components/entity-lists/compute.d.cts +10 -8
- package/dist/report/components/entity-lists/compute.d.mts +10 -8
- package/dist/report/components/entity-lists/compute.d.ts +10 -8
- package/dist/report/components/entity-lists/content.cjs +31 -6
- package/dist/report/components/entity-lists/content.cjs.map +1 -1
- package/dist/report/components/entity-lists/index.cjs +9 -4
- package/dist/report/components/entity-lists/index.cjs.map +1 -1
- package/dist/report/components/entity-lists/index.d.cts +4 -7
- package/dist/report/components/entity-lists/index.d.mts +4 -7
- package/dist/report/components/entity-lists/index.d.ts +4 -7
- package/dist/report/components/summaries/compute.cjs +8 -1
- package/dist/report/components/summaries/compute.cjs.map +1 -1
- package/dist/report/components/summaries/compute.d.cts +9 -1
- package/dist/report/components/summaries/compute.d.mts +9 -1
- package/dist/report/components/summaries/compute.d.ts +9 -1
- package/dist/report/components/summaries/index.cjs +48 -4
- package/dist/report/components/summaries/index.cjs.map +1 -1
- package/dist/report/components/summaries/index.d.cts +7 -4
- package/dist/report/components/summaries/index.d.mts +7 -4
- package/dist/report/components/summaries/index.d.ts +7 -4
- package/dist/report/definition/cell.cjs +3 -0
- package/dist/report/definition/cell.cjs.map +1 -1
- package/dist/report/definition/cell.d.cts +2 -0
- package/dist/report/definition/cell.d.mts +2 -0
- package/dist/report/definition/cell.d.ts +2 -0
- package/dist/report/definition/primitives/chart.cjs +8 -5
- package/dist/report/definition/primitives/chart.cjs.map +1 -1
- package/dist/report/definition/primitives/points-dataset.cjs +1 -0
- package/dist/report/definition/primitives/points-dataset.cjs.map +1 -1
- package/dist/report/definition/primitives.cjs +15 -2
- package/dist/report/definition/primitives.cjs.map +1 -1
- package/dist/report/definition/report.cjs +27 -3
- package/dist/report/definition/report.cjs.map +1 -1
- package/dist/report/definition/report.d.cts +8 -0
- package/dist/report/definition/report.d.mts +8 -0
- package/dist/report/definition/report.d.ts +8 -0
- package/dist/report/execution/machine.cjs +30 -14
- package/dist/report/execution/machine.cjs.map +1 -1
- package/dist/report/execution/machine.d.cts +20 -10
- package/dist/report/execution/machine.d.mts +20 -10
- package/dist/report/execution/machine.d.ts +20 -10
- package/dist/report/execution/machine.js +6 -6
- package/dist/report/host/execute.cjs +25 -16
- package/dist/report/host/execute.cjs.map +1 -1
- package/dist/report/host/execute.d.cts +4 -0
- package/dist/report/host/execute.d.mts +4 -0
- package/dist/report/host/execute.d.ts +4 -0
- package/dist/report/host/from-record.cjs +22 -5
- package/dist/report/host/from-record.cjs.map +1 -1
- package/dist/report/host/machine.cjs +53 -11
- package/dist/report/host/machine.cjs.map +1 -1
- package/dist/report/host/site-runtime.cjs +9 -8
- package/dist/report/host/site-runtime.cjs.map +1 -1
- package/dist/report/host/site-runtime.d.cts +1 -1
- package/dist/report/host/site-runtime.d.mts +1 -1
- package/dist/report/host/site-runtime.d.ts +1 -1
- package/dist/report/host/static.cjs +56 -9
- package/dist/report/host/static.cjs.map +1 -1
- package/dist/report/index.cjs.map +1 -1
- package/dist/report/index.d.cts +1 -1
- package/dist/report/index.d.mts +1 -1
- package/dist/report/index.d.ts +1 -1
- package/dist/report/model/dataset.cjs +7 -2
- package/dist/report/model/dataset.cjs.map +1 -1
- package/dist/report/model/locale.cjs +2 -0
- package/dist/report/model/locale.cjs.map +1 -1
- package/dist/report/model/locale.d.cts +1 -0
- package/dist/report/model/locale.d.mts +1 -0
- package/dist/report/model/locale.d.ts +1 -0
- package/dist/report/model/metrics.cjs +3 -1
- package/dist/report/model/metrics.cjs.map +1 -1
- package/dist/report/model/pricing.cjs +16 -11
- package/dist/report/model/pricing.cjs.map +1 -1
- package/dist/report/react/styles.css +53 -29
- package/dist/runner/adoption.cjs +1 -0
- package/dist/runner/adoption.cjs.map +1 -1
- package/dist/runner/attempt.cjs +44 -5
- package/dist/runner/attempt.cjs.map +1 -1
- package/dist/runner/attempt.d.cts +13 -1
- package/dist/runner/attempt.d.mts +13 -1
- package/dist/runner/attempt.d.ts +13 -1
- package/dist/runner/config-identity.cjs +10 -2
- package/dist/runner/config-identity.cjs.map +1 -1
- package/dist/runner/config-identity.d.cts +4 -2
- package/dist/runner/config-identity.d.mts +4 -2
- package/dist/runner/config-identity.d.ts +4 -2
- package/dist/runner/feedback/human.cjs +298 -59
- package/dist/runner/feedback/human.cjs.map +1 -1
- package/dist/runner/feedback/json.cjs +14 -6
- package/dist/runner/feedback/json.cjs.map +1 -1
- package/dist/runner/feedback/json.d.cts +3 -2
- package/dist/runner/feedback/json.d.mts +3 -2
- package/dist/runner/feedback/json.d.ts +3 -2
- package/dist/runner/feedback/reducer.cjs +2 -0
- package/dist/runner/feedback/reducer.cjs.map +1 -1
- package/dist/runner/judge-config.cjs +1 -1
- package/dist/runner/judge-config.cjs.map +1 -1
- package/dist/runner/ledger.cjs +124 -12
- package/dist/runner/ledger.cjs.map +1 -1
- package/dist/runner/project-current.cjs +13 -5
- package/dist/runner/project-current.cjs.map +1 -1
- package/dist/runner/run.cjs +564 -256
- package/dist/runner/run.cjs.map +1 -1
- package/dist/runner/sandbox-pool.cjs +161 -30
- package/dist/runner/sandbox-pool.cjs.map +1 -1
- package/dist/runner/sandbox-pool.d.cts +34 -2
- package/dist/runner/sandbox-pool.d.mts +34 -2
- package/dist/runner/sandbox-pool.d.ts +34 -2
- package/dist/runner/shared-state-lease.cjs +803 -0
- package/dist/runner/shared-state-lease.cjs.map +1 -0
- package/dist/runner/shared-state-lease.d.cts +121 -0
- package/dist/runner/shared-state-lease.d.mts +121 -0
- package/dist/runner/shared-state-lease.d.ts +121 -0
- package/dist/runner/shared-state-lease.js +24 -0
- package/dist/runner/shared-state-lease.js.map +1 -0
- package/dist/runner/teardown-registry.cjs +97 -34
- package/dist/runner/teardown-registry.cjs.map +1 -1
- package/dist/runner/teardown-registry.d.cts +27 -1
- package/dist/runner/teardown-registry.d.mts +27 -1
- package/dist/runner/teardown-registry.d.ts +27 -1
- package/dist/runner/teardown-registry.js +20 -16
- package/dist/runner/types.cjs.map +1 -1
- package/dist/runner/types.d.cts +31 -5
- package/dist/runner/types.d.mts +31 -5
- package/dist/runner/types.d.ts +31 -5
- package/dist/sample/analysis.cjs +249 -459
- package/dist/sample/analysis.cjs.map +1 -1
- package/dist/sample/capability.cjs +138 -7
- package/dist/sample/capability.cjs.map +1 -1
- package/dist/sample/capability.d.cts +3 -1
- package/dist/sample/capability.d.mts +3 -1
- package/dist/sample/capability.d.ts +3 -1
- package/dist/sample/capability.js +8 -6
- package/dist/sandbox/docker-agent-image.cjs +5 -1
- package/dist/sandbox/docker-agent-image.cjs.map +1 -1
- package/dist/sandbox/docker-agent-image.d.cts +2 -0
- package/dist/sandbox/docker-agent-image.d.mts +2 -0
- package/dist/sandbox/docker-agent-image.d.ts +2 -0
- package/dist/sandbox/docker-agent-image.js +14 -10
- package/dist/sandbox/docker-profile/schema.cjs +283 -534
- package/dist/sandbox/docker-profile/schema.cjs.map +1 -1
- package/dist/sandbox/docker-profile/schema.d.cts +10 -27
- package/dist/sandbox/docker-profile/schema.d.mts +10 -27
- package/dist/sandbox/docker-profile/schema.d.ts +10 -27
- package/dist/sandbox/e2b-agent-template.cjs +5 -2
- package/dist/sandbox/e2b-agent-template.cjs.map +1 -1
- package/dist/sandbox/index.cjs +4 -2
- package/dist/sandbox/index.cjs.map +1 -1
- package/dist/sandbox/index.d.cts +1 -1
- package/dist/sandbox/index.d.mts +1 -1
- package/dist/sandbox/index.d.ts +1 -1
- package/dist/sandbox/index.js +120 -116
- package/dist/sandbox/index.mjs +120 -116
- package/dist/sandbox/keep-registry.cjs +28 -66
- package/dist/sandbox/keep-registry.cjs.map +1 -1
- package/dist/sandbox/types.cjs.map +1 -1
- package/dist/sandbox/types.d.cts +1 -1
- package/dist/sandbox/types.d.mts +1 -1
- package/dist/sandbox/types.d.ts +1 -1
- package/dist/shared/aggregate.cjs +16 -6
- package/dist/shared/aggregate.cjs.map +1 -1
- package/dist/shared/aggregate.d.cts +12 -4
- package/dist/shared/aggregate.d.mts +12 -4
- package/dist/shared/aggregate.d.ts +12 -4
- package/dist/shared/turn-label.cjs +20 -2
- package/dist/shared/turn-label.cjs.map +1 -1
- package/dist/shared/turn-label.d.cts +2 -0
- package/dist/shared/turn-label.d.mts +2 -0
- package/dist/shared/turn-label.d.ts +2 -0
- package/dist/shared/turn-label.js +4 -2
- package/dist/vendor/remark/remark-gfm.cjs +7 -7
- package/dist/vendor/remark/remark-gfm.cjs.map +1 -1
- package/dist/vendor/remark/remark-parse.cjs +12 -12
- package/dist/vendor/remark/remark-parse.cjs.map +1 -1
- package/dist/vendor/remark/remark.cjs +13 -13
- package/dist/vendor/remark/remark.cjs.map +1 -1
- package/dist/view/server.cjs +6 -0
- package/dist/view/server.cjs.map +1 -1
- package/docs-site/images/hitl-handshake-zh.svg +3 -3
- package/docs-site/zh/examples/ai-agent-application.mdx +1 -1
- package/docs-site/zh/examples/coding-agent-extensions.mdx +1 -1
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +4 -3
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +5 -5
- package/docs-site/zh/examples/integrations/langgraph.mdx +4 -4
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +5 -5
- package/docs-site/zh/explanation/adapter.mdx +5 -5
- package/docs-site/zh/explanation/assert.mdx +7 -7
- package/docs-site/zh/explanation/drive.mdx +4 -2
- package/docs-site/zh/explanation/evals.mdx +10 -8
- package/docs-site/zh/explanation/experiment.mdx +2 -2
- package/docs-site/zh/explanation/hitl.mdx +9 -9
- package/docs-site/zh/explanation/judge.mdx +5 -5
- package/docs-site/zh/explanation/overview.mdx +3 -3
- package/docs-site/zh/explanation/runner.mdx +1 -1
- package/docs-site/zh/index.mdx +3 -3
- package/docs-site/zh/reference/builtin-agents.mdx +13 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +18 -12
- package/docs-site/zh/reference/define-agent.mdx +130 -5
- package/docs-site/zh/reference/define-config.mdx +2 -2
- package/docs-site/zh/reference/define-eval.mdx +4 -4
- package/docs-site/zh/reference/events.mdx +11 -11
- package/docs-site/zh/reference/official-adapters.mdx +6 -6
- package/docs-site/zh/reference/results-data.mdx +6 -0
- package/docs-site/zh/troubleshooting/debugging.mdx +1 -1
- package/docs-site/zh/troubleshooting/recover-after-kill.mdx +37 -5
- package/docs-site/zh/tutorials/accept-results.mdx +72 -0
- package/docs-site/zh/tutorials/agent-onboarding.mdx +6 -6
- package/docs-site/zh/tutorials/authoring.mdx +9 -13
- package/docs-site/zh/tutorials/ci-integration.mdx +6 -6
- package/docs-site/zh/tutorials/compare-runs.mdx +61 -0
- package/docs-site/zh/tutorials/concurrency.mdx +13 -12
- package/docs-site/zh/tutorials/configuration.mdx +4 -6
- package/docs-site/zh/tutorials/connect-your-agent.mdx +1 -1
- package/docs-site/zh/tutorials/control-run-cost.mdx +58 -0
- package/docs-site/zh/tutorials/criteria-files.mdx +2 -2
- package/docs-site/zh/tutorials/custom-reports.mdx +2 -2
- package/docs-site/zh/tutorials/debug-lifecycle-plan.mdx +46 -0
- package/docs-site/zh/tutorials/docker-in-docker.mdx +52 -43
- package/docs-site/zh/tutorials/eval-groups.mdx +6 -6
- package/docs-site/zh/tutorials/evaluation-kinds.mdx +3 -3
- package/docs-site/zh/tutorials/experiments.mdx +2 -3
- package/docs-site/zh/tutorials/handle-execution-failures.mdx +18 -10
- package/docs-site/zh/tutorials/install-custom-sandbox-agent.mdx +16 -8
- package/docs-site/zh/tutorials/nixos-managed-dind.mdx +160 -0
- package/docs-site/zh/tutorials/plugins.mdx +14 -0
- package/docs-site/zh/tutorials/rerun-and-cache.mdx +4 -2
- package/docs-site/zh/tutorials/sandbox-agent.mdx +17 -11
- package/docs-site/zh/tutorials/sandbox-providers.mdx +17 -13
- package/docs-site/zh/tutorials/sandbox-reuse.mdx +12 -19
- package/docs-site/zh/tutorials/select-and-preview.mdx +65 -0
- package/docs-site/zh/tutorials/verify-judge.mdx +78 -0
- package/docs-site/zh/tutorials/viewing-results.mdx +38 -1
- package/docs-site/zh/tutorials/write-experiment.mdx +7 -5
- package/docs-site/zh/tutorials/write-send.mdx +9 -8
- package/package.json +9 -5
- package/dist/report/built-in/overview.cjs +0 -44
- package/dist/report/built-in/overview.cjs.map +0 -1
- package/dist/report/built-in/overview.d.cts +0 -6
- package/dist/report/built-in/overview.d.mts +0 -6
- package/dist/report/built-in/overview.d.ts +0 -6
- package/dist/report/built-in/overview.js +0 -9
- package/dist/report/built-in/overview.js.map +0 -1
- package/dist/runner/gate-lease.cjs +0 -283
- package/dist/runner/gate-lease.cjs.map +0 -1
- package/dist/runner/gate-lease.d.cts +0 -62
- package/dist/runner/gate-lease.d.mts +0 -62
- package/dist/runner/gate-lease.d.ts +0 -62
- package/dist/runner/gate-lease.js +0 -22
- package/dist/runner/gate-lease.js.map +0 -1
|
@@ -29,7 +29,7 @@ export type InputProjection<Value> = {
|
|
|
29
29
|
readonly state: "value";
|
|
30
30
|
readonly value: Value;
|
|
31
31
|
} | {
|
|
32
|
-
readonly state: "missing" | "unsupported" | "failed";
|
|
32
|
+
readonly state: "missing" | "migration-required" | "unsupported" | "failed";
|
|
33
33
|
readonly message: string;
|
|
34
34
|
};
|
|
35
35
|
/** One semantic input owns an immutable owner/family requirement and projector. */
|
|
@@ -413,15 +413,23 @@ export declare const attemptObservabilityFamily: FixedFamilyBinding<"attempt", {
|
|
|
413
413
|
readonly commandId: string;
|
|
414
414
|
})[]];
|
|
415
415
|
};
|
|
416
|
-
readonly intervals: readonly {
|
|
416
|
+
readonly intervals: readonly ({
|
|
417
417
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
418
418
|
readonly label: string;
|
|
419
|
-
readonly phase: "agent.
|
|
419
|
+
readonly phase: "agent.send";
|
|
420
420
|
readonly startOffsetMs: number;
|
|
421
421
|
readonly durationMs: number;
|
|
422
422
|
readonly intervalId: string;
|
|
423
423
|
readonly parentIntervalId: string | null;
|
|
424
|
-
}
|
|
424
|
+
} | {
|
|
425
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
426
|
+
readonly label: string;
|
|
427
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
428
|
+
readonly startOffsetMs: number;
|
|
429
|
+
readonly durationMs: number;
|
|
430
|
+
readonly intervalId: string;
|
|
431
|
+
readonly parentIntervalId: string | null;
|
|
432
|
+
})[];
|
|
425
433
|
};
|
|
426
434
|
readonly diagnostics: {
|
|
427
435
|
readonly collection: {
|
|
@@ -862,15 +870,23 @@ export declare const attemptObservabilityFamily: FixedFamilyBinding<"attempt", {
|
|
|
862
870
|
readonly commandId: string;
|
|
863
871
|
})[]];
|
|
864
872
|
};
|
|
865
|
-
readonly intervals: readonly {
|
|
873
|
+
readonly intervals: readonly ({
|
|
866
874
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
867
875
|
readonly label: string;
|
|
868
|
-
readonly phase: "agent.
|
|
876
|
+
readonly phase: "agent.send";
|
|
869
877
|
readonly startOffsetMs: number;
|
|
870
878
|
readonly durationMs: number;
|
|
871
879
|
readonly intervalId: string;
|
|
872
880
|
readonly parentIntervalId: string | null;
|
|
873
|
-
}
|
|
881
|
+
} | {
|
|
882
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
883
|
+
readonly label: string;
|
|
884
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
885
|
+
readonly startOffsetMs: number;
|
|
886
|
+
readonly durationMs: number;
|
|
887
|
+
readonly intervalId: string;
|
|
888
|
+
readonly parentIntervalId: string | null;
|
|
889
|
+
})[];
|
|
874
890
|
};
|
|
875
891
|
readonly diagnostics: {
|
|
876
892
|
readonly collection: {
|
|
@@ -1417,15 +1433,23 @@ export declare const publishedAnalysisInputBindings: Readonly<{
|
|
|
1417
1433
|
readonly commandId: string;
|
|
1418
1434
|
})[]];
|
|
1419
1435
|
};
|
|
1420
|
-
readonly intervals: readonly {
|
|
1436
|
+
readonly intervals: readonly ({
|
|
1421
1437
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
1422
1438
|
readonly label: string;
|
|
1423
|
-
readonly phase: "agent.
|
|
1439
|
+
readonly phase: "agent.send";
|
|
1424
1440
|
readonly startOffsetMs: number;
|
|
1425
1441
|
readonly durationMs: number;
|
|
1426
1442
|
readonly intervalId: string;
|
|
1427
1443
|
readonly parentIntervalId: string | null;
|
|
1428
|
-
}
|
|
1444
|
+
} | {
|
|
1445
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
1446
|
+
readonly label: string;
|
|
1447
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
1448
|
+
readonly startOffsetMs: number;
|
|
1449
|
+
readonly durationMs: number;
|
|
1450
|
+
readonly intervalId: string;
|
|
1451
|
+
readonly parentIntervalId: string | null;
|
|
1452
|
+
})[];
|
|
1429
1453
|
};
|
|
1430
1454
|
readonly diagnostics: {
|
|
1431
1455
|
readonly collection: {
|
|
@@ -1866,15 +1890,23 @@ export declare const publishedAnalysisInputBindings: Readonly<{
|
|
|
1866
1890
|
readonly commandId: string;
|
|
1867
1891
|
})[]];
|
|
1868
1892
|
};
|
|
1869
|
-
readonly intervals: readonly {
|
|
1893
|
+
readonly intervals: readonly ({
|
|
1870
1894
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
1871
1895
|
readonly label: string;
|
|
1872
|
-
readonly phase: "agent.
|
|
1896
|
+
readonly phase: "agent.send";
|
|
1873
1897
|
readonly startOffsetMs: number;
|
|
1874
1898
|
readonly durationMs: number;
|
|
1875
1899
|
readonly intervalId: string;
|
|
1876
1900
|
readonly parentIntervalId: string | null;
|
|
1877
|
-
}
|
|
1901
|
+
} | {
|
|
1902
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
1903
|
+
readonly label: string;
|
|
1904
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
1905
|
+
readonly startOffsetMs: number;
|
|
1906
|
+
readonly durationMs: number;
|
|
1907
|
+
readonly intervalId: string;
|
|
1908
|
+
readonly parentIntervalId: string | null;
|
|
1909
|
+
})[];
|
|
1878
1910
|
};
|
|
1879
1911
|
readonly diagnostics: {
|
|
1880
1912
|
readonly collection: {
|
|
@@ -2330,15 +2362,23 @@ export declare const publishedAnalysisInputBindings: Readonly<{
|
|
|
2330
2362
|
readonly commandId: string;
|
|
2331
2363
|
})[]];
|
|
2332
2364
|
};
|
|
2333
|
-
readonly intervals: readonly {
|
|
2365
|
+
readonly intervals: readonly ({
|
|
2334
2366
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
2335
2367
|
readonly label: string;
|
|
2336
|
-
readonly phase: "agent.
|
|
2368
|
+
readonly phase: "agent.send";
|
|
2337
2369
|
readonly startOffsetMs: number;
|
|
2338
2370
|
readonly durationMs: number;
|
|
2339
2371
|
readonly intervalId: string;
|
|
2340
2372
|
readonly parentIntervalId: string | null;
|
|
2341
|
-
}
|
|
2373
|
+
} | {
|
|
2374
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
2375
|
+
readonly label: string;
|
|
2376
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
2377
|
+
readonly startOffsetMs: number;
|
|
2378
|
+
readonly durationMs: number;
|
|
2379
|
+
readonly intervalId: string;
|
|
2380
|
+
readonly parentIntervalId: string | null;
|
|
2381
|
+
})[];
|
|
2342
2382
|
};
|
|
2343
2383
|
readonly diagnostics: {
|
|
2344
2384
|
readonly collection: {
|
|
@@ -2779,15 +2819,23 @@ export declare const publishedAnalysisInputBindings: Readonly<{
|
|
|
2779
2819
|
readonly commandId: string;
|
|
2780
2820
|
})[]];
|
|
2781
2821
|
};
|
|
2782
|
-
readonly intervals: readonly {
|
|
2822
|
+
readonly intervals: readonly ({
|
|
2783
2823
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
2784
2824
|
readonly label: string;
|
|
2785
|
-
readonly phase: "agent.
|
|
2825
|
+
readonly phase: "agent.send";
|
|
2786
2826
|
readonly startOffsetMs: number;
|
|
2787
2827
|
readonly durationMs: number;
|
|
2788
2828
|
readonly intervalId: string;
|
|
2789
2829
|
readonly parentIntervalId: string | null;
|
|
2790
|
-
}
|
|
2830
|
+
} | {
|
|
2831
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
2832
|
+
readonly label: string;
|
|
2833
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
2834
|
+
readonly startOffsetMs: number;
|
|
2835
|
+
readonly durationMs: number;
|
|
2836
|
+
readonly intervalId: string;
|
|
2837
|
+
readonly parentIntervalId: string | null;
|
|
2838
|
+
})[];
|
|
2791
2839
|
};
|
|
2792
2840
|
readonly diagnostics: {
|
|
2793
2841
|
readonly collection: {
|
|
@@ -3237,15 +3285,23 @@ export declare const publishedAnalysisInputBindings: Readonly<{
|
|
|
3237
3285
|
readonly commandId: string;
|
|
3238
3286
|
})[]];
|
|
3239
3287
|
};
|
|
3240
|
-
readonly intervals: readonly {
|
|
3288
|
+
readonly intervals: readonly ({
|
|
3241
3289
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
3242
3290
|
readonly label: string;
|
|
3243
|
-
readonly phase: "agent.
|
|
3291
|
+
readonly phase: "agent.send";
|
|
3244
3292
|
readonly startOffsetMs: number;
|
|
3245
3293
|
readonly durationMs: number;
|
|
3246
3294
|
readonly intervalId: string;
|
|
3247
3295
|
readonly parentIntervalId: string | null;
|
|
3248
|
-
}
|
|
3296
|
+
} | {
|
|
3297
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
3298
|
+
readonly label: string;
|
|
3299
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
3300
|
+
readonly startOffsetMs: number;
|
|
3301
|
+
readonly durationMs: number;
|
|
3302
|
+
readonly intervalId: string;
|
|
3303
|
+
readonly parentIntervalId: string | null;
|
|
3304
|
+
})[];
|
|
3249
3305
|
};
|
|
3250
3306
|
readonly diagnostics: {
|
|
3251
3307
|
readonly collection: {
|
|
@@ -3686,15 +3742,23 @@ export declare const publishedAnalysisInputBindings: Readonly<{
|
|
|
3686
3742
|
readonly commandId: string;
|
|
3687
3743
|
})[]];
|
|
3688
3744
|
};
|
|
3689
|
-
readonly intervals: readonly {
|
|
3745
|
+
readonly intervals: readonly ({
|
|
3690
3746
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
3691
3747
|
readonly label: string;
|
|
3692
|
-
readonly phase: "agent.
|
|
3748
|
+
readonly phase: "agent.send";
|
|
3693
3749
|
readonly startOffsetMs: number;
|
|
3694
3750
|
readonly durationMs: number;
|
|
3695
3751
|
readonly intervalId: string;
|
|
3696
3752
|
readonly parentIntervalId: string | null;
|
|
3697
|
-
}
|
|
3753
|
+
} | {
|
|
3754
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
3755
|
+
readonly label: string;
|
|
3756
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
3757
|
+
readonly startOffsetMs: number;
|
|
3758
|
+
readonly durationMs: number;
|
|
3759
|
+
readonly intervalId: string;
|
|
3760
|
+
readonly parentIntervalId: string | null;
|
|
3761
|
+
})[];
|
|
3698
3762
|
};
|
|
3699
3763
|
readonly diagnostics: {
|
|
3700
3764
|
readonly collection: {
|
|
@@ -4248,15 +4312,23 @@ export declare const attemptObservabilityDomainBinding: Readonly<{
|
|
|
4248
4312
|
readonly commandId: string;
|
|
4249
4313
|
})[]];
|
|
4250
4314
|
};
|
|
4251
|
-
readonly intervals: readonly {
|
|
4315
|
+
readonly intervals: readonly ({
|
|
4252
4316
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
4253
4317
|
readonly label: string;
|
|
4254
|
-
readonly phase: "agent.
|
|
4318
|
+
readonly phase: "agent.send";
|
|
4255
4319
|
readonly startOffsetMs: number;
|
|
4256
4320
|
readonly durationMs: number;
|
|
4257
4321
|
readonly intervalId: string;
|
|
4258
4322
|
readonly parentIntervalId: string | null;
|
|
4259
|
-
}
|
|
4323
|
+
} | {
|
|
4324
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
4325
|
+
readonly label: string;
|
|
4326
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
4327
|
+
readonly startOffsetMs: number;
|
|
4328
|
+
readonly durationMs: number;
|
|
4329
|
+
readonly intervalId: string;
|
|
4330
|
+
readonly parentIntervalId: string | null;
|
|
4331
|
+
})[];
|
|
4260
4332
|
};
|
|
4261
4333
|
readonly diagnostics: {
|
|
4262
4334
|
readonly collection: {
|
|
@@ -4697,15 +4769,23 @@ export declare const attemptObservabilityDomainBinding: Readonly<{
|
|
|
4697
4769
|
readonly commandId: string;
|
|
4698
4770
|
})[]];
|
|
4699
4771
|
};
|
|
4700
|
-
readonly intervals: readonly {
|
|
4772
|
+
readonly intervals: readonly ({
|
|
4701
4773
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
4702
4774
|
readonly label: string;
|
|
4703
|
-
readonly phase: "agent.
|
|
4775
|
+
readonly phase: "agent.send";
|
|
4704
4776
|
readonly startOffsetMs: number;
|
|
4705
4777
|
readonly durationMs: number;
|
|
4706
4778
|
readonly intervalId: string;
|
|
4707
4779
|
readonly parentIntervalId: string | null;
|
|
4708
|
-
}
|
|
4780
|
+
} | {
|
|
4781
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
4782
|
+
readonly label: string;
|
|
4783
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
4784
|
+
readonly startOffsetMs: number;
|
|
4785
|
+
readonly durationMs: number;
|
|
4786
|
+
readonly intervalId: string;
|
|
4787
|
+
readonly parentIntervalId: string | null;
|
|
4788
|
+
})[];
|
|
4709
4789
|
};
|
|
4710
4790
|
readonly diagnostics: {
|
|
4711
4791
|
readonly collection: {
|
|
@@ -5498,15 +5578,23 @@ export declare const sandboxHistoryDomainBinding: Readonly<{
|
|
|
5498
5578
|
readonly commandId: string;
|
|
5499
5579
|
})[]];
|
|
5500
5580
|
};
|
|
5501
|
-
readonly intervals: readonly {
|
|
5581
|
+
readonly intervals: readonly ({
|
|
5502
5582
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
5503
5583
|
readonly label: string;
|
|
5504
|
-
readonly phase: "agent.
|
|
5584
|
+
readonly phase: "agent.send";
|
|
5505
5585
|
readonly startOffsetMs: number;
|
|
5506
5586
|
readonly durationMs: number;
|
|
5507
5587
|
readonly intervalId: string;
|
|
5508
5588
|
readonly parentIntervalId: string | null;
|
|
5509
|
-
}
|
|
5589
|
+
} | {
|
|
5590
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
5591
|
+
readonly label: string;
|
|
5592
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
5593
|
+
readonly startOffsetMs: number;
|
|
5594
|
+
readonly durationMs: number;
|
|
5595
|
+
readonly intervalId: string;
|
|
5596
|
+
readonly parentIntervalId: string | null;
|
|
5597
|
+
})[];
|
|
5510
5598
|
};
|
|
5511
5599
|
readonly diagnostics: {
|
|
5512
5600
|
readonly collection: {
|
|
@@ -5947,15 +6035,23 @@ export declare const sandboxHistoryDomainBinding: Readonly<{
|
|
|
5947
6035
|
readonly commandId: string;
|
|
5948
6036
|
})[]];
|
|
5949
6037
|
};
|
|
5950
|
-
readonly intervals: readonly {
|
|
6038
|
+
readonly intervals: readonly ({
|
|
5951
6039
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
5952
6040
|
readonly label: string;
|
|
5953
|
-
readonly phase: "agent.
|
|
6041
|
+
readonly phase: "agent.send";
|
|
5954
6042
|
readonly startOffsetMs: number;
|
|
5955
6043
|
readonly durationMs: number;
|
|
5956
6044
|
readonly intervalId: string;
|
|
5957
6045
|
readonly parentIntervalId: string | null;
|
|
5958
|
-
}
|
|
6046
|
+
} | {
|
|
6047
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
6048
|
+
readonly label: string;
|
|
6049
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
6050
|
+
readonly startOffsetMs: number;
|
|
6051
|
+
readonly durationMs: number;
|
|
6052
|
+
readonly intervalId: string;
|
|
6053
|
+
readonly parentIntervalId: string | null;
|
|
6054
|
+
})[];
|
|
5959
6055
|
};
|
|
5960
6056
|
readonly diagnostics: {
|
|
5961
6057
|
readonly collection: {
|
|
@@ -6508,15 +6604,23 @@ export declare const builtinDomainViewBindings: Readonly<{
|
|
|
6508
6604
|
readonly commandId: string;
|
|
6509
6605
|
})[]];
|
|
6510
6606
|
};
|
|
6511
|
-
readonly intervals: readonly {
|
|
6607
|
+
readonly intervals: readonly ({
|
|
6512
6608
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
6513
6609
|
readonly label: string;
|
|
6514
|
-
readonly phase: "agent.
|
|
6610
|
+
readonly phase: "agent.send";
|
|
6515
6611
|
readonly startOffsetMs: number;
|
|
6516
6612
|
readonly durationMs: number;
|
|
6517
6613
|
readonly intervalId: string;
|
|
6518
6614
|
readonly parentIntervalId: string | null;
|
|
6519
|
-
}
|
|
6615
|
+
} | {
|
|
6616
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
6617
|
+
readonly label: string;
|
|
6618
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
6619
|
+
readonly startOffsetMs: number;
|
|
6620
|
+
readonly durationMs: number;
|
|
6621
|
+
readonly intervalId: string;
|
|
6622
|
+
readonly parentIntervalId: string | null;
|
|
6623
|
+
})[];
|
|
6520
6624
|
};
|
|
6521
6625
|
readonly diagnostics: {
|
|
6522
6626
|
readonly collection: {
|
|
@@ -6957,15 +7061,23 @@ export declare const builtinDomainViewBindings: Readonly<{
|
|
|
6957
7061
|
readonly commandId: string;
|
|
6958
7062
|
})[]];
|
|
6959
7063
|
};
|
|
6960
|
-
readonly intervals: readonly {
|
|
7064
|
+
readonly intervals: readonly ({
|
|
6961
7065
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
6962
7066
|
readonly label: string;
|
|
6963
|
-
readonly phase: "agent.
|
|
7067
|
+
readonly phase: "agent.send";
|
|
6964
7068
|
readonly startOffsetMs: number;
|
|
6965
7069
|
readonly durationMs: number;
|
|
6966
7070
|
readonly intervalId: string;
|
|
6967
7071
|
readonly parentIntervalId: string | null;
|
|
6968
|
-
}
|
|
7072
|
+
} | {
|
|
7073
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
7074
|
+
readonly label: string;
|
|
7075
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
7076
|
+
readonly startOffsetMs: number;
|
|
7077
|
+
readonly durationMs: number;
|
|
7078
|
+
readonly intervalId: string;
|
|
7079
|
+
readonly parentIntervalId: string | null;
|
|
7080
|
+
})[];
|
|
6969
7081
|
};
|
|
6970
7082
|
readonly diagnostics: {
|
|
6971
7083
|
readonly collection: {
|
|
@@ -7758,15 +7870,23 @@ export declare const builtinDomainViewBindings: Readonly<{
|
|
|
7758
7870
|
readonly commandId: string;
|
|
7759
7871
|
})[]];
|
|
7760
7872
|
};
|
|
7761
|
-
readonly intervals: readonly {
|
|
7873
|
+
readonly intervals: readonly ({
|
|
7762
7874
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
7763
7875
|
readonly label: string;
|
|
7764
|
-
readonly phase: "agent.
|
|
7876
|
+
readonly phase: "agent.send";
|
|
7765
7877
|
readonly startOffsetMs: number;
|
|
7766
7878
|
readonly durationMs: number;
|
|
7767
7879
|
readonly intervalId: string;
|
|
7768
7880
|
readonly parentIntervalId: string | null;
|
|
7769
|
-
}
|
|
7881
|
+
} | {
|
|
7882
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
7883
|
+
readonly label: string;
|
|
7884
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
7885
|
+
readonly startOffsetMs: number;
|
|
7886
|
+
readonly durationMs: number;
|
|
7887
|
+
readonly intervalId: string;
|
|
7888
|
+
readonly parentIntervalId: string | null;
|
|
7889
|
+
})[];
|
|
7770
7890
|
};
|
|
7771
7891
|
readonly diagnostics: {
|
|
7772
7892
|
readonly collection: {
|
|
@@ -8207,15 +8327,23 @@ export declare const builtinDomainViewBindings: Readonly<{
|
|
|
8207
8327
|
readonly commandId: string;
|
|
8208
8328
|
})[]];
|
|
8209
8329
|
};
|
|
8210
|
-
readonly intervals: readonly {
|
|
8330
|
+
readonly intervals: readonly ({
|
|
8211
8331
|
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
8212
8332
|
readonly label: string;
|
|
8213
|
-
readonly phase: "agent.
|
|
8333
|
+
readonly phase: "agent.send";
|
|
8214
8334
|
readonly startOffsetMs: number;
|
|
8215
8335
|
readonly durationMs: number;
|
|
8216
8336
|
readonly intervalId: string;
|
|
8217
8337
|
readonly parentIntervalId: string | null;
|
|
8218
|
-
}
|
|
8338
|
+
} | {
|
|
8339
|
+
readonly outcome: "completed" | "cancelled" | "interrupted" | "failed" | "unknown";
|
|
8340
|
+
readonly label: string;
|
|
8341
|
+
readonly phase: "agent.ensure" | "sandbox.prepare" | "eval.run" | "sandbox.command" | "attempt.setup" | "attempt.teardown" | "assertion.evaluate" | "verdict.fold";
|
|
8342
|
+
readonly startOffsetMs: number;
|
|
8343
|
+
readonly durationMs: number;
|
|
8344
|
+
readonly intervalId: string;
|
|
8345
|
+
readonly parentIntervalId: string | null;
|
|
8346
|
+
})[];
|
|
8219
8347
|
};
|
|
8220
8348
|
readonly diagnostics: {
|
|
8221
8349
|
readonly collection: {
|
|
@@ -5,6 +5,7 @@ exports.makeClosedRows = makeClosedRows;
|
|
|
5
5
|
exports.isClosedRows = isClosedRows;
|
|
6
6
|
exports.closedRowsMetadata = closedRowsMetadata;
|
|
7
7
|
exports.sampleCapabilityTypeId = Symbol("niceeval.analysis.Sample");
|
|
8
|
+
const experimentComparisonScopeTypeId = Symbol("niceeval.analysis.ExperimentComparisonScope");
|
|
8
9
|
const closedRowsTypeId = Symbol("niceeval.analysis.ClosedRows");
|
|
9
10
|
const closedRowsBrands = new WeakSet();
|
|
10
11
|
const closedRowsMetadataByValue = new WeakMap();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contracts.js","sourceRoot":"","sources":["contracts.ts"],"names":[],"mappings":";;;AA+QA,wCAqBC;AAED,oCAIC;AAED,gDAEC;AAjIY,QAAA,sBAAsB,GAAkB,MAAM,CAAC,0BAA0B,CAAC,CAAC;AAiFxF,MAAM,gBAAgB,GAAkB,MAAM,CAAC,8BAA8B,CAAC,CAAC;AAO/E,MAAM,gBAAgB,GAAG,IAAI,OAAO,EAAU,CAAC;AAC/C,MAAM,yBAAyB,GAAG,IAAI,OAAO,EAA8B,CAAC;AAQ5E,sEAAsE;AACtE,SAAgB,cAAc,CAAM,KAKnC;IACG,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAA+B,CAAC;IAC3D,MAAM,QAAQ,GAAuB,MAAM,CAAC,MAAM,CAAC;QAC/C,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAC7E,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;KACvC,CAAC,CAAC;IACH,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;QAC1B,CAAC,gBAAgB,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE;QACtD,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE;QACxD,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE;QACpD,IAAI,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;KACnD,CAAC,CAAC;IACH,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,yBAAyB,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC/B,CAAC;AACD,+EAA+E;AAC/E,SAAgB,YAAY,CAAC,KAAc;IACvC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;WACpB,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC;WAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,gBAAgB,CAAC,KAAK,IAAI,CAAC;AACzD,CAAC;AACD,8EAA8E;AAC9E,SAAgB,kBAAkB,CAAC,KAAc;IAC7C,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAClF,CAAC","sourcesContent":["import type { MembershipAction } from \"../record/model/core.ts\";\nimport type { AttemptId, EvalId, ExecutionIdentityDigest, ExperimentId, RunId, SlotId, UtcMillis, } from \"../record/model/identifiers.ts\";\nimport type { AttemptLocator } from \"../attempt-locator.ts\";\nexport type { AttemptId, EvalId, ExecutionIdentityDigest, ExperimentId, RunId, SlotId, UtcMillis, };\n/** JSON is the only wire shape accepted by the Sample snapshot codec. */\nexport type JsonValue = null | boolean | number | string | readonly JsonValue[] | {\n readonly [key: string]: JsonValue;\n};\n/** Stable identity for one Sample capability and its portable snapshot. */\nexport interface SampleIdentity {\n readonly kind: \"analysis-sample\";\n readonly id: string;\n}\n/**\n * A public identity for evidence from one immutable Attempt. It deliberately\n * carries a locator rather than Record's nominal selected-reference capability\n * or an AttemptId that could be used as an ad-hoc storage lookup.\n */\nexport interface AttemptEvidenceIdentity {\n readonly kind: \"attempt\";\n readonly locator: AttemptLocator;\n readonly originRunId: RunId;\n}\nexport type AnalysisSelectionProblem = {\n readonly code: \"incomplete-run\";\n readonly runId: RunId;\n} | {\n readonly code: \"record-core-invalid\";\n readonly runId: RunId;\n} | {\n readonly code: \"selection-run-missing\";\n readonly runId: RunId;\n} | {\n readonly code: \"selection-run-unreadable\";\n readonly runId: RunId;\n};\n/**\n * One current target Slot identity. `project-current` keeps only Sample\n * members whose experiment/eval/attempt ordinal aligns, then whose digest\n * still narrows that aligned identity.\n */\nexport interface AnalysisCurrentSlotIdentity {\n readonly experimentId: ExperimentId;\n readonly evalId: EvalId;\n /** Logical attempt identity; never inferred from expectedSlots array order. */\n readonly attemptOrdinal: number;\n /** Narrows an already aligned logical identity; it is not the alignment key. */\n readonly executionIdentityDigest: ExecutionIdentityDigest;\n}\n/** Exact occurrence retained after project-current logical alignment. */\nexport interface AnalysisSlotOccurrenceIdentity {\n readonly runId: RunId;\n readonly slotId: SlotId;\n}\n/** Host-issued selection used by Application, CLI, and Report from-record. */\nexport type AnalysisSelectionRequest = {\n readonly policy: \"explicit-runs\";\n readonly runIds: readonly RunId[];\n} | {\n readonly policy: \"project-current\";\n readonly experimentIds?: readonly ExperimentId[];\n readonly currentSlots: readonly AnalysisCurrentSlotIdentity[];\n};\n/**\n * The exact selection audit retained by a closed SampleSnapshot. `runIds` are\n * the explicit request; `selectedRunIds` are recomputed from the current,\n * non-excluded Sample frame after every monotonic narrowing.\n */\nexport type AnalysisSelectionSummary = {\n readonly policy: \"explicit-runs\";\n readonly runIds: readonly RunId[];\n readonly selectedRunIds: readonly RunId[];\n readonly problems: readonly AnalysisSelectionProblem[];\n} | {\n readonly policy: \"project-current\";\n readonly experimentIds: readonly ExperimentId[] | \"all\";\n readonly selectedRunIds: readonly RunId[];\n readonly problems: readonly AnalysisSelectionProblem[];\n};\nexport interface AnalysisRun {\n readonly runId: RunId;\n readonly experimentId: ExperimentId;\n /**\n * The safe, immutable portion of the selected Run's Core context. A later\n * Core read can be unavailable after selection, in which case no context is\n * invented for that Run.\n */\n readonly context: AnalysisRunContext | null;\n readonly startedAt: UtcMillis;\n readonly completedAt: UtcMillis;\n readonly expectedSlots: readonly SlotId[];\n}\n/**\n * Display- and grouping-safe Run configuration retained by a Sample. It has\n * no Record capability and is recursively JSON-only.\n */\nexport interface AnalysisRunContext {\n readonly execution: AnalysisRunExecution;\n readonly labels: Readonly<Record<string, string>>;\n}\nexport interface AnalysisRunExecution {\n readonly agentId: string;\n readonly model: string | null;\n readonly reasoningEffort: string | null;\n readonly flags: Readonly<Record<string, JsonValue>>;\n}\nexport interface AnalysisSlotRef {\n /** Occurrence identity is always the exact selected Run/Slot pair. */\n readonly runId: RunId;\n readonly slotId: SlotId;\n /** Derived from the associated selected AnalysisRun, never a Record Slot field. */\n readonly experimentId: ExperimentId;\n /** Logical alignment is experiment + eval + durable attempt ordinal. */\n readonly evalId: EvalId;\n readonly attemptOrdinal: number;\n /** This remains an identity-narrowing value, never a logical alignment key. */\n readonly executionIdentityDigest: ExecutionIdentityDigest;\n}\n/** A Member with an exact origin Attempt. */\nexport interface IncludedAnalysisSlot extends AnalysisSlotRef {\n readonly state: \"included\";\n /** Core provenance is not inferred from whether the Attempt is local. */\n readonly action: \"executed\" | \"carried\" | \"accepted\";\n readonly relation: \"origin\" | \"reference\";\n readonly attempt: AttemptEvidenceIdentity;\n}\n/**\n * An expected Slot without an Attempt. `null` means no Member document was\n * recorded; the two actions retain a recorded null Member without suggesting\n * that it is pending work.\n */\nexport interface NotRecordedAnalysisSlot extends AnalysisSlotRef {\n readonly state: \"not-recorded\";\n readonly action: \"not-dispatched\" | \"interrupted\" | null;\n readonly attempt: null;\n}\n/** Core could be read far enough to identify the frame but not its member. */\nexport interface CoreInvalidAnalysisSlot extends AnalysisSlotRef {\n readonly state: \"core-invalid\";\n readonly action: MembershipAction | null;\n readonly attempt: null;\n readonly issues: readonly AnalysisIssue[];\n}\nexport type ActiveAnalysisSlot = IncludedAnalysisSlot | NotRecordedAnalysisSlot | CoreInvalidAnalysisSlot;\n/** A monotonic Sample narrowing preserves the previous, non-nested state. */\nexport interface ExcludedAnalysisSlot extends AnalysisSlotRef {\n readonly state: \"excluded\";\n readonly base: ActiveAnalysisSlot;\n /** Present when `project-current` dropped this Slot for identity mismatch. */\n readonly reason?: \"identity-mismatch\";\n}\nexport type AnalysisSlot = ActiveAnalysisSlot | ExcludedAnalysisSlot;\nexport interface SampleCoverage {\n readonly frameTotal: number;\n readonly selected: number;\n readonly included: number;\n readonly notRecorded: number;\n readonly coreInvalid: number;\n readonly excluded: number;\n}\n/**\n * The closed, portable representation of a Record selection. It contains\n * only selection identity, Core-derived Run/Slot denominator facts, and\n * selection problems. Decoding it can never reopen a Record.\n */\nexport interface SampleSnapshot {\n readonly version: 1;\n readonly identity: SampleIdentity;\n readonly selection: AnalysisSelectionSummary;\n readonly runs: readonly AnalysisRun[];\n readonly slots: readonly AnalysisSlot[];\n readonly coverage: SampleCoverage;\n}\nexport const sampleCapabilityTypeId: unique symbol = Symbol(\"niceeval.analysis.Sample\");\n/** A Host-issued, Scope-bound lazy Record-reading capability. */\nexport interface Sample {\n readonly kind: \"analysis-sample\";\n readonly snapshot: SampleSnapshot;\n readonly [sampleCapabilityTypeId]: true;\n}\nexport interface SampleSelector {\n readonly runIds?: readonly RunId[];\n readonly slotIds?: readonly SlotId[];\n}\nexport interface SampleClosedError {\n readonly code: \"analysis-sample-closed\";\n readonly sample: SampleIdentity;\n}\nexport interface SampleSnapshotCodecError {\n readonly code: \"sample-snapshot-invalid\";\n readonly path: readonly string[];\n readonly reason: string;\n}\nexport interface AnalysisRequestError {\n readonly code: \"analysis-request-invalid\";\n readonly reason: string;\n}\nexport type AnalysisInputIssueCode = \"missing\" | \"unsupported\" | \"producer-incompatible\" | \"input-invalid\" | \"reduction-failed\" | \"relation-unmatched\";\nexport interface EvidenceIdentity {\n readonly kind: \"attempt\";\n readonly locator: AttemptLocator;\n}\nexport interface EvidenceRef {\n readonly identity: EvidenceIdentity;\n}\nexport interface AnalysisIssue {\n readonly code: AnalysisInputIssueCode;\n readonly message: string;\n readonly refs: readonly EvidenceRef[];\n}\nexport interface ProducerIdentity {\n readonly id: string;\n}\n/** Internal result of a published AnalysisInput read. It never reaches Report. */\nexport type SampleInputObservation<Value> = {\n readonly state: \"value\";\n readonly value: Value;\n readonly refs: readonly EvidenceRef[];\n readonly producer?: ProducerIdentity;\n} | {\n readonly state: \"missing\" | \"unsupported\" | \"failed\";\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n readonly producer?: ProducerIdentity;\n};\nexport type MetricState = \"available\" | \"partial\"\n/** A specialized closed metric may have no reportable value at all. */\n | \"unavailable\" | \"empty\" | \"unsupported\" | \"failed\";\nexport type MetricBasis = \"attempt\" | \"eval\" | \"run\" | \"pair\" | \"slot\";\nexport type MeasureFormat = string | {\n readonly kind: string;\n readonly options?: JsonValue;\n};\n/** A closed Measure cell: never a selected Record handle or reader callback. */\nexport interface MetricValue<Value = number> {\n readonly value: Value | null;\n readonly state: MetricState;\n readonly samples: number;\n readonly total: number;\n readonly basis: MetricBasis;\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n readonly unit?: string;\n readonly format?: MeasureFormat;\n readonly better?: \"higher\" | \"lower\" | \"neutral\";\n readonly bounds?: {\n readonly min?: number;\n readonly max?: number;\n };\n}\nexport interface ClosedRowsIdentity {\n readonly kind: \"closed-rows\";\n readonly id: string;\n}\nconst closedRowsTypeId: unique symbol = Symbol(\"niceeval.analysis.ClosedRows\");\n/** @internal Metadata is retained out-of-band so an ordinary array cannot forge it. */\nexport interface ClosedRowsMetadata {\n readonly identity: ClosedRowsIdentity;\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n}\nconst closedRowsBrands = new WeakSet<object>();\nconst closedRowsMetadataByValue = new WeakMap<object, ClosedRowsMetadata>();\n/** A branded, frozen array produced only by Analysis query execution. */\nexport interface ClosedRows<Row> extends ReadonlyArray<Readonly<Row>> {\n readonly [closedRowsTypeId]: true;\n readonly identity: ClosedRowsIdentity;\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n}\n/** @internal Keeps the brand unforgeable to normal public callers. */\nexport function makeClosedRows<Row>(input: {\n readonly rows: readonly Readonly<Row>[];\n readonly identity: ClosedRowsIdentity;\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n}): ClosedRows<Row> {\n const rows = [...input.rows] as unknown as ClosedRows<Row>;\n const metadata: ClosedRowsMetadata = Object.freeze({\n identity: Object.freeze({ kind: input.identity.kind, id: input.identity.id }),\n issues: Object.freeze([...input.issues]),\n refs: Object.freeze([...input.refs]),\n });\n Object.defineProperties(rows, {\n [closedRowsTypeId]: { value: true, enumerable: false },\n identity: { value: metadata.identity, enumerable: true },\n issues: { value: metadata.issues, enumerable: true },\n refs: { value: metadata.refs, enumerable: true },\n });\n closedRowsBrands.add(rows);\n closedRowsMetadataByValue.set(rows, metadata);\n return Object.freeze(rows);\n}\n/** @internal Checks actual Analysis issuance, rather than structural shape. */\nexport function isClosedRows(value: unknown): value is ClosedRows<unknown> {\n return Array.isArray(value)\n && closedRowsBrands.has(value)\n && Reflect.get(value, closedRowsTypeId) === true;\n}\n/** @internal Returns immutable metadata registered at Analysis close time. */\nexport function closedRowsMetadata(value: unknown): ClosedRowsMetadata | undefined {\n return isClosedRows(value) ? closedRowsMetadataByValue.get(value) : undefined;\n}\nexport interface PopulationIdentity {\n readonly kind: \"population\";\n readonly id: string;\n}\n/** The common, entirely closed base of every published DomainView. */\nexport interface DomainView {\n readonly kind: \"domain-view\";\n readonly identity: {\n readonly kind: \"domain-view\";\n readonly id: string;\n };\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n}\n"]}
|
|
1
|
+
{"version":3,"file":"contracts.js","sourceRoot":"","sources":["contracts.ts"],"names":[],"mappings":";;;AAuTA,wCAqBC;AAED,oCAIC;AAED,gDAEC;AAzKY,QAAA,sBAAsB,GAAkB,MAAM,CAAC,0BAA0B,CAAC,CAAC;AAoCxF,MAAM,+BAA+B,GAAkB,MAAM,CAAC,6CAA6C,CAAC,CAAC;AAqF7G,MAAM,gBAAgB,GAAkB,MAAM,CAAC,8BAA8B,CAAC,CAAC;AAO/E,MAAM,gBAAgB,GAAG,IAAI,OAAO,EAAU,CAAC;AAC/C,MAAM,yBAAyB,GAAG,IAAI,OAAO,EAA8B,CAAC;AAQ5E,sEAAsE;AACtE,SAAgB,cAAc,CAAM,KAKnC;IACG,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,CAA+B,CAAC;IAC3D,MAAM,QAAQ,GAAuB,MAAM,CAAC,MAAM,CAAC;QAC/C,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QAC7E,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QACxC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;KACvC,CAAC,CAAC;IACH,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE;QAC1B,CAAC,gBAAgB,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE;QACtD,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE;QACxD,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE;QACpD,IAAI,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE;KACnD,CAAC,CAAC;IACH,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC3B,yBAAyB,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC/B,CAAC;AACD,+EAA+E;AAC/E,SAAgB,YAAY,CAAC,KAAc;IACvC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;WACpB,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC;WAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,gBAAgB,CAAC,KAAK,IAAI,CAAC;AACzD,CAAC;AACD,8EAA8E;AAC9E,SAAgB,kBAAkB,CAAC,KAAc;IAC7C,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,yBAAyB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAClF,CAAC","sourcesContent":["import type { MembershipAction } from \"../record/model/core.ts\";\nimport type { AttemptId, EvalId, ExecutionIdentityDigest, ExperimentId, RunId, SlotId, UtcMillis, } from \"../record/model/identifiers.ts\";\nimport type { AttemptLocator } from \"../attempt-locator.ts\";\nexport type { AttemptId, EvalId, ExecutionIdentityDigest, ExperimentId, RunId, SlotId, UtcMillis, };\n/** JSON is the only wire shape accepted by the Sample snapshot codec. */\nexport type JsonValue = null | boolean | number | string | readonly JsonValue[] | {\n readonly [key: string]: JsonValue;\n};\n/** Stable identity for one Sample capability and its portable snapshot. */\nexport interface SampleIdentity {\n readonly kind: \"analysis-sample\";\n readonly id: string;\n}\n/**\n * A public identity for evidence from one immutable Attempt. It deliberately\n * carries a locator rather than Record's nominal selected-reference capability\n * or an AttemptId that could be used as an ad-hoc storage lookup.\n */\nexport interface AttemptEvidenceIdentity {\n readonly kind: \"attempt\";\n readonly locator: AttemptLocator;\n readonly originRunId: RunId;\n}\nexport type AnalysisSelectionProblem = {\n readonly code: \"incomplete-run\";\n readonly runId: RunId;\n} | {\n readonly code: \"record-core-invalid\";\n readonly runId: RunId;\n} | {\n readonly code: \"selection-run-missing\";\n readonly runId: RunId;\n} | {\n readonly code: \"selection-run-unreadable\";\n readonly runId: RunId;\n};\n/**\n * One current target Slot identity. `project-current` keeps only Sample\n * members whose experiment/eval/attempt ordinal aligns, then whose digest\n * still narrows that aligned identity.\n */\nexport interface AnalysisCurrentSlotIdentity {\n readonly experimentId: ExperimentId;\n readonly evalId: EvalId;\n /** Logical attempt identity; never inferred from expectedSlots array order. */\n readonly attemptOrdinal: number;\n /** Narrows an already aligned logical identity; it is not the alignment key. */\n readonly executionIdentityDigest: ExecutionIdentityDigest;\n}\n/** Exact occurrence retained after project-current logical alignment. */\nexport interface AnalysisSlotOccurrenceIdentity {\n readonly runId: RunId;\n readonly slotId: SlotId;\n}\n/** Host-issued selection used by Application, CLI, and Report from-record. */\nexport type AnalysisSelectionRequest = {\n readonly policy: \"explicit-runs\";\n readonly runIds: readonly RunId[];\n} | {\n readonly policy: \"project-current\";\n readonly experimentIds?: readonly ExperimentId[];\n readonly currentSlots: readonly AnalysisCurrentSlotIdentity[];\n};\n/**\n * The exact selection audit retained by a closed SampleSnapshot. `runIds` are\n * the explicit request; `selectedRunIds` are recomputed from the current,\n * non-excluded Sample frame after every monotonic narrowing.\n */\nexport type AnalysisSelectionSummary = {\n readonly policy: \"explicit-runs\";\n readonly runIds: readonly RunId[];\n readonly selectedRunIds: readonly RunId[];\n readonly problems: readonly AnalysisSelectionProblem[];\n} | {\n readonly policy: \"project-current\";\n readonly experimentIds: readonly ExperimentId[] | \"all\";\n readonly selectedRunIds: readonly RunId[];\n readonly problems: readonly AnalysisSelectionProblem[];\n};\nexport interface AnalysisRun {\n readonly runId: RunId;\n readonly experimentId: ExperimentId;\n /**\n * The safe, immutable portion of the selected Run's Core context. A later\n * Core read can be unavailable after selection, in which case no context is\n * invented for that Run.\n */\n readonly context: AnalysisRunContext | null;\n readonly startedAt: UtcMillis;\n readonly completedAt: UtcMillis;\n readonly expectedSlots: readonly SlotId[];\n}\n/**\n * Display- and grouping-safe Run configuration retained by a Sample. It has\n * no Record capability and is recursively JSON-only.\n */\nexport interface AnalysisRunContext {\n readonly execution: AnalysisRunExecution;\n readonly labels: Readonly<Record<string, string>>;\n}\nexport interface AnalysisRunExecution {\n readonly agentId: string;\n readonly model: string | null;\n readonly reasoningEffort: string | null;\n readonly flags: Readonly<Record<string, JsonValue>>;\n}\nexport interface AnalysisSlotRef {\n /** Occurrence identity is always the exact selected Run/Slot pair. */\n readonly runId: RunId;\n readonly slotId: SlotId;\n /** Derived from the associated selected AnalysisRun, never a Record Slot field. */\n readonly experimentId: ExperimentId;\n /** Logical alignment is experiment + eval + durable attempt ordinal. */\n readonly evalId: EvalId;\n readonly attemptOrdinal: number;\n /** This remains an identity-narrowing value, never a logical alignment key. */\n readonly executionIdentityDigest: ExecutionIdentityDigest;\n}\n/** A Member with an exact origin Attempt. */\nexport interface IncludedAnalysisSlot extends AnalysisSlotRef {\n readonly state: \"included\";\n /** Core provenance is not inferred from whether the Attempt is local. */\n readonly action: \"executed\" | \"carried\" | \"accepted\";\n readonly relation: \"origin\" | \"reference\";\n readonly attempt: AttemptEvidenceIdentity;\n}\n/**\n * An expected Slot without an Attempt. `null` means no Member document was\n * recorded; the two actions retain a recorded null Member without suggesting\n * that it is pending work.\n */\nexport interface NotRecordedAnalysisSlot extends AnalysisSlotRef {\n readonly state: \"not-recorded\";\n readonly action: \"not-dispatched\" | \"interrupted\" | null;\n readonly attempt: null;\n}\n/** Core could be read far enough to identify the frame but not its member. */\nexport interface CoreInvalidAnalysisSlot extends AnalysisSlotRef {\n readonly state: \"core-invalid\";\n readonly action: MembershipAction | null;\n readonly attempt: null;\n readonly issues: readonly AnalysisIssue[];\n}\nexport type ActiveAnalysisSlot = IncludedAnalysisSlot | NotRecordedAnalysisSlot | CoreInvalidAnalysisSlot;\n/** A monotonic Sample narrowing preserves the previous, non-nested state. */\nexport interface ExcludedAnalysisSlot extends AnalysisSlotRef {\n readonly state: \"excluded\";\n readonly base: ActiveAnalysisSlot;\n /** Present when `project-current` dropped this Slot for identity mismatch. */\n readonly reason?: \"identity-mismatch\";\n}\nexport type AnalysisSlot = ActiveAnalysisSlot | ExcludedAnalysisSlot;\nexport interface SampleCoverage {\n readonly frameTotal: number;\n readonly selected: number;\n readonly included: number;\n readonly notRecorded: number;\n readonly coreInvalid: number;\n readonly excluded: number;\n}\n/**\n * The closed, portable representation of a Record selection. It contains\n * only selection identity, Core-derived Run/Slot denominator facts, and\n * selection problems. Decoding it can never reopen a Record.\n */\nexport interface SampleSnapshot {\n readonly version: 1;\n readonly identity: SampleIdentity;\n readonly selection: AnalysisSelectionSummary;\n readonly runs: readonly AnalysisRun[];\n readonly slots: readonly AnalysisSlot[];\n readonly coverage: SampleCoverage;\n}\nexport const sampleCapabilityTypeId: unique symbol = Symbol(\"niceeval.analysis.Sample\");\n/** A Host-issued, Scope-bound lazy Record-reading capability. */\nexport interface Sample {\n readonly kind: \"analysis-sample\";\n readonly snapshot: SampleSnapshot;\n readonly [sampleCapabilityTypeId]: true;\n}\nexport type ExperimentGroupIdentity = {\n readonly kind: \"named\";\n readonly groupId: string;\n readonly key: `named/${string}`;\n} | {\n readonly kind: \"singleton\";\n readonly experimentId: ExperimentId;\n readonly key: `singleton/${string}`;\n};\nexport type NonComparableReason = \"eval-population-mismatch\";\nexport interface NonComparableIssue {\n readonly reason: NonComparableReason;\n readonly members: readonly ExperimentId[];\n readonly actual: readonly {\n readonly member: ExperimentId;\n readonly population: JsonValue | null;\n readonly basis: JsonValue | null;\n }[];\n readonly refs: readonly EvidenceRef[];\n readonly params: Readonly<Record<string, JsonValue>>;\n}\nexport type ExperimentComparisonState = {\n readonly state: \"comparable\";\n readonly members: readonly ExperimentId[];\n} | {\n readonly state: \"non-comparable\";\n readonly members: readonly ExperimentId[];\n readonly issues: readonly NonComparableIssue[];\n};\nconst experimentComparisonScopeTypeId: unique symbol = Symbol(\"niceeval.analysis.ExperimentComparisonScope\");\n/** Analysis-issued comparison capability; the backing Sample remains private. */\nexport interface ExperimentComparisonScope {\n readonly group: ExperimentGroupIdentity;\n readonly comparison: ExperimentComparisonState;\n readonly [experimentComparisonScopeTypeId]: true;\n}\nexport interface AnalysisComparisonGroupMismatchError {\n readonly code: \"analysis-comparison-group-mismatch\";\n readonly groups: readonly ExperimentGroupIdentity[];\n}\nexport interface SampleSelector {\n readonly runIds?: readonly RunId[];\n readonly slotIds?: readonly SlotId[];\n}\nexport interface SampleClosedError {\n readonly code: \"analysis-sample-closed\";\n readonly sample: SampleIdentity;\n}\nexport interface SampleSnapshotCodecError {\n readonly code: \"sample-snapshot-invalid\";\n readonly path: readonly string[];\n readonly reason: string;\n}\nexport interface AnalysisRequestError {\n readonly code: \"analysis-request-invalid\";\n readonly reason: string;\n}\nexport type AnalysisInputIssueCode = \"missing\" | \"migration-required\" | \"unsupported\" | \"producer-incompatible\" | \"input-invalid\" | \"reduction-failed\" | \"relation-unmatched\";\nexport interface EvidenceIdentity {\n readonly kind: \"attempt\";\n readonly locator: AttemptLocator;\n}\nexport interface EvidenceRef {\n readonly identity: EvidenceIdentity;\n}\nexport interface AnalysisIssue {\n readonly code: AnalysisInputIssueCode;\n readonly message: string;\n readonly refs: readonly EvidenceRef[];\n}\nexport interface ProducerIdentity {\n readonly id: string;\n}\n/** Internal result of a published AnalysisInput read. It never reaches Report. */\nexport type SampleInputObservation<Value> = {\n readonly state: \"value\";\n readonly value: Value;\n readonly refs: readonly EvidenceRef[];\n readonly producer?: ProducerIdentity;\n} | {\n readonly state: \"missing\" | \"migration-required\" | \"unsupported\" | \"failed\";\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n readonly producer?: ProducerIdentity;\n};\nexport type MetricState = \"available\" | \"partial\"\n/** A specialized closed metric may have no reportable value at all. */\n | \"unavailable\" | \"empty\" | \"migration-required\" | \"unsupported\" | \"failed\";\nexport type MetricBasis = \"attempt\" | \"eval\" | \"run\" | \"pair\" | \"slot\";\nexport type MeasureFormat = string | {\n readonly kind: string;\n readonly options?: JsonValue;\n};\n/** A closed Measure cell: never a selected Record handle or reader callback. */\nexport interface MetricValue<Value = number> {\n readonly value: Value | null;\n readonly state: MetricState;\n readonly samples: number;\n readonly total: number;\n readonly basis: MetricBasis;\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n readonly unit?: string;\n readonly format?: MeasureFormat;\n readonly better?: \"higher\" | \"lower\" | \"neutral\";\n readonly bounds?: {\n readonly min?: number;\n readonly max?: number;\n };\n}\nexport interface ClosedRowsIdentity {\n readonly kind: \"closed-rows\";\n readonly id: string;\n}\nconst closedRowsTypeId: unique symbol = Symbol(\"niceeval.analysis.ClosedRows\");\n/** @internal Metadata is retained out-of-band so an ordinary array cannot forge it. */\nexport interface ClosedRowsMetadata {\n readonly identity: ClosedRowsIdentity;\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n}\nconst closedRowsBrands = new WeakSet<object>();\nconst closedRowsMetadataByValue = new WeakMap<object, ClosedRowsMetadata>();\n/** A branded, frozen array produced only by Analysis query execution. */\nexport interface ClosedRows<Row> extends ReadonlyArray<Readonly<Row>> {\n readonly [closedRowsTypeId]: true;\n readonly identity: ClosedRowsIdentity;\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n}\n/** @internal Keeps the brand unforgeable to normal public callers. */\nexport function makeClosedRows<Row>(input: {\n readonly rows: readonly Readonly<Row>[];\n readonly identity: ClosedRowsIdentity;\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n}): ClosedRows<Row> {\n const rows = [...input.rows] as unknown as ClosedRows<Row>;\n const metadata: ClosedRowsMetadata = Object.freeze({\n identity: Object.freeze({ kind: input.identity.kind, id: input.identity.id }),\n issues: Object.freeze([...input.issues]),\n refs: Object.freeze([...input.refs]),\n });\n Object.defineProperties(rows, {\n [closedRowsTypeId]: { value: true, enumerable: false },\n identity: { value: metadata.identity, enumerable: true },\n issues: { value: metadata.issues, enumerable: true },\n refs: { value: metadata.refs, enumerable: true },\n });\n closedRowsBrands.add(rows);\n closedRowsMetadataByValue.set(rows, metadata);\n return Object.freeze(rows);\n}\n/** @internal Checks actual Analysis issuance, rather than structural shape. */\nexport function isClosedRows(value: unknown): value is ClosedRows<unknown> {\n return Array.isArray(value)\n && closedRowsBrands.has(value)\n && Reflect.get(value, closedRowsTypeId) === true;\n}\n/** @internal Returns immutable metadata registered at Analysis close time. */\nexport function closedRowsMetadata(value: unknown): ClosedRowsMetadata | undefined {\n return isClosedRows(value) ? closedRowsMetadataByValue.get(value) : undefined;\n}\nexport interface PopulationIdentity {\n readonly kind: \"population\";\n readonly id: string;\n}\n/** The common, entirely closed base of every published DomainView. */\nexport interface DomainView {\n readonly kind: \"domain-view\";\n readonly identity: {\n readonly kind: \"domain-view\";\n readonly id: string;\n };\n readonly issues: readonly AnalysisIssue[];\n readonly refs: readonly EvidenceRef[];\n}\n"]}
|
|
@@ -178,6 +178,46 @@ export interface Sample {
|
|
|
178
178
|
readonly snapshot: SampleSnapshot;
|
|
179
179
|
readonly [sampleCapabilityTypeId]: true;
|
|
180
180
|
}
|
|
181
|
+
export type ExperimentGroupIdentity = {
|
|
182
|
+
readonly kind: "named";
|
|
183
|
+
readonly groupId: string;
|
|
184
|
+
readonly key: `named/${string}`;
|
|
185
|
+
} | {
|
|
186
|
+
readonly kind: "singleton";
|
|
187
|
+
readonly experimentId: ExperimentId;
|
|
188
|
+
readonly key: `singleton/${string}`;
|
|
189
|
+
};
|
|
190
|
+
export type NonComparableReason = "eval-population-mismatch";
|
|
191
|
+
export interface NonComparableIssue {
|
|
192
|
+
readonly reason: NonComparableReason;
|
|
193
|
+
readonly members: readonly ExperimentId[];
|
|
194
|
+
readonly actual: readonly {
|
|
195
|
+
readonly member: ExperimentId;
|
|
196
|
+
readonly population: JsonValue | null;
|
|
197
|
+
readonly basis: JsonValue | null;
|
|
198
|
+
}[];
|
|
199
|
+
readonly refs: readonly EvidenceRef[];
|
|
200
|
+
readonly params: Readonly<Record<string, JsonValue>>;
|
|
201
|
+
}
|
|
202
|
+
export type ExperimentComparisonState = {
|
|
203
|
+
readonly state: "comparable";
|
|
204
|
+
readonly members: readonly ExperimentId[];
|
|
205
|
+
} | {
|
|
206
|
+
readonly state: "non-comparable";
|
|
207
|
+
readonly members: readonly ExperimentId[];
|
|
208
|
+
readonly issues: readonly NonComparableIssue[];
|
|
209
|
+
};
|
|
210
|
+
declare const experimentComparisonScopeTypeId: unique symbol;
|
|
211
|
+
/** Analysis-issued comparison capability; the backing Sample remains private. */
|
|
212
|
+
export interface ExperimentComparisonScope {
|
|
213
|
+
readonly group: ExperimentGroupIdentity;
|
|
214
|
+
readonly comparison: ExperimentComparisonState;
|
|
215
|
+
readonly [experimentComparisonScopeTypeId]: true;
|
|
216
|
+
}
|
|
217
|
+
export interface AnalysisComparisonGroupMismatchError {
|
|
218
|
+
readonly code: "analysis-comparison-group-mismatch";
|
|
219
|
+
readonly groups: readonly ExperimentGroupIdentity[];
|
|
220
|
+
}
|
|
181
221
|
export interface SampleSelector {
|
|
182
222
|
readonly runIds?: readonly RunId[];
|
|
183
223
|
readonly slotIds?: readonly SlotId[];
|
|
@@ -195,7 +235,7 @@ export interface AnalysisRequestError {
|
|
|
195
235
|
readonly code: "analysis-request-invalid";
|
|
196
236
|
readonly reason: string;
|
|
197
237
|
}
|
|
198
|
-
export type AnalysisInputIssueCode = "missing" | "unsupported" | "producer-incompatible" | "input-invalid" | "reduction-failed" | "relation-unmatched";
|
|
238
|
+
export type AnalysisInputIssueCode = "missing" | "migration-required" | "unsupported" | "producer-incompatible" | "input-invalid" | "reduction-failed" | "relation-unmatched";
|
|
199
239
|
export interface EvidenceIdentity {
|
|
200
240
|
readonly kind: "attempt";
|
|
201
241
|
readonly locator: AttemptLocator;
|
|
@@ -218,14 +258,14 @@ export type SampleInputObservation<Value> = {
|
|
|
218
258
|
readonly refs: readonly EvidenceRef[];
|
|
219
259
|
readonly producer?: ProducerIdentity;
|
|
220
260
|
} | {
|
|
221
|
-
readonly state: "missing" | "unsupported" | "failed";
|
|
261
|
+
readonly state: "missing" | "migration-required" | "unsupported" | "failed";
|
|
222
262
|
readonly issues: readonly AnalysisIssue[];
|
|
223
263
|
readonly refs: readonly EvidenceRef[];
|
|
224
264
|
readonly producer?: ProducerIdentity;
|
|
225
265
|
};
|
|
226
266
|
export type MetricState = "available" | "partial"
|
|
227
267
|
/** A specialized closed metric may have no reportable value at all. */
|
|
228
|
-
| "unavailable" | "empty" | "unsupported" | "failed";
|
|
268
|
+
| "unavailable" | "empty" | "migration-required" | "unsupported" | "failed";
|
|
229
269
|
export type MetricBasis = "attempt" | "eval" | "run" | "pair" | "slot";
|
|
230
270
|
export type MeasureFormat = string | {
|
|
231
271
|
readonly kind: string;
|