niceeval 0.13.1 → 0.13.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +25 -2
- package/README.md +9 -10
- package/README.zh.md +9 -10
- package/dist/agents/ai-sdk.cjs.map +1 -1
- package/dist/agents/ai-sdk.d.cts +1 -1
- package/dist/agents/ai-sdk.d.mts +1 -1
- package/dist/agents/ai-sdk.d.ts +1 -1
- package/dist/agents/bub-install-spec.cjs +19 -5
- package/dist/agents/bub-install-spec.cjs.map +1 -1
- package/dist/agents/bub-install-spec.d.cts +9 -2
- package/dist/agents/bub-install-spec.d.mts +9 -2
- package/dist/agents/bub-install-spec.d.ts +9 -2
- package/dist/agents/bub-install-spec.js +14 -12
- package/dist/agents/bub.cjs +10 -6
- package/dist/agents/bub.cjs.map +1 -1
- package/dist/agents/bub.d.cts +2 -2
- package/dist/agents/bub.d.mts +2 -2
- package/dist/agents/bub.d.ts +2 -2
- package/dist/agents/claude-code.cjs.map +1 -1
- package/dist/agents/claude-code.d.cts +2 -2
- package/dist/agents/claude-code.d.mts +2 -2
- package/dist/agents/claude-code.d.ts +2 -2
- package/dist/agents/codex.cjs +4 -4
- package/dist/agents/codex.cjs.map +1 -1
- package/dist/agents/codex.d.cts +3 -3
- package/dist/agents/codex.d.mts +3 -3
- package/dist/agents/codex.d.ts +3 -3
- package/dist/agents/coding-cli-versions.cjs +10 -1
- package/dist/agents/coding-cli-versions.cjs.map +1 -1
- package/dist/agents/coding-cli-versions.d.cts +8 -3
- package/dist/agents/coding-cli-versions.d.mts +8 -3
- package/dist/agents/coding-cli-versions.d.ts +8 -3
- package/dist/agents/coding-cli-versions.js +18 -12
- package/dist/agents/deepseek-harness.cjs +151 -0
- package/dist/agents/deepseek-harness.cjs.map +1 -0
- package/dist/agents/deepseek-harness.d.cts +14 -0
- package/dist/agents/deepseek-harness.d.mts +14 -0
- package/dist/agents/deepseek-harness.d.ts +14 -0
- package/dist/agents/deepseek-harness.js +7 -0
- package/dist/agents/deepseek-harness.js.map +1 -0
- package/dist/agents/exact-npm-plugins.cjs +28 -0
- package/dist/agents/exact-npm-plugins.cjs.map +1 -0
- package/dist/agents/exact-npm-plugins.d.cts +10 -0
- package/dist/agents/exact-npm-plugins.d.mts +10 -0
- package/dist/agents/exact-npm-plugins.d.ts +10 -0
- package/dist/agents/exact-npm-plugins.js +8 -0
- package/dist/agents/exact-npm-plugins.js.map +1 -0
- package/dist/agents/index.cjs +5 -1
- package/dist/agents/index.cjs.map +1 -1
- package/dist/agents/index.d.cts +4 -0
- package/dist/agents/index.d.mts +4 -0
- package/dist/agents/index.d.ts +4 -0
- package/dist/agents/index.js +54 -50
- package/dist/agents/index.mjs +54 -50
- package/dist/agents/omp.cjs +170 -0
- package/dist/agents/omp.cjs.map +1 -0
- package/dist/agents/omp.d.cts +14 -0
- package/dist/agents/omp.d.mts +14 -0
- package/dist/agents/omp.d.ts +14 -0
- package/dist/agents/omp.js +7 -0
- package/dist/agents/omp.js.map +1 -0
- package/dist/agents/openclaw.cjs +51 -2
- package/dist/agents/openclaw.cjs.map +1 -1
- package/dist/agents/openclaw.d.cts +2 -0
- package/dist/agents/openclaw.d.mts +2 -0
- package/dist/agents/openclaw.d.ts +2 -0
- package/dist/agents/types.cjs.map +1 -1
- package/dist/agents/types.d.cts +4 -4
- package/dist/agents/types.d.mts +4 -4
- package/dist/agents/types.d.ts +4 -4
- package/dist/agents/ui-message-stream.cjs +24 -7
- package/dist/agents/ui-message-stream.cjs.map +1 -1
- package/dist/analysis/api.cjs +28 -2
- package/dist/analysis/api.cjs.map +1 -1
- package/dist/analysis/api.d.cts +22 -1
- package/dist/analysis/api.d.mts +22 -1
- package/dist/analysis/api.d.ts +22 -1
- package/dist/analysis/api.js +8 -6
- package/dist/analysis/bindings.cjs +11 -18
- package/dist/analysis/bindings.cjs.map +1 -1
- package/dist/analysis/bindings.d.cts +177 -49
- package/dist/analysis/bindings.d.mts +177 -49
- package/dist/analysis/bindings.d.ts +177 -49
- package/dist/analysis/contracts.cjs +1 -0
- package/dist/analysis/contracts.cjs.map +1 -1
- package/dist/analysis/contracts.d.cts +43 -3
- package/dist/analysis/contracts.d.mts +43 -3
- package/dist/analysis/contracts.d.ts +43 -3
- package/dist/analysis/cost-projection.cjs +2 -1
- package/dist/analysis/cost-projection.cjs.map +1 -1
- package/dist/analysis/cost-projection.d.cts +1 -1
- package/dist/analysis/cost-projection.d.mts +1 -1
- package/dist/analysis/cost-projection.d.ts +1 -1
- package/dist/analysis/cost.cjs +13 -3
- package/dist/analysis/cost.cjs.map +1 -1
- package/dist/analysis/cost.d.cts +10 -3
- package/dist/analysis/cost.d.mts +10 -3
- package/dist/analysis/cost.d.ts +10 -3
- package/dist/analysis/experiment-groups.cjs +96 -0
- package/dist/analysis/experiment-groups.cjs.map +1 -0
- package/dist/analysis/experiment-groups.d.cts +9 -0
- package/dist/analysis/experiment-groups.d.mts +9 -0
- package/dist/analysis/experiment-groups.d.ts +9 -0
- package/dist/analysis/experiment-groups.js +10 -0
- package/dist/analysis/experiment-groups.js.map +1 -0
- package/dist/analysis/index.cjs +4 -1
- package/dist/analysis/index.cjs.map +1 -1
- package/dist/analysis/index.d.cts +4 -2
- package/dist/analysis/index.d.mts +4 -2
- package/dist/analysis/index.d.ts +4 -2
- package/dist/analysis/index.js +40 -36
- package/dist/analysis/index.mjs +40 -36
- package/dist/assertions/limits.cjs +4 -0
- package/dist/assertions/limits.cjs.map +1 -1
- package/dist/assertions/limits.d.cts +4 -0
- package/dist/assertions/limits.d.mts +4 -0
- package/dist/assertions/limits.d.ts +4 -0
- package/dist/assertions/match.cjs +63 -25
- package/dist/assertions/match.cjs.map +1 -1
- package/dist/assertions/match.d.cts +7 -0
- package/dist/assertions/match.d.mts +7 -0
- package/dist/assertions/match.d.ts +7 -0
- package/dist/assertions/record/attachment.cjs +32 -4
- package/dist/assertions/record/attachment.cjs.map +1 -1
- package/dist/assertions/record/diff.cjs +10 -4
- package/dist/assertions/record/diff.cjs.map +1 -1
- package/dist/assertions/record/producer.cjs +101 -4
- package/dist/assertions/record/producer.cjs.map +1 -1
- package/dist/assertions/record/producer.d.cts +4 -1
- package/dist/assertions/record/producer.d.mts +4 -1
- package/dist/assertions/record/producer.d.ts +4 -1
- package/dist/assertions/runtime.cjs +45 -5
- package/dist/assertions/runtime.cjs.map +1 -1
- package/dist/cli/record.cjs +113 -8
- package/dist/cli/record.cjs.map +1 -1
- package/dist/cli/record.d.cts +20 -1
- package/dist/cli/record.d.mts +20 -1
- package/dist/cli/record.d.ts +20 -1
- package/dist/cli/record.js +4 -2
- package/dist/cli.cjs +386 -31
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.cts +10 -2
- package/dist/cli.d.mts +10 -2
- package/dist/cli.d.ts +10 -2
- package/dist/context/send-retry.cjs +4 -4
- package/dist/context/send-retry.cjs.map +1 -1
- package/dist/coordination/host/runtime.cjs +8 -30
- package/dist/coordination/host/runtime.cjs.map +1 -1
- package/dist/coordination/host/runtime.d.cts +3 -3
- package/dist/coordination/host/runtime.d.mts +3 -3
- package/dist/coordination/host/runtime.d.ts +3 -3
- package/dist/coordination/host/types.cjs.map +1 -1
- package/dist/coordination/host/types.d.cts +0 -5
- package/dist/coordination/host/types.d.mts +0 -5
- package/dist/coordination/host/types.d.ts +0 -5
- package/dist/coordination/platform/node.cjs +90 -9
- package/dist/coordination/platform/node.cjs.map +1 -1
- package/dist/define.cjs +25 -1
- package/dist/define.cjs.map +1 -1
- package/dist/experiment/host/index.cjs +1 -0
- package/dist/experiment/host/index.cjs.map +1 -1
- package/dist/i18n/en.cjs +26 -8
- package/dist/i18n/en.cjs.map +1 -1
- package/dist/i18n/en.d.cts +19 -6
- package/dist/i18n/en.d.mts +19 -6
- package/dist/i18n/en.d.ts +19 -6
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/o11y/parsers/hermes.cjs +1 -1
- package/dist/o11y/parsers/hermes.cjs.map +1 -1
- package/dist/o11y/prices.json +1 -1
- package/dist/o11y/record/families.cjs +12 -3
- package/dist/o11y/record/families.cjs.map +1 -1
- package/dist/o11y/record/families.d.cts +8 -8
- package/dist/o11y/record/families.d.mts +8 -8
- package/dist/o11y/record/families.d.ts +8 -8
- package/dist/o11y/record/family-projectors.d.cts +66 -18
- package/dist/o11y/record/family-projectors.d.mts +66 -18
- package/dist/o11y/record/family-projectors.d.ts +66 -18
- package/dist/o11y/record/family-writers.d.cts +11 -3
- package/dist/o11y/record/family-writers.d.mts +11 -3
- package/dist/o11y/record/family-writers.d.ts +11 -3
- package/dist/o11y/record/runner-producer.cjs +14 -4
- package/dist/o11y/record/runner-producer.cjs.map +1 -1
- package/dist/record/codec/core.d.cts +2 -2
- package/dist/record/codec/core.d.mts +2 -2
- package/dist/record/codec/core.d.ts +2 -2
- package/dist/record/family/catalog.d.cts +44 -12
- package/dist/record/family/catalog.d.mts +44 -12
- package/dist/record/family/catalog.d.ts +44 -12
- package/dist/record/family/index.js +128 -124
- package/dist/record/family/observability-v1.cjs +35 -0
- package/dist/record/family/observability-v1.cjs.map +1 -0
- package/dist/record/family/observability-v1.d.cts +3 -0
- package/dist/record/family/observability-v1.d.mts +3 -0
- package/dist/record/family/observability-v1.d.ts +3 -0
- package/dist/record/family/observability-v1.js +6 -0
- package/dist/record/family/observability-v1.js.map +1 -0
- package/dist/record/family/observability.cjs +58 -9
- package/dist/record/family/observability.cjs.map +1 -1
- package/dist/record/family/observability.d.cts +1014 -15
- package/dist/record/family/observability.d.mts +1014 -15
- package/dist/record/family/observability.d.ts +1014 -15
- package/dist/record/family/observability.js +22 -18
- package/dist/record/host/runtime.cjs +600 -31
- package/dist/record/host/runtime.cjs.map +1 -1
- package/dist/record/host/types.cjs.map +1 -1
- package/dist/record/host/types.d.cts +33 -9
- package/dist/record/host/types.d.mts +33 -9
- package/dist/record/host/types.d.ts +33 -9
- package/dist/record/maintenance/runtime.cjs +1 -2
- package/dist/record/maintenance/runtime.cjs.map +1 -1
- package/dist/record/model/core.cjs.map +1 -1
- package/dist/record/model/core.d.cts +1 -1
- package/dist/record/model/core.d.mts +1 -1
- package/dist/record/model/core.d.ts +1 -1
- package/dist/record/model/definition.d.cts +6 -6
- package/dist/record/model/definition.d.mts +6 -6
- package/dist/record/model/definition.d.ts +6 -6
- package/dist/record/model/identifiers.cjs +1 -1
- package/dist/record/model/identifiers.cjs.map +1 -1
- package/dist/record/model/identifiers.d.cts +1 -1
- package/dist/record/model/identifiers.d.mts +1 -1
- package/dist/record/model/identifiers.d.ts +1 -1
- package/dist/record/platform/node.cjs +159 -23
- package/dist/record/platform/node.cjs.map +1 -1
- package/dist/record/platform/services.cjs.map +1 -1
- package/dist/record/platform/services.d.cts +15 -2
- package/dist/record/platform/services.d.mts +15 -2
- package/dist/record/platform/services.d.ts +15 -2
- package/dist/record/reader/errors.cjs +37 -1
- package/dist/record/reader/errors.cjs.map +1 -1
- package/dist/record/reader/errors.d.cts +49 -1
- package/dist/record/reader/errors.d.mts +49 -1
- package/dist/record/reader/errors.d.ts +49 -1
- package/dist/record/reader/errors.js +22 -12
- package/dist/record/reader/format.cjs +76 -17
- package/dist/record/reader/format.cjs.map +1 -1
- package/dist/record/reader/format.d.cts +11 -4
- package/dist/record/reader/format.d.mts +11 -4
- package/dist/record/reader/format.d.ts +11 -4
- package/dist/record/reader/format.js +2 -0
- package/dist/record/reader/index.js +32 -18
- package/dist/record/reader/runtime.cjs +64 -0
- package/dist/record/reader/runtime.cjs.map +1 -1
- package/dist/record/reader/runtime.d.cts +15 -0
- package/dist/record/reader/runtime.d.mts +15 -0
- package/dist/record/reader/runtime.d.ts +15 -0
- package/dist/record/reader/runtime.js +2 -0
- package/dist/report/assets/styles.css +53 -29
- package/dist/report/built-in/analysis-values.cjs +21 -13
- package/dist/report/built-in/analysis-values.cjs.map +1 -1
- package/dist/report/built-in/analysis-values.d.cts +11 -22
- package/dist/report/built-in/analysis-values.d.mts +11 -22
- package/dist/report/built-in/analysis-values.d.ts +11 -22
- package/dist/report/built-in/analysis-values.js +4 -6
- package/dist/report/built-in/index.cjs +1 -3
- package/dist/report/built-in/index.cjs.map +1 -1
- package/dist/report/built-in/index.d.cts +1 -1
- package/dist/report/built-in/index.d.mts +1 -1
- package/dist/report/built-in/index.d.ts +1 -1
- package/dist/report/built-in/index.js +4 -8
- package/dist/report/built-in/index.mjs +4 -8
- package/dist/report/built-in/machine.cjs +0 -2
- package/dist/report/built-in/machine.cjs.map +1 -1
- package/dist/report/built-in/machine.d.cts +0 -2
- package/dist/report/built-in/machine.d.mts +0 -2
- package/dist/report/built-in/machine.d.ts +0 -2
- package/dist/report/built-in/result-components.cjs +49 -51
- package/dist/report/built-in/result-components.cjs.map +1 -1
- package/dist/report/built-in/result-components.d.cts +16 -17
- package/dist/report/built-in/result-components.d.mts +16 -17
- package/dist/report/built-in/result-components.d.ts +16 -17
- package/dist/report/built-in/result-components.js +0 -4
- package/dist/report/built-in/run-membership-overview.cjs +126 -17
- package/dist/report/built-in/run-membership-overview.cjs.map +1 -1
- package/dist/report/built-in/run-membership-overview.d.cts +14 -0
- package/dist/report/built-in/run-membership-overview.d.mts +14 -0
- package/dist/report/built-in/run-membership-overview.d.ts +14 -0
- package/dist/report/built-in/run-membership-overview.js +4 -2
- package/dist/report/built-in/standard.cjs +80 -45
- package/dist/report/built-in/standard.cjs.map +1 -1
- package/dist/report/built-in/standard.d.cts +19 -41
- package/dist/report/built-in/standard.d.mts +19 -41
- package/dist/report/built-in/standard.d.ts +19 -41
- package/dist/report/built-in/standard.js +6 -6
- package/dist/report/components/attempt-detail/compute.cjs +78 -2
- package/dist/report/components/attempt-detail/compute.cjs.map +1 -1
- package/dist/report/components/attempt-detail/compute.d.cts +1 -1
- package/dist/report/components/attempt-detail/compute.d.mts +1 -1
- package/dist/report/components/attempt-detail/compute.d.ts +1 -1
- package/dist/report/components/attempt-detail/content.cjs +1 -1
- package/dist/report/components/attempt-detail/content.cjs.map +1 -1
- package/dist/report/components/entity-lists/compute.cjs +42 -5
- package/dist/report/components/entity-lists/compute.cjs.map +1 -1
- package/dist/report/components/entity-lists/compute.d.cts +10 -8
- package/dist/report/components/entity-lists/compute.d.mts +10 -8
- package/dist/report/components/entity-lists/compute.d.ts +10 -8
- package/dist/report/components/entity-lists/content.cjs +31 -6
- package/dist/report/components/entity-lists/content.cjs.map +1 -1
- package/dist/report/components/entity-lists/index.cjs +9 -4
- package/dist/report/components/entity-lists/index.cjs.map +1 -1
- package/dist/report/components/entity-lists/index.d.cts +4 -7
- package/dist/report/components/entity-lists/index.d.mts +4 -7
- package/dist/report/components/entity-lists/index.d.ts +4 -7
- package/dist/report/components/summaries/compute.cjs +8 -1
- package/dist/report/components/summaries/compute.cjs.map +1 -1
- package/dist/report/components/summaries/compute.d.cts +9 -1
- package/dist/report/components/summaries/compute.d.mts +9 -1
- package/dist/report/components/summaries/compute.d.ts +9 -1
- package/dist/report/components/summaries/index.cjs +48 -4
- package/dist/report/components/summaries/index.cjs.map +1 -1
- package/dist/report/components/summaries/index.d.cts +7 -4
- package/dist/report/components/summaries/index.d.mts +7 -4
- package/dist/report/components/summaries/index.d.ts +7 -4
- package/dist/report/definition/cell.cjs +3 -0
- package/dist/report/definition/cell.cjs.map +1 -1
- package/dist/report/definition/cell.d.cts +2 -0
- package/dist/report/definition/cell.d.mts +2 -0
- package/dist/report/definition/cell.d.ts +2 -0
- package/dist/report/definition/primitives/chart.cjs +8 -5
- package/dist/report/definition/primitives/chart.cjs.map +1 -1
- package/dist/report/definition/primitives/points-dataset.cjs +1 -0
- package/dist/report/definition/primitives/points-dataset.cjs.map +1 -1
- package/dist/report/definition/primitives.cjs +15 -2
- package/dist/report/definition/primitives.cjs.map +1 -1
- package/dist/report/definition/report.cjs +27 -3
- package/dist/report/definition/report.cjs.map +1 -1
- package/dist/report/definition/report.d.cts +8 -0
- package/dist/report/definition/report.d.mts +8 -0
- package/dist/report/definition/report.d.ts +8 -0
- package/dist/report/execution/machine.cjs +30 -14
- package/dist/report/execution/machine.cjs.map +1 -1
- package/dist/report/execution/machine.d.cts +20 -10
- package/dist/report/execution/machine.d.mts +20 -10
- package/dist/report/execution/machine.d.ts +20 -10
- package/dist/report/execution/machine.js +6 -6
- package/dist/report/host/execute.cjs +25 -16
- package/dist/report/host/execute.cjs.map +1 -1
- package/dist/report/host/execute.d.cts +4 -0
- package/dist/report/host/execute.d.mts +4 -0
- package/dist/report/host/execute.d.ts +4 -0
- package/dist/report/host/from-record.cjs +22 -5
- package/dist/report/host/from-record.cjs.map +1 -1
- package/dist/report/host/machine.cjs +53 -11
- package/dist/report/host/machine.cjs.map +1 -1
- package/dist/report/host/site-runtime.cjs +9 -8
- package/dist/report/host/site-runtime.cjs.map +1 -1
- package/dist/report/host/site-runtime.d.cts +1 -1
- package/dist/report/host/site-runtime.d.mts +1 -1
- package/dist/report/host/site-runtime.d.ts +1 -1
- package/dist/report/host/static.cjs +56 -9
- package/dist/report/host/static.cjs.map +1 -1
- package/dist/report/index.cjs.map +1 -1
- package/dist/report/index.d.cts +1 -1
- package/dist/report/index.d.mts +1 -1
- package/dist/report/index.d.ts +1 -1
- package/dist/report/model/dataset.cjs +7 -2
- package/dist/report/model/dataset.cjs.map +1 -1
- package/dist/report/model/locale.cjs +2 -0
- package/dist/report/model/locale.cjs.map +1 -1
- package/dist/report/model/locale.d.cts +1 -0
- package/dist/report/model/locale.d.mts +1 -0
- package/dist/report/model/locale.d.ts +1 -0
- package/dist/report/model/metrics.cjs +3 -1
- package/dist/report/model/metrics.cjs.map +1 -1
- package/dist/report/model/pricing.cjs +16 -11
- package/dist/report/model/pricing.cjs.map +1 -1
- package/dist/report/react/styles.css +53 -29
- package/dist/runner/adoption.cjs +1 -0
- package/dist/runner/adoption.cjs.map +1 -1
- package/dist/runner/attempt.cjs +44 -5
- package/dist/runner/attempt.cjs.map +1 -1
- package/dist/runner/attempt.d.cts +13 -1
- package/dist/runner/attempt.d.mts +13 -1
- package/dist/runner/attempt.d.ts +13 -1
- package/dist/runner/config-identity.cjs +10 -2
- package/dist/runner/config-identity.cjs.map +1 -1
- package/dist/runner/config-identity.d.cts +4 -2
- package/dist/runner/config-identity.d.mts +4 -2
- package/dist/runner/config-identity.d.ts +4 -2
- package/dist/runner/feedback/human.cjs +298 -59
- package/dist/runner/feedback/human.cjs.map +1 -1
- package/dist/runner/feedback/json.cjs +14 -6
- package/dist/runner/feedback/json.cjs.map +1 -1
- package/dist/runner/feedback/json.d.cts +3 -2
- package/dist/runner/feedback/json.d.mts +3 -2
- package/dist/runner/feedback/json.d.ts +3 -2
- package/dist/runner/feedback/reducer.cjs +2 -0
- package/dist/runner/feedback/reducer.cjs.map +1 -1
- package/dist/runner/judge-config.cjs +1 -1
- package/dist/runner/judge-config.cjs.map +1 -1
- package/dist/runner/ledger.cjs +124 -12
- package/dist/runner/ledger.cjs.map +1 -1
- package/dist/runner/project-current.cjs +13 -5
- package/dist/runner/project-current.cjs.map +1 -1
- package/dist/runner/run.cjs +564 -256
- package/dist/runner/run.cjs.map +1 -1
- package/dist/runner/sandbox-pool.cjs +161 -30
- package/dist/runner/sandbox-pool.cjs.map +1 -1
- package/dist/runner/sandbox-pool.d.cts +34 -2
- package/dist/runner/sandbox-pool.d.mts +34 -2
- package/dist/runner/sandbox-pool.d.ts +34 -2
- package/dist/runner/shared-state-lease.cjs +803 -0
- package/dist/runner/shared-state-lease.cjs.map +1 -0
- package/dist/runner/shared-state-lease.d.cts +121 -0
- package/dist/runner/shared-state-lease.d.mts +121 -0
- package/dist/runner/shared-state-lease.d.ts +121 -0
- package/dist/runner/shared-state-lease.js +24 -0
- package/dist/runner/shared-state-lease.js.map +1 -0
- package/dist/runner/teardown-registry.cjs +97 -34
- package/dist/runner/teardown-registry.cjs.map +1 -1
- package/dist/runner/teardown-registry.d.cts +27 -1
- package/dist/runner/teardown-registry.d.mts +27 -1
- package/dist/runner/teardown-registry.d.ts +27 -1
- package/dist/runner/teardown-registry.js +20 -16
- package/dist/runner/types.cjs.map +1 -1
- package/dist/runner/types.d.cts +31 -5
- package/dist/runner/types.d.mts +31 -5
- package/dist/runner/types.d.ts +31 -5
- package/dist/sample/analysis.cjs +249 -459
- package/dist/sample/analysis.cjs.map +1 -1
- package/dist/sample/capability.cjs +138 -7
- package/dist/sample/capability.cjs.map +1 -1
- package/dist/sample/capability.d.cts +3 -1
- package/dist/sample/capability.d.mts +3 -1
- package/dist/sample/capability.d.ts +3 -1
- package/dist/sample/capability.js +8 -6
- package/dist/sandbox/docker-agent-image.cjs +5 -1
- package/dist/sandbox/docker-agent-image.cjs.map +1 -1
- package/dist/sandbox/docker-agent-image.d.cts +2 -0
- package/dist/sandbox/docker-agent-image.d.mts +2 -0
- package/dist/sandbox/docker-agent-image.d.ts +2 -0
- package/dist/sandbox/docker-agent-image.js +14 -10
- package/dist/sandbox/docker-profile/schema.cjs +283 -534
- package/dist/sandbox/docker-profile/schema.cjs.map +1 -1
- package/dist/sandbox/docker-profile/schema.d.cts +10 -27
- package/dist/sandbox/docker-profile/schema.d.mts +10 -27
- package/dist/sandbox/docker-profile/schema.d.ts +10 -27
- package/dist/sandbox/e2b-agent-template.cjs +5 -2
- package/dist/sandbox/e2b-agent-template.cjs.map +1 -1
- package/dist/sandbox/index.cjs +4 -2
- package/dist/sandbox/index.cjs.map +1 -1
- package/dist/sandbox/index.d.cts +1 -1
- package/dist/sandbox/index.d.mts +1 -1
- package/dist/sandbox/index.d.ts +1 -1
- package/dist/sandbox/index.js +120 -116
- package/dist/sandbox/index.mjs +120 -116
- package/dist/sandbox/keep-registry.cjs +28 -66
- package/dist/sandbox/keep-registry.cjs.map +1 -1
- package/dist/sandbox/types.cjs.map +1 -1
- package/dist/sandbox/types.d.cts +1 -1
- package/dist/sandbox/types.d.mts +1 -1
- package/dist/sandbox/types.d.ts +1 -1
- package/dist/shared/aggregate.cjs +16 -6
- package/dist/shared/aggregate.cjs.map +1 -1
- package/dist/shared/aggregate.d.cts +12 -4
- package/dist/shared/aggregate.d.mts +12 -4
- package/dist/shared/aggregate.d.ts +12 -4
- package/dist/shared/turn-label.cjs +20 -2
- package/dist/shared/turn-label.cjs.map +1 -1
- package/dist/shared/turn-label.d.cts +2 -0
- package/dist/shared/turn-label.d.mts +2 -0
- package/dist/shared/turn-label.d.ts +2 -0
- package/dist/shared/turn-label.js +4 -2
- package/dist/vendor/remark/remark-gfm.cjs +7 -7
- package/dist/vendor/remark/remark-gfm.cjs.map +1 -1
- package/dist/vendor/remark/remark-parse.cjs +12 -12
- package/dist/vendor/remark/remark-parse.cjs.map +1 -1
- package/dist/vendor/remark/remark.cjs +13 -13
- package/dist/vendor/remark/remark.cjs.map +1 -1
- package/dist/view/server.cjs +6 -0
- package/dist/view/server.cjs.map +1 -1
- package/docs-site/images/hitl-handshake-zh.svg +3 -3
- package/docs-site/zh/examples/ai-agent-application.mdx +1 -1
- package/docs-site/zh/examples/coding-agent-extensions.mdx +1 -1
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +4 -3
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +3 -3
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +5 -5
- package/docs-site/zh/examples/integrations/langgraph.mdx +4 -4
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +5 -5
- package/docs-site/zh/explanation/adapter.mdx +5 -5
- package/docs-site/zh/explanation/assert.mdx +7 -7
- package/docs-site/zh/explanation/drive.mdx +4 -2
- package/docs-site/zh/explanation/evals.mdx +10 -8
- package/docs-site/zh/explanation/experiment.mdx +2 -2
- package/docs-site/zh/explanation/hitl.mdx +9 -9
- package/docs-site/zh/explanation/judge.mdx +5 -5
- package/docs-site/zh/explanation/overview.mdx +3 -3
- package/docs-site/zh/explanation/runner.mdx +1 -1
- package/docs-site/zh/index.mdx +3 -3
- package/docs-site/zh/reference/builtin-agents.mdx +13 -13
- package/docs-site/zh/reference/capabilities.mdx +1 -1
- package/docs-site/zh/reference/cli.mdx +18 -12
- package/docs-site/zh/reference/define-agent.mdx +130 -5
- package/docs-site/zh/reference/define-config.mdx +2 -2
- package/docs-site/zh/reference/define-eval.mdx +4 -4
- package/docs-site/zh/reference/events.mdx +11 -11
- package/docs-site/zh/reference/official-adapters.mdx +6 -6
- package/docs-site/zh/reference/results-data.mdx +6 -0
- package/docs-site/zh/troubleshooting/debugging.mdx +1 -1
- package/docs-site/zh/troubleshooting/recover-after-kill.mdx +37 -5
- package/docs-site/zh/tutorials/accept-results.mdx +72 -0
- package/docs-site/zh/tutorials/agent-onboarding.mdx +6 -6
- package/docs-site/zh/tutorials/authoring.mdx +9 -13
- package/docs-site/zh/tutorials/ci-integration.mdx +6 -6
- package/docs-site/zh/tutorials/compare-runs.mdx +61 -0
- package/docs-site/zh/tutorials/concurrency.mdx +13 -12
- package/docs-site/zh/tutorials/configuration.mdx +4 -6
- package/docs-site/zh/tutorials/connect-your-agent.mdx +1 -1
- package/docs-site/zh/tutorials/control-run-cost.mdx +58 -0
- package/docs-site/zh/tutorials/criteria-files.mdx +2 -2
- package/docs-site/zh/tutorials/custom-reports.mdx +2 -2
- package/docs-site/zh/tutorials/debug-lifecycle-plan.mdx +46 -0
- package/docs-site/zh/tutorials/docker-in-docker.mdx +52 -43
- package/docs-site/zh/tutorials/eval-groups.mdx +6 -6
- package/docs-site/zh/tutorials/evaluation-kinds.mdx +3 -3
- package/docs-site/zh/tutorials/experiments.mdx +2 -3
- package/docs-site/zh/tutorials/handle-execution-failures.mdx +18 -10
- package/docs-site/zh/tutorials/install-custom-sandbox-agent.mdx +16 -8
- package/docs-site/zh/tutorials/nixos-managed-dind.mdx +160 -0
- package/docs-site/zh/tutorials/plugins.mdx +14 -0
- package/docs-site/zh/tutorials/rerun-and-cache.mdx +4 -2
- package/docs-site/zh/tutorials/sandbox-agent.mdx +17 -11
- package/docs-site/zh/tutorials/sandbox-providers.mdx +17 -13
- package/docs-site/zh/tutorials/sandbox-reuse.mdx +12 -19
- package/docs-site/zh/tutorials/select-and-preview.mdx +65 -0
- package/docs-site/zh/tutorials/verify-judge.mdx +78 -0
- package/docs-site/zh/tutorials/viewing-results.mdx +38 -1
- package/docs-site/zh/tutorials/write-experiment.mdx +7 -5
- package/docs-site/zh/tutorials/write-send.mdx +9 -8
- package/package.json +9 -5
- package/dist/report/built-in/overview.cjs +0 -44
- package/dist/report/built-in/overview.cjs.map +0 -1
- package/dist/report/built-in/overview.d.cts +0 -6
- package/dist/report/built-in/overview.d.mts +0 -6
- package/dist/report/built-in/overview.d.ts +0 -6
- package/dist/report/built-in/overview.js +0 -9
- package/dist/report/built-in/overview.js.map +0 -1
- package/dist/runner/gate-lease.cjs +0 -283
- package/dist/runner/gate-lease.cjs.map +0 -1
- package/dist/runner/gate-lease.d.cts +0 -62
- package/dist/runner/gate-lease.d.mts +0 -62
- package/dist/runner/gate-lease.d.ts +0 -62
- package/dist/runner/gate-lease.js +0 -22
- package/dist/runner/gate-lease.js.map +0 -1
|
@@ -41,6 +41,7 @@ exports.commandMatch = commandMatch;
|
|
|
41
41
|
exports.eventMatch = eventMatch;
|
|
42
42
|
const json_match_ts_1 = require("../shared/json-match.cjs");
|
|
43
43
|
const util_ts_1 = require("../util.cjs");
|
|
44
|
+
const limits_ts_1 = require("./limits.cjs");
|
|
44
45
|
// 这些 symbol 不导出。外部作者无法构造看似可消费的 Match;输入 variance 与 refinement
|
|
45
46
|
// variance 也保持分离,ScoreMatch 不会伪造一个没有含义的输出 refinement。
|
|
46
47
|
const matchInputBrand = Symbol("niceeval.match.input");
|
|
@@ -67,6 +68,7 @@ function diagnostic(code, message, options = {}) {
|
|
|
67
68
|
...(options.reason === undefined ? {} : { reason: options.reason }),
|
|
68
69
|
...(options.locator === undefined ? {} : { locator: options.locator }),
|
|
69
70
|
...(options.children === undefined ? {} : { children: Object.freeze([...options.children]) }),
|
|
71
|
+
...(options.truncation === undefined ? {} : { truncation: Object.freeze({ ...options.truncation }) }),
|
|
70
72
|
});
|
|
71
73
|
}
|
|
72
74
|
function matched(value, detail) {
|
|
@@ -1081,30 +1083,66 @@ function assertManagedEventMatch(value, label = "match") {
|
|
|
1081
1083
|
* cannot manufacture a positive count or an absence proof.
|
|
1082
1084
|
*/
|
|
1083
1085
|
async function evaluateToolMatchCollection(match, occurrences, options) {
|
|
1084
|
-
const
|
|
1085
|
-
|
|
1086
|
-
|
|
1086
|
+
const sampledCandidates = [];
|
|
1087
|
+
let matchCount = 0;
|
|
1088
|
+
let firstMatch;
|
|
1089
|
+
let firstUnavailable;
|
|
1090
|
+
let exactOverage;
|
|
1091
|
+
for (const [index, occurrence] of occurrences.entries()) {
|
|
1092
|
+
const candidate = Object.freeze({
|
|
1093
|
+
index,
|
|
1087
1094
|
candidate: occurrence,
|
|
1088
1095
|
result: await evaluateBooleanMatch(match, occurrence),
|
|
1089
|
-
})
|
|
1096
|
+
});
|
|
1097
|
+
if (sampledCandidates.length < limits_ts_1.assertionRuntimeLimits.collectionDiagnosticCandidates) {
|
|
1098
|
+
sampledCandidates.push(candidate);
|
|
1099
|
+
}
|
|
1100
|
+
if (candidate.result.state === "matched") {
|
|
1101
|
+
matchCount += 1;
|
|
1102
|
+
firstMatch ??= candidate;
|
|
1103
|
+
if (options.quantifier.kind === "exact" && matchCount === options.quantifier.count + 1) {
|
|
1104
|
+
exactOverage = candidate;
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
else if (candidate.result.state === "unavailable") {
|
|
1108
|
+
firstUnavailable ??= candidate;
|
|
1109
|
+
}
|
|
1090
1110
|
}
|
|
1091
|
-
const
|
|
1092
|
-
|
|
1093
|
-
const unavailableCandidate = candidates.find((candidate) => candidate.result.state === "unavailable");
|
|
1094
|
-
const unavailableReason = unavailableCandidate?.result.state === "unavailable"
|
|
1095
|
-
? unavailableCandidate.result.reason
|
|
1111
|
+
const unavailableReason = firstUnavailable?.result.state === "unavailable"
|
|
1112
|
+
? firstUnavailable.result.reason
|
|
1096
1113
|
: options.coverageReason;
|
|
1097
1114
|
const incomplete = unavailableReason !== undefined;
|
|
1098
|
-
const received = `${
|
|
1115
|
+
const received = `${matchCount} definite match${matchCount === 1 ? "" : "es"}`;
|
|
1099
1116
|
const candidateLocator = (candidate) => candidate === undefined ? undefined : { kind: "tool-occurrence", id: candidate.candidate.id };
|
|
1117
|
+
const diagnosticSample = (...decisive) => {
|
|
1118
|
+
const byIndex = new Map();
|
|
1119
|
+
for (const candidate of [...sampledCandidates, ...decisive]) {
|
|
1120
|
+
if (candidate !== undefined)
|
|
1121
|
+
byIndex.set(candidate.index, candidate);
|
|
1122
|
+
}
|
|
1123
|
+
const captured = [...byIndex.values()].sort((left, right) => left.index - right.index);
|
|
1124
|
+
return Object.freeze({
|
|
1125
|
+
children: Object.freeze(captured.map((candidate) => resultChild(candidate.result, candidate.index, `candidate ${candidate.candidate.id}`))),
|
|
1126
|
+
...(captured.length === occurrences.length
|
|
1127
|
+
? {}
|
|
1128
|
+
: {
|
|
1129
|
+
truncation: Object.freeze({
|
|
1130
|
+
code: "diagnostic-truncated",
|
|
1131
|
+
reason: "sampled",
|
|
1132
|
+
captured: captured.length,
|
|
1133
|
+
knownTotal: occurrences.length,
|
|
1134
|
+
}),
|
|
1135
|
+
}),
|
|
1136
|
+
});
|
|
1137
|
+
};
|
|
1100
1138
|
if (options.quantifier.kind === "absent") {
|
|
1101
|
-
const first =
|
|
1139
|
+
const first = firstMatch;
|
|
1102
1140
|
if (first !== undefined) {
|
|
1103
1141
|
return mismatched(diagnostic("tool-absence-mismatch", `notCalledTool(${match.name}) observed a matching tool`, {
|
|
1104
1142
|
expected: `no ${match.name}`,
|
|
1105
1143
|
received,
|
|
1106
1144
|
locator: candidateLocator(first),
|
|
1107
|
-
|
|
1145
|
+
...diagnosticSample(first),
|
|
1108
1146
|
}));
|
|
1109
1147
|
}
|
|
1110
1148
|
if (incomplete) {
|
|
@@ -1112,38 +1150,38 @@ async function evaluateToolMatchCollection(match, occurrences, options) {
|
|
|
1112
1150
|
expected: `no ${match.name}`,
|
|
1113
1151
|
received,
|
|
1114
1152
|
reason: unavailableReason,
|
|
1115
|
-
|
|
1153
|
+
...diagnosticSample(firstUnavailable),
|
|
1116
1154
|
}));
|
|
1117
1155
|
}
|
|
1118
1156
|
return matched(undefined, diagnostic("tool-absence-match", `notCalledTool(${match.name}) observed no matching tool`, {
|
|
1119
1157
|
expected: `no ${match.name}`,
|
|
1120
1158
|
received,
|
|
1121
|
-
|
|
1159
|
+
...diagnosticSample(),
|
|
1122
1160
|
}));
|
|
1123
1161
|
}
|
|
1124
1162
|
const { kind, count } = options.quantifier;
|
|
1125
1163
|
const expected = kind === "exact" ? `exactly ${count} × ${match.name}` : `at least ${count} × ${match.name}`;
|
|
1126
|
-
if (kind === "exact" &&
|
|
1127
|
-
const overage =
|
|
1164
|
+
if (kind === "exact" && matchCount > count) {
|
|
1165
|
+
const overage = exactOverage;
|
|
1128
1166
|
return mismatched(diagnostic("tool-count-exceeded", `calledTool(${match.name}) exceeded its exact count`, {
|
|
1129
1167
|
expected,
|
|
1130
1168
|
received,
|
|
1131
1169
|
locator: candidateLocator(overage),
|
|
1132
|
-
|
|
1170
|
+
...diagnosticSample(overage),
|
|
1133
1171
|
}));
|
|
1134
1172
|
}
|
|
1135
|
-
if (kind === "at-least" &&
|
|
1136
|
-
return matched(
|
|
1173
|
+
if (kind === "at-least" && matchCount >= count) {
|
|
1174
|
+
return matched(firstMatch?.candidate, diagnostic("tool-count-match", `calledTool(${match.name}) satisfied its lower bound`, {
|
|
1137
1175
|
expected,
|
|
1138
1176
|
received,
|
|
1139
|
-
|
|
1177
|
+
...diagnosticSample(firstMatch),
|
|
1140
1178
|
}));
|
|
1141
1179
|
}
|
|
1142
|
-
if (kind === "exact" &&
|
|
1143
|
-
return matched(
|
|
1180
|
+
if (kind === "exact" && matchCount === count && !incomplete) {
|
|
1181
|
+
return matched(firstMatch?.candidate, diagnostic("tool-count-match", `calledTool(${match.name}) satisfied its exact count`, {
|
|
1144
1182
|
expected,
|
|
1145
1183
|
received,
|
|
1146
|
-
|
|
1184
|
+
...diagnosticSample(firstMatch),
|
|
1147
1185
|
}));
|
|
1148
1186
|
}
|
|
1149
1187
|
if (incomplete) {
|
|
@@ -1151,13 +1189,13 @@ async function evaluateToolMatchCollection(match, occurrences, options) {
|
|
|
1151
1189
|
expected,
|
|
1152
1190
|
received,
|
|
1153
1191
|
reason: unavailableReason,
|
|
1154
|
-
|
|
1192
|
+
...diagnosticSample(firstUnavailable),
|
|
1155
1193
|
}));
|
|
1156
1194
|
}
|
|
1157
1195
|
return mismatched(diagnostic("tool-count-mismatch", `calledTool(${match.name}) did not satisfy its count`, {
|
|
1158
1196
|
expected,
|
|
1159
1197
|
received,
|
|
1160
|
-
|
|
1198
|
+
...diagnosticSample(),
|
|
1161
1199
|
}));
|
|
1162
1200
|
}
|
|
1163
1201
|
function commandResult(occurrence, executable, argsStart, excludes) {
|