niceeval 0.12.1 → 0.12.2-canary.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +33 -27
- package/bin/niceeval.js +8 -6
- package/dist/agents/ai-sdk-otel.cjs +54 -0
- package/dist/agents/ai-sdk-otel.cjs.map +1 -0
- package/dist/agents/ai-sdk-otel.d.cts +27 -0
- package/dist/agents/ai-sdk-otel.d.mts +27 -0
- package/dist/agents/ai-sdk-otel.d.ts +27 -0
- package/dist/agents/ai-sdk-otel.js +6 -0
- package/dist/agents/ai-sdk-otel.js.map +1 -0
- package/dist/agents/ai-sdk-otel.mjs +6 -0
- package/dist/agents/ai-sdk-otel.mjs.map +1 -0
- package/dist/agents/ai-sdk.cjs +412 -0
- package/dist/agents/ai-sdk.cjs.map +1 -0
- package/dist/agents/ai-sdk.d.cts +214 -0
- package/dist/agents/ai-sdk.d.mts +214 -0
- package/dist/agents/ai-sdk.d.ts +214 -0
- package/dist/agents/ai-sdk.js +8 -0
- package/dist/agents/ai-sdk.js.map +1 -0
- package/dist/agents/bub-install-spec.cjs +42 -0
- package/dist/agents/bub-install-spec.cjs.map +1 -0
- package/dist/agents/bub-install-spec.d.cts +22 -0
- package/dist/agents/bub-install-spec.d.mts +22 -0
- package/dist/agents/bub-install-spec.d.ts +22 -0
- package/dist/agents/bub-install-spec.js +20 -0
- package/dist/agents/bub-install-spec.js.map +1 -0
- package/dist/agents/bub.cjs +204 -0
- package/dist/agents/bub.cjs.map +1 -0
- package/dist/agents/bub.d.cts +57 -0
- package/dist/agents/bub.d.mts +57 -0
- package/dist/agents/bub.d.ts +57 -0
- package/dist/agents/bub.js +7 -0
- package/dist/agents/bub.js.map +1 -0
- package/dist/agents/builtin.cjs +21 -0
- package/dist/agents/builtin.cjs.map +1 -0
- package/dist/agents/builtin.d.cts +2 -0
- package/dist/agents/builtin.d.mts +2 -0
- package/dist/agents/builtin.d.ts +2 -0
- package/dist/agents/builtin.js +6 -0
- package/dist/agents/builtin.js.map +1 -0
- package/dist/agents/cache-marker.cjs +6 -0
- package/dist/agents/cache-marker.cjs.map +1 -0
- package/dist/agents/cache-marker.d.cts +5 -0
- package/dist/agents/cache-marker.d.mts +5 -0
- package/dist/agents/cache-marker.js +6 -2
- package/dist/agents/cache-marker.js.map +1 -0
- package/dist/agents/claude-code.cjs +372 -0
- package/dist/agents/claude-code.cjs.map +1 -0
- package/dist/agents/claude-code.d.cts +91 -0
- package/dist/agents/claude-code.d.mts +91 -0
- package/dist/agents/claude-code.d.ts +91 -0
- package/dist/agents/claude-code.js +9 -0
- package/dist/agents/claude-code.js.map +1 -0
- package/dist/agents/codex.cjs +658 -0
- package/dist/agents/codex.cjs.map +1 -0
- package/dist/agents/codex.d.cts +96 -0
- package/dist/agents/codex.d.mts +96 -0
- package/dist/agents/codex.d.ts +96 -0
- package/dist/agents/codex.js +9 -0
- package/dist/agents/codex.js.map +1 -0
- package/dist/agents/coding-cli-versions.cjs +71 -0
- package/dist/agents/coding-cli-versions.cjs.map +1 -0
- package/dist/agents/coding-cli-versions.d.cts +40 -0
- package/dist/agents/coding-cli-versions.d.mts +40 -0
- package/dist/agents/coding-cli-versions.d.ts +40 -0
- package/dist/agents/coding-cli-versions.js +22 -0
- package/dist/agents/coding-cli-versions.js.map +1 -0
- package/dist/agents/hermes.cjs +237 -0
- package/dist/agents/hermes.cjs.map +1 -0
- package/dist/agents/hermes.d.cts +17 -0
- package/dist/agents/hermes.d.mts +17 -0
- package/dist/agents/hermes.d.ts +17 -0
- package/dist/agents/hermes.js +7 -0
- package/dist/agents/hermes.js.map +1 -0
- package/dist/agents/index.cjs +92 -0
- package/dist/agents/index.cjs.map +1 -0
- package/dist/agents/index.d.cts +44 -0
- package/dist/agents/index.d.mts +44 -0
- package/dist/agents/index.d.ts +44 -0
- package/dist/agents/index.js +86 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/index.mjs +86 -0
- package/dist/agents/index.mjs.map +1 -0
- package/dist/agents/langgraph.cjs +426 -0
- package/dist/agents/langgraph.cjs.map +1 -0
- package/dist/agents/langgraph.d.cts +43 -0
- package/dist/agents/langgraph.d.mts +43 -0
- package/dist/agents/langgraph.d.ts +43 -0
- package/dist/agents/langgraph.js +6 -0
- package/dist/agents/langgraph.js.map +1 -0
- package/dist/agents/marketplace.cjs +81 -0
- package/dist/agents/marketplace.cjs.map +1 -0
- package/dist/agents/marketplace.d.cts +27 -0
- package/dist/agents/marketplace.d.mts +27 -0
- package/dist/agents/marketplace.d.ts +27 -0
- package/dist/agents/marketplace.js +8 -0
- package/dist/agents/marketplace.js.map +1 -0
- package/dist/agents/mcp.cjs +28 -0
- package/dist/agents/mcp.cjs.map +1 -0
- package/dist/agents/mcp.d.cts +7 -0
- package/dist/agents/mcp.d.mts +7 -0
- package/dist/agents/mcp.d.ts +7 -0
- package/dist/agents/mcp.js +10 -0
- package/dist/agents/mcp.js.map +1 -0
- package/dist/agents/native-config.cjs +288 -0
- package/dist/agents/native-config.cjs.map +1 -0
- package/dist/agents/native-config.d.cts +71 -0
- package/dist/agents/native-config.d.mts +71 -0
- package/dist/agents/native-config.d.ts +71 -0
- package/dist/agents/native-config.js +18 -0
- package/dist/agents/native-config.js.map +1 -0
- package/dist/agents/npm-staged.cjs +378 -0
- package/dist/agents/npm-staged.cjs.map +1 -0
- package/dist/agents/npm-staged.d.cts +52 -0
- package/dist/agents/npm-staged.d.mts +52 -0
- package/dist/agents/npm-staged.d.ts +52 -0
- package/dist/agents/npm-staged.js +12 -0
- package/dist/agents/npm-staged.js.map +1 -0
- package/dist/agents/openai-compat.cjs +158 -0
- package/dist/agents/openai-compat.cjs.map +1 -0
- package/dist/agents/openai-compat.d.cts +88 -0
- package/dist/agents/openai-compat.d.mts +88 -0
- package/dist/agents/openai-compat.d.ts +88 -0
- package/dist/agents/openai-compat.js +12 -0
- package/dist/agents/openai-compat.js.map +1 -0
- package/dist/agents/openclaw.cjs +236 -0
- package/dist/agents/openclaw.cjs.map +1 -0
- package/dist/agents/openclaw.d.cts +26 -0
- package/dist/agents/openclaw.d.mts +26 -0
- package/dist/agents/openclaw.d.ts +26 -0
- package/dist/agents/openclaw.js +7 -0
- package/dist/agents/openclaw.js.map +1 -0
- package/dist/agents/opencode.cjs +228 -0
- package/dist/agents/opencode.cjs.map +1 -0
- package/dist/agents/opencode.d.cts +17 -0
- package/dist/agents/opencode.d.mts +17 -0
- package/dist/agents/opencode.d.ts +17 -0
- package/dist/agents/opencode.js +7 -0
- package/dist/agents/opencode.js.map +1 -0
- package/dist/agents/post-setup.cjs +119 -0
- package/dist/agents/post-setup.cjs.map +1 -0
- package/dist/agents/post-setup.d.cts +20 -0
- package/dist/agents/post-setup.d.mts +20 -0
- package/dist/agents/post-setup.d.ts +20 -0
- package/dist/agents/post-setup.js +12 -0
- package/dist/agents/post-setup.js.map +1 -0
- package/dist/agents/provisioner.cjs +446 -0
- package/dist/agents/provisioner.cjs.map +1 -0
- package/dist/agents/provisioner.d.cts +116 -0
- package/dist/agents/provisioner.d.mts +116 -0
- package/dist/agents/provisioner.d.ts +8 -6
- package/dist/agents/provisioner.js +32 -0
- package/dist/agents/provisioner.js.map +1 -0
- package/dist/agents/sdk-streams.cjs +447 -0
- package/dist/agents/sdk-streams.cjs.map +1 -0
- package/dist/agents/sdk-streams.d.cts +135 -0
- package/dist/agents/sdk-streams.d.mts +135 -0
- package/dist/agents/sdk-streams.d.ts +135 -0
- package/dist/agents/sdk-streams.js +12 -0
- package/dist/agents/sdk-streams.js.map +1 -0
- package/dist/agents/session-slot.cjs +8 -0
- package/dist/agents/session-slot.cjs.map +1 -0
- package/dist/agents/session-slot.d.cts +12 -0
- package/dist/agents/session-slot.d.mts +12 -0
- package/dist/agents/session-slot.js +6 -0
- package/dist/agents/session-slot.js.map +1 -0
- package/dist/agents/shared.cjs +202 -0
- package/dist/agents/shared.cjs.map +1 -0
- package/dist/agents/shared.d.cts +95 -0
- package/dist/agents/shared.d.mts +95 -0
- package/dist/agents/shared.d.ts +95 -0
- package/dist/agents/shared.js +6 -0
- package/dist/agents/shared.js.map +1 -0
- package/dist/agents/skills.cjs +262 -0
- package/dist/agents/skills.cjs.map +1 -0
- package/dist/agents/skills.d.cts +60 -0
- package/dist/agents/skills.d.mts +60 -0
- package/dist/agents/skills.d.ts +60 -0
- package/dist/agents/skills.js +22 -0
- package/dist/agents/skills.js.map +1 -0
- package/dist/agents/streaming.cjs +142 -0
- package/dist/agents/streaming.cjs.map +1 -0
- package/dist/agents/streaming.d.cts +82 -0
- package/dist/agents/streaming.d.mts +82 -0
- package/dist/agents/streaming.d.ts +82 -0
- package/dist/agents/streaming.js +8 -0
- package/dist/agents/streaming.js.map +1 -0
- package/dist/agents/types.cjs +5 -0
- package/dist/agents/types.cjs.map +1 -0
- package/dist/agents/types.d.cts +592 -0
- package/dist/agents/types.d.mts +592 -0
- package/dist/agents/types.d.ts +26 -20
- package/dist/agents/types.js +4 -2
- package/dist/agents/types.js.map +1 -0
- package/dist/agents/ui-message-stream.cjs +426 -0
- package/dist/agents/ui-message-stream.cjs.map +1 -0
- package/dist/agents/ui-message-stream.d.cts +68 -0
- package/dist/agents/ui-message-stream.d.mts +68 -0
- package/dist/agents/ui-message-stream.d.ts +68 -0
- package/dist/agents/ui-message-stream.js +6 -0
- package/dist/agents/ui-message-stream.js.map +1 -0
- package/dist/analysis/index.cjs +11 -0
- package/dist/analysis/index.cjs.map +1 -0
- package/dist/analysis/index.d.cts +2 -0
- package/dist/analysis/index.d.mts +2 -0
- package/dist/analysis/index.d.ts +2 -0
- package/dist/analysis/index.js +12 -0
- package/dist/analysis/index.js.map +1 -0
- package/dist/analysis/index.mjs +12 -0
- package/dist/analysis/index.mjs.map +1 -0
- package/dist/assertions/api.cjs +26 -0
- package/dist/assertions/api.cjs.map +1 -0
- package/dist/assertions/api.d.cts +395 -0
- package/dist/assertions/api.d.mts +395 -0
- package/dist/assertions/api.d.ts +395 -0
- package/dist/assertions/api.js +8 -0
- package/dist/assertions/api.js.map +1 -0
- package/dist/assertions/coverage.cjs +97 -0
- package/dist/assertions/coverage.cjs.map +1 -0
- package/dist/assertions/coverage.d.cts +23 -0
- package/dist/assertions/coverage.d.mts +23 -0
- package/dist/assertions/coverage.d.ts +1 -1
- package/dist/assertions/coverage.js +14 -94
- package/dist/assertions/coverage.js.map +1 -0
- package/dist/assertions/diff.cjs +249 -0
- package/dist/assertions/diff.cjs.map +1 -0
- package/dist/assertions/diff.d.cts +88 -0
- package/dist/assertions/diff.d.mts +88 -0
- package/dist/assertions/diff.d.ts +70 -2
- package/dist/assertions/diff.js +36 -116
- package/dist/assertions/diff.js.map +1 -0
- package/dist/assertions/display.cjs +79 -0
- package/dist/assertions/display.cjs.map +1 -0
- package/dist/assertions/display.d.cts +36 -0
- package/dist/assertions/display.d.mts +36 -0
- package/dist/assertions/display.d.ts +32 -32
- package/dist/assertions/display.js +14 -200
- package/dist/assertions/display.js.map +1 -0
- package/dist/assertions/identity.cjs +6 -0
- package/dist/assertions/identity.cjs.map +1 -0
- package/dist/assertions/identity.d.cts +4 -0
- package/dist/assertions/identity.d.mts +4 -0
- package/dist/assertions/identity.d.ts +4 -0
- package/dist/assertions/identity.js +6 -0
- package/dist/assertions/identity.js.map +1 -0
- package/dist/assertions/judge.cjs +342 -0
- package/dist/assertions/judge.cjs.map +1 -0
- package/dist/assertions/judge.d.cts +22 -0
- package/dist/assertions/judge.d.mts +22 -0
- package/dist/assertions/judge.d.ts +22 -0
- package/dist/assertions/judge.js +12 -0
- package/dist/assertions/judge.js.map +1 -0
- package/dist/assertions/limits.cjs +14 -0
- package/dist/assertions/limits.cjs.map +1 -0
- package/dist/assertions/limits.d.cts +10 -0
- package/dist/assertions/limits.d.mts +10 -0
- package/dist/assertions/limits.d.ts +10 -0
- package/dist/assertions/limits.js +6 -0
- package/dist/assertions/limits.js.map +1 -0
- package/dist/assertions/match.cjs +1314 -0
- package/dist/assertions/match.cjs.map +1 -0
- package/dist/assertions/match.d.cts +276 -0
- package/dist/assertions/match.d.mts +276 -0
- package/dist/assertions/match.d.ts +276 -0
- package/dist/assertions/match.js +82 -0
- package/dist/assertions/match.js.map +1 -0
- package/dist/assertions/record/attachment.cjs +448 -0
- package/dist/assertions/record/attachment.cjs.map +1 -0
- package/dist/assertions/record/attachment.d.cts +220 -0
- package/dist/assertions/record/attachment.d.mts +220 -0
- package/dist/assertions/record/attachment.d.ts +220 -0
- package/dist/assertions/record/attachment.js +20 -0
- package/dist/assertions/record/attachment.js.map +1 -0
- package/dist/assertions/record/codec.cjs +541 -0
- package/dist/assertions/record/codec.cjs.map +1 -0
- package/dist/assertions/record/codec.d.cts +146 -0
- package/dist/assertions/record/codec.d.mts +146 -0
- package/dist/assertions/record/codec.d.ts +146 -0
- package/dist/assertions/record/codec.js +42 -0
- package/dist/assertions/record/codec.js.map +1 -0
- package/dist/assertions/record/diff-model.cjs +120 -0
- package/dist/assertions/record/diff-model.cjs.map +1 -0
- package/dist/assertions/record/diff-model.d.cts +47 -0
- package/dist/assertions/record/diff-model.d.mts +47 -0
- package/dist/assertions/record/diff-model.d.ts +47 -0
- package/dist/assertions/record/diff-model.js +12 -0
- package/dist/assertions/record/diff-model.js.map +1 -0
- package/dist/assertions/record/diff.cjs +149 -0
- package/dist/assertions/record/diff.cjs.map +1 -0
- package/dist/assertions/record/diff.d.cts +27 -0
- package/dist/assertions/record/diff.d.mts +27 -0
- package/dist/assertions/record/diff.d.ts +27 -0
- package/dist/assertions/record/diff.js +18 -0
- package/dist/assertions/record/diff.js.map +1 -0
- package/dist/assertions/record/index.cjs +22 -0
- package/dist/assertions/record/index.cjs.map +1 -0
- package/dist/assertions/record/index.d.cts +5 -0
- package/dist/assertions/record/index.d.mts +5 -0
- package/dist/assertions/record/index.d.ts +5 -0
- package/dist/assertions/record/index.js +76 -0
- package/dist/assertions/record/index.js.map +1 -0
- package/dist/assertions/record/model.cjs +10 -0
- package/dist/assertions/record/model.cjs.map +1 -0
- package/dist/assertions/record/model.d.cts +257 -0
- package/dist/assertions/record/model.d.mts +257 -0
- package/dist/assertions/record/model.d.ts +257 -0
- package/dist/assertions/record/model.js +6 -0
- package/dist/assertions/record/model.js.map +1 -0
- package/dist/assertions/record/producer.cjs +55 -0
- package/dist/assertions/record/producer.cjs.map +1 -0
- package/dist/assertions/record/producer.d.cts +45 -0
- package/dist/assertions/record/producer.d.mts +45 -0
- package/dist/assertions/record/producer.d.ts +45 -0
- package/dist/assertions/record/producer.js +6 -0
- package/dist/assertions/record/producer.js.map +1 -0
- package/dist/assertions/runtime.cjs +1163 -0
- package/dist/assertions/runtime.cjs.map +1 -0
- package/dist/assertions/runtime.d.cts +45 -0
- package/dist/assertions/runtime.d.mts +45 -0
- package/dist/assertions/runtime.d.ts +45 -0
- package/dist/assertions/runtime.js +16 -0
- package/dist/assertions/runtime.js.map +1 -0
- package/dist/assertions/types.cjs +3 -0
- package/dist/assertions/types.cjs.map +1 -0
- package/dist/assertions/types.d.cts +238 -0
- package/dist/assertions/types.d.mts +238 -0
- package/dist/assertions/types.d.ts +166 -186
- package/dist/assertions/types.js +4 -2
- package/dist/assertions/types.js.map +1 -0
- package/dist/assertions/workspace-diff.cjs +237 -0
- package/dist/assertions/workspace-diff.cjs.map +1 -0
- package/dist/assertions/workspace-diff.d.cts +77 -0
- package/dist/assertions/workspace-diff.d.mts +77 -0
- package/dist/assertions/workspace-diff.d.ts +77 -0
- package/dist/assertions/workspace-diff.js +16 -0
- package/dist/assertions/workspace-diff.js.map +1 -0
- package/dist/attempt-locator-resolution.cjs +51 -0
- package/dist/attempt-locator-resolution.cjs.map +1 -0
- package/dist/attempt-locator-resolution.d.cts +18 -0
- package/dist/attempt-locator-resolution.d.mts +18 -0
- package/dist/attempt-locator-resolution.d.ts +18 -0
- package/dist/attempt-locator-resolution.js +6 -0
- package/dist/attempt-locator-resolution.js.map +1 -0
- package/dist/attempt-locator.cjs +122 -0
- package/dist/attempt-locator.cjs.map +1 -0
- package/dist/attempt-locator.d.cts +25 -0
- package/dist/attempt-locator.d.mts +25 -0
- package/dist/attempt-locator.d.ts +25 -0
- package/dist/attempt-locator.js +10 -0
- package/dist/attempt-locator.js.map +1 -0
- package/dist/cli/bootstrap.cjs +87 -0
- package/dist/cli/bootstrap.cjs.map +1 -0
- package/dist/cli/bootstrap.d.mts +1 -0
- package/dist/cli/bootstrap.d.ts +1 -0
- package/dist/cli/bootstrap.js +4 -0
- package/dist/cli/bootstrap.js.map +1 -0
- package/dist/cli/record.cjs +198 -0
- package/dist/cli/record.cjs.map +1 -0
- package/dist/cli/record.d.cts +28 -0
- package/dist/cli/record.d.mts +28 -0
- package/dist/cli/record.d.ts +28 -0
- package/dist/cli/record.js +12 -0
- package/dist/cli/record.js.map +1 -0
- package/dist/cli.cjs +2405 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.d.cts +139 -0
- package/dist/cli.d.mts +139 -0
- package/dist/cli.d.ts +139 -0
- package/dist/cli.js +34 -0
- package/dist/cli.js.map +1 -0
- package/dist/context/assert-first.cjs +1766 -0
- package/dist/context/assert-first.cjs.map +1 -0
- package/dist/context/assert-first.d.cts +204 -0
- package/dist/context/assert-first.d.mts +204 -0
- package/dist/context/assert-first.d.ts +204 -0
- package/dist/context/assert-first.js +6 -0
- package/dist/context/assert-first.js.map +1 -0
- package/dist/context/control-flow.cjs +15 -0
- package/dist/context/control-flow.cjs.map +1 -0
- package/dist/context/control-flow.d.cts +5 -0
- package/dist/context/control-flow.d.mts +5 -0
- package/dist/context/control-flow.d.ts +5 -0
- package/dist/context/control-flow.js +6 -0
- package/dist/context/control-flow.js.map +1 -0
- package/dist/context/send-failures.cjs +119 -0
- package/dist/context/send-failures.cjs.map +1 -0
- package/dist/context/send-failures.d.cts +45 -0
- package/dist/context/send-failures.d.mts +45 -0
- package/dist/context/send-failures.d.ts +3 -3
- package/dist/context/send-failures.js +18 -0
- package/dist/context/send-failures.js.map +1 -0
- package/dist/context/send-retry.cjs +105 -0
- package/dist/context/send-retry.cjs.map +1 -0
- package/dist/context/send-retry.d.cts +41 -0
- package/dist/context/send-retry.d.mts +41 -0
- package/dist/context/send-retry.d.ts +41 -0
- package/dist/context/send-retry.js +12 -0
- package/dist/context/send-retry.js.map +1 -0
- package/dist/context/session.cjs +503 -0
- package/dist/context/session.cjs.map +1 -0
- package/dist/context/session.d.cts +179 -0
- package/dist/context/session.d.mts +179 -0
- package/dist/context/session.d.ts +179 -0
- package/dist/context/session.js +12 -0
- package/dist/context/session.js.map +1 -0
- package/dist/context/types.cjs +3 -0
- package/dist/context/types.cjs.map +1 -0
- package/dist/context/types.d.cts +37 -0
- package/dist/context/types.d.mts +37 -0
- package/dist/context/types.d.ts +30 -387
- package/dist/context/types.js +4 -3
- package/dist/context/types.js.map +1 -0
- package/dist/define.cjs +246 -0
- package/dist/define.cjs.map +1 -0
- package/dist/define.d.cts +26 -0
- package/dist/define.d.mts +26 -0
- package/dist/define.d.ts +26 -0
- package/dist/define.js +22 -0
- package/dist/define.js.map +1 -0
- package/dist/eval/experiment-id.cjs +18 -0
- package/dist/eval/experiment-id.cjs.map +1 -0
- package/dist/eval/experiment-id.d.cts +10 -0
- package/dist/eval/experiment-id.d.mts +10 -0
- package/dist/eval/experiment-id.d.ts +10 -0
- package/dist/eval/experiment-id.js +10 -0
- package/dist/eval/experiment-id.js.map +1 -0
- package/dist/eval/record/attachment.cjs +60 -0
- package/dist/eval/record/attachment.cjs.map +1 -0
- package/dist/eval/record/attachment.d.cts +30 -0
- package/dist/eval/record/attachment.d.mts +30 -0
- package/dist/eval/record/attachment.d.ts +30 -0
- package/dist/eval/record/attachment.js +16 -0
- package/dist/eval/record/attachment.js.map +1 -0
- package/dist/eval/record/coherence.cjs +118 -0
- package/dist/eval/record/coherence.cjs.map +1 -0
- package/dist/eval/record/coherence.d.cts +61 -0
- package/dist/eval/record/coherence.d.mts +61 -0
- package/dist/eval/record/coherence.d.ts +61 -0
- package/dist/eval/record/coherence.js +6 -0
- package/dist/eval/record/coherence.js.map +1 -0
- package/dist/eval/record/eligibility.cjs +157 -0
- package/dist/eval/record/eligibility.cjs.map +1 -0
- package/dist/eval/record/eligibility.d.cts +151 -0
- package/dist/eval/record/eligibility.d.mts +151 -0
- package/dist/eval/record/eligibility.d.ts +151 -0
- package/dist/eval/record/eligibility.js +42 -0
- package/dist/eval/record/eligibility.js.map +1 -0
- package/dist/eval/record/evaluation-plan.cjs +26 -0
- package/dist/eval/record/evaluation-plan.cjs.map +1 -0
- package/dist/eval/record/evaluation-plan.d.cts +16 -0
- package/dist/eval/record/evaluation-plan.d.mts +16 -0
- package/dist/eval/record/evaluation-plan.d.ts +16 -0
- package/dist/eval/record/evaluation-plan.js +6 -0
- package/dist/eval/record/evaluation-plan.js.map +1 -0
- package/dist/eval/record/evaluation-record.cjs +540 -0
- package/dist/eval/record/evaluation-record.cjs.map +1 -0
- package/dist/eval/record/evaluation-record.d.cts +168 -0
- package/dist/eval/record/evaluation-record.d.mts +168 -0
- package/dist/eval/record/evaluation-record.d.ts +168 -0
- package/dist/eval/record/evaluation-record.js +22 -0
- package/dist/eval/record/evaluation-record.js.map +1 -0
- package/dist/eval/record/evaluation.cjs +220 -0
- package/dist/eval/record/evaluation.cjs.map +1 -0
- package/dist/eval/record/evaluation.d.cts +175 -0
- package/dist/eval/record/evaluation.d.mts +175 -0
- package/dist/eval/record/evaluation.d.ts +175 -0
- package/dist/eval/record/evaluation.js +36 -0
- package/dist/eval/record/evaluation.js.map +1 -0
- package/dist/eval/record/index.cjs +26 -0
- package/dist/eval/record/index.cjs.map +1 -0
- package/dist/eval/record/index.d.cts +9 -0
- package/dist/eval/record/index.d.mts +9 -0
- package/dist/eval/record/index.d.ts +9 -0
- package/dist/eval/record/index.js +242 -0
- package/dist/eval/record/index.js.map +1 -0
- package/dist/eval/record/membership-provenance.cjs +213 -0
- package/dist/eval/record/membership-provenance.cjs.map +1 -0
- package/dist/eval/record/membership-provenance.d.cts +855 -0
- package/dist/eval/record/membership-provenance.d.mts +855 -0
- package/dist/eval/record/membership-provenance.d.ts +855 -0
- package/dist/eval/record/membership-provenance.js +58 -0
- package/dist/eval/record/membership-provenance.js.map +1 -0
- package/dist/eval/record/score.cjs +248 -0
- package/dist/eval/record/score.cjs.map +1 -0
- package/dist/eval/record/score.d.cts +165 -0
- package/dist/eval/record/score.d.mts +165 -0
- package/dist/eval/record/score.d.ts +165 -0
- package/dist/eval/record/score.js +44 -0
- package/dist/eval/record/score.js.map +1 -0
- package/dist/eval/record/sealed-assertion.cjs +31 -0
- package/dist/eval/record/sealed-assertion.cjs.map +1 -0
- package/dist/eval/record/sealed-assertion.d.cts +32 -0
- package/dist/eval/record/sealed-assertion.d.mts +32 -0
- package/dist/eval/record/sealed-assertion.d.ts +32 -0
- package/dist/eval/record/sealed-assertion.js +14 -0
- package/dist/eval/record/sealed-assertion.js.map +1 -0
- package/dist/eval/record/verdict.cjs +89 -0
- package/dist/eval/record/verdict.cjs.map +1 -0
- package/dist/eval/record/verdict.d.cts +60 -0
- package/dist/eval/record/verdict.d.mts +60 -0
- package/dist/eval/record/verdict.d.ts +60 -0
- package/dist/eval/record/verdict.js +36 -0
- package/dist/eval/record/verdict.js.map +1 -0
- package/dist/expect/index.cjs +29 -0
- package/dist/expect/index.cjs.map +1 -0
- package/dist/expect/index.d.cts +2 -0
- package/dist/expect/index.d.mts +2 -0
- package/dist/expect/index.d.ts +2 -0
- package/dist/expect/index.js +50 -0
- package/dist/expect/index.js.map +1 -0
- package/dist/expect/index.mjs +50 -0
- package/dist/expect/index.mjs.map +1 -0
- package/dist/fresh-import.cjs +288 -0
- package/dist/fresh-import.cjs.map +1 -0
- package/dist/fresh-import.d.cts +20 -0
- package/dist/fresh-import.d.mts +20 -0
- package/dist/fresh-import.d.ts +20 -0
- package/dist/fresh-import.js +8 -0
- package/dist/fresh-import.js.map +1 -0
- package/dist/i18n/core.cjs +21 -0
- package/dist/i18n/core.cjs.map +1 -0
- package/dist/i18n/core.d.cts +11 -0
- package/dist/i18n/core.d.mts +11 -0
- package/dist/i18n/core.js +8 -19
- package/dist/i18n/core.js.map +1 -0
- package/dist/i18n/en.cjs +474 -0
- package/dist/i18n/en.cjs.map +1 -0
- package/dist/i18n/en.d.cts +391 -0
- package/dist/i18n/en.d.mts +391 -0
- package/dist/i18n/en.d.ts +389 -363
- package/dist/i18n/en.js +6 -455
- package/dist/i18n/en.js.map +1 -0
- package/dist/i18n/index.cjs +12 -0
- package/dist/i18n/index.cjs.map +1 -0
- package/dist/i18n/index.d.cts +5 -0
- package/dist/i18n/index.d.mts +5 -0
- package/dist/i18n/index.d.ts +4 -9
- package/dist/i18n/index.js +6 -30
- package/dist/i18n/index.js.map +1 -0
- package/dist/index.cjs +25 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +10 -0
- package/dist/index.d.mts +10 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +28 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +28 -0
- package/dist/index.mjs.map +1 -0
- package/dist/load-config.cjs +52 -0
- package/dist/load-config.cjs.map +1 -0
- package/dist/load-config.d.cts +4 -0
- package/dist/load-config.d.mts +4 -0
- package/dist/load-config.d.ts +4 -0
- package/dist/load-config.js +6 -0
- package/dist/load-config.js.map +1 -0
- package/dist/loaders/glob.cjs +150 -0
- package/dist/loaders/glob.cjs.map +1 -0
- package/dist/loaders/glob.d.cts +28 -0
- package/dist/loaders/glob.d.mts +28 -0
- package/dist/loaders/glob.d.ts +28 -0
- package/dist/loaders/glob.js +12 -0
- package/dist/loaders/glob.js.map +1 -0
- package/dist/loaders/index.cjs +281 -0
- package/dist/loaders/index.cjs.map +1 -0
- package/dist/loaders/index.d.cts +95 -0
- package/dist/loaders/index.d.mts +95 -0
- package/dist/loaders/index.d.ts +95 -0
- package/dist/loaders/index.js +16 -0
- package/dist/loaders/index.js.map +1 -0
- package/dist/loaders/index.mjs +16 -0
- package/dist/loaders/index.mjs.map +1 -0
- package/dist/o11y/command-projection.cjs +127 -0
- package/dist/o11y/command-projection.cjs.map +1 -0
- package/dist/o11y/command-projection.d.cts +29 -0
- package/dist/o11y/command-projection.d.mts +29 -0
- package/dist/o11y/command-projection.d.ts +29 -0
- package/dist/o11y/command-projection.js +18 -0
- package/dist/o11y/command-projection.js.map +1 -0
- package/dist/o11y/cost.cjs +76 -0
- package/dist/o11y/cost.cjs.map +1 -0
- package/dist/o11y/cost.d.cts +8 -0
- package/dist/o11y/cost.d.mts +8 -0
- package/dist/o11y/cost.d.ts +8 -0
- package/dist/o11y/cost.js +6 -0
- package/dist/o11y/cost.js.map +1 -0
- package/dist/o11y/derive.cjs +572 -0
- package/dist/o11y/derive.cjs.map +1 -0
- package/dist/o11y/derive.d.cts +24 -0
- package/dist/o11y/derive.d.mts +24 -0
- package/dist/o11y/derive.d.ts +16 -1
- package/dist/o11y/derive.js +14 -281
- package/dist/o11y/derive.js.map +1 -0
- package/dist/o11y/execution-tree.cjs +213 -0
- package/dist/o11y/execution-tree.cjs.map +1 -0
- package/dist/o11y/execution-tree.d.cts +113 -0
- package/dist/o11y/execution-tree.d.mts +113 -0
- package/dist/o11y/execution-tree.d.ts +1 -1
- package/dist/o11y/execution-tree.js +6 -254
- package/dist/o11y/execution-tree.js.map +1 -0
- package/dist/o11y/otlp/canonical.cjs +94 -0
- package/dist/o11y/otlp/canonical.cjs.map +1 -0
- package/dist/o11y/otlp/canonical.d.cts +34 -0
- package/dist/o11y/otlp/canonical.d.mts +34 -0
- package/dist/o11y/otlp/canonical.d.ts +34 -0
- package/dist/o11y/otlp/canonical.js +26 -0
- package/dist/o11y/otlp/canonical.js.map +1 -0
- package/dist/o11y/otlp/mappers/bub.cjs +24 -0
- package/dist/o11y/otlp/mappers/bub.cjs.map +1 -0
- package/dist/o11y/otlp/mappers/bub.d.cts +2 -0
- package/dist/o11y/otlp/mappers/bub.d.mts +2 -0
- package/dist/o11y/otlp/mappers/bub.d.ts +2 -0
- package/dist/o11y/otlp/mappers/bub.js +6 -0
- package/dist/o11y/otlp/mappers/bub.js.map +1 -0
- package/dist/o11y/otlp/mappers/claude-code.cjs +16 -0
- package/dist/o11y/otlp/mappers/claude-code.cjs.map +1 -0
- package/dist/o11y/otlp/mappers/claude-code.d.cts +2 -0
- package/dist/o11y/otlp/mappers/claude-code.d.mts +2 -0
- package/dist/o11y/otlp/mappers/claude-code.d.ts +2 -0
- package/dist/o11y/otlp/mappers/claude-code.js +6 -0
- package/dist/o11y/otlp/mappers/claude-code.js.map +1 -0
- package/dist/o11y/otlp/mappers/codex.cjs +31 -0
- package/dist/o11y/otlp/mappers/codex.cjs.map +1 -0
- package/dist/o11y/otlp/mappers/codex.d.cts +2 -0
- package/dist/o11y/otlp/mappers/codex.d.mts +2 -0
- package/dist/o11y/otlp/mappers/codex.d.ts +2 -0
- package/dist/o11y/otlp/mappers/codex.js +6 -0
- package/dist/o11y/otlp/mappers/codex.js.map +1 -0
- package/dist/o11y/otlp/mappers/index.cjs +21 -0
- package/dist/o11y/otlp/mappers/index.cjs.map +1 -0
- package/dist/o11y/otlp/mappers/index.d.cts +4 -0
- package/dist/o11y/otlp/mappers/index.d.mts +4 -0
- package/dist/o11y/otlp/mappers/index.d.ts +4 -0
- package/dist/o11y/otlp/mappers/index.js +12 -0
- package/dist/o11y/otlp/mappers/index.js.map +1 -0
- package/dist/o11y/otlp/parse.cjs +369 -0
- package/dist/o11y/otlp/parse.cjs.map +1 -0
- package/dist/o11y/otlp/parse.d.cts +3 -0
- package/dist/o11y/otlp/parse.d.mts +3 -0
- package/dist/o11y/otlp/parse.d.ts +1 -1
- package/dist/o11y/otlp/parse.js +6 -0
- package/dist/o11y/otlp/parse.js.map +1 -0
- package/dist/o11y/otlp/receiver.cjs +326 -0
- package/dist/o11y/otlp/receiver.cjs.map +1 -0
- package/dist/o11y/otlp/receiver.d.cts +27 -0
- package/dist/o11y/otlp/receiver.d.mts +27 -0
- package/dist/o11y/otlp/receiver.d.ts +16 -10
- package/dist/o11y/otlp/receiver.js +10 -0
- package/dist/o11y/otlp/receiver.js.map +1 -0
- package/dist/o11y/otlp/sandbox-receiver.cjs +226 -0
- package/dist/o11y/otlp/sandbox-receiver.cjs.map +1 -0
- package/dist/o11y/otlp/sandbox-receiver.d.cts +4 -0
- package/dist/o11y/otlp/sandbox-receiver.d.mts +4 -0
- package/dist/o11y/otlp/sandbox-receiver.d.ts +4 -0
- package/dist/o11y/otlp/sandbox-receiver.js +6 -0
- package/dist/o11y/otlp/sandbox-receiver.js.map +1 -0
- package/dist/o11y/otlp/select.cjs +82 -0
- package/dist/o11y/otlp/select.cjs.map +1 -0
- package/dist/o11y/otlp/select.d.cts +22 -0
- package/dist/o11y/otlp/select.d.mts +22 -0
- package/dist/o11y/otlp/select.d.ts +1 -1
- package/dist/o11y/otlp/select.js +12 -86
- package/dist/o11y/otlp/select.js.map +1 -0
- package/dist/o11y/otlp/turn-otel.cjs +180 -0
- package/dist/o11y/otlp/turn-otel.cjs.map +1 -0
- package/dist/o11y/otlp/turn-otel.d.cts +59 -0
- package/dist/o11y/otlp/turn-otel.d.mts +59 -0
- package/dist/o11y/otlp/turn-otel.d.ts +26 -10
- package/dist/o11y/otlp/turn-otel.js +8 -0
- package/dist/o11y/otlp/turn-otel.js.map +1 -0
- package/dist/o11y/parsers/bub.cjs +277 -0
- package/dist/o11y/parsers/bub.cjs.map +1 -0
- package/dist/o11y/parsers/bub.d.cts +7 -0
- package/dist/o11y/parsers/bub.d.mts +7 -0
- package/dist/o11y/parsers/bub.d.ts +7 -0
- package/dist/o11y/parsers/bub.js +10 -0
- package/dist/o11y/parsers/bub.js.map +1 -0
- package/dist/o11y/parsers/claude-code.cjs +309 -0
- package/dist/o11y/parsers/claude-code.cjs.map +1 -0
- package/dist/o11y/parsers/claude-code.d.cts +11 -0
- package/dist/o11y/parsers/claude-code.d.mts +11 -0
- package/dist/o11y/parsers/claude-code.d.ts +11 -0
- package/dist/o11y/parsers/claude-code.js +10 -0
- package/dist/o11y/parsers/claude-code.js.map +1 -0
- package/dist/o11y/parsers/codex.cjs +439 -0
- package/dist/o11y/parsers/codex.cjs.map +1 -0
- package/dist/o11y/parsers/codex.d.cts +7 -0
- package/dist/o11y/parsers/codex.d.mts +7 -0
- package/dist/o11y/parsers/codex.d.ts +7 -0
- package/dist/o11y/parsers/codex.js +10 -0
- package/dist/o11y/parsers/codex.js.map +1 -0
- package/dist/o11y/parsers/hermes.cjs +234 -0
- package/dist/o11y/parsers/hermes.cjs.map +1 -0
- package/dist/o11y/parsers/hermes.d.cts +9 -0
- package/dist/o11y/parsers/hermes.d.mts +9 -0
- package/dist/o11y/parsers/hermes.d.ts +9 -0
- package/dist/o11y/parsers/hermes.js +10 -0
- package/dist/o11y/parsers/hermes.js.map +1 -0
- package/dist/o11y/parsers/index.cjs +13 -0
- package/dist/o11y/parsers/index.cjs.map +1 -0
- package/dist/o11y/parsers/index.d.cts +12 -0
- package/dist/o11y/parsers/index.d.mts +12 -0
- package/dist/o11y/parsers/index.d.ts +12 -0
- package/dist/o11y/parsers/index.js +16 -0
- package/dist/o11y/parsers/index.js.map +1 -0
- package/dist/o11y/parsers/openclaw.cjs +320 -0
- package/dist/o11y/parsers/openclaw.cjs.map +1 -0
- package/dist/o11y/parsers/openclaw.d.cts +39 -0
- package/dist/o11y/parsers/openclaw.d.mts +39 -0
- package/dist/o11y/parsers/openclaw.d.ts +39 -0
- package/dist/o11y/parsers/openclaw.js +12 -0
- package/dist/o11y/parsers/openclaw.js.map +1 -0
- package/dist/o11y/parsers/opencode.cjs +337 -0
- package/dist/o11y/parsers/opencode.cjs.map +1 -0
- package/dist/o11y/parsers/opencode.d.cts +14 -0
- package/dist/o11y/parsers/opencode.d.mts +14 -0
- package/dist/o11y/parsers/opencode.d.ts +14 -0
- package/dist/o11y/parsers/opencode.js +12 -0
- package/dist/o11y/parsers/opencode.js.map +1 -0
- package/dist/o11y/prices.json +11649 -0
- package/dist/o11y/record/capture.cjs +232 -0
- package/dist/o11y/record/capture.cjs.map +1 -0
- package/dist/o11y/record/capture.d.cts +81 -0
- package/dist/o11y/record/capture.d.mts +81 -0
- package/dist/o11y/record/capture.d.ts +81 -0
- package/dist/o11y/record/capture.js +30 -0
- package/dist/o11y/record/capture.js.map +1 -0
- package/dist/o11y/record/codec.cjs +222 -0
- package/dist/o11y/record/codec.cjs.map +1 -0
- package/dist/o11y/record/codec.d.cts +661 -0
- package/dist/o11y/record/codec.d.mts +661 -0
- package/dist/o11y/record/codec.d.ts +661 -0
- package/dist/o11y/record/codec.js +66 -0
- package/dist/o11y/record/codec.js.map +1 -0
- package/dist/o11y/record/errors.cjs +71 -0
- package/dist/o11y/record/errors.cjs.map +1 -0
- package/dist/o11y/record/errors.d.cts +53 -0
- package/dist/o11y/record/errors.d.mts +53 -0
- package/dist/o11y/record/errors.d.ts +53 -0
- package/dist/o11y/record/errors.js +24 -0
- package/dist/o11y/record/errors.js.map +1 -0
- package/dist/o11y/record/families.cjs +699 -0
- package/dist/o11y/record/families.cjs.map +1 -0
- package/dist/o11y/record/families.d.cts +4298 -0
- package/dist/o11y/record/families.d.mts +4298 -0
- package/dist/o11y/record/families.d.ts +4298 -0
- package/dist/o11y/record/families.js +68 -0
- package/dist/o11y/record/families.js.map +1 -0
- package/dist/o11y/record/family-projectors.cjs +83 -0
- package/dist/o11y/record/family-projectors.cjs.map +1 -0
- package/dist/o11y/record/family-projectors.d.cts +46 -0
- package/dist/o11y/record/family-projectors.d.mts +46 -0
- package/dist/o11y/record/family-projectors.d.ts +46 -0
- package/dist/o11y/record/family-projectors.js +18 -0
- package/dist/o11y/record/family-projectors.js.map +1 -0
- package/dist/o11y/record/family-writers.cjs +371 -0
- package/dist/o11y/record/family-writers.cjs.map +1 -0
- package/dist/o11y/record/family-writers.d.cts +5293 -0
- package/dist/o11y/record/family-writers.d.mts +5293 -0
- package/dist/o11y/record/family-writers.d.ts +5293 -0
- package/dist/o11y/record/family-writers.js +60 -0
- package/dist/o11y/record/family-writers.js.map +1 -0
- package/dist/o11y/record/index.cjs +31 -0
- package/dist/o11y/record/index.cjs.map +1 -0
- package/dist/o11y/record/index.d.cts +14 -0
- package/dist/o11y/record/index.d.mts +14 -0
- package/dist/o11y/record/index.d.ts +14 -0
- package/dist/o11y/record/index.js +488 -0
- package/dist/o11y/record/index.js.map +1 -0
- package/dist/o11y/record/limits.cjs +69 -0
- package/dist/o11y/record/limits.cjs.map +1 -0
- package/dist/o11y/record/limits.d.cts +65 -0
- package/dist/o11y/record/limits.d.mts +65 -0
- package/dist/o11y/record/limits.d.ts +65 -0
- package/dist/o11y/record/limits.js +68 -0
- package/dist/o11y/record/limits.js.map +1 -0
- package/dist/o11y/record/model.cjs +590 -0
- package/dist/o11y/record/model.cjs.map +1 -0
- package/dist/o11y/record/model.d.cts +275 -0
- package/dist/o11y/record/model.d.mts +275 -0
- package/dist/o11y/record/model.d.ts +275 -0
- package/dist/o11y/record/model.js +158 -0
- package/dist/o11y/record/model.js.map +1 -0
- package/dist/o11y/record/runner-producer.cjs +1575 -0
- package/dist/o11y/record/runner-producer.cjs.map +1 -0
- package/dist/o11y/record/runner-producer.d.cts +103 -0
- package/dist/o11y/record/runner-producer.d.mts +103 -0
- package/dist/o11y/record/runner-producer.d.ts +103 -0
- package/dist/o11y/record/runner-producer.js +26 -0
- package/dist/o11y/record/runner-producer.js.map +1 -0
- package/dist/o11y/record/validation.cjs +323 -0
- package/dist/o11y/record/validation.cjs.map +1 -0
- package/dist/o11y/record/validation.d.cts +62 -0
- package/dist/o11y/record/validation.d.mts +62 -0
- package/dist/o11y/record/validation.d.ts +62 -0
- package/dist/o11y/record/validation.js +28 -0
- package/dist/o11y/record/validation.js.map +1 -0
- package/dist/o11y/tool-names.cjs +68 -0
- package/dist/o11y/tool-names.cjs.map +1 -0
- package/dist/o11y/tool-names.d.cts +11 -0
- package/dist/o11y/tool-names.d.mts +11 -0
- package/dist/o11y/tool-names.d.ts +11 -0
- package/dist/o11y/tool-names.js +8 -0
- package/dist/o11y/tool-names.js.map +1 -0
- package/dist/o11y/types.cjs +3 -0
- package/dist/o11y/types.cjs.map +1 -0
- package/dist/o11y/types.d.cts +387 -0
- package/dist/o11y/types.d.mts +387 -0
- package/dist/o11y/types.d.ts +142 -1
- package/dist/o11y/types.js +4 -3
- package/dist/o11y/types.js.map +1 -0
- package/dist/plugin/contracts.cjs +187 -0
- package/dist/plugin/contracts.cjs.map +1 -0
- package/dist/plugin/contracts.d.cts +109 -0
- package/dist/plugin/contracts.d.mts +109 -0
- package/dist/plugin/contracts.d.ts +109 -0
- package/dist/plugin/contracts.js +18 -0
- package/dist/plugin/contracts.js.map +1 -0
- package/dist/plugin/index.cjs +7 -0
- package/dist/plugin/index.cjs.map +1 -0
- package/dist/plugin/index.d.cts +2 -0
- package/dist/plugin/index.d.mts +2 -0
- package/dist/plugin/index.d.ts +2 -0
- package/dist/plugin/index.js +6 -0
- package/dist/plugin/index.js.map +1 -0
- package/dist/plugin/index.mjs +6 -0
- package/dist/plugin/index.mjs.map +1 -0
- package/dist/plugin/link.cjs +204 -0
- package/dist/plugin/link.cjs.map +1 -0
- package/dist/plugin/link.d.cts +68 -0
- package/dist/plugin/link.d.mts +68 -0
- package/dist/plugin/link.d.ts +68 -0
- package/dist/plugin/link.js +14 -0
- package/dist/plugin/link.js.map +1 -0
- package/dist/plugins/record/attachment.cjs +62 -0
- package/dist/plugins/record/attachment.cjs.map +1 -0
- package/dist/plugins/record/attachment.d.cts +14 -0
- package/dist/plugins/record/attachment.d.mts +14 -0
- package/dist/plugins/record/attachment.d.ts +14 -0
- package/dist/plugins/record/attachment.js +20 -0
- package/dist/plugins/record/attachment.js.map +1 -0
- package/dist/plugins/record/capability.cjs +258 -0
- package/dist/plugins/record/capability.cjs.map +1 -0
- package/dist/plugins/record/capability.d.cts +94 -0
- package/dist/plugins/record/capability.d.mts +94 -0
- package/dist/plugins/record/capability.d.ts +94 -0
- package/dist/plugins/record/capability.js +26 -0
- package/dist/plugins/record/capability.js.map +1 -0
- package/dist/plugins/record/codec.cjs +244 -0
- package/dist/plugins/record/codec.cjs.map +1 -0
- package/dist/plugins/record/codec.d.cts +28 -0
- package/dist/plugins/record/codec.d.mts +28 -0
- package/dist/plugins/record/codec.d.ts +28 -0
- package/dist/plugins/record/codec.js +48 -0
- package/dist/plugins/record/codec.js.map +1 -0
- package/dist/plugins/record/index.cjs +54 -0
- package/dist/plugins/record/index.cjs.map +1 -0
- package/dist/plugins/record/index.d.cts +9 -0
- package/dist/plugins/record/index.d.mts +9 -0
- package/dist/plugins/record/index.d.ts +9 -0
- package/dist/plugins/record/index.js +94 -0
- package/dist/plugins/record/index.js.map +1 -0
- package/dist/plugins/record/migration.cjs +33 -0
- package/dist/plugins/record/migration.cjs.map +1 -0
- package/dist/plugins/record/migration.d.cts +22 -0
- package/dist/plugins/record/migration.d.mts +22 -0
- package/dist/plugins/record/migration.d.ts +22 -0
- package/dist/plugins/record/migration.js +6 -0
- package/dist/plugins/record/migration.js.map +1 -0
- package/dist/plugins/record/model.cjs +3 -0
- package/dist/plugins/record/model.cjs.map +1 -0
- package/dist/plugins/record/model.d.cts +93 -0
- package/dist/plugins/record/model.d.mts +93 -0
- package/dist/plugins/record/model.d.ts +93 -0
- package/dist/plugins/record/model.js +4 -0
- package/dist/plugins/record/model.js.map +1 -0
- package/dist/plugins/record/provenance.cjs +319 -0
- package/dist/plugins/record/provenance.cjs.map +1 -0
- package/dist/plugins/record/provenance.d.cts +79 -0
- package/dist/plugins/record/provenance.d.mts +79 -0
- package/dist/plugins/record/provenance.d.ts +79 -0
- package/dist/plugins/record/provenance.js +24 -0
- package/dist/plugins/record/provenance.js.map +1 -0
- package/dist/projection/attachment-result.cjs +50 -0
- package/dist/projection/attachment-result.cjs.map +1 -0
- package/dist/projection/attachment-result.d.cts +38 -0
- package/dist/projection/attachment-result.d.mts +38 -0
- package/dist/projection/attachment-result.d.ts +38 -0
- package/dist/projection/attachment-result.js +6 -0
- package/dist/projection/attachment-result.js.map +1 -0
- package/dist/projection/attempt-selection.cjs +87 -0
- package/dist/projection/attempt-selection.cjs.map +1 -0
- package/dist/projection/attempt-selection.d.cts +40 -0
- package/dist/projection/attempt-selection.d.mts +40 -0
- package/dist/projection/attempt-selection.d.ts +40 -0
- package/dist/projection/attempt-selection.js +8 -0
- package/dist/projection/attempt-selection.js.map +1 -0
- package/dist/projection/coverage.cjs +106 -0
- package/dist/projection/coverage.cjs.map +1 -0
- package/dist/projection/coverage.d.cts +43 -0
- package/dist/projection/coverage.d.mts +43 -0
- package/dist/projection/coverage.d.ts +43 -0
- package/dist/projection/coverage.js +6 -0
- package/dist/projection/coverage.js.map +1 -0
- package/dist/projection/index.cjs +36 -0
- package/dist/projection/index.cjs.map +1 -0
- package/dist/projection/index.d.cts +25 -0
- package/dist/projection/index.d.mts +25 -0
- package/dist/projection/index.d.ts +25 -0
- package/dist/projection/index.js +46 -0
- package/dist/projection/index.js.map +1 -0
- package/dist/projection/index.mjs +46 -0
- package/dist/projection/index.mjs.map +1 -0
- package/dist/projection/model.cjs +3 -0
- package/dist/projection/model.cjs.map +1 -0
- package/dist/projection/model.d.cts +49 -0
- package/dist/projection/model.d.mts +49 -0
- package/dist/projection/model.d.ts +49 -0
- package/dist/projection/model.js +4 -0
- package/dist/projection/model.js.map +1 -0
- package/dist/projection/projector.cjs +111 -0
- package/dist/projection/projector.cjs.map +1 -0
- package/dist/projection/projector.d.cts +51 -0
- package/dist/projection/projector.d.mts +51 -0
- package/dist/projection/projector.d.ts +51 -0
- package/dist/projection/projector.js +16 -0
- package/dist/projection/projector.js.map +1 -0
- package/dist/projection/runtime.cjs +246 -0
- package/dist/projection/runtime.cjs.map +1 -0
- package/dist/projection/runtime.d.cts +14 -0
- package/dist/projection/runtime.d.mts +14 -0
- package/dist/projection/runtime.d.ts +14 -0
- package/dist/projection/runtime.js +6 -0
- package/dist/projection/runtime.js.map +1 -0
- package/dist/record/attachment/errors.cjs +71 -0
- package/dist/record/attachment/errors.cjs.map +1 -0
- package/dist/record/attachment/errors.d.cts +57 -0
- package/dist/record/attachment/errors.d.mts +57 -0
- package/dist/record/attachment/errors.d.ts +57 -0
- package/dist/record/attachment/errors.js +22 -0
- package/dist/record/attachment/errors.js.map +1 -0
- package/dist/record/attachment/index.cjs +14 -0
- package/dist/record/attachment/index.cjs.map +1 -0
- package/dist/record/attachment/index.d.cts +3 -0
- package/dist/record/attachment/index.d.mts +3 -0
- package/dist/record/attachment/index.d.ts +3 -0
- package/dist/record/attachment/index.js +22 -0
- package/dist/record/attachment/index.js.map +1 -0
- package/dist/record/attachment/internal.cjs +24 -0
- package/dist/record/attachment/internal.cjs.map +1 -0
- package/dist/record/attachment/internal.d.cts +6 -0
- package/dist/record/attachment/internal.d.mts +6 -0
- package/dist/record/attachment/internal.d.ts +6 -0
- package/dist/record/attachment/internal.js +34 -0
- package/dist/record/attachment/internal.js.map +1 -0
- package/dist/record/attachment/runtime.cjs +1017 -0
- package/dist/record/attachment/runtime.cjs.map +1 -0
- package/dist/record/attachment/runtime.d.cts +86 -0
- package/dist/record/attachment/runtime.d.mts +86 -0
- package/dist/record/attachment/runtime.d.ts +86 -0
- package/dist/record/attachment/runtime.js +52 -0
- package/dist/record/attachment/runtime.js.map +1 -0
- package/dist/record/attachment/types.cjs +47 -0
- package/dist/record/attachment/types.cjs.map +1 -0
- package/dist/record/attachment/types.d.cts +194 -0
- package/dist/record/attachment/types.d.mts +194 -0
- package/dist/record/attachment/types.d.ts +194 -0
- package/dist/record/attachment/types.js +28 -0
- package/dist/record/attachment/types.js.map +1 -0
- package/dist/record/codec/core.cjs +134 -0
- package/dist/record/codec/core.cjs.map +1 -0
- package/dist/record/codec/core.d.cts +63 -0
- package/dist/record/codec/core.d.mts +63 -0
- package/dist/record/codec/core.d.ts +63 -0
- package/dist/record/codec/core.js +46 -0
- package/dist/record/codec/core.js.map +1 -0
- package/dist/record/codec/identifiers.cjs +31 -0
- package/dist/record/codec/identifiers.cjs.map +1 -0
- package/dist/record/codec/identifiers.d.cts +11 -0
- package/dist/record/codec/identifiers.d.mts +11 -0
- package/dist/record/codec/identifiers.d.ts +11 -0
- package/dist/record/codec/identifiers.js +22 -0
- package/dist/record/codec/identifiers.js.map +1 -0
- package/dist/record/codec/index.cjs +19 -0
- package/dist/record/codec/index.cjs.map +1 -0
- package/dist/record/codec/index.d.cts +2 -0
- package/dist/record/codec/index.d.mts +2 -0
- package/dist/record/codec/index.d.ts +2 -0
- package/dist/record/codec/index.js +64 -0
- package/dist/record/codec/index.js.map +1 -0
- package/dist/record/errors/index.cjs +18 -0
- package/dist/record/errors/index.cjs.map +1 -0
- package/dist/record/errors/index.d.cts +1 -0
- package/dist/record/errors/index.d.mts +1 -0
- package/dist/record/errors/index.d.ts +1 -0
- package/dist/record/errors/index.js +28 -0
- package/dist/record/errors/index.js.map +1 -0
- package/dist/record/errors/record-errors.cjs +58 -0
- package/dist/record/errors/record-errors.cjs.map +1 -0
- package/dist/record/errors/record-errors.d.cts +64 -0
- package/dist/record/errors/record-errors.d.mts +64 -0
- package/dist/record/errors/record-errors.d.ts +64 -0
- package/dist/record/errors/record-errors.js +28 -0
- package/dist/record/errors/record-errors.js.map +1 -0
- package/dist/record/fact-record.cjs +199 -0
- package/dist/record/fact-record.cjs.map +1 -0
- package/dist/record/fact-record.d.cts +22 -0
- package/dist/record/fact-record.d.mts +22 -0
- package/dist/record/fact-record.d.ts +22 -0
- package/dist/record/fact-record.js +18 -0
- package/dist/record/fact-record.js.map +1 -0
- package/dist/record/index.cjs +55 -0
- package/dist/record/index.cjs.map +1 -0
- package/dist/record/index.d.cts +11 -0
- package/dist/record/index.d.mts +11 -0
- package/dist/record/index.d.ts +11 -0
- package/dist/record/index.js +314 -0
- package/dist/record/index.js.map +1 -0
- package/dist/record/locator.cjs +146 -0
- package/dist/record/locator.cjs.map +1 -0
- package/dist/record/locator.d.cts +71 -0
- package/dist/record/locator.d.mts +71 -0
- package/dist/record/locator.js +20 -136
- package/dist/record/locator.js.map +1 -0
- package/dist/record/maintenance/index.cjs +19 -0
- package/dist/record/maintenance/index.cjs.map +1 -0
- package/dist/record/maintenance/index.d.cts +2 -0
- package/dist/record/maintenance/index.d.mts +2 -0
- package/dist/record/maintenance/index.d.ts +2 -0
- package/dist/record/maintenance/index.js +14 -0
- package/dist/record/maintenance/index.js.map +1 -0
- package/dist/record/maintenance/runtime.cjs +98 -0
- package/dist/record/maintenance/runtime.cjs.map +1 -0
- package/dist/record/maintenance/runtime.d.cts +24 -0
- package/dist/record/maintenance/runtime.d.mts +24 -0
- package/dist/record/maintenance/runtime.d.ts +24 -0
- package/dist/record/maintenance/runtime.js +12 -0
- package/dist/record/maintenance/runtime.js.map +1 -0
- package/dist/record/maintenance/types.cjs +9 -0
- package/dist/record/maintenance/types.cjs.map +1 -0
- package/dist/record/maintenance/types.d.cts +34 -0
- package/dist/record/maintenance/types.d.mts +34 -0
- package/dist/record/maintenance/types.d.ts +34 -0
- package/dist/record/maintenance/types.js +6 -0
- package/dist/record/maintenance/types.js.map +1 -0
- package/dist/record/manifest.cjs +44 -0
- package/dist/record/manifest.cjs.map +1 -0
- package/dist/record/manifest.d.cts +39 -0
- package/dist/record/manifest.d.mts +39 -0
- package/dist/record/manifest.d.ts +5 -3
- package/dist/record/manifest.js +14 -0
- package/dist/record/manifest.js.map +1 -0
- package/dist/record/migration/errors.cjs +68 -0
- package/dist/record/migration/errors.cjs.map +1 -0
- package/dist/record/migration/errors.d.cts +73 -0
- package/dist/record/migration/errors.d.mts +73 -0
- package/dist/record/migration/errors.d.ts +73 -0
- package/dist/record/migration/errors.js +22 -0
- package/dist/record/migration/errors.js.map +1 -0
- package/dist/record/migration/identity.cjs +27 -0
- package/dist/record/migration/identity.cjs.map +1 -0
- package/dist/record/migration/identity.d.cts +5 -0
- package/dist/record/migration/identity.d.mts +5 -0
- package/dist/record/migration/identity.d.ts +5 -0
- package/dist/record/migration/identity.js +8 -0
- package/dist/record/migration/identity.js.map +1 -0
- package/dist/record/migration/index.cjs +28 -0
- package/dist/record/migration/index.cjs.map +1 -0
- package/dist/record/migration/index.d.cts +8 -0
- package/dist/record/migration/index.d.mts +8 -0
- package/dist/record/migration/index.d.ts +8 -0
- package/dist/record/migration/index.js +48 -0
- package/dist/record/migration/index.js.map +1 -0
- package/dist/record/migration/internal.cjs +3 -0
- package/dist/record/migration/internal.cjs.map +1 -0
- package/dist/record/migration/internal.d.cts +111 -0
- package/dist/record/migration/internal.d.mts +111 -0
- package/dist/record/migration/internal.d.ts +111 -0
- package/dist/record/migration/internal.js +4 -0
- package/dist/record/migration/internal.js.map +1 -0
- package/dist/record/migration/orchestrate.cjs +368 -0
- package/dist/record/migration/orchestrate.cjs.map +1 -0
- package/dist/record/migration/orchestrate.d.cts +113 -0
- package/dist/record/migration/orchestrate.d.mts +113 -0
- package/dist/record/migration/orchestrate.d.ts +113 -0
- package/dist/record/migration/orchestrate.js +16 -0
- package/dist/record/migration/orchestrate.js.map +1 -0
- package/dist/record/migration/plan.cjs +63 -0
- package/dist/record/migration/plan.cjs.map +1 -0
- package/dist/record/migration/plan.d.cts +32 -0
- package/dist/record/migration/plan.d.mts +32 -0
- package/dist/record/migration/plan.d.ts +32 -0
- package/dist/record/migration/plan.js +10 -0
- package/dist/record/migration/plan.js.map +1 -0
- package/dist/record/migration/registry.cjs +150 -0
- package/dist/record/migration/registry.cjs.map +1 -0
- package/dist/record/migration/registry.d.cts +61 -0
- package/dist/record/migration/registry.d.mts +61 -0
- package/dist/record/migration/registry.d.ts +61 -0
- package/dist/record/migration/registry.js +14 -0
- package/dist/record/migration/registry.js.map +1 -0
- package/dist/record/migration/storage.cjs +382 -0
- package/dist/record/migration/storage.cjs.map +1 -0
- package/dist/record/migration/storage.d.cts +18 -0
- package/dist/record/migration/storage.d.mts +18 -0
- package/dist/record/migration/storage.d.ts +18 -0
- package/dist/record/migration/storage.js +6 -0
- package/dist/record/migration/storage.js.map +1 -0
- package/dist/record/migration/types.cjs +3 -0
- package/dist/record/migration/types.cjs.map +1 -0
- package/dist/record/migration/types.d.cts +16 -0
- package/dist/record/migration/types.d.mts +16 -0
- package/dist/record/migration/types.d.ts +16 -0
- package/dist/record/migration/types.js +4 -0
- package/dist/record/migration/types.js.map +1 -0
- package/dist/record/model/core.cjs +11 -0
- package/dist/record/model/core.cjs.map +1 -0
- package/dist/record/model/core.d.cts +53 -0
- package/dist/record/model/core.d.mts +53 -0
- package/dist/record/model/core.d.ts +53 -0
- package/dist/record/model/core.js +6 -0
- package/dist/record/model/core.js.map +1 -0
- package/dist/record/model/identifiers.cjs +78 -0
- package/dist/record/model/identifiers.cjs.map +1 -0
- package/dist/record/model/identifiers.d.cts +44 -0
- package/dist/record/model/identifiers.d.mts +44 -0
- package/dist/record/model/identifiers.d.ts +44 -0
- package/dist/record/model/identifiers.js +40 -0
- package/dist/record/model/identifiers.js.map +1 -0
- package/dist/record/model/index.cjs +21 -0
- package/dist/record/model/index.cjs.map +1 -0
- package/dist/record/model/index.d.cts +4 -0
- package/dist/record/model/index.d.mts +4 -0
- package/dist/record/model/index.d.ts +4 -0
- package/dist/record/model/index.js +52 -0
- package/dist/record/model/index.js.map +1 -0
- package/dist/record/model/read-state.cjs +3 -0
- package/dist/record/model/read-state.cjs.map +1 -0
- package/dist/record/model/read-state.d.cts +45 -0
- package/dist/record/model/read-state.d.mts +45 -0
- package/dist/record/model/read-state.d.ts +45 -0
- package/dist/record/model/read-state.js +4 -0
- package/dist/record/model/read-state.js.map +1 -0
- package/dist/record/model/validation.cjs +143 -0
- package/dist/record/model/validation.cjs.map +1 -0
- package/dist/record/model/validation.d.cts +16 -0
- package/dist/record/model/validation.d.mts +16 -0
- package/dist/record/model/validation.d.ts +16 -0
- package/dist/record/model/validation.js +14 -0
- package/dist/record/model/validation.js.map +1 -0
- package/dist/record/platform/errors.cjs +101 -0
- package/dist/record/platform/errors.cjs.map +1 -0
- package/dist/record/platform/errors.d.cts +123 -0
- package/dist/record/platform/errors.d.mts +123 -0
- package/dist/record/platform/errors.d.ts +123 -0
- package/dist/record/platform/errors.js +32 -0
- package/dist/record/platform/errors.js.map +1 -0
- package/dist/record/platform/index.cjs +21 -0
- package/dist/record/platform/index.cjs.map +1 -0
- package/dist/record/platform/index.d.cts +4 -0
- package/dist/record/platform/index.d.mts +4 -0
- package/dist/record/platform/index.d.ts +4 -0
- package/dist/record/platform/index.js +60 -0
- package/dist/record/platform/index.js.map +1 -0
- package/dist/record/platform/node.cjs +715 -0
- package/dist/record/platform/node.cjs.map +1 -0
- package/dist/record/platform/node.d.cts +12 -0
- package/dist/record/platform/node.d.mts +12 -0
- package/dist/record/platform/node.d.ts +12 -0
- package/dist/record/platform/node.js +16 -0
- package/dist/record/platform/node.js.map +1 -0
- package/dist/record/platform/root.cjs +79 -0
- package/dist/record/platform/root.cjs.map +1 -0
- package/dist/record/platform/root.d.cts +35 -0
- package/dist/record/platform/root.d.mts +35 -0
- package/dist/record/platform/root.d.ts +35 -0
- package/dist/record/platform/root.js +8 -0
- package/dist/record/platform/root.js.map +1 -0
- package/dist/record/platform/services.cjs +24 -0
- package/dist/record/platform/services.cjs.map +1 -0
- package/dist/record/platform/services.d.cts +139 -0
- package/dist/record/platform/services.d.mts +139 -0
- package/dist/record/platform/services.d.ts +139 -0
- package/dist/record/platform/services.js +16 -0
- package/dist/record/platform/services.js.map +1 -0
- package/dist/record/reader/errors.cjs +46 -0
- package/dist/record/reader/errors.cjs.map +1 -0
- package/dist/record/reader/errors.d.cts +58 -0
- package/dist/record/reader/errors.d.mts +58 -0
- package/dist/record/reader/errors.d.ts +58 -0
- package/dist/record/reader/errors.js +16 -0
- package/dist/record/reader/errors.js.map +1 -0
- package/dist/record/reader/format.cjs +95 -0
- package/dist/record/reader/format.cjs.map +1 -0
- package/dist/record/reader/format.d.cts +17 -0
- package/dist/record/reader/format.d.mts +17 -0
- package/dist/record/reader/format.d.ts +17 -0
- package/dist/record/reader/format.js +8 -0
- package/dist/record/reader/format.js.map +1 -0
- package/dist/record/reader/identity.cjs +47 -0
- package/dist/record/reader/identity.cjs.map +1 -0
- package/dist/record/reader/identity.d.cts +31 -0
- package/dist/record/reader/identity.d.mts +31 -0
- package/dist/record/reader/identity.d.ts +31 -0
- package/dist/record/reader/identity.js +8 -0
- package/dist/record/reader/identity.js.map +1 -0
- package/dist/record/reader/index.cjs +21 -0
- package/dist/record/reader/index.cjs.map +1 -0
- package/dist/record/reader/index.d.cts +4 -0
- package/dist/record/reader/index.d.mts +4 -0
- package/dist/record/reader/index.d.ts +4 -0
- package/dist/record/reader/index.js +42 -0
- package/dist/record/reader/index.js.map +1 -0
- package/dist/record/reader/internal.cjs +17 -0
- package/dist/record/reader/internal.cjs.map +1 -0
- package/dist/record/reader/internal.d.cts +26 -0
- package/dist/record/reader/internal.d.mts +26 -0
- package/dist/record/reader/internal.d.ts +26 -0
- package/dist/record/reader/internal.js +8 -0
- package/dist/record/reader/internal.js.map +1 -0
- package/dist/record/reader/runtime.cjs +728 -0
- package/dist/record/reader/runtime.cjs.map +1 -0
- package/dist/record/reader/runtime.d.cts +22 -0
- package/dist/record/reader/runtime.d.mts +22 -0
- package/dist/record/reader/runtime.d.ts +22 -0
- package/dist/record/reader/runtime.js +20 -0
- package/dist/record/reader/runtime.js.map +1 -0
- package/dist/record/reader/types.cjs +12 -0
- package/dist/record/reader/types.cjs.map +1 -0
- package/dist/record/reader/types.d.cts +50 -0
- package/dist/record/reader/types.d.mts +50 -0
- package/dist/record/reader/types.d.ts +50 -0
- package/dist/record/reader/types.js +10 -0
- package/dist/record/reader/types.js.map +1 -0
- package/dist/record/source-hash.cjs +30 -0
- package/dist/record/source-hash.cjs.map +1 -0
- package/dist/record/source-hash.d.cts +9 -0
- package/dist/record/source-hash.d.mts +9 -0
- package/dist/record/source-hash.js +8 -25
- package/dist/record/source-hash.js.map +1 -0
- package/dist/record/writer/attachment-payload.cjs +81 -0
- package/dist/record/writer/attachment-payload.cjs.map +1 -0
- package/dist/record/writer/attachment-payload.d.cts +19 -0
- package/dist/record/writer/attachment-payload.d.mts +19 -0
- package/dist/record/writer/attachment-payload.d.ts +19 -0
- package/dist/record/writer/attachment-payload.js +10 -0
- package/dist/record/writer/attachment-payload.js.map +1 -0
- package/dist/record/writer/errors.cjs +35 -0
- package/dist/record/writer/errors.cjs.map +1 -0
- package/dist/record/writer/errors.d.cts +35 -0
- package/dist/record/writer/errors.d.mts +35 -0
- package/dist/record/writer/errors.d.ts +35 -0
- package/dist/record/writer/errors.js +14 -0
- package/dist/record/writer/errors.js.map +1 -0
- package/dist/record/writer/index.cjs +20 -0
- package/dist/record/writer/index.cjs.map +1 -0
- package/dist/record/writer/index.d.cts +3 -0
- package/dist/record/writer/index.d.mts +3 -0
- package/dist/record/writer/index.d.ts +3 -0
- package/dist/record/writer/index.js +22 -0
- package/dist/record/writer/index.js.map +1 -0
- package/dist/record/writer/limits.cjs +16 -0
- package/dist/record/writer/limits.cjs.map +1 -0
- package/dist/record/writer/limits.d.cts +4 -0
- package/dist/record/writer/limits.d.mts +4 -0
- package/dist/record/writer/limits.d.ts +4 -0
- package/dist/record/writer/limits.js +8 -0
- package/dist/record/writer/limits.js.map +1 -0
- package/dist/record/writer/runtime.cjs +656 -0
- package/dist/record/writer/runtime.cjs.map +1 -0
- package/dist/record/writer/runtime.d.cts +7 -0
- package/dist/record/writer/runtime.d.mts +7 -0
- package/dist/record/writer/runtime.d.ts +7 -0
- package/dist/record/writer/runtime.js +6 -0
- package/dist/record/writer/runtime.js.map +1 -0
- package/dist/record/writer/types.cjs +7 -0
- package/dist/record/writer/types.cjs.map +1 -0
- package/dist/record/writer/types.d.cts +56 -0
- package/dist/record/writer/types.d.mts +56 -0
- package/dist/record/writer/types.d.ts +56 -0
- package/dist/record/writer/types.js +10 -0
- package/dist/record/writer/types.js.map +1 -0
- package/dist/report/author/identity.cjs +268 -0
- package/dist/report/author/identity.cjs.map +1 -0
- package/dist/report/author/identity.d.cts +75 -0
- package/dist/report/author/identity.d.mts +75 -0
- package/dist/report/author/identity.d.ts +75 -0
- package/dist/report/author/identity.js +34 -0
- package/dist/report/author/identity.js.map +1 -0
- package/dist/report/author/index.cjs +19 -0
- package/dist/report/author/index.cjs.map +1 -0
- package/dist/report/author/index.d.cts +4 -0
- package/dist/report/author/index.d.mts +4 -0
- package/dist/report/author/index.d.ts +4 -0
- package/dist/report/author/index.js +30 -0
- package/dist/report/author/index.js.map +1 -0
- package/dist/report/author/internal.cjs +98 -0
- package/dist/report/author/internal.cjs.map +1 -0
- package/dist/report/author/internal.d.cts +91 -0
- package/dist/report/author/internal.d.mts +91 -0
- package/dist/report/author/internal.d.ts +91 -0
- package/dist/report/author/internal.js +18 -0
- package/dist/report/author/internal.js.map +1 -0
- package/dist/report/author/model.cjs +467 -0
- package/dist/report/author/model.cjs.map +1 -0
- package/dist/report/author/model.d.cts +241 -0
- package/dist/report/author/model.d.mts +241 -0
- package/dist/report/author/model.d.ts +241 -0
- package/dist/report/author/model.js +32 -0
- package/dist/report/author/model.js.map +1 -0
- package/dist/report/built-in/attempt-overview.cjs +395 -0
- package/dist/report/built-in/attempt-overview.cjs.map +1 -0
- package/dist/report/built-in/attempt-overview.d.cts +10 -0
- package/dist/report/built-in/attempt-overview.d.mts +10 -0
- package/dist/report/built-in/attempt-overview.d.ts +10 -0
- package/dist/report/built-in/attempt-overview.js +9 -0
- package/dist/report/built-in/attempt-overview.js.map +1 -0
- package/dist/report/built-in/execution.cjs +766 -0
- package/dist/report/built-in/execution.cjs.map +1 -0
- package/dist/report/built-in/execution.d.cts +25 -0
- package/dist/report/built-in/execution.d.mts +25 -0
- package/dist/report/built-in/execution.d.ts +25 -0
- package/dist/report/built-in/execution.js +12 -0
- package/dist/report/built-in/execution.js.map +1 -0
- package/dist/report/built-in/index.cjs +31 -0
- package/dist/report/built-in/index.cjs.map +1 -0
- package/dist/report/built-in/index.d.cts +6 -0
- package/dist/report/built-in/index.d.mts +6 -0
- package/dist/report/built-in/index.d.ts +6 -8
- package/dist/report/built-in/index.js +35 -13
- package/dist/report/built-in/index.js.map +1 -0
- package/dist/report/built-in/index.mjs +35 -0
- package/dist/report/built-in/index.mjs.map +1 -0
- package/dist/report/built-in/overview.cjs +206 -0
- package/dist/report/built-in/overview.cjs.map +1 -0
- package/dist/report/built-in/overview.d.cts +6 -0
- package/dist/report/built-in/overview.d.mts +6 -0
- package/dist/report/built-in/overview.d.ts +6 -0
- package/dist/report/built-in/overview.js +11 -0
- package/dist/report/built-in/overview.js.map +1 -0
- package/dist/report/built-in/run-membership-overview.cjs +194 -0
- package/dist/report/built-in/run-membership-overview.cjs.map +1 -0
- package/dist/report/built-in/run-membership-overview.d.cts +6 -0
- package/dist/report/built-in/run-membership-overview.d.mts +6 -0
- package/dist/report/built-in/run-membership-overview.d.ts +6 -0
- package/dist/report/built-in/run-membership-overview.js +9 -0
- package/dist/report/built-in/run-membership-overview.js.map +1 -0
- package/dist/report/built-in/sandbox-history.cjs +384 -0
- package/dist/report/built-in/sandbox-history.cjs.map +1 -0
- package/dist/report/built-in/sandbox-history.d.cts +9 -0
- package/dist/report/built-in/sandbox-history.d.mts +9 -0
- package/dist/report/built-in/sandbox-history.d.ts +9 -0
- package/dist/report/built-in/sandbox-history.js +9 -0
- package/dist/report/built-in/sandbox-history.js.map +1 -0
- package/dist/report/built-in/source.cjs +162 -0
- package/dist/report/built-in/source.cjs.map +1 -0
- package/dist/report/built-in/source.d.cts +11 -0
- package/dist/report/built-in/source.d.mts +11 -0
- package/dist/report/built-in/source.d.ts +11 -0
- package/dist/report/built-in/source.js +8 -0
- package/dist/report/built-in/source.js.map +1 -0
- package/dist/report/execution/index.cjs +20 -0
- package/dist/report/execution/index.cjs.map +1 -0
- package/dist/report/execution/index.d.cts +3 -0
- package/dist/report/execution/index.d.mts +3 -0
- package/dist/report/execution/index.d.ts +3 -0
- package/dist/report/execution/index.js +30 -0
- package/dist/report/execution/index.js.map +1 -0
- package/dist/report/execution/model.cjs +483 -0
- package/dist/report/execution/model.cjs.map +1 -0
- package/dist/report/execution/model.d.cts +50 -0
- package/dist/report/execution/model.d.mts +50 -0
- package/dist/report/execution/model.d.ts +50 -0
- package/dist/report/execution/model.js +16 -0
- package/dist/report/execution/model.js.map +1 -0
- package/dist/report/execution/problems.cjs +210 -0
- package/dist/report/execution/problems.cjs.map +1 -0
- package/dist/report/execution/problems.d.cts +54 -0
- package/dist/report/execution/problems.d.mts +54 -0
- package/dist/report/execution/problems.d.ts +54 -0
- package/dist/report/execution/problems.js +14 -0
- package/dist/report/execution/problems.js.map +1 -0
- package/dist/report/execution/results.cjs +26 -0
- package/dist/report/execution/results.cjs.map +1 -0
- package/dist/report/execution/results.d.cts +102 -0
- package/dist/report/execution/results.d.mts +102 -0
- package/dist/report/execution/results.d.ts +102 -0
- package/dist/report/execution/results.js +8 -0
- package/dist/report/execution/results.js.map +1 -0
- package/dist/report/host/execute.cjs +1287 -0
- package/dist/report/host/execute.cjs.map +1 -0
- package/dist/report/host/execute.d.cts +23 -0
- package/dist/report/host/execute.d.mts +23 -0
- package/dist/report/host/execute.d.ts +23 -0
- package/dist/report/host/execute.js +6 -0
- package/dist/report/host/execute.js.map +1 -0
- package/dist/report/host/from-record.cjs +47 -0
- package/dist/report/host/from-record.cjs.map +1 -0
- package/dist/report/host/from-record.d.cts +36 -0
- package/dist/report/host/from-record.d.mts +36 -0
- package/dist/report/host/from-record.d.ts +36 -0
- package/dist/report/host/from-record.js +8 -0
- package/dist/report/host/from-record.js.map +1 -0
- package/dist/report/host/html.cjs +424 -0
- package/dist/report/host/html.cjs.map +1 -0
- package/dist/report/host/html.d.cts +28 -0
- package/dist/report/host/html.d.mts +28 -0
- package/dist/report/host/html.d.ts +28 -0
- package/dist/report/host/html.js +6 -0
- package/dist/report/host/html.js.map +1 -0
- package/dist/report/host/index.cjs +21 -0
- package/dist/report/host/index.cjs.map +1 -0
- package/dist/report/host/index.d.cts +11 -0
- package/dist/report/host/index.d.mts +11 -0
- package/dist/report/host/index.d.ts +11 -0
- package/dist/report/host/index.js +24 -0
- package/dist/report/host/index.js.map +1 -0
- package/dist/report/host/node/loader.cjs +210 -0
- package/dist/report/host/node/loader.cjs.map +1 -0
- package/dist/report/host/node/loader.d.cts +45 -0
- package/dist/report/host/node/loader.d.mts +45 -0
- package/dist/report/host/node/loader.d.ts +45 -0
- package/dist/report/host/node/loader.js +14 -0
- package/dist/report/host/node/loader.js.map +1 -0
- package/dist/report/host/node/theme.cjs +21 -0
- package/dist/report/host/node/theme.cjs.map +1 -0
- package/dist/report/host/node/theme.d.cts +1 -0
- package/dist/report/host/node/theme.d.mts +1 -0
- package/dist/report/host/node/theme.d.ts +1 -0
- package/dist/report/host/node/theme.js +14 -0
- package/dist/report/host/node/theme.js.map +1 -0
- package/dist/report/host/node.cjs +147 -0
- package/dist/report/host/node.cjs.map +1 -0
- package/dist/report/host/node.d.cts +36 -0
- package/dist/report/host/node.d.mts +36 -0
- package/dist/report/host/node.d.ts +36 -0
- package/dist/report/host/node.js +36 -0
- package/dist/report/host/node.js.map +1 -0
- package/dist/report/host/presentation.cjs +423 -0
- package/dist/report/host/presentation.cjs.map +1 -0
- package/dist/report/host/presentation.d.cts +42 -0
- package/dist/report/host/presentation.d.mts +42 -0
- package/dist/report/host/presentation.d.ts +42 -0
- package/dist/report/host/presentation.js +16 -0
- package/dist/report/host/presentation.js.map +1 -0
- package/dist/report/host/static.cjs +126 -0
- package/dist/report/host/static.cjs.map +1 -0
- package/dist/report/host/static.d.cts +55 -0
- package/dist/report/host/static.d.mts +55 -0
- package/dist/report/host/static.d.ts +55 -0
- package/dist/report/host/static.js +8 -0
- package/dist/report/host/static.js.map +1 -0
- package/dist/report/host/theme.cjs +223 -0
- package/dist/report/host/theme.cjs.map +1 -0
- package/dist/report/host/theme.d.cts +64 -0
- package/dist/report/host/theme.d.mts +64 -0
- package/dist/report/host/theme.d.ts +64 -0
- package/dist/report/host/theme.js +14 -0
- package/dist/report/host/theme.js.map +1 -0
- package/dist/report/host/view-session.cjs +82 -0
- package/dist/report/host/view-session.cjs.map +1 -0
- package/dist/report/host/view-session.d.cts +82 -0
- package/dist/report/host/view-session.d.mts +82 -0
- package/dist/report/host/view-session.d.ts +82 -0
- package/dist/report/host/view-session.js +6 -0
- package/dist/report/host/view-session.js.map +1 -0
- package/dist/report/index.cjs +58 -0
- package/dist/report/index.cjs.map +1 -0
- package/dist/report/index.d.cts +11 -0
- package/dist/report/index.d.mts +11 -0
- package/dist/report/index.d.ts +11 -43
- package/dist/report/index.js +120 -44
- package/dist/report/index.js.map +1 -0
- package/dist/report/index.mjs +120 -0
- package/dist/report/index.mjs.map +1 -0
- package/dist/report/model/panel.cjs +271 -0
- package/dist/report/model/panel.cjs.map +1 -0
- package/dist/report/model/panel.d.cts +109 -0
- package/dist/report/model/panel.d.mts +109 -0
- package/dist/report/model/panel.d.ts +2 -2
- package/dist/report/model/panel.js +22 -260
- package/dist/report/model/panel.js.map +1 -0
- package/dist/report/model/text-layout.cjs +185 -0
- package/dist/report/model/text-layout.cjs.map +1 -0
- package/dist/report/model/text-layout.d.cts +75 -0
- package/dist/report/model/text-layout.d.mts +75 -0
- package/dist/report/model/text-layout.js +22 -173
- package/dist/report/model/text-layout.js.map +1 -0
- package/dist/report/semantic/document.cjs +593 -0
- package/dist/report/semantic/document.cjs.map +1 -0
- package/dist/report/semantic/document.d.cts +162 -0
- package/dist/report/semantic/document.d.mts +162 -0
- package/dist/report/semantic/document.d.ts +162 -0
- package/dist/report/semantic/document.js +38 -0
- package/dist/report/semantic/document.js.map +1 -0
- package/dist/report/semantic/index.cjs +18 -0
- package/dist/report/semantic/index.cjs.map +1 -0
- package/dist/report/semantic/index.d.cts +1 -0
- package/dist/report/semantic/index.d.mts +1 -0
- package/dist/report/semantic/index.d.ts +1 -0
- package/dist/report/semantic/index.js +38 -0
- package/dist/report/semantic/index.js.map +1 -0
- package/dist/runner/accept-prompt.cjs +50 -0
- package/dist/runner/accept-prompt.cjs.map +1 -0
- package/dist/runner/accept-prompt.d.cts +27 -0
- package/dist/runner/accept-prompt.d.mts +27 -0
- package/dist/runner/accept-prompt.d.ts +27 -0
- package/dist/runner/accept-prompt.js +10 -0
- package/dist/runner/accept-prompt.js.map +1 -0
- package/dist/runner/accept.cjs +280 -0
- package/dist/runner/accept.cjs.map +1 -0
- package/dist/runner/accept.d.cts +75 -0
- package/dist/runner/accept.d.mts +75 -0
- package/dist/runner/accept.d.ts +75 -0
- package/dist/runner/accept.js +16 -0
- package/dist/runner/accept.js.map +1 -0
- package/dist/runner/adoption.cjs +859 -0
- package/dist/runner/adoption.cjs.map +1 -0
- package/dist/runner/adoption.d.cts +214 -0
- package/dist/runner/adoption.d.mts +214 -0
- package/dist/runner/adoption.d.ts +214 -0
- package/dist/runner/adoption.js +50 -0
- package/dist/runner/adoption.js.map +1 -0
- package/dist/runner/assert-first-bridge.cjs +203 -0
- package/dist/runner/assert-first-bridge.cjs.map +1 -0
- package/dist/runner/assert-first-bridge.d.cts +51 -0
- package/dist/runner/assert-first-bridge.d.mts +51 -0
- package/dist/runner/assert-first-bridge.d.ts +51 -0
- package/dist/runner/assert-first-bridge.js +6 -0
- package/dist/runner/assert-first-bridge.js.map +1 -0
- package/dist/runner/assertions.cjs +77 -0
- package/dist/runner/assertions.cjs.map +1 -0
- package/dist/runner/assertions.d.cts +8 -0
- package/dist/runner/assertions.d.mts +8 -0
- package/dist/runner/assertions.d.ts +8 -0
- package/dist/runner/assertions.js +6 -0
- package/dist/runner/assertions.js.map +1 -0
- package/dist/runner/attempt.cjs +2057 -0
- package/dist/runner/attempt.cjs.map +1 -0
- package/dist/runner/attempt.d.cts +90 -0
- package/dist/runner/attempt.d.mts +90 -0
- package/dist/runner/attempt.d.ts +90 -0
- package/dist/runner/attempt.js +14 -0
- package/dist/runner/attempt.js.map +1 -0
- package/dist/runner/build-preparation.cjs +87 -0
- package/dist/runner/build-preparation.cjs.map +1 -0
- package/dist/runner/build-preparation.d.cts +23 -0
- package/dist/runner/build-preparation.d.mts +23 -0
- package/dist/runner/build-preparation.d.ts +23 -0
- package/dist/runner/build-preparation.js +8 -0
- package/dist/runner/build-preparation.js.map +1 -0
- package/dist/runner/cleanup-timeout.cjs +40 -0
- package/dist/runner/cleanup-timeout.cjs.map +1 -0
- package/dist/runner/cleanup-timeout.d.cts +21 -0
- package/dist/runner/cleanup-timeout.d.mts +21 -0
- package/dist/runner/cleanup-timeout.d.ts +19 -1
- package/dist/runner/cleanup-timeout.js +12 -0
- package/dist/runner/cleanup-timeout.js.map +1 -0
- package/dist/runner/command-plan.cjs +475 -0
- package/dist/runner/command-plan.cjs.map +1 -0
- package/dist/runner/command-plan.d.cts +106 -0
- package/dist/runner/command-plan.d.mts +106 -0
- package/dist/runner/command-plan.d.ts +106 -0
- package/dist/runner/command-plan.js +6 -0
- package/dist/runner/command-plan.js.map +1 -0
- package/dist/runner/config-identity.cjs +259 -0
- package/dist/runner/config-identity.cjs.map +1 -0
- package/dist/runner/config-identity.d.cts +88 -0
- package/dist/runner/config-identity.d.mts +88 -0
- package/dist/runner/config-identity.d.ts +7 -5
- package/dist/runner/config-identity.js +22 -0
- package/dist/runner/config-identity.js.map +1 -0
- package/dist/runner/direct-agent-sandbox.cjs +64 -0
- package/dist/runner/direct-agent-sandbox.cjs.map +1 -0
- package/dist/runner/direct-agent-sandbox.d.cts +5 -0
- package/dist/runner/direct-agent-sandbox.d.mts +5 -0
- package/dist/runner/direct-agent-sandbox.d.ts +5 -0
- package/dist/runner/direct-agent-sandbox.js +8 -0
- package/dist/runner/direct-agent-sandbox.js.map +1 -0
- package/dist/runner/discover.cjs +628 -0
- package/dist/runner/discover.cjs.map +1 -0
- package/dist/runner/discover.d.cts +29 -0
- package/dist/runner/discover.d.mts +29 -0
- package/dist/runner/discover.d.ts +29 -0
- package/dist/runner/discover.js +12 -0
- package/dist/runner/discover.js.map +1 -0
- package/dist/runner/eval-selection.cjs +95 -0
- package/dist/runner/eval-selection.cjs.map +1 -0
- package/dist/runner/eval-selection.d.cts +37 -0
- package/dist/runner/eval-selection.d.mts +37 -0
- package/dist/runner/eval-selection.d.ts +2 -7
- package/dist/runner/eval-selection.js +12 -0
- package/dist/runner/eval-selection.js.map +1 -0
- package/dist/runner/eval-source.cjs +64 -0
- package/dist/runner/eval-source.cjs.map +1 -0
- package/dist/runner/eval-source.d.cts +28 -0
- package/dist/runner/eval-source.d.mts +28 -0
- package/dist/runner/eval-source.d.ts +1 -1
- package/dist/runner/eval-source.js +26 -0
- package/dist/runner/eval-source.js.map +1 -0
- package/dist/runner/experiment-cleanup-registry.cjs +70 -0
- package/dist/runner/experiment-cleanup-registry.cjs.map +1 -0
- package/dist/runner/experiment-cleanup-registry.d.cts +19 -0
- package/dist/runner/experiment-cleanup-registry.d.mts +19 -0
- package/dist/runner/experiment-cleanup-registry.d.ts +19 -0
- package/dist/runner/experiment-cleanup-registry.js +14 -0
- package/dist/runner/experiment-cleanup-registry.js.map +1 -0
- package/dist/runner/feedback/coordinator.cjs +404 -0
- package/dist/runner/feedback/coordinator.cjs.map +1 -0
- package/dist/runner/feedback/coordinator.d.cts +61 -0
- package/dist/runner/feedback/coordinator.d.mts +61 -0
- package/dist/runner/feedback/coordinator.d.ts +61 -0
- package/dist/runner/feedback/coordinator.js +6 -0
- package/dist/runner/feedback/coordinator.js.map +1 -0
- package/dist/runner/feedback/eval-conclusions.cjs +75 -0
- package/dist/runner/feedback/eval-conclusions.cjs.map +1 -0
- package/dist/runner/feedback/eval-conclusions.d.cts +30 -0
- package/dist/runner/feedback/eval-conclusions.d.mts +30 -0
- package/dist/runner/feedback/eval-conclusions.d.ts +30 -0
- package/dist/runner/feedback/eval-conclusions.js +8 -0
- package/dist/runner/feedback/eval-conclusions.js.map +1 -0
- package/dist/runner/feedback/failure.cjs +194 -0
- package/dist/runner/feedback/failure.cjs.map +1 -0
- package/dist/runner/feedback/failure.d.cts +13 -0
- package/dist/runner/feedback/failure.d.mts +13 -0
- package/dist/runner/feedback/failure.d.ts +13 -0
- package/dist/runner/feedback/failure.js +8 -0
- package/dist/runner/feedback/failure.js.map +1 -0
- package/dist/runner/feedback/human.cjs +1469 -0
- package/dist/runner/feedback/human.cjs.map +1 -0
- package/dist/runner/feedback/human.d.cts +139 -0
- package/dist/runner/feedback/human.d.mts +139 -0
- package/dist/runner/feedback/human.d.ts +139 -0
- package/dist/runner/feedback/human.js +18 -0
- package/dist/runner/feedback/human.js.map +1 -0
- package/dist/runner/feedback/index.cjs +51 -0
- package/dist/runner/feedback/index.cjs.map +1 -0
- package/dist/runner/feedback/index.d.cts +10 -0
- package/dist/runner/feedback/index.d.mts +10 -0
- package/dist/runner/feedback/index.d.ts +10 -0
- package/dist/runner/feedback/index.js +58 -0
- package/dist/runner/feedback/index.js.map +1 -0
- package/dist/runner/feedback/input-guard.cjs +106 -0
- package/dist/runner/feedback/input-guard.cjs.map +1 -0
- package/dist/runner/feedback/input-guard.d.cts +43 -0
- package/dist/runner/feedback/input-guard.d.mts +43 -0
- package/dist/runner/feedback/input-guard.d.ts +43 -0
- package/dist/runner/feedback/input-guard.js +8 -0
- package/dist/runner/feedback/input-guard.js.map +1 -0
- package/dist/runner/feedback/io.cjs +43 -0
- package/dist/runner/feedback/io.cjs.map +1 -0
- package/dist/runner/feedback/io.d.cts +38 -0
- package/dist/runner/feedback/io.d.mts +38 -0
- package/dist/runner/feedback/io.d.ts +38 -0
- package/dist/runner/feedback/io.js +6 -0
- package/dist/runner/feedback/io.js.map +1 -0
- package/dist/runner/feedback/json.cjs +371 -0
- package/dist/runner/feedback/json.cjs.map +1 -0
- package/dist/runner/feedback/json.d.cts +242 -0
- package/dist/runner/feedback/json.d.mts +242 -0
- package/dist/runner/feedback/json.d.ts +242 -0
- package/dist/runner/feedback/json.js +10 -0
- package/dist/runner/feedback/json.js.map +1 -0
- package/dist/runner/feedback/profile.cjs +8 -0
- package/dist/runner/feedback/profile.cjs.map +1 -0
- package/dist/runner/feedback/profile.d.cts +14 -0
- package/dist/runner/feedback/profile.d.mts +14 -0
- package/dist/runner/feedback/profile.d.ts +14 -0
- package/dist/runner/feedback/profile.js +6 -0
- package/dist/runner/feedback/profile.js.map +1 -0
- package/dist/runner/feedback/reducer.cjs +372 -0
- package/dist/runner/feedback/reducer.cjs.map +1 -0
- package/dist/runner/feedback/reducer.d.cts +4 -0
- package/dist/runner/feedback/reducer.d.mts +4 -0
- package/dist/runner/feedback/reducer.d.ts +4 -0
- package/dist/runner/feedback/reducer.js +8 -0
- package/dist/runner/feedback/reducer.js.map +1 -0
- package/dist/runner/feedback/renderer.cjs +3 -0
- package/dist/runner/feedback/renderer.cjs.map +1 -0
- package/dist/runner/feedback/renderer.d.cts +63 -0
- package/dist/runner/feedback/renderer.d.mts +63 -0
- package/dist/runner/feedback/renderer.d.ts +63 -0
- package/dist/runner/feedback/renderer.js +4 -0
- package/dist/runner/feedback/renderer.js.map +1 -0
- package/dist/runner/feedback/sink.cjs +194 -0
- package/dist/runner/feedback/sink.cjs.map +1 -0
- package/dist/runner/feedback/sink.d.cts +187 -0
- package/dist/runner/feedback/sink.d.mts +187 -0
- package/dist/runner/feedback/sink.d.ts +7 -7
- package/dist/runner/feedback/sink.js +34 -0
- package/dist/runner/feedback/sink.js.map +1 -0
- package/dist/runner/feedback/testing.cjs +69 -0
- package/dist/runner/feedback/testing.cjs.map +1 -0
- package/dist/runner/feedback/testing.d.cts +26 -0
- package/dist/runner/feedback/testing.d.mts +26 -0
- package/dist/runner/feedback/testing.d.ts +26 -0
- package/dist/runner/feedback/testing.js +6 -0
- package/dist/runner/feedback/testing.js.map +1 -0
- package/dist/runner/fingerprint.cjs +379 -0
- package/dist/runner/fingerprint.cjs.map +1 -0
- package/dist/runner/fingerprint.d.cts +71 -0
- package/dist/runner/fingerprint.d.mts +71 -0
- package/dist/runner/fingerprint.d.ts +49 -167
- package/dist/runner/fingerprint.js +22 -0
- package/dist/runner/fingerprint.js.map +1 -0
- package/dist/runner/gate-lease.cjs +271 -0
- package/dist/runner/gate-lease.cjs.map +1 -0
- package/dist/runner/gate-lease.d.cts +62 -0
- package/dist/runner/gate-lease.d.mts +62 -0
- package/dist/runner/gate-lease.d.ts +62 -0
- package/dist/runner/gate-lease.js +22 -0
- package/dist/runner/gate-lease.js.map +1 -0
- package/dist/runner/judge-config.cjs +26 -0
- package/dist/runner/judge-config.cjs.map +1 -0
- package/dist/runner/judge-config.d.cts +8 -0
- package/dist/runner/judge-config.d.mts +8 -0
- package/dist/runner/judge-config.d.ts +6 -4
- package/dist/runner/judge-config.js +6 -0
- package/dist/runner/judge-config.js.map +1 -0
- package/dist/runner/leak-gate.cjs +381 -0
- package/dist/runner/leak-gate.cjs.map +1 -0
- package/dist/runner/leak-gate.d.cts +124 -0
- package/dist/runner/leak-gate.d.mts +124 -0
- package/dist/runner/leak-gate.d.ts +36 -8
- package/dist/runner/leak-gate.js +32 -0
- package/dist/runner/leak-gate.js.map +1 -0
- package/dist/runner/ledger.cjs +470 -0
- package/dist/runner/ledger.cjs.map +1 -0
- package/dist/runner/ledger.d.cts +35 -0
- package/dist/runner/ledger.d.mts +35 -0
- package/dist/runner/ledger.d.ts +35 -0
- package/dist/runner/ledger.js +8 -0
- package/dist/runner/ledger.js.map +1 -0
- package/dist/runner/lock.cjs +236 -0
- package/dist/runner/lock.cjs.map +1 -0
- package/dist/runner/lock.d.cts +56 -0
- package/dist/runner/lock.d.mts +56 -0
- package/dist/runner/lock.d.ts +56 -0
- package/dist/runner/lock.js +22 -0
- package/dist/runner/lock.js.map +1 -0
- package/dist/runner/manifest.cjs +179 -0
- package/dist/runner/manifest.cjs.map +1 -0
- package/dist/runner/manifest.d.cts +61 -0
- package/dist/runner/manifest.d.mts +61 -0
- package/dist/runner/manifest.d.ts +4 -4
- package/dist/runner/manifest.js +22 -0
- package/dist/runner/manifest.js.map +1 -0
- package/dist/runner/project-current.cjs +117 -0
- package/dist/runner/project-current.cjs.map +1 -0
- package/dist/runner/project-current.d.cts +29 -0
- package/dist/runner/project-current.d.mts +29 -0
- package/dist/runner/project-current.d.ts +29 -0
- package/dist/runner/project-current.js +8 -0
- package/dist/runner/project-current.js.map +1 -0
- package/dist/runner/project-target.cjs +3 -0
- package/dist/runner/project-target.cjs.map +1 -0
- package/dist/runner/project-target.d.cts +30 -0
- package/dist/runner/project-target.d.mts +30 -0
- package/dist/runner/project-target.d.ts +30 -0
- package/dist/runner/project-target.js +4 -0
- package/dist/runner/project-target.js.map +1 -0
- package/dist/runner/record.cjs +883 -0
- package/dist/runner/record.cjs.map +1 -0
- package/dist/runner/record.d.cts +151 -0
- package/dist/runner/record.d.mts +151 -0
- package/dist/runner/record.d.ts +151 -0
- package/dist/runner/record.js +10 -0
- package/dist/runner/record.js.map +1 -0
- package/dist/runner/rename-experiment.cjs +273 -0
- package/dist/runner/rename-experiment.cjs.map +1 -0
- package/dist/runner/rename-experiment.d.cts +123 -0
- package/dist/runner/rename-experiment.d.mts +123 -0
- package/dist/runner/rename-experiment.d.ts +123 -0
- package/dist/runner/rename-experiment.js +14 -0
- package/dist/runner/rename-experiment.js.map +1 -0
- package/dist/runner/report.cjs +123 -0
- package/dist/runner/report.cjs.map +1 -0
- package/dist/runner/report.d.cts +27 -0
- package/dist/runner/report.d.mts +27 -0
- package/dist/runner/report.d.ts +27 -0
- package/dist/runner/report.js +14 -0
- package/dist/runner/report.js.map +1 -0
- package/dist/runner/reporters/braintrust.cjs +302 -0
- package/dist/runner/reporters/braintrust.cjs.map +1 -0
- package/dist/runner/reporters/braintrust.d.cts +59 -0
- package/dist/runner/reporters/braintrust.d.mts +59 -0
- package/dist/runner/reporters/braintrust.d.ts +59 -0
- package/dist/runner/reporters/braintrust.js +8 -0
- package/dist/runner/reporters/braintrust.js.map +1 -0
- package/dist/runner/reporters/index.cjs +12 -0
- package/dist/runner/reporters/index.cjs.map +1 -0
- package/dist/runner/reporters/index.d.cts +2 -0
- package/dist/runner/reporters/index.d.mts +2 -0
- package/dist/runner/reporters/index.d.ts +2 -0
- package/dist/runner/reporters/index.js +10 -0
- package/dist/runner/reporters/index.js.map +1 -0
- package/dist/runner/reporters/index.mjs +10 -0
- package/dist/runner/reporters/index.mjs.map +1 -0
- package/dist/runner/reporters/json.cjs +116 -0
- package/dist/runner/reporters/json.cjs.map +1 -0
- package/dist/runner/reporters/json.d.cts +3 -0
- package/dist/runner/reporters/json.d.mts +3 -0
- package/dist/runner/reporters/json.d.ts +3 -0
- package/dist/runner/reporters/json.js +8 -0
- package/dist/runner/reporters/json.js.map +1 -0
- package/dist/runner/reporters/shared.cjs +14 -0
- package/dist/runner/reporters/shared.cjs.map +1 -0
- package/dist/runner/reporters/shared.d.cts +3 -0
- package/dist/runner/reporters/shared.d.mts +3 -0
- package/dist/runner/reporters/shared.d.ts +3 -0
- package/dist/runner/reporters/shared.js +8 -0
- package/dist/runner/reporters/shared.js.map +1 -0
- package/dist/runner/reuse-diagnostics.cjs +71 -0
- package/dist/runner/reuse-diagnostics.cjs.map +1 -0
- package/dist/runner/reuse-diagnostics.d.cts +21 -0
- package/dist/runner/reuse-diagnostics.d.mts +21 -0
- package/dist/runner/reuse-diagnostics.d.ts +21 -0
- package/dist/runner/reuse-diagnostics.js +8 -0
- package/dist/runner/reuse-diagnostics.js.map +1 -0
- package/dist/runner/reuse-plan.cjs +739 -0
- package/dist/runner/reuse-plan.cjs.map +1 -0
- package/dist/runner/reuse-plan.d.cts +185 -0
- package/dist/runner/reuse-plan.d.mts +185 -0
- package/dist/runner/reuse-plan.d.ts +185 -0
- package/dist/runner/reuse-plan.js +26 -0
- package/dist/runner/reuse-plan.js.map +1 -0
- package/dist/runner/reuse-readback.cjs +275 -0
- package/dist/runner/reuse-readback.cjs.map +1 -0
- package/dist/runner/reuse-readback.d.cts +169 -0
- package/dist/runner/reuse-readback.d.mts +169 -0
- package/dist/runner/reuse-readback.d.ts +169 -0
- package/dist/runner/reuse-readback.js +20 -0
- package/dist/runner/reuse-readback.js.map +1 -0
- package/dist/runner/run.cjs +2362 -0
- package/dist/runner/run.cjs.map +1 -0
- package/dist/runner/run.d.cts +44 -0
- package/dist/runner/run.d.mts +44 -0
- package/dist/runner/run.d.ts +44 -0
- package/dist/runner/run.js +12 -0
- package/dist/runner/run.js.map +1 -0
- package/dist/runner/sandbox-pool.cjs +312 -0
- package/dist/runner/sandbox-pool.cjs.map +1 -0
- package/dist/runner/sandbox-pool.d.cts +70 -0
- package/dist/runner/sandbox-pool.d.mts +70 -0
- package/dist/runner/sandbox-pool.d.ts +70 -0
- package/dist/runner/sandbox-pool.js +6 -0
- package/dist/runner/sandbox-pool.js.map +1 -0
- package/dist/runner/sandbox-prep.cjs +57 -0
- package/dist/runner/sandbox-prep.cjs.map +1 -0
- package/dist/runner/sandbox-prep.d.cts +8 -0
- package/dist/runner/sandbox-prep.d.mts +8 -0
- package/dist/runner/sandbox-prep.d.ts +8 -0
- package/dist/runner/sandbox-prep.js +8 -0
- package/dist/runner/sandbox-prep.js.map +1 -0
- package/dist/runner/sandbox-record-producer.cjs +142 -0
- package/dist/runner/sandbox-record-producer.cjs.map +1 -0
- package/dist/runner/sandbox-record-producer.d.cts +30 -0
- package/dist/runner/sandbox-record-producer.d.mts +30 -0
- package/dist/runner/sandbox-record-producer.d.ts +30 -0
- package/dist/runner/sandbox-record-producer.js +6 -0
- package/dist/runner/sandbox-record-producer.js.map +1 -0
- package/dist/runner/sandbox-reuse.cjs +57 -0
- package/dist/runner/sandbox-reuse.cjs.map +1 -0
- package/dist/runner/sandbox-reuse.d.cts +43 -0
- package/dist/runner/sandbox-reuse.d.mts +43 -0
- package/dist/runner/sandbox-reuse.d.ts +43 -0
- package/dist/runner/sandbox-reuse.js +8 -0
- package/dist/runner/sandbox-reuse.js.map +1 -0
- package/dist/runner/sandbox-selection.cjs +362 -0
- package/dist/runner/sandbox-selection.cjs.map +1 -0
- package/dist/runner/sandbox-selection.d.cts +87 -0
- package/dist/runner/sandbox-selection.d.mts +87 -0
- package/dist/runner/sandbox-selection.d.ts +17 -6
- package/dist/runner/sandbox-selection.js +22 -0
- package/dist/runner/sandbox-selection.js.map +1 -0
- package/dist/runner/session.cjs +535 -0
- package/dist/runner/session.cjs.map +1 -0
- package/dist/runner/session.d.cts +116 -0
- package/dist/runner/session.d.mts +116 -0
- package/dist/runner/session.d.ts +44 -24
- package/dist/runner/session.js +28 -0
- package/dist/runner/session.js.map +1 -0
- package/dist/runner/source-closure.cjs +51 -0
- package/dist/runner/source-closure.cjs.map +1 -0
- package/dist/runner/source-closure.d.cts +6 -0
- package/dist/runner/source-closure.d.mts +6 -0
- package/dist/runner/source-closure.d.ts +6 -0
- package/dist/runner/source-closure.js +6 -0
- package/dist/runner/source-closure.js.map +1 -0
- package/dist/runner/source-producer.cjs +417 -0
- package/dist/runner/source-producer.cjs.map +1 -0
- package/dist/runner/source-producer.d.cts +70 -0
- package/dist/runner/source-producer.d.mts +70 -0
- package/dist/runner/source-producer.d.ts +70 -0
- package/dist/runner/source-producer.js +10 -0
- package/dist/runner/source-producer.js.map +1 -0
- package/dist/runner/teardown-registry.cjs +112 -0
- package/dist/runner/teardown-registry.cjs.map +1 -0
- package/dist/runner/teardown-registry.d.cts +35 -0
- package/dist/runner/teardown-registry.d.mts +35 -0
- package/dist/runner/teardown-registry.d.ts +35 -0
- package/dist/runner/teardown-registry.js +20 -0
- package/dist/runner/teardown-registry.js.map +1 -0
- package/dist/runner/timeout.cjs +31 -0
- package/dist/runner/timeout.cjs.map +1 -0
- package/dist/runner/timeout.d.cts +30 -0
- package/dist/runner/timeout.d.mts +30 -0
- package/dist/runner/timeout.d.ts +30 -0
- package/dist/runner/timeout.js +8 -0
- package/dist/runner/timeout.js.map +1 -0
- package/dist/runner/timing.cjs +279 -0
- package/dist/runner/timing.cjs.map +1 -0
- package/dist/runner/timing.d.cts +97 -0
- package/dist/runner/timing.d.mts +97 -0
- package/dist/runner/timing.d.ts +2 -2
- package/dist/runner/timing.js +26 -0
- package/dist/runner/timing.js.map +1 -0
- package/dist/runner/types.cjs +135 -0
- package/dist/runner/types.cjs.map +1 -0
- package/dist/runner/types.d.cts +1703 -0
- package/dist/runner/types.d.mts +1703 -0
- package/dist/runner/types.d.ts +214 -166
- package/dist/runner/types.js +32 -109
- package/dist/runner/types.js.map +1 -0
- package/dist/sample/analysis.cjs +1068 -0
- package/dist/sample/analysis.cjs.map +1 -0
- package/dist/sample/analysis.d.cts +153 -0
- package/dist/sample/analysis.d.mts +153 -0
- package/dist/sample/analysis.d.ts +153 -0
- package/dist/sample/analysis.js +30 -0
- package/dist/sample/analysis.js.map +1 -0
- package/dist/sample/index.cjs +16 -0
- package/dist/sample/index.cjs.map +1 -0
- package/dist/sample/index.d.cts +2 -0
- package/dist/sample/index.d.mts +2 -0
- package/dist/sample/index.d.ts +2 -102
- package/dist/sample/index.js +28 -386
- package/dist/sample/index.js.map +1 -0
- package/dist/sandbox/backend.cjs +116 -0
- package/dist/sandbox/backend.cjs.map +1 -0
- package/dist/sandbox/backend.d.cts +58 -0
- package/dist/sandbox/backend.d.mts +58 -0
- package/dist/sandbox/backend.d.ts +9 -3
- package/dist/sandbox/backend.js +26 -0
- package/dist/sandbox/backend.js.map +1 -0
- package/dist/sandbox/build-coordinator.cjs +309 -0
- package/dist/sandbox/build-coordinator.cjs.map +1 -0
- package/dist/sandbox/build-coordinator.d.cts +111 -0
- package/dist/sandbox/build-coordinator.d.mts +111 -0
- package/dist/sandbox/build-coordinator.d.ts +4 -4
- package/dist/sandbox/build-coordinator.js +12 -0
- package/dist/sandbox/build-coordinator.js.map +1 -0
- package/dist/sandbox/case-types.cjs +3 -0
- package/dist/sandbox/case-types.cjs.map +1 -0
- package/dist/sandbox/case-types.d.cts +55 -0
- package/dist/sandbox/case-types.d.mts +55 -0
- package/dist/sandbox/case-types.d.ts +3 -3
- package/dist/sandbox/case-types.js +4 -0
- package/dist/sandbox/case-types.js.map +1 -0
- package/dist/sandbox/checkpoint.cjs +65 -0
- package/dist/sandbox/checkpoint.cjs.map +1 -0
- package/dist/sandbox/checkpoint.d.cts +7 -0
- package/dist/sandbox/checkpoint.d.mts +7 -0
- package/dist/sandbox/checkpoint.d.ts +7 -0
- package/dist/sandbox/checkpoint.js +8 -0
- package/dist/sandbox/checkpoint.js.map +1 -0
- package/dist/sandbox/cli-commands.cjs +535 -0
- package/dist/sandbox/cli-commands.cjs.map +1 -0
- package/dist/sandbox/cli-commands.d.cts +41 -0
- package/dist/sandbox/cli-commands.d.mts +41 -0
- package/dist/sandbox/cli-commands.d.ts +41 -0
- package/dist/sandbox/cli-commands.js +14 -0
- package/dist/sandbox/cli-commands.js.map +1 -0
- package/dist/sandbox/commands.cjs +281 -0
- package/dist/sandbox/commands.cjs.map +1 -0
- package/dist/sandbox/commands.d.cts +141 -0
- package/dist/sandbox/commands.d.mts +141 -0
- package/dist/sandbox/commands.d.ts +64 -3
- package/dist/sandbox/commands.js +20 -0
- package/dist/sandbox/commands.js.map +1 -0
- package/dist/sandbox/compose.cjs +1293 -0
- package/dist/sandbox/compose.cjs.map +1 -0
- package/dist/sandbox/compose.d.cts +215 -0
- package/dist/sandbox/compose.d.mts +215 -0
- package/dist/sandbox/compose.d.ts +32 -16
- package/dist/sandbox/compose.js +38 -0
- package/dist/sandbox/compose.js.map +1 -0
- package/dist/sandbox/content.cjs +153 -0
- package/dist/sandbox/content.cjs.map +1 -0
- package/dist/sandbox/content.d.cts +35 -0
- package/dist/sandbox/content.d.mts +35 -0
- package/dist/sandbox/content.js +10 -0
- package/dist/sandbox/content.js.map +1 -0
- package/dist/sandbox/deadline.cjs +44 -0
- package/dist/sandbox/deadline.cjs.map +1 -0
- package/dist/sandbox/deadline.d.cts +45 -0
- package/dist/sandbox/deadline.d.mts +45 -0
- package/dist/sandbox/deadline.d.ts +2 -2
- package/dist/sandbox/deadline.js +10 -0
- package/dist/sandbox/deadline.js.map +1 -0
- package/dist/sandbox/dind-supervisor.cjs +124 -0
- package/dist/sandbox/dind-supervisor.cjs.map +1 -0
- package/dist/sandbox/dind-supervisor.d.cts +12 -0
- package/dist/sandbox/dind-supervisor.d.mts +12 -0
- package/dist/sandbox/dind-supervisor.d.ts +12 -0
- package/dist/sandbox/dind-supervisor.js +14 -0
- package/dist/sandbox/dind-supervisor.js.map +1 -0
- package/dist/sandbox/docker-agent-image.cjs +36 -0
- package/dist/sandbox/docker-agent-image.cjs.map +1 -0
- package/dist/sandbox/docker-agent-image.d.cts +22 -0
- package/dist/sandbox/docker-agent-image.d.mts +22 -0
- package/dist/sandbox/docker-agent-image.d.ts +22 -0
- package/dist/sandbox/docker-agent-image.js +20 -0
- package/dist/sandbox/docker-agent-image.js.map +1 -0
- package/dist/sandbox/docker-profile/cli.cjs +193 -0
- package/dist/sandbox/docker-profile/cli.cjs.map +1 -0
- package/dist/sandbox/docker-profile/cli.d.cts +12 -0
- package/dist/sandbox/docker-profile/cli.d.mts +12 -0
- package/dist/sandbox/docker-profile/cli.d.ts +12 -0
- package/dist/sandbox/docker-profile/cli.js +12 -0
- package/dist/sandbox/docker-profile/cli.js.map +1 -0
- package/dist/sandbox/docker-profile/errors.cjs +16 -0
- package/dist/sandbox/docker-profile/errors.cjs.map +1 -0
- package/dist/sandbox/docker-profile/errors.d.cts +25 -0
- package/dist/sandbox/docker-profile/errors.d.mts +25 -0
- package/dist/sandbox/docker-profile/errors.d.ts +25 -0
- package/dist/sandbox/docker-profile/errors.js +8 -0
- package/dist/sandbox/docker-profile/errors.js.map +1 -0
- package/dist/sandbox/docker-profile/registry.cjs +377 -0
- package/dist/sandbox/docker-profile/registry.cjs.map +1 -0
- package/dist/sandbox/docker-profile/registry.d.cts +43 -0
- package/dist/sandbox/docker-profile/registry.d.mts +43 -0
- package/dist/sandbox/docker-profile/registry.d.ts +43 -0
- package/dist/sandbox/docker-profile/registry.js +18 -0
- package/dist/sandbox/docker-profile/registry.js.map +1 -0
- package/dist/sandbox/docker-profile/runtime.cjs +198 -0
- package/dist/sandbox/docker-profile/runtime.cjs.map +1 -0
- package/dist/sandbox/docker-profile/runtime.d.cts +36 -0
- package/dist/sandbox/docker-profile/runtime.d.mts +36 -0
- package/dist/sandbox/docker-profile/runtime.d.ts +36 -0
- package/dist/sandbox/docker-profile/runtime.js +20 -0
- package/dist/sandbox/docker-profile/runtime.js.map +1 -0
- package/dist/sandbox/docker-profile/schema.cjs +650 -0
- package/dist/sandbox/docker-profile/schema.cjs.map +1 -0
- package/dist/sandbox/docker-profile/schema.d.cts +129 -0
- package/dist/sandbox/docker-profile/schema.d.mts +129 -0
- package/dist/sandbox/docker-profile/schema.d.ts +129 -0
- package/dist/sandbox/docker-profile/schema.js +38 -0
- package/dist/sandbox/docker-profile/schema.js.map +1 -0
- package/dist/sandbox/docker-stream.cjs +153 -0
- package/dist/sandbox/docker-stream.cjs.map +1 -0
- package/dist/sandbox/docker-stream.d.cts +33 -0
- package/dist/sandbox/docker-stream.d.mts +33 -0
- package/dist/sandbox/docker-stream.js +14 -0
- package/dist/sandbox/docker-stream.js.map +1 -0
- package/dist/sandbox/docker.cjs +1161 -0
- package/dist/sandbox/docker.cjs.map +1 -0
- package/dist/sandbox/docker.d.cts +288 -0
- package/dist/sandbox/docker.d.mts +288 -0
- package/dist/sandbox/docker.d.ts +84 -12
- package/dist/sandbox/docker.js +20 -0
- package/dist/sandbox/docker.js.map +1 -0
- package/dist/sandbox/dockerfile-agent-cache.cjs +104 -0
- package/dist/sandbox/dockerfile-agent-cache.cjs.map +1 -0
- package/dist/sandbox/dockerfile-agent-cache.d.cts +45 -0
- package/dist/sandbox/dockerfile-agent-cache.d.mts +45 -0
- package/dist/sandbox/dockerfile-agent-cache.d.ts +8 -7
- package/dist/sandbox/dockerfile-agent-cache.js +14 -0
- package/dist/sandbox/dockerfile-agent-cache.js.map +1 -0
- package/dist/sandbox/dockerfile-build.cjs +256 -0
- package/dist/sandbox/dockerfile-build.cjs.map +1 -0
- package/dist/sandbox/dockerfile-build.d.cts +59 -0
- package/dist/sandbox/dockerfile-build.d.mts +59 -0
- package/dist/sandbox/dockerfile-build.d.ts +21 -8
- package/dist/sandbox/dockerfile-build.js +16 -0
- package/dist/sandbox/dockerfile-build.js.map +1 -0
- package/dist/sandbox/dockerfile-identity.cjs +124 -0
- package/dist/sandbox/dockerfile-identity.cjs.map +1 -0
- package/dist/sandbox/dockerfile-identity.d.cts +37 -0
- package/dist/sandbox/dockerfile-identity.d.mts +37 -0
- package/dist/sandbox/dockerfile-identity.d.ts +13 -3
- package/dist/sandbox/dockerfile-identity.js +12 -0
- package/dist/sandbox/dockerfile-identity.js.map +1 -0
- package/dist/sandbox/download-directory.cjs +26 -0
- package/dist/sandbox/download-directory.cjs.map +1 -0
- package/dist/sandbox/download-directory.d.cts +9 -0
- package/dist/sandbox/download-directory.d.mts +9 -0
- package/dist/sandbox/download-directory.d.ts +1 -1
- package/dist/sandbox/download-directory.js +6 -0
- package/dist/sandbox/download-directory.js.map +1 -0
- package/dist/sandbox/e2b-agent-template.cjs +140 -0
- package/dist/sandbox/e2b-agent-template.cjs.map +1 -0
- package/dist/sandbox/e2b-agent-template.d.cts +65 -0
- package/dist/sandbox/e2b-agent-template.d.mts +65 -0
- package/dist/sandbox/e2b-agent-template.d.ts +65 -0
- package/dist/sandbox/e2b-agent-template.js +24 -0
- package/dist/sandbox/e2b-agent-template.js.map +1 -0
- package/dist/sandbox/e2b-agent-template.mjs +24 -0
- package/dist/sandbox/e2b-agent-template.mjs.map +1 -0
- package/dist/sandbox/e2b.cjs +664 -0
- package/dist/sandbox/e2b.cjs.map +1 -0
- package/dist/sandbox/e2b.d.cts +124 -0
- package/dist/sandbox/e2b.d.mts +124 -0
- package/dist/sandbox/e2b.d.ts +9 -9
- package/dist/sandbox/e2b.js +10 -0
- package/dist/sandbox/e2b.js.map +1 -0
- package/dist/sandbox/errors.cjs +191 -0
- package/dist/sandbox/errors.cjs.map +1 -0
- package/dist/sandbox/errors.d.cts +60 -0
- package/dist/sandbox/errors.d.mts +60 -0
- package/dist/sandbox/errors.d.ts +1 -1
- package/dist/sandbox/errors.js +20 -0
- package/dist/sandbox/errors.js.map +1 -0
- package/dist/sandbox/identity.cjs +140 -0
- package/dist/sandbox/identity.cjs.map +1 -0
- package/dist/sandbox/identity.d.cts +97 -0
- package/dist/sandbox/identity.d.mts +97 -0
- package/dist/sandbox/identity.d.ts +1 -1
- package/dist/sandbox/identity.js +26 -0
- package/dist/sandbox/identity.js.map +1 -0
- package/dist/sandbox/index.cjs +93 -0
- package/dist/sandbox/index.cjs.map +1 -0
- package/dist/sandbox/index.d.cts +28 -0
- package/dist/sandbox/index.d.mts +28 -0
- package/dist/sandbox/index.d.ts +28 -0
- package/dist/sandbox/index.js +144 -0
- package/dist/sandbox/index.js.map +1 -0
- package/dist/sandbox/index.mjs +144 -0
- package/dist/sandbox/index.mjs.map +1 -0
- package/dist/sandbox/io-retry.cjs +29 -0
- package/dist/sandbox/io-retry.cjs.map +1 -0
- package/dist/sandbox/io-retry.d.cts +17 -0
- package/dist/sandbox/io-retry.d.mts +17 -0
- package/dist/sandbox/io-retry.d.ts +1 -1
- package/dist/sandbox/io-retry.js +6 -0
- package/dist/sandbox/io-retry.js.map +1 -0
- package/dist/sandbox/keep-registry.cjs +269 -0
- package/dist/sandbox/keep-registry.cjs.map +1 -0
- package/dist/sandbox/keep-registry.d.cts +78 -0
- package/dist/sandbox/keep-registry.d.mts +78 -0
- package/dist/sandbox/keep-registry.d.ts +78 -0
- package/dist/sandbox/keep-registry.js +26 -0
- package/dist/sandbox/keep-registry.js.map +1 -0
- package/dist/sandbox/keep.cjs +448 -0
- package/dist/sandbox/keep.cjs.map +1 -0
- package/dist/sandbox/keep.d.cts +64 -0
- package/dist/sandbox/keep.d.mts +64 -0
- package/dist/sandbox/keep.d.ts +64 -0
- package/dist/sandbox/keep.js +28 -0
- package/dist/sandbox/keep.js.map +1 -0
- package/dist/sandbox/layer.cjs +1639 -0
- package/dist/sandbox/layer.cjs.map +1 -0
- package/dist/sandbox/layer.d.cts +565 -0
- package/dist/sandbox/layer.d.mts +565 -0
- package/dist/sandbox/layer.d.ts +179 -32
- package/dist/sandbox/layer.js +42 -0
- package/dist/sandbox/layer.js.map +1 -0
- package/dist/sandbox/ledger-paths.cjs +22 -0
- package/dist/sandbox/ledger-paths.cjs.map +1 -0
- package/dist/sandbox/ledger-paths.d.cts +14 -0
- package/dist/sandbox/ledger-paths.d.mts +14 -0
- package/dist/sandbox/ledger-paths.js +12 -0
- package/dist/sandbox/ledger-paths.js.map +1 -0
- package/dist/sandbox/link.cjs +395 -0
- package/dist/sandbox/link.cjs.map +1 -0
- package/dist/sandbox/link.d.cts +160 -0
- package/dist/sandbox/link.d.mts +160 -0
- package/dist/sandbox/link.d.ts +25 -8
- package/dist/sandbox/link.js +18 -0
- package/dist/sandbox/link.js.map +1 -0
- package/dist/sandbox/local-files.cjs +31 -0
- package/dist/sandbox/local-files.cjs.map +1 -0
- package/dist/sandbox/local-files.d.cts +6 -0
- package/dist/sandbox/local-files.d.mts +6 -0
- package/dist/sandbox/local-files.js +6 -0
- package/dist/sandbox/local-files.js.map +1 -0
- package/dist/sandbox/local.cjs +297 -0
- package/dist/sandbox/local.cjs.map +1 -0
- package/dist/sandbox/local.d.cts +78 -0
- package/dist/sandbox/local.d.mts +78 -0
- package/dist/sandbox/local.d.ts +7 -7
- package/dist/sandbox/local.js +6 -0
- package/dist/sandbox/local.js.map +1 -0
- package/dist/sandbox/operations.cjs +160 -0
- package/dist/sandbox/operations.cjs.map +1 -0
- package/dist/sandbox/operations.d.cts +10 -0
- package/dist/sandbox/operations.d.mts +10 -0
- package/dist/sandbox/operations.d.ts +2 -2
- package/dist/sandbox/operations.js +10 -0
- package/dist/sandbox/operations.js.map +1 -0
- package/dist/sandbox/orphans.cjs +199 -0
- package/dist/sandbox/orphans.cjs.map +1 -0
- package/dist/sandbox/orphans.d.cts +45 -0
- package/dist/sandbox/orphans.d.mts +45 -0
- package/dist/sandbox/orphans.d.ts +45 -0
- package/dist/sandbox/orphans.js +10 -0
- package/dist/sandbox/orphans.js.map +1 -0
- package/dist/sandbox/paths.cjs +102 -0
- package/dist/sandbox/paths.cjs.map +1 -0
- package/dist/sandbox/paths.d.cts +11 -0
- package/dist/sandbox/paths.d.mts +11 -0
- package/dist/sandbox/paths.d.ts +2 -2
- package/dist/sandbox/paths.js +12 -0
- package/dist/sandbox/paths.js.map +1 -0
- package/dist/sandbox/plan.cjs +223 -0
- package/dist/sandbox/plan.cjs.map +1 -0
- package/dist/sandbox/plan.d.cts +88 -0
- package/dist/sandbox/plan.d.mts +88 -0
- package/dist/sandbox/plan.d.ts +4 -3
- package/dist/sandbox/plan.js +20 -0
- package/dist/sandbox/plan.js.map +1 -0
- package/dist/sandbox/prepare-commands.cjs +280 -0
- package/dist/sandbox/prepare-commands.cjs.map +1 -0
- package/dist/sandbox/prepare-commands.d.cts +29 -0
- package/dist/sandbox/prepare-commands.d.mts +29 -0
- package/dist/sandbox/prepare-commands.d.ts +29 -0
- package/dist/sandbox/prepare-commands.js +8 -0
- package/dist/sandbox/prepare-commands.js.map +1 -0
- package/dist/sandbox/record/attachment.cjs +72 -0
- package/dist/sandbox/record/attachment.cjs.map +1 -0
- package/dist/sandbox/record/attachment.d.cts +30 -0
- package/dist/sandbox/record/attachment.d.mts +30 -0
- package/dist/sandbox/record/attachment.d.ts +30 -0
- package/dist/sandbox/record/attachment.js +10 -0
- package/dist/sandbox/record/attachment.js.map +1 -0
- package/dist/sandbox/record/codec.cjs +36 -0
- package/dist/sandbox/record/codec.cjs.map +1 -0
- package/dist/sandbox/record/codec.d.cts +30 -0
- package/dist/sandbox/record/codec.d.mts +30 -0
- package/dist/sandbox/record/codec.d.ts +30 -0
- package/dist/sandbox/record/codec.js +10 -0
- package/dist/sandbox/record/codec.js.map +1 -0
- package/dist/sandbox/record/model.cjs +20 -0
- package/dist/sandbox/record/model.cjs.map +1 -0
- package/dist/sandbox/record/model.d.cts +33 -0
- package/dist/sandbox/record/model.d.mts +33 -0
- package/dist/sandbox/record/model.d.ts +33 -0
- package/dist/sandbox/record/model.js +10 -0
- package/dist/sandbox/record/model.js.map +1 -0
- package/dist/sandbox/record/projector.cjs +29 -0
- package/dist/sandbox/record/projector.cjs.map +1 -0
- package/dist/sandbox/record/projector.d.cts +18 -0
- package/dist/sandbox/record/projector.d.mts +18 -0
- package/dist/sandbox/record/projector.d.ts +18 -0
- package/dist/sandbox/record/projector.js +6 -0
- package/dist/sandbox/record/projector.js.map +1 -0
- package/dist/sandbox/redaction.cjs +59 -0
- package/dist/sandbox/redaction.cjs.map +1 -0
- package/dist/sandbox/redaction.d.cts +19 -0
- package/dist/sandbox/redaction.d.mts +19 -0
- package/dist/sandbox/redaction.js +14 -0
- package/dist/sandbox/redaction.js.map +1 -0
- package/dist/sandbox/registry.cjs +76 -0
- package/dist/sandbox/registry.cjs.map +1 -0
- package/dist/sandbox/registry.d.cts +16 -0
- package/dist/sandbox/registry.d.mts +16 -0
- package/dist/sandbox/registry.d.ts +4 -3
- package/dist/sandbox/registry.js +14 -0
- package/dist/sandbox/registry.js.map +1 -0
- package/dist/sandbox/retry.cjs +82 -0
- package/dist/sandbox/retry.cjs.map +1 -0
- package/dist/sandbox/retry.d.cts +20 -0
- package/dist/sandbox/retry.d.mts +20 -0
- package/dist/sandbox/retry.d.ts +2 -2
- package/dist/sandbox/retry.js +6 -0
- package/dist/sandbox/retry.js.map +1 -0
- package/dist/sandbox/run-identity.cjs +109 -0
- package/dist/sandbox/run-identity.cjs.map +1 -0
- package/dist/sandbox/run-identity.d.cts +38 -0
- package/dist/sandbox/run-identity.d.mts +38 -0
- package/dist/sandbox/run-identity.js +36 -0
- package/dist/sandbox/run-identity.js.map +1 -0
- package/dist/sandbox/runtime.cjs +873 -0
- package/dist/sandbox/runtime.cjs.map +1 -0
- package/dist/sandbox/runtime.d.cts +116 -0
- package/dist/sandbox/runtime.d.mts +116 -0
- package/dist/sandbox/runtime.d.ts +16 -16
- package/dist/sandbox/runtime.js +40 -0
- package/dist/sandbox/runtime.js.map +1 -0
- package/dist/sandbox/shell.cjs +22 -0
- package/dist/sandbox/shell.cjs.map +1 -0
- package/dist/sandbox/shell.d.cts +11 -0
- package/dist/sandbox/shell.d.mts +11 -0
- package/dist/sandbox/shell.js +8 -0
- package/dist/sandbox/shell.js.map +1 -0
- package/dist/sandbox/transfer-errors.cjs +116 -0
- package/dist/sandbox/transfer-errors.cjs.map +1 -0
- package/dist/sandbox/transfer-errors.d.cts +25 -0
- package/dist/sandbox/transfer-errors.d.mts +25 -0
- package/dist/sandbox/transfer-errors.js +14 -0
- package/dist/sandbox/transfer-errors.js.map +1 -0
- package/dist/sandbox/types.cjs +3 -0
- package/dist/sandbox/types.cjs.map +1 -0
- package/dist/sandbox/types.d.cts +177 -0
- package/dist/sandbox/types.d.mts +177 -0
- package/dist/sandbox/types.d.ts +15 -10
- package/dist/sandbox/types.js +4 -3
- package/dist/sandbox/types.js.map +1 -0
- package/dist/sandbox/vercel.cjs +335 -0
- package/dist/sandbox/vercel.cjs.map +1 -0
- package/dist/sandbox/vercel.d.cts +97 -0
- package/dist/sandbox/vercel.d.mts +97 -0
- package/dist/sandbox/vercel.d.ts +9 -9
- package/dist/sandbox/vercel.js +8 -0
- package/dist/sandbox/vercel.js.map +1 -0
- package/dist/shared/aggregate.cjs +97 -0
- package/dist/shared/aggregate.cjs.map +1 -0
- package/dist/shared/aggregate.d.cts +45 -0
- package/dist/shared/aggregate.d.mts +45 -0
- package/dist/shared/aggregate.d.ts +4 -4
- package/dist/shared/aggregate.js +24 -87
- package/dist/shared/aggregate.js.map +1 -0
- package/dist/shared/entry-file-store.cjs +126 -0
- package/dist/shared/entry-file-store.cjs.map +1 -0
- package/dist/shared/entry-file-store.d.cts +46 -0
- package/dist/shared/entry-file-store.d.mts +46 -0
- package/dist/shared/entry-file-store.d.ts +9 -6
- package/dist/shared/entry-file-store.js +18 -0
- package/dist/shared/entry-file-store.js.map +1 -0
- package/dist/shared/external-cause.cjs +278 -0
- package/dist/shared/external-cause.cjs.map +1 -0
- package/dist/shared/external-cause.d.cts +51 -0
- package/dist/shared/external-cause.d.mts +51 -0
- package/dist/shared/external-cause.d.ts +1 -1
- package/dist/shared/external-cause.js +12 -0
- package/dist/shared/external-cause.js.map +1 -0
- package/dist/shared/facts.cjs +24 -0
- package/dist/shared/facts.cjs.map +1 -0
- package/dist/shared/facts.d.cts +7 -0
- package/dist/shared/facts.d.mts +7 -0
- package/dist/shared/facts.d.ts +7 -0
- package/dist/shared/facts.js +6 -0
- package/dist/shared/facts.js.map +1 -0
- package/dist/shared/failure-class.cjs +166 -0
- package/dist/shared/failure-class.cjs.map +1 -0
- package/dist/shared/failure-class.d.cts +92 -0
- package/dist/shared/failure-class.d.mts +92 -0
- package/dist/shared/failure-class.d.ts +2 -2
- package/dist/shared/failure-class.js +20 -0
- package/dist/shared/failure-class.js.map +1 -0
- package/dist/shared/format.cjs +36 -0
- package/dist/shared/format.cjs.map +1 -0
- package/dist/shared/format.d.cts +3 -0
- package/dist/shared/format.d.mts +3 -0
- package/dist/shared/format.d.ts +3 -0
- package/dist/shared/format.js +10 -0
- package/dist/shared/format.js.map +1 -0
- package/dist/shared/json-match.cjs +44 -0
- package/dist/shared/json-match.cjs.map +1 -0
- package/dist/shared/json-match.d.cts +3 -0
- package/dist/shared/json-match.d.mts +3 -0
- package/dist/shared/json-match.d.ts +3 -0
- package/dist/shared/json-match.js +6 -0
- package/dist/shared/json-match.js.map +1 -0
- package/dist/shared/json-value.cjs +45 -0
- package/dist/shared/json-value.cjs.map +1 -0
- package/dist/shared/json-value.d.cts +11 -0
- package/dist/shared/json-value.d.mts +11 -0
- package/dist/shared/json-value.d.ts +11 -0
- package/dist/shared/json-value.js +8 -0
- package/dist/shared/json-value.js.map +1 -0
- package/dist/shared/turn-label.cjs +19 -0
- package/dist/shared/turn-label.cjs.map +1 -0
- package/dist/shared/turn-label.d.cts +9 -0
- package/dist/shared/turn-label.d.mts +9 -0
- package/dist/shared/turn-label.js +8 -14
- package/dist/shared/turn-label.js.map +1 -0
- package/dist/shared/types.cjs +3 -0
- package/dist/shared/types.cjs.map +1 -0
- package/dist/shared/types.d.cts +80 -0
- package/dist/shared/types.d.mts +80 -0
- package/dist/shared/types.d.ts +0 -2
- package/dist/shared/types.js +4 -4
- package/dist/shared/types.js.map +1 -0
- package/dist/shared/verdict.cjs +70 -0
- package/dist/shared/verdict.cjs.map +1 -0
- package/dist/shared/verdict.d.cts +36 -0
- package/dist/shared/verdict.d.mts +36 -0
- package/dist/shared/verdict.d.ts +21 -32
- package/dist/shared/verdict.js +14 -53
- package/dist/shared/verdict.js.map +1 -0
- package/dist/show/command.cjs +20 -0
- package/dist/show/command.cjs.map +1 -0
- package/dist/show/command.d.cts +13 -0
- package/dist/show/command.d.mts +13 -0
- package/dist/show/command.d.ts +13 -0
- package/dist/show/command.js +6 -0
- package/dist/show/command.js.map +1 -0
- package/dist/show/json.cjs +8 -0
- package/dist/show/json.cjs.map +1 -0
- package/dist/show/json.d.cts +2 -0
- package/dist/show/json.d.mts +2 -0
- package/dist/show/json.d.ts +2 -0
- package/dist/show/json.js +8 -0
- package/dist/show/json.js.map +1 -0
- package/dist/show/render.cjs +7 -0
- package/dist/show/render.cjs.map +1 -0
- package/dist/show/render.d.cts +2 -0
- package/dist/show/render.d.mts +2 -0
- package/dist/show/render.d.ts +2 -0
- package/dist/show/render.js +6 -0
- package/dist/show/render.js.map +1 -0
- package/dist/source-loc.cjs +137 -0
- package/dist/source-loc.cjs.map +1 -0
- package/dist/source-loc.d.cts +22 -0
- package/dist/source-loc.d.mts +22 -0
- package/dist/source-loc.d.ts +22 -0
- package/dist/source-loc.js +10 -0
- package/dist/source-loc.js.map +1 -0
- package/dist/sources/assemble.cjs +634 -0
- package/dist/sources/assemble.cjs.map +1 -0
- package/dist/sources/assemble.d.cts +6 -0
- package/dist/sources/assemble.d.mts +6 -0
- package/dist/sources/assemble.d.ts +6 -0
- package/dist/sources/assemble.js +6 -0
- package/dist/sources/assemble.js.map +1 -0
- package/dist/sources/attachment.cjs +272 -0
- package/dist/sources/attachment.cjs.map +1 -0
- package/dist/sources/attachment.d.cts +331 -0
- package/dist/sources/attachment.d.mts +331 -0
- package/dist/sources/attachment.d.ts +331 -0
- package/dist/sources/attachment.js +22 -0
- package/dist/sources/attachment.js.map +1 -0
- package/dist/sources/codec.cjs +220 -0
- package/dist/sources/codec.cjs.map +1 -0
- package/dist/sources/codec.d.cts +408 -0
- package/dist/sources/codec.d.mts +408 -0
- package/dist/sources/codec.d.ts +408 -0
- package/dist/sources/codec.js +56 -0
- package/dist/sources/codec.js.map +1 -0
- package/dist/sources/identity.cjs +10 -0
- package/dist/sources/identity.cjs.map +1 -0
- package/dist/sources/identity.d.cts +10 -0
- package/dist/sources/identity.d.mts +10 -0
- package/dist/sources/identity.d.ts +10 -0
- package/dist/sources/identity.js +10 -0
- package/dist/sources/identity.js.map +1 -0
- package/dist/sources/index.cjs +24 -0
- package/dist/sources/index.cjs.map +1 -0
- package/dist/sources/index.d.cts +7 -0
- package/dist/sources/index.d.mts +7 -0
- package/dist/sources/index.d.ts +7 -0
- package/dist/sources/index.js +90 -0
- package/dist/sources/index.js.map +1 -0
- package/dist/sources/model.cjs +6 -0
- package/dist/sources/model.cjs.map +1 -0
- package/dist/sources/model.d.cts +92 -0
- package/dist/sources/model.d.mts +92 -0
- package/dist/sources/model.d.ts +92 -0
- package/dist/sources/model.js +6 -0
- package/dist/sources/model.js.map +1 -0
- package/dist/sources/projection-model.cjs +3 -0
- package/dist/sources/projection-model.cjs.map +1 -0
- package/dist/sources/projection-model.d.cts +327 -0
- package/dist/sources/projection-model.d.mts +327 -0
- package/dist/sources/projection-model.d.ts +327 -0
- package/dist/sources/projection-model.js +4 -0
- package/dist/sources/projection-model.js.map +1 -0
- package/dist/sources/projector.cjs +282 -0
- package/dist/sources/projector.cjs.map +1 -0
- package/dist/sources/projector.d.cts +8 -0
- package/dist/sources/projector.d.mts +8 -0
- package/dist/sources/projector.d.ts +8 -0
- package/dist/sources/projector.js +10 -0
- package/dist/sources/projector.js.map +1 -0
- package/dist/tty-line.cjs +17 -0
- package/dist/tty-line.cjs.map +1 -0
- package/dist/tty-line.d.cts +2 -0
- package/dist/tty-line.d.mts +2 -0
- package/dist/tty-line.js +6 -0
- package/dist/tty-line.js.map +1 -0
- package/dist/types.cjs +38 -0
- package/dist/types.cjs.map +1 -0
- package/dist/types.d.cts +9 -0
- package/dist/types.d.mts +9 -0
- package/dist/types.d.ts +9 -9
- package/dist/types.js +32 -21
- package/dist/types.js.map +1 -0
- package/dist/util.cjs +193 -0
- package/dist/util.cjs.map +1 -0
- package/dist/util.d.cts +46 -0
- package/dist/util.d.mts +46 -0
- package/dist/util.js +22 -132
- package/dist/util.js.map +1 -0
- package/dist/vendor/remark/remark-gfm.cjs +3484 -0
- package/dist/vendor/remark/remark-gfm.cjs.map +7 -0
- package/dist/vendor/remark/remark-parse.cjs +6975 -0
- package/dist/vendor/remark/remark-parse.cjs.map +7 -0
- package/dist/vendor/remark/remark.cjs +10151 -0
- package/dist/vendor/remark/remark.cjs.map +7 -0
- package/dist/view/data.cjs +3 -0
- package/dist/view/data.cjs.map +1 -0
- package/dist/view/data.d.cts +8 -0
- package/dist/view/data.d.mts +8 -0
- package/dist/view/data.d.ts +8 -0
- package/dist/view/data.js +4 -0
- package/dist/view/data.js.map +1 -0
- package/dist/view/index.cjs +25 -0
- package/dist/view/index.cjs.map +1 -0
- package/dist/view/index.d.cts +9 -0
- package/dist/view/index.d.mts +9 -0
- package/dist/view/index.d.ts +9 -0
- package/dist/view/index.js +30 -0
- package/dist/view/index.js.map +1 -0
- package/dist/view/server.cjs +658 -0
- package/dist/view/server.cjs.map +1 -0
- package/dist/view/server.d.cts +49 -0
- package/dist/view/server.d.mts +49 -0
- package/dist/view/server.d.ts +49 -0
- package/dist/view/server.js +6 -0
- package/dist/view/server.js.map +1 -0
- package/dist/view/shared/types.cjs +3 -0
- package/dist/view/shared/types.cjs.map +1 -0
- package/dist/view/shared/types.d.cts +51 -0
- package/dist/view/shared/types.d.mts +51 -0
- package/dist/view/shared/types.d.ts +51 -0
- package/dist/view/shared/types.js +4 -0
- package/dist/view/shared/types.js.map +1 -0
- package/dist/view/site.cjs +40 -0
- package/dist/view/site.cjs.map +1 -0
- package/dist/view/site.d.cts +26 -0
- package/dist/view/site.d.mts +26 -0
- package/dist/view/site.d.ts +26 -0
- package/dist/view/site.js +10 -0
- package/dist/view/site.js.map +1 -0
- package/dist/view/styles.css +170 -0
- package/docs-site/zh/README.md +6 -6
- package/docs-site/zh/examples/ai-agent-application.mdx +3 -3
- package/docs-site/zh/examples/coding-agent-extensions.mdx +3 -3
- package/docs-site/zh/examples/index.mdx +10 -3
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +10 -9
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +11 -9
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +14 -13
- package/docs-site/zh/examples/integrations/langgraph.mdx +13 -11
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +11 -9
- package/docs-site/zh/explanation/adapter.mdx +27 -27
- package/docs-site/zh/explanation/assert.mdx +51 -198
- package/docs-site/zh/explanation/drive.mdx +14 -12
- package/docs-site/zh/explanation/evals.mdx +16 -12
- package/docs-site/zh/explanation/experiment.mdx +7 -3
- package/docs-site/zh/explanation/hitl.mdx +7 -7
- package/docs-site/zh/explanation/judge.mdx +67 -96
- package/docs-site/zh/explanation/overview.mdx +59 -100
- package/docs-site/zh/explanation/record.mdx +60 -0
- package/docs-site/zh/explanation/runner.mdx +47 -123
- package/docs-site/zh/explanation/tier.mdx +2 -2
- package/docs-site/zh/index.mdx +15 -13
- package/docs-site/zh/introduction.mdx +15 -13
- package/docs-site/zh/reference/builtin-agents.mdx +60 -37
- package/docs-site/zh/reference/capabilities.mdx +7 -7
- package/docs-site/zh/reference/cli.mdx +146 -80
- package/docs-site/zh/reference/define-agent.mdx +29 -20
- package/docs-site/zh/reference/define-config.mdx +6 -15
- package/docs-site/zh/reference/define-eval.mdx +78 -326
- package/docs-site/zh/reference/events.mdx +70 -21
- package/docs-site/zh/reference/expect.mdx +148 -117
- package/docs-site/zh/reference/official-adapters.mdx +33 -22
- package/docs-site/zh/reference/report-components.mdx +78 -362
- package/docs-site/zh/reference/results-data.mdx +53 -270
- package/docs-site/zh/troubleshooting/debug-sandbox.mdx +8 -6
- package/docs-site/zh/troubleshooting/debugging.mdx +48 -218
- package/docs-site/zh/troubleshooting/recover-after-kill.mdx +3 -3
- package/docs-site/zh/tutorials/agent-feedback-loop.mdx +48 -290
- package/docs-site/zh/tutorials/agent-onboarding.mdx +22 -20
- package/docs-site/zh/tutorials/authoring.mdx +9 -9
- package/docs-site/zh/tutorials/ci-integration.mdx +40 -117
- package/docs-site/zh/tutorials/concurrency.mdx +44 -56
- package/docs-site/zh/tutorials/configuration.mdx +10 -14
- package/docs-site/zh/tutorials/connect-otel.mdx +21 -21
- package/docs-site/zh/tutorials/connect-your-agent.mdx +29 -24
- package/docs-site/zh/tutorials/criteria-files.mdx +2 -2
- package/docs-site/zh/tutorials/custom-reports.mdx +69 -343
- package/docs-site/zh/tutorials/dataset-fanout.mdx +1 -1
- package/docs-site/zh/tutorials/deploy-report-site.mdx +34 -136
- package/docs-site/zh/tutorials/docker-in-docker.mdx +295 -0
- package/docs-site/zh/tutorials/eval-groups.mdx +207 -0
- package/docs-site/zh/tutorials/evaluation-kinds.mdx +67 -103
- package/docs-site/zh/tutorials/experiments.mdx +4 -4
- package/docs-site/zh/tutorials/fixtures.mdx +3 -3
- package/docs-site/zh/tutorials/handle-execution-failures.mdx +114 -0
- package/docs-site/zh/tutorials/install-custom-sandbox-agent.mdx +102 -0
- package/docs-site/zh/tutorials/plugins.mdx +110 -0
- package/docs-site/zh/tutorials/publish-report.mdx +40 -121
- package/docs-site/zh/tutorials/quickstart.mdx +15 -9
- package/docs-site/zh/tutorials/reporters.mdx +40 -90
- package/docs-site/zh/tutorials/rerun-and-cache.mdx +44 -225
- package/docs-site/zh/tutorials/sandbox-agent.mdx +47 -40
- package/docs-site/zh/tutorials/sandbox-providers.mdx +49 -39
- package/docs-site/zh/tutorials/sandbox-reuse.mdx +37 -15
- package/docs-site/zh/tutorials/theming.mdx +21 -156
- package/docs-site/zh/tutorials/viewing-results.mdx +73 -666
- package/docs-site/zh/tutorials/write-experiment.mdx +22 -21
- package/docs-site/zh/tutorials/write-send.mdx +38 -35
- package/package.json +127 -65
- package/dist/context/deferred-file-content.d.ts +0 -15
- package/dist/i18n/zh-CN.d.ts +0 -367
- package/dist/i18n/zh-CN.js +0 -458
- package/dist/record/annotated-source.d.ts +0 -126
- package/dist/record/annotated-source.js +0 -511
- package/dist/record/attempt-evidence.d.ts +0 -84
- package/dist/record/attempt-evidence.js +0 -108
- package/dist/record/attempt-source.d.ts +0 -4
- package/dist/record/attempt-source.js +0 -28
- package/dist/record/format.d.ts +0 -39
- package/dist/record/format.js +0 -75
- package/dist/record/open.d.ts +0 -62
- package/dist/record/open.js +0 -683
- package/dist/record/types.d.ts +0 -297
- package/dist/record/types.js +0 -9
- package/dist/report/assets/colors.d.ts +0 -22
- package/dist/report/assets/colors.js +0 -61
- package/dist/report/assets/series-encoding.d.ts +0 -36
- package/dist/report/assets/series-encoding.js +0 -104
- package/dist/report/built-in/failures.d.ts +0 -1
- package/dist/report/built-in/failures.js +0 -18
- package/dist/report/built-in/result-components.d.ts +0 -10
- package/dist/report/built-in/result-components.js +0 -199
- package/dist/report/built-in/stability.d.ts +0 -1
- package/dist/report/built-in/stability.js +0 -20
- package/dist/report/built-in/standard.d.ts +0 -50
- package/dist/report/built-in/standard.js +0 -93
- package/dist/report/components/attempt-detail/compute.d.ts +0 -51
- package/dist/report/components/attempt-detail/compute.js +0 -512
- package/dist/report/components/attempt-detail/content.d.ts +0 -28
- package/dist/report/components/attempt-detail/content.js +0 -575
- package/dist/report/components/attempt-detail/index.d.ts +0 -15
- package/dist/report/components/attempt-detail/index.js +0 -154
- package/dist/report/components/attempt-detail/validate.d.ts +0 -11
- package/dist/report/components/attempt-detail/validate.js +0 -417
- package/dist/report/components/cell.d.ts +0 -12
- package/dist/report/components/cell.js +0 -10
- package/dist/report/components/entity-lists/compute.d.ts +0 -35
- package/dist/report/components/entity-lists/compute.js +0 -333
- package/dist/report/components/entity-lists/content.d.ts +0 -7
- package/dist/report/components/entity-lists/content.js +0 -483
- package/dist/report/components/entity-lists/index.d.ts +0 -48
- package/dist/report/components/entity-lists/index.js +0 -81
- package/dist/report/components/entity-lists/validate.d.ts +0 -4
- package/dist/report/components/entity-lists/validate.js +0 -155
- package/dist/report/components/experiment-detail/compute.d.ts +0 -7
- package/dist/report/components/experiment-detail/compute.js +0 -32
- package/dist/report/components/experiment-detail/index.d.ts +0 -9
- package/dist/report/components/experiment-detail/index.js +0 -41
- package/dist/report/components/fixtures.d.ts +0 -10
- package/dist/report/components/fixtures.js +0 -341
- package/dist/report/components/shared-compute.d.ts +0 -23
- package/dist/report/components/shared-compute.js +0 -48
- package/dist/report/components/shared-faces.d.ts +0 -8
- package/dist/report/components/shared-faces.js +0 -21
- package/dist/report/components/shared.d.ts +0 -52
- package/dist/report/components/shared.js +0 -95
- package/dist/report/components/site-components/HeroCard.d.ts +0 -14
- package/dist/report/components/site-components/HeroCard.js +0 -35
- package/dist/report/components/site-components/PoweredBy.d.ts +0 -5
- package/dist/report/components/site-components/PoweredBy.js +0 -7
- package/dist/report/components/site-components/compute.d.ts +0 -34
- package/dist/report/components/site-components/compute.js +0 -144
- package/dist/report/components/site-components/content.d.ts +0 -1
- package/dist/report/components/site-components/content.js +0 -2
- package/dist/report/components/site-components/faces.d.ts +0 -9
- package/dist/report/components/site-components/faces.js +0 -34
- package/dist/report/components/site-components/hero-types.d.ts +0 -17
- package/dist/report/components/site-components/index.d.ts +0 -34
- package/dist/report/components/site-components/index.js +0 -55
- package/dist/report/components/site-components/projections.d.ts +0 -12
- package/dist/report/components/site-components/projections.js +0 -67
- package/dist/report/components/site-components/scope-warnings.d.ts +0 -44
- package/dist/report/components/site-components/scope-warnings.js +0 -115
- package/dist/report/components/site-components/snapshot-diagnostics.d.ts +0 -18
- package/dist/report/components/site-components/snapshot-diagnostics.js +0 -42
- package/dist/report/components/site-components/validators.d.ts +0 -6
- package/dist/report/components/site-components/validators.js +0 -115
- package/dist/report/components/summaries/compute.d.ts +0 -7
- package/dist/report/components/summaries/compute.js +0 -60
- package/dist/report/components/summaries/index.d.ts +0 -29
- package/dist/report/components/summaries/index.js +0 -122
- package/dist/report/components/summaries/stability-overview.d.ts +0 -13
- package/dist/report/components/summaries/stability-overview.js +0 -73
- package/dist/report/components/summaries/validate.d.ts +0 -2
- package/dist/report/components/summaries/validate.js +0 -29
- package/dist/report/definition/cell.d.ts +0 -118
- package/dist/report/definition/cell.js +0 -96
- package/dist/report/definition/grid-layout.d.ts +0 -77
- package/dist/report/definition/grid-layout.js +0 -147
- package/dist/report/definition/primitives/callouts-logic.d.ts +0 -26
- package/dist/report/definition/primitives/callouts-logic.js +0 -102
- package/dist/report/definition/primitives/callouts.d.ts +0 -9
- package/dist/report/definition/primitives/callouts.js +0 -101
- package/dist/report/definition/primitives/chart-map.d.ts +0 -74
- package/dist/report/definition/primitives/chart-map.js +0 -124
- package/dist/report/definition/primitives/chart.d.ts +0 -43
- package/dist/report/definition/primitives/chart.js +0 -619
- package/dist/report/definition/primitives/conversation.d.ts +0 -55
- package/dist/report/definition/primitives/conversation.js +0 -216
- package/dist/report/definition/primitives/copy-block.d.ts +0 -19
- package/dist/report/definition/primitives/copy-block.js +0 -51
- package/dist/report/definition/primitives/diff-lines.d.ts +0 -68
- package/dist/report/definition/primitives/diff-lines.js +0 -149
- package/dist/report/definition/primitives/diff-view.d.ts +0 -14
- package/dist/report/definition/primitives/diff-view.js +0 -156
- package/dist/report/definition/primitives/markdown.d.ts +0 -29
- package/dist/report/definition/primitives/markdown.js +0 -278
- package/dist/report/definition/primitives/marks.d.ts +0 -124
- package/dist/report/definition/primitives/marks.js +0 -91
- package/dist/report/definition/primitives/points-dataset.d.ts +0 -17
- package/dist/report/definition/primitives/points-dataset.js +0 -121
- package/dist/report/definition/primitives/source-view.d.ts +0 -43
- package/dist/report/definition/primitives/source-view.js +0 -179
- package/dist/report/definition/primitives/waterfall.d.ts +0 -34
- package/dist/report/definition/primitives/waterfall.js +0 -201
- package/dist/report/definition/primitives.d.ts +0 -154
- package/dist/report/definition/primitives.js +0 -691
- package/dist/report/definition/report.d.ts +0 -200
- package/dist/report/definition/report.js +0 -275
- package/dist/report/definition/table-text.d.ts +0 -14
- package/dist/report/definition/table-text.js +0 -187
- package/dist/report/definition/tree.d.ts +0 -233
- package/dist/report/definition/tree.js +0 -566
- package/dist/report/extension/assets.d.ts +0 -16
- package/dist/report/extension/assets.js +0 -124
- package/dist/report/extension/define.d.ts +0 -9
- package/dist/report/extension/define.js +0 -137
- package/dist/report/extension/index.d.ts +0 -6
- package/dist/report/extension/index.js +0 -5
- package/dist/report/extension/meta.d.ts +0 -9
- package/dist/report/extension/meta.js +0 -11
- package/dist/report/extension/types.d.ts +0 -52
- package/dist/report/extension/types.js +0 -3
- package/dist/report/model/aggregate.d.ts +0 -111
- package/dist/report/model/aggregate.js +0 -396
- package/dist/report/model/calculation.d.ts +0 -139
- package/dist/report/model/calculation.js +0 -447
- package/dist/report/model/chart/math.d.ts +0 -73
- package/dist/report/model/chart/math.js +0 -198
- package/dist/report/model/chart/plot.d.ts +0 -52
- package/dist/report/model/chart/plot.js +0 -89
- package/dist/report/model/conversions.d.ts +0 -40
- package/dist/report/model/conversions.js +0 -88
- package/dist/report/model/dataset.d.ts +0 -18
- package/dist/report/model/dataset.js +0 -74
- package/dist/report/model/evaluation-kind.d.ts +0 -19
- package/dist/report/model/evaluation-kind.js +0 -32
- package/dist/report/model/flag.d.ts +0 -52
- package/dist/report/model/flag.js +0 -157
- package/dist/report/model/format.d.ts +0 -96
- package/dist/report/model/format.js +0 -339
- package/dist/report/model/locale.d.ts +0 -219
- package/dist/report/model/locale.js +0 -426
- package/dist/report/model/metrics.d.ts +0 -44
- package/dist/report/model/metrics.js +0 -255
- package/dist/report/model/types.d.ts +0 -784
- package/dist/report/model/types.js +0 -5
- package/dist/report/presentation.d.ts +0 -114
- package/dist/report/presentation.js +0 -210
- package/dist/report/react/index.d.ts +0 -13
- package/dist/report/react/index.js +0 -18
- package/dist/report/runtime/load.d.ts +0 -17
- package/dist/report/runtime/load.js +0 -146
- package/dist/report/runtime/page-render.d.ts +0 -47
- package/dist/report/runtime/page-render.js +0 -95
- package/dist/report/runtime/resolved-page.d.ts +0 -35
- package/dist/report/runtime/resolved-page.js +0 -122
- package/dist/report/runtime/target.d.ts +0 -19
- package/dist/report/runtime/target.js +0 -42
- package/dist/report/runtime/text.d.ts +0 -76
- package/dist/report/runtime/text.js +0 -78
- package/dist/report/runtime/web.d.ts +0 -37
- package/dist/report/runtime/web.js +0 -34
- package/dist/report/slices/compute.d.ts +0 -83
- package/dist/report/slices/compute.js +0 -808
- package/dist/report/slices/content.d.ts +0 -17
- package/dist/report/slices/content.js +0 -132
- package/dist/report/slices/validate.d.ts +0 -6
- package/dist/report/slices/validate.js +0 -296
- package/dist/report/tasks.d.ts +0 -144
- package/dist/report/tasks.js +0 -313
- package/dist/report/theme.d.ts +0 -51
- package/dist/report/theme.js +0 -112
- package/src/agents/ai-sdk-otel.ts +0 -62
- package/src/agents/ai-sdk.test.ts +0 -35
- package/src/agents/ai-sdk.ts +0 -603
- package/src/agents/bub-install-spec.ts +0 -50
- package/src/agents/bub.ts +0 -293
- package/src/agents/builtin.ts +0 -16
- package/src/agents/cache-marker.ts +0 -6
- package/src/agents/claude-code.ts +0 -439
- package/src/agents/codex-env.test.ts +0 -153
- package/src/agents/codex.ts +0 -727
- package/src/agents/coding-cli-versions.ts +0 -87
- package/src/agents/command-redaction.test.ts +0 -96
- package/src/agents/hermes.ts +0 -269
- package/src/agents/index.ts +0 -149
- package/src/agents/langgraph.test.ts +0 -30
- package/src/agents/langgraph.ts +0 -508
- package/src/agents/marketplace.ts +0 -96
- package/src/agents/mcp.test.ts +0 -41
- package/src/agents/mcp.ts +0 -31
- package/src/agents/native-config.ts +0 -267
- package/src/agents/npm-staged.ts +0 -370
- package/src/agents/openai-compat.test.ts +0 -69
- package/src/agents/openai-compat.ts +0 -205
- package/src/agents/openclaw.ts +0 -275
- package/src/agents/opencode.ts +0 -198
- package/src/agents/post-setup.test.ts +0 -118
- package/src/agents/post-setup.ts +0 -116
- package/src/agents/provisioner.test.ts +0 -291
- package/src/agents/provisioner.ts +0 -593
- package/src/agents/sdk-streams.test.ts +0 -66
- package/src/agents/sdk-streams.ts +0 -542
- package/src/agents/session-slot.ts +0 -15
- package/src/agents/shared.ts +0 -250
- package/src/agents/skills.ts +0 -309
- package/src/agents/streaming.ts +0 -203
- package/src/agents/types.ts +0 -612
- package/src/agents/ui-message-stream.ts +0 -398
- package/src/assertions/collector.test.ts +0 -457
- package/src/assertions/collector.ts +0 -452
- package/src/assertions/coverage.ts +0 -118
- package/src/assertions/diff.ts +0 -120
- package/src/assertions/display.test.ts +0 -175
- package/src/assertions/display.ts +0 -213
- package/src/assertions/evidence.test.ts +0 -194
- package/src/assertions/judge.test.ts +0 -623
- package/src/assertions/judge.ts +0 -448
- package/src/assertions/match.ts +0 -79
- package/src/assertions/scoped.test.ts +0 -359
- package/src/assertions/scoped.ts +0 -551
- package/src/assertions/types.ts +0 -286
- package/src/cli.ts +0 -1762
- package/src/context/context.test.ts +0 -759
- package/src/context/context.ts +0 -816
- package/src/context/control-flow.ts +0 -17
- package/src/context/deferred-file-content.ts +0 -16
- package/src/context/send-failures.test.ts +0 -122
- package/src/context/send-failures.ts +0 -164
- package/src/context/send-retry.test.ts +0 -112
- package/src/context/send-retry.ts +0 -145
- package/src/context/session.test.ts +0 -477
- package/src/context/session.ts +0 -516
- package/src/context/types.ts +0 -409
- package/src/define-agent.test.ts +0 -99
- package/src/define-eval.test.ts +0 -213
- package/src/define-experiment.test.ts +0 -44
- package/src/define.test.ts +0 -25
- package/src/define.ts +0 -238
- package/src/expect/index.test.ts +0 -39
- package/src/expect/index.ts +0 -252
- package/src/fresh-import.ts +0 -34
- package/src/i18n/core.ts +0 -28
- package/src/i18n/en.ts +0 -532
- package/src/i18n/index.ts +0 -40
- package/src/i18n/locale.test.ts +0 -41
- package/src/i18n/zh-CN.ts +0 -533
- package/src/index.ts +0 -91
- package/src/load-config.ts +0 -39
- package/src/loaders/criteria.test.ts +0 -305
- package/src/loaders/glob.ts +0 -149
- package/src/loaders/index.test.ts +0 -166
- package/src/loaders/index.ts +0 -256
- package/src/loaders/private.test.ts +0 -110
- package/src/o11y/cost.test.ts +0 -41
- package/src/o11y/cost.ts +0 -81
- package/src/o11y/derive.test.ts +0 -161
- package/src/o11y/derive.ts +0 -301
- package/src/o11y/execution-tree.test.ts +0 -390
- package/src/o11y/execution-tree.ts +0 -391
- package/src/o11y/otlp/canonical.ts +0 -113
- package/src/o11y/otlp/mappers/bub.ts +0 -30
- package/src/o11y/otlp/mappers/claude-code.ts +0 -24
- package/src/o11y/otlp/mappers/codex.ts +0 -40
- package/src/o11y/otlp/mappers/index.ts +0 -14
- package/src/o11y/otlp/parse.ts +0 -334
- package/src/o11y/otlp/receiver.ts +0 -118
- package/src/o11y/otlp/sandbox-receiver.test.ts +0 -301
- package/src/o11y/otlp/sandbox-receiver.ts +0 -255
- package/src/o11y/otlp/select.ts +0 -86
- package/src/o11y/otlp/turn-otel.ts +0 -132
- package/src/o11y/parsers/bub.test.ts +0 -30
- package/src/o11y/parsers/bub.ts +0 -264
- package/src/o11y/parsers/claude-code.ts +0 -302
- package/src/o11y/parsers/codex.test.ts +0 -19
- package/src/o11y/parsers/codex.ts +0 -476
- package/src/o11y/parsers/hermes.ts +0 -199
- package/src/o11y/parsers/index.ts +0 -28
- package/src/o11y/parsers/openclaw.ts +0 -340
- package/src/o11y/parsers/opencode.ts +0 -300
- package/src/o11y/prices.json +0 -10812
- package/src/o11y/tool-names.ts +0 -74
- package/src/o11y/types.ts +0 -221
- package/src/record/annotated-source.test.ts +0 -329
- package/src/record/annotated-source.ts +0 -643
- package/src/record/attempt-evidence.test.ts +0 -208
- package/src/record/attempt-evidence.ts +0 -184
- package/src/record/attempt-source.ts +0 -37
- package/src/record/copy.ts +0 -303
- package/src/record/evidence-schema.test.ts +0 -85
- package/src/record/format.ts +0 -91
- package/src/record/host-equivalence.test.ts +0 -539
- package/src/record/index.ts +0 -119
- package/src/record/locator.test.ts +0 -125
- package/src/record/locator.ts +0 -178
- package/src/record/manifest.ts +0 -72
- package/src/record/open.ts +0 -758
- package/src/record/publish.ts +0 -7
- package/src/record/results.test.ts +0 -1563
- package/src/record/skipped-notice.ts +0 -44
- package/src/record/source-hash.ts +0 -28
- package/src/record/timing-activity.test.ts +0 -395
- package/src/record/truncate.ts +0 -116
- package/src/record/types.ts +0 -319
- package/src/record/writer.ts +0 -549
- package/src/report/assets/colors.ts +0 -69
- package/src/report/assets/enhance.js +0 -214
- package/src/report/assets/series-encoding.tsx +0 -177
- package/src/report/assets/styles.css +0 -1780
- package/src/report/built-in/failures.tsx +0 -29
- package/src/report/built-in/index.tsx +0 -66
- package/src/report/built-in/result-components.tsx +0 -404
- package/src/report/built-in/stability.tsx +0 -30
- package/src/report/built-in/standard.tsx +0 -132
- package/src/report/components/attempt-detail/attempt-components.test.tsx +0 -706
- package/src/report/components/attempt-detail/compute.ts +0 -544
- package/src/report/components/attempt-detail/content.tsx +0 -633
- package/src/report/components/attempt-detail/index.tsx +0 -285
- package/src/report/components/attempt-detail/validate.test.ts +0 -253
- package/src/report/components/attempt-detail/validate.tsx +0 -361
- package/src/report/components/cell.tsx +0 -79
- package/src/report/components/compute.test.ts +0 -1540
- package/src/report/components/entity-lists/compute.test.ts +0 -38
- package/src/report/components/entity-lists/compute.ts +0 -383
- package/src/report/components/entity-lists/content.test.ts +0 -611
- package/src/report/components/entity-lists/content.ts +0 -591
- package/src/report/components/entity-lists/index.tsx +0 -169
- package/src/report/components/entity-lists/validate.test.ts +0 -161
- package/src/report/components/entity-lists/validate.ts +0 -121
- package/src/report/components/experiment-detail/compute.ts +0 -40
- package/src/report/components/experiment-detail/experiment-detail.test.tsx +0 -181
- package/src/report/components/experiment-detail/index.tsx +0 -89
- package/src/report/components/fixtures.ts +0 -366
- package/src/report/components/scope.harness.ts +0 -83
- package/src/report/components/shared-compute.ts +0 -60
- package/src/report/components/shared-faces.ts +0 -25
- package/src/report/components/shared.test.ts +0 -62
- package/src/report/components/shared.ts +0 -127
- package/src/report/components/site-components/HeroCard.tsx +0 -87
- package/src/report/components/site-components/PoweredBy.tsx +0 -20
- package/src/report/components/site-components/compute.ts +0 -165
- package/src/report/components/site-components/content.ts +0 -8
- package/src/report/components/site-components/faces.ts +0 -46
- package/src/report/components/site-components/hero-types.ts +0 -20
- package/src/report/components/site-components/index.tsx +0 -109
- package/src/report/components/site-components/projections.ts +0 -74
- package/src/report/components/site-components/scope-warnings.ts +0 -160
- package/src/report/components/site-components/site-components.test.tsx +0 -460
- package/src/report/components/site-components/snapshot-diagnostics.ts +0 -69
- package/src/report/components/site-components/validate.test.ts +0 -121
- package/src/report/components/site-components/validators.ts +0 -97
- package/src/report/components/summaries/compute.ts +0 -66
- package/src/report/components/summaries/index.tsx +0 -255
- package/src/report/components/summaries/stability-overview.test.tsx +0 -141
- package/src/report/components/summaries/stability-overview.tsx +0 -105
- package/src/report/components/summaries/validate.test.ts +0 -49
- package/src/report/components/summaries/validate.ts +0 -21
- package/src/report/definition/callouts.test.tsx +0 -146
- package/src/report/definition/cell.ts +0 -203
- package/src/report/definition/chart.test.tsx +0 -285
- package/src/report/definition/conversation.test.tsx +0 -150
- package/src/report/definition/copy-block.test.tsx +0 -71
- package/src/report/definition/diff-view.test.tsx +0 -209
- package/src/report/definition/grid-layout.test.tsx +0 -262
- package/src/report/definition/grid-layout.ts +0 -203
- package/src/report/definition/markdown.test.tsx +0 -179
- package/src/report/definition/primitives/callouts-logic.ts +0 -117
- package/src/report/definition/primitives/callouts.tsx +0 -185
- package/src/report/definition/primitives/chart-map.ts +0 -214
- package/src/report/definition/primitives/chart.tsx +0 -1012
- package/src/report/definition/primitives/conversation.tsx +0 -345
- package/src/report/definition/primitives/copy-block.tsx +0 -87
- package/src/report/definition/primitives/diff-lines.ts +0 -212
- package/src/report/definition/primitives/diff-view.tsx +0 -332
- package/src/report/definition/primitives/markdown.tsx +0 -365
- package/src/report/definition/primitives/marks-contract.test.tsx +0 -45
- package/src/report/definition/primitives/marks.tsx +0 -254
- package/src/report/definition/primitives/points-dataset.test.ts +0 -117
- package/src/report/definition/primitives/points-dataset.ts +0 -168
- package/src/report/definition/primitives/source-view.tsx +0 -359
- package/src/report/definition/primitives/waterfall.tsx +0 -386
- package/src/report/definition/primitives.test.tsx +0 -186
- package/src/report/definition/primitives.tsx +0 -1150
- package/src/report/definition/report-contract.test.ts +0 -119
- package/src/report/definition/report.ts +0 -562
- package/src/report/definition/shell-head.test.ts +0 -100
- package/src/report/definition/source-view.test.tsx +0 -119
- package/src/report/definition/stat.test.tsx +0 -45
- package/src/report/definition/table-text.ts +0 -228
- package/src/report/definition/table.test.tsx +0 -633
- package/src/report/definition/tree.ts +0 -852
- package/src/report/definition/waterfall.test.tsx +0 -581
- package/src/report/extension/assets.ts +0 -154
- package/src/report/extension/define.test.tsx +0 -228
- package/src/report/extension/define.ts +0 -182
- package/src/report/extension/index.ts +0 -23
- package/src/report/extension/meta.ts +0 -21
- package/src/report/extension/types.ts +0 -64
- package/src/report/index.ts +0 -410
- package/src/report/model/aggregate.ts +0 -463
- package/src/report/model/calculation.test.ts +0 -354
- package/src/report/model/calculation.ts +0 -659
- package/src/report/model/chart/math.test.ts +0 -189
- package/src/report/model/chart/math.ts +0 -245
- package/src/report/model/chart/plot.ts +0 -144
- package/src/report/model/conversions.ts +0 -150
- package/src/report/model/dataset.ts +0 -80
- package/src/report/model/evaluation-kind.ts +0 -35
- package/src/report/model/flag.ts +0 -174
- package/src/report/model/format.ts +0 -375
- package/src/report/model/locale.ts +0 -510
- package/src/report/model/metrics.ts +0 -256
- package/src/report/model/panel.test.ts +0 -272
- package/src/report/model/panel.ts +0 -340
- package/src/report/model/presentation.test.ts +0 -141
- package/src/report/model/text-layout.ts +0 -190
- package/src/report/model/types.ts +0 -824
- package/src/report/presentation.test.tsx +0 -474
- package/src/report/presentation.ts +0 -354
- package/src/report/react/index.tsx +0 -96
- package/src/report/runtime/dual-render.test.tsx +0 -1088
- package/src/report/runtime/host.test.ts +0 -86
- package/src/report/runtime/host.ts +0 -317
- package/src/report/runtime/load.test.ts +0 -34
- package/src/report/runtime/load.ts +0 -159
- package/src/report/runtime/page-render.test.tsx +0 -247
- package/src/report/runtime/page-render.ts +0 -147
- package/src/report/runtime/resolved-page.test.tsx +0 -81
- package/src/report/runtime/resolved-page.ts +0 -169
- package/src/report/runtime/target.ts +0 -50
- package/src/report/runtime/text.ts +0 -155
- package/src/report/runtime/web.ts +0 -65
- package/src/report/slices/compute.ts +0 -979
- package/src/report/slices/content.ts +0 -157
- package/src/report/slices/delta-table.test.ts +0 -423
- package/src/report/slices/stability-matrix.test.ts +0 -169
- package/src/report/slices/validate.test.ts +0 -236
- package/src/report/slices/validate.ts +0 -216
- package/src/report/tasks.test.ts +0 -82
- package/src/report/tasks.ts +0 -553
- package/src/report/theme.test.ts +0 -32
- package/src/report/theme.ts +0 -141
- package/src/runner/accept-prompt.test.ts +0 -51
- package/src/runner/accept-prompt.ts +0 -67
- package/src/runner/accept.test.ts +0 -592
- package/src/runner/accept.ts +0 -705
- package/src/runner/attempt.test.ts +0 -1989
- package/src/runner/attempt.ts +0 -1912
- package/src/runner/build-preparation-plan.test.ts +0 -119
- package/src/runner/build-preparation.ts +0 -104
- package/src/runner/cleanup-timeout.test.ts +0 -16
- package/src/runner/cleanup-timeout.ts +0 -23
- package/src/runner/config-identity.test.ts +0 -277
- package/src/runner/config-identity.ts +0 -318
- package/src/runner/direct-agent-sandbox.test.ts +0 -48
- package/src/runner/direct-agent-sandbox.ts +0 -68
- package/src/runner/discover.test.ts +0 -262
- package/src/runner/discover.ts +0 -469
- package/src/runner/eval-selection.test.ts +0 -241
- package/src/runner/eval-selection.ts +0 -139
- package/src/runner/eval-source.test.ts +0 -108
- package/src/runner/eval-source.ts +0 -82
- package/src/runner/experiment-cleanup-registry.test.ts +0 -89
- package/src/runner/experiment-cleanup-registry.ts +0 -39
- package/src/runner/experiment-labels.test.ts +0 -100
- package/src/runner/feedback/coordinator.test.ts +0 -367
- package/src/runner/feedback/coordinator.ts +0 -511
- package/src/runner/feedback/eval-conclusions.test.ts +0 -135
- package/src/runner/feedback/eval-conclusions.ts +0 -115
- package/src/runner/feedback/failure.test.ts +0 -104
- package/src/runner/feedback/failure.ts +0 -60
- package/src/runner/feedback/human.test.ts +0 -1722
- package/src/runner/feedback/human.ts +0 -1499
- package/src/runner/feedback/index.ts +0 -94
- package/src/runner/feedback/input-guard.test.ts +0 -251
- package/src/runner/feedback/input-guard.ts +0 -171
- package/src/runner/feedback/io.ts +0 -82
- package/src/runner/feedback/json.test.ts +0 -617
- package/src/runner/feedback/json.ts +0 -708
- package/src/runner/feedback/profile.test.ts +0 -28
- package/src/runner/feedback/profile.ts +0 -23
- package/src/runner/feedback/reducer.test.ts +0 -908
- package/src/runner/feedback/reducer.ts +0 -411
- package/src/runner/feedback/renderer.ts +0 -83
- package/src/runner/feedback/sink.ts +0 -332
- package/src/runner/feedback/testing.ts +0 -109
- package/src/runner/fingerprint.test.ts +0 -1406
- package/src/runner/fingerprint.ts +0 -924
- package/src/runner/gate-lease.test.ts +0 -555
- package/src/runner/gate-lease.ts +0 -398
- package/src/runner/judge-config.test.ts +0 -32
- package/src/runner/judge-config.ts +0 -26
- package/src/runner/leak-gate.test.ts +0 -195
- package/src/runner/leak-gate.ts +0 -364
- package/src/runner/ledger.test.ts +0 -577
- package/src/runner/ledger.ts +0 -533
- package/src/runner/lock.test.ts +0 -498
- package/src/runner/lock.ts +0 -334
- package/src/runner/manifest.ts +0 -282
- package/src/runner/report.test.ts +0 -238
- package/src/runner/report.ts +0 -127
- package/src/runner/reporters/artifacts.ts +0 -120
- package/src/runner/reporters/braintrust.test.ts +0 -109
- package/src/runner/reporters/braintrust.ts +0 -211
- package/src/runner/reporters/index.ts +0 -7
- package/src/runner/reporters/json.test.ts +0 -163
- package/src/runner/reporters/json.ts +0 -79
- package/src/runner/reporters/shared.ts +0 -14
- package/src/runner/reuse-diagnostics.test.ts +0 -114
- package/src/runner/reuse-diagnostics.ts +0 -92
- package/src/runner/reuse-pool-key.test.ts +0 -22
- package/src/runner/run.test.ts +0 -5319
- package/src/runner/run.ts +0 -2656
- package/src/runner/sandbox-pool-plan.test.ts +0 -180
- package/src/runner/sandbox-pool.test.ts +0 -278
- package/src/runner/sandbox-pool.ts +0 -337
- package/src/runner/sandbox-prep.ts +0 -55
- package/src/runner/sandbox-selection.test.ts +0 -288
- package/src/runner/sandbox-selection.ts +0 -351
- package/src/runner/session.test.ts +0 -112
- package/src/runner/session.ts +0 -466
- package/src/runner/teardown-registry.test.ts +0 -156
- package/src/runner/teardown-registry.ts +0 -146
- package/src/runner/timeout-attribution.test.ts +0 -34
- package/src/runner/timeout.ts +0 -54
- package/src/runner/timing.ts +0 -379
- package/src/runner/types.ts +0 -1733
- package/src/sample/index.ts +0 -483
- package/src/sample/sample.test.ts +0 -622
- package/src/sandbox/backend.ts +0 -140
- package/src/sandbox/build-coordinator.test.ts +0 -333
- package/src/sandbox/build-coordinator.ts +0 -444
- package/src/sandbox/case-types.ts +0 -67
- package/src/sandbox/checkpoint.test.ts +0 -55
- package/src/sandbox/checkpoint.ts +0 -64
- package/src/sandbox/cli-commands.test.ts +0 -524
- package/src/sandbox/cli-commands.ts +0 -593
- package/src/sandbox/commands.ts +0 -287
- package/src/sandbox/compose.test.ts +0 -625
- package/src/sandbox/compose.ts +0 -1291
- package/src/sandbox/content.ts +0 -202
- package/src/sandbox/deadline.ts +0 -73
- package/src/sandbox/docker-agent-image.ts +0 -36
- package/src/sandbox/docker-stream.ts +0 -116
- package/src/sandbox/docker.test.ts +0 -263
- package/src/sandbox/docker.ts +0 -836
- package/src/sandbox/dockerfile-agent-cache.test.ts +0 -157
- package/src/sandbox/dockerfile-agent-cache.ts +0 -155
- package/src/sandbox/dockerfile-agent-runtime.test.ts +0 -166
- package/src/sandbox/dockerfile-build.test.ts +0 -27
- package/src/sandbox/dockerfile-build.ts +0 -269
- package/src/sandbox/dockerfile-identity.ts +0 -129
- package/src/sandbox/download-directory.test.ts +0 -68
- package/src/sandbox/download-directory.ts +0 -36
- package/src/sandbox/e2b-agent-template.test.ts +0 -107
- package/src/sandbox/e2b-agent-template.ts +0 -194
- package/src/sandbox/e2b-reconcile.test.ts +0 -126
- package/src/sandbox/e2b.test.ts +0 -629
- package/src/sandbox/e2b.ts +0 -781
- package/src/sandbox/errors.test.ts +0 -72
- package/src/sandbox/errors.ts +0 -203
- package/src/sandbox/identity.ts +0 -203
- package/src/sandbox/index.ts +0 -138
- package/src/sandbox/io-retry.test.ts +0 -58
- package/src/sandbox/io-retry.ts +0 -45
- package/src/sandbox/keep-registry.test.ts +0 -141
- package/src/sandbox/keep-registry.ts +0 -271
- package/src/sandbox/keep.test.ts +0 -538
- package/src/sandbox/keep.ts +0 -405
- package/src/sandbox/layer.test.ts +0 -343
- package/src/sandbox/layer.ts +0 -1641
- package/src/sandbox/ledger-paths.ts +0 -37
- package/src/sandbox/link.test.ts +0 -321
- package/src/sandbox/link.ts +0 -484
- package/src/sandbox/local-files.ts +0 -31
- package/src/sandbox/local.test.ts +0 -216
- package/src/sandbox/local.ts +0 -364
- package/src/sandbox/official-baselines.test.ts +0 -175
- package/src/sandbox/operations.test.ts +0 -391
- package/src/sandbox/operations.ts +0 -187
- package/src/sandbox/orphans.test.ts +0 -470
- package/src/sandbox/orphans.ts +0 -261
- package/src/sandbox/paths.test.ts +0 -141
- package/src/sandbox/paths.ts +0 -132
- package/src/sandbox/plan.test.ts +0 -487
- package/src/sandbox/plan.ts +0 -353
- package/src/sandbox/prepare-commands.test.ts +0 -168
- package/src/sandbox/prepare-commands.ts +0 -342
- package/src/sandbox/redaction.test.ts +0 -48
- package/src/sandbox/redaction.ts +0 -50
- package/src/sandbox/registry.ts +0 -78
- package/src/sandbox/retry.test.ts +0 -138
- package/src/sandbox/retry.ts +0 -86
- package/src/sandbox/run-identity.ts +0 -120
- package/src/sandbox/runtime.test.ts +0 -551
- package/src/sandbox/runtime.ts +0 -984
- package/src/sandbox/shell.test.ts +0 -23
- package/src/sandbox/shell.ts +0 -18
- package/src/sandbox/transfer-errors.test.ts +0 -116
- package/src/sandbox/transfer-errors.ts +0 -118
- package/src/sandbox/types.ts +0 -196
- package/src/sandbox/vercel.test.ts +0 -180
- package/src/sandbox/vercel.ts +0 -400
- package/src/shared/aggregate.test.ts +0 -52
- package/src/shared/aggregate.ts +0 -99
- package/src/shared/entry-file-store.test.ts +0 -193
- package/src/shared/entry-file-store.ts +0 -145
- package/src/shared/external-cause.ts +0 -322
- package/src/shared/facts.ts +0 -36
- package/src/shared/failure-class.test.ts +0 -195
- package/src/shared/failure-class.ts +0 -206
- package/src/shared/format.ts +0 -26
- package/src/shared/json-match.ts +0 -42
- package/src/shared/json-value.test.ts +0 -24
- package/src/shared/json-value.ts +0 -36
- package/src/shared/turn-label.test.ts +0 -20
- package/src/shared/turn-label.ts +0 -15
- package/src/shared/types.ts +0 -101
- package/src/shared/verdict.ts +0 -82
- package/src/show/command.test.ts +0 -36
- package/src/show/command.ts +0 -18
- package/src/show/compose.ts +0 -89
- package/src/show/index.test.ts +0 -95
- package/src/show/index.ts +0 -1226
- package/src/show/json.test.ts +0 -412
- package/src/show/json.ts +0 -106
- package/src/show/render.test.ts +0 -857
- package/src/show/render.ts +0 -1510
- package/src/source-loc.ts +0 -134
- package/src/tty-line.ts +0 -14
- package/src/types.ts +0 -37
- package/src/util.test.ts +0 -77
- package/src/util.ts +0 -138
- package/src/view/app/App.tsx +0 -351
- package/src/view/app/components/ui/dialog.tsx +0 -36
- package/src/view/app/components/ui/tabs.tsx +0 -30
- package/src/view/app/css.d.ts +0 -3
- package/src/view/app/i18n.ts +0 -65
- package/src/view/app/index.html +0 -13
- package/src/view/app/lib/cn.ts +0 -7
- package/src/view/app/lib/target-dialog.test.ts +0 -105
- package/src/view/app/lib/target-dialog.ts +0 -110
- package/src/view/app/main.tsx +0 -109
- package/src/view/app/types.ts +0 -15
- package/src/view/app/vite.config.ts +0 -26
- package/src/view/data.test.ts +0 -541
- package/src/view/data.ts +0 -950
- package/src/view/index.ts +0 -91
- package/src/view/server-rebuild.test.ts +0 -323
- package/src/view/server-watch.test.ts +0 -106
- package/src/view/server.ts +0 -484
- package/src/view/shared/types.ts +0 -93
- package/src/view/site-base.test.ts +0 -51
- package/src/view/site-head.test.ts +0 -142
- package/src/view/site-param-pages.test.ts +0 -186
- package/src/view/site-renderer-assets.test.ts +0 -68
- package/src/view/site.ts +0 -484
- package/src/view/styles.css +0 -170
- /package/dist/{report/components/site-components/hero-types.js → cli/bootstrap.d.cts} +0 -0
- /package/{src → dist}/view/client-dist/app.css +0 -0
- /package/{src → dist}/view/client-dist/app.js +0 -0
- /package/{src → dist}/view/template.html +0 -0
|
@@ -1,1540 +0,0 @@
|
|
|
1
|
-
// cases: docs/engineering/testing/unit/reports.md
|
|
2
|
-
// niceeval/report 计算层的单元测试:全部用内存 fake(Run / AttemptHandle 按
|
|
3
|
-
// niceeval/record 的读取契约手工构造)。覆盖登记行:两级聚合 vs 平铺、errored=0 口径、
|
|
4
|
-
// skipped=null、null≠0、Scoreboard 固定分母(notRun/unscorable 分开)、权重最长前缀、
|
|
5
|
-
// 身份键去重、现刻水位、自定义指标 where/aggregate、evalGroup 完整父路径、verdict 权威、
|
|
6
|
-
// MetricValue 诚实、durationMs 超时删失(线值不进均值、samples<total 覆盖率缺口)、缺 artifact 指标、repeatedFailedCommands、实体列表 failureSummary、
|
|
7
|
-
// sampleSummary 两级计票、experimentListData/sampleSummary 的 selectedEvalIds 投影、conditionsByFlag、
|
|
8
|
-
// MetricLine 点身份、空数组反馈、measureRowsData sort、series 配色的确定性索引计算(colorIndexForKey /
|
|
9
|
-
// colorIndicesForKeys,纯函数,不断言渲染出的颜色值)。Scatter 的 text 面(`scatterText`)本身是
|
|
10
|
-
// 终端排版渲染——图例分配、connect 位移摘要的字符串产物归 E2E 报告域(docs/engineering/testing/e2e/
|
|
11
|
-
// report.md §5 终端排版),不在本文件内。
|
|
12
|
-
|
|
13
|
-
import { describe, expect, it } from "vitest";
|
|
14
|
-
|
|
15
|
-
import type { AssertionResult, AttemptError, EvalResult, O11ySummary, Verdict } from "../../types.ts";
|
|
16
|
-
import { completeEvidenceCoverage } from "../../assertions/coverage.ts";
|
|
17
|
-
import type { AttemptHandle, Sample, SampleIssue, Run } from "../../record/index.ts";
|
|
18
|
-
import { attemptHandleOf, scopeOf } from "./scope.harness.ts";
|
|
19
|
-
import { makeSample } from "../../sample/index.ts";
|
|
20
|
-
import { encodeAttemptLocator } from "../../record/locator.ts";
|
|
21
|
-
import { COVERAGE_ROW_PREFIX, experimentListContent } from "./entity-lists/content.ts";
|
|
22
|
-
import { formatCellText, type Cell } from "../definition/cell.ts";
|
|
23
|
-
import type { Record } from "../../record/types.ts";
|
|
24
|
-
import {
|
|
25
|
-
assistantTurns,
|
|
26
|
-
costUSD,
|
|
27
|
-
durationMs,
|
|
28
|
-
passRate,
|
|
29
|
-
examScore,
|
|
30
|
-
executionReliability,
|
|
31
|
-
repeatedFailedCommands,
|
|
32
|
-
taskPassRate,
|
|
33
|
-
totalScore,
|
|
34
|
-
} from "../model/metrics.ts";
|
|
35
|
-
import { flag, label, numericFlag, numericLabel } from "../model/flag.ts";
|
|
36
|
-
import { colorIndexForKey, colorIndicesForKeys } from "../assets/colors.ts";
|
|
37
|
-
import { attemptListData, evalListData, experimentListData } from "./entity-lists/compute.ts";
|
|
38
|
-
import { validateAttemptListData, validateEvalListData, validateExperimentListData } from "./entity-lists/validate.ts";
|
|
39
|
-
import {
|
|
40
|
-
conditionsByFlag,
|
|
41
|
-
deltaTableData,
|
|
42
|
-
metricLineData,
|
|
43
|
-
metricMatrixData,
|
|
44
|
-
measureRowsData,
|
|
45
|
-
scoreboardData,
|
|
46
|
-
stabilityMatrixData,
|
|
47
|
-
} from "../slices/compute.ts";
|
|
48
|
-
import {
|
|
49
|
-
validateDeltaData,
|
|
50
|
-
validateLineData,
|
|
51
|
-
validateMatrixData,
|
|
52
|
-
validateScoreboardData,
|
|
53
|
-
validateStabilityMatrixData,
|
|
54
|
-
} from "../slices/validate.ts";
|
|
55
|
-
import { sampleSummary } from "./summaries/compute.ts";
|
|
56
|
-
import { validateSampleSummaryContent } from "./summaries/validate.ts";
|
|
57
|
-
import { evalGroupOf } from "../model/aggregate.ts";
|
|
58
|
-
import { evaluationKindComposition } from "../model/evaluation-kind.ts";
|
|
59
|
-
import { isDataset, isMetricValue } from "../model/dataset.ts";
|
|
60
|
-
import type { AttemptMetric, DatasetRow, MetricValue } from "../model/types.ts";
|
|
61
|
-
|
|
62
|
-
function metricCell(row: DatasetRow, name: string): MetricValue {
|
|
63
|
-
const value = row.values[name];
|
|
64
|
-
if (!isMetricValue(value)) throw new Error(`expected metric cell ${JSON.stringify(name)} in row ${JSON.stringify(row.key)}`);
|
|
65
|
-
return value;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
// ───────────────────────── fake 数据(按 results 读取契约造)─────────────────────────
|
|
69
|
-
|
|
70
|
-
/** 结构化 `AttemptError` 的最小构造(测试用)。 */
|
|
71
|
-
function erroredWith(message: string): AttemptError {
|
|
72
|
-
return { code: "unexpected-error", message, origin: { scope: "attempt" as const, phase: "eval.run" } };
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
let seq = 0;
|
|
76
|
-
|
|
77
|
-
/** 造一条结果;默认给每条唯一 startedAt —— 身份键含 startedAt,免得普通样本被去重误伤。 */
|
|
78
|
-
function res(id: string, verdict: Verdict, extra: Partial<EvalResult> = {}): EvalResult {
|
|
79
|
-
seq += 1;
|
|
80
|
-
return {
|
|
81
|
-
id,
|
|
82
|
-
agent: "agent-x",
|
|
83
|
-
verdict,
|
|
84
|
-
attempt: 0,
|
|
85
|
-
startedAt: `2026-07-01T00:00:00.${String(seq).padStart(6, "0")}Z`,
|
|
86
|
-
durationMs: 1000,
|
|
87
|
-
assertions: [],
|
|
88
|
-
evidenceCoverage: completeEvidenceCoverage,
|
|
89
|
-
...extra,
|
|
90
|
-
};
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
function softAssertion(name: string, score: number, extra: Partial<AssertionResult> = {}): AssertionResult {
|
|
94
|
-
return { name, severity: "soft", score, outcome: "passed" as const, ...extra } as AssertionResult;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
/** 计分制断言:.points(n) 挂了才有的 points 字段(n × score,由调用方直接传最终挣分)。 */
|
|
98
|
-
function pointsAssertion(name: string, points: number, extra: Partial<AssertionResult> = {}): AssertionResult {
|
|
99
|
-
return { name, severity: "gate", score: 1, outcome: "passed" as const, points, ...extra } as AssertionResult;
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/** 最小合规 O11ySummary;shellCommands / totalTurns 按需变。 */
|
|
103
|
-
function o11ySummary(partial: Partial<O11ySummary> = {}): O11ySummary {
|
|
104
|
-
return {
|
|
105
|
-
totalTurns: 0,
|
|
106
|
-
toolCalls: {},
|
|
107
|
-
totalToolCalls: 0,
|
|
108
|
-
filesRead: [],
|
|
109
|
-
filesModified: [],
|
|
110
|
-
shellCommands: [],
|
|
111
|
-
webFetches: [],
|
|
112
|
-
errors: [],
|
|
113
|
-
thinkingBlocks: 0,
|
|
114
|
-
compactions: 0,
|
|
115
|
-
durationMs: 0,
|
|
116
|
-
usage: { inputTokens: 0, outputTokens: 0 },
|
|
117
|
-
...partial,
|
|
118
|
-
} as O11ySummary;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
interface SnapSpec {
|
|
122
|
-
experimentId: string;
|
|
123
|
-
results: EvalResult[];
|
|
124
|
-
agent?: string;
|
|
125
|
-
model?: string;
|
|
126
|
-
runStartedAt?: string;
|
|
127
|
-
knownEvalIds?: string[];
|
|
128
|
-
experiment?: Omit<NonNullable<Run["experiment"]>, "sandboxLayer" | "sandboxPlansByEval" | "agentInstalls"> &
|
|
129
|
-
Partial<Pick<NonNullable<Run["experiment"]>, "sandboxLayer" | "sandboxPlansByEval" | "agentInstalls">>;
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
function experimentInfo(
|
|
133
|
-
info: NonNullable<SnapSpec["experiment"]>,
|
|
134
|
-
): NonNullable<Run["experiment"]> {
|
|
135
|
-
return {
|
|
136
|
-
sandboxLayer: {},
|
|
137
|
-
sandboxPlansByEval: {},
|
|
138
|
-
agentInstalls: [],
|
|
139
|
-
...info,
|
|
140
|
-
};
|
|
141
|
-
}
|
|
142
|
-
|
|
143
|
-
let runSeq = 0;
|
|
144
|
-
|
|
145
|
-
/** 最小构造:一个快照目录装一个快照。runStartedAt 决定去重时谁是「最新快照」。 */
|
|
146
|
-
function snap(spec: SnapSpec): Run {
|
|
147
|
-
runSeq += 1;
|
|
148
|
-
const startedAt = spec.runStartedAt ?? `2026-06-01T00:00:00.${String(runSeq).padStart(3, "0")}Z`;
|
|
149
|
-
const dir = `/results/exp/snap-${runSeq}`;
|
|
150
|
-
const run = {
|
|
151
|
-
runId: `run-${runSeq}`,
|
|
152
|
-
experimentId: spec.experimentId,
|
|
153
|
-
startedAt,
|
|
154
|
-
completedAt: startedAt,
|
|
155
|
-
agent: spec.agent ?? "agent-x",
|
|
156
|
-
model: spec.model,
|
|
157
|
-
experiment: spec.experiment === undefined ? undefined : experimentInfo(spec.experiment),
|
|
158
|
-
schemaVersion: 1,
|
|
159
|
-
dir,
|
|
160
|
-
knownEvalIds: spec.knownEvalIds,
|
|
161
|
-
} as Run;
|
|
162
|
-
const attempts: AttemptHandle[] = spec.results.map((r) =>
|
|
163
|
-
attemptHandleOf(
|
|
164
|
-
run,
|
|
165
|
-
r,
|
|
166
|
-
{ run: `exp/snap-${runSeq}`, attempt: `${r.id}/a${r.attempt}` },
|
|
167
|
-
{ o11y: async () => r.o11y ?? null },
|
|
168
|
-
),
|
|
169
|
-
);
|
|
170
|
-
const evals = new Map<string, AttemptHandle[]>();
|
|
171
|
-
for (const attempt of attempts) {
|
|
172
|
-
const list = evals.get(attempt.evalId);
|
|
173
|
-
if (list) list.push(attempt);
|
|
174
|
-
else evals.set(attempt.evalId, [attempt]);
|
|
175
|
-
}
|
|
176
|
-
run.evals = [...evals.entries()].map(([id, list]) => ({ id, attempts: list }));
|
|
177
|
-
run.attempts = attempts;
|
|
178
|
-
return run;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
// ───────────────────────── 指标聚合口径 ─────────────────────────
|
|
182
|
-
|
|
183
|
-
describe("两级聚合口径", () => {
|
|
184
|
-
// 区分力 fixture:题级值 [1, 2/3, 0] → 两级 5/9;attempt 平铺 3/5;
|
|
185
|
-
// 条件任务通过率 5/6(errored 不进分母);「任一轮通过」2/3。四种口径互不相等。
|
|
186
|
-
const discriminating = snap({
|
|
187
|
-
experimentId: "exp/a",
|
|
188
|
-
results: [
|
|
189
|
-
res("e1", "passed"),
|
|
190
|
-
res("e2", "passed", { attempt: 0 }),
|
|
191
|
-
res("e2", "passed", { attempt: 1 }),
|
|
192
|
-
res("e2", "failed", { attempt: 2 }),
|
|
193
|
-
res("e3", "errored", { error: erroredWith("boom") }),
|
|
194
|
-
],
|
|
195
|
-
});
|
|
196
|
-
|
|
197
|
-
it("passRate 先题内折叠再跨题折叠:5/9,不是平铺 3/5、条件 5/6 或任一轮 2/3", async () => {
|
|
198
|
-
const data = await sampleSummary([discriminating]);
|
|
199
|
-
expect(data.endToEndPassRate.value).toBeCloseTo(5 / 9);
|
|
200
|
-
expect(data.endToEndPassRate.value).not.toBeCloseTo(3 / 5);
|
|
201
|
-
expect(data.endToEndPassRate.value).not.toBeCloseTo(5 / 6);
|
|
202
|
-
expect(data.endToEndPassRate.value).not.toBeCloseTo(2 / 3);
|
|
203
|
-
expect(data.endToEndPassRate.unit).toBe("%");
|
|
204
|
-
expect(data.endToEndPassRate.value).toBeCloseTo(5 / 9);
|
|
205
|
-
expect(data.endToEndPassRate.samples).toBe(5);
|
|
206
|
-
expect(data.endToEndPassRate.total).toBe(5);
|
|
207
|
-
});
|
|
208
|
-
|
|
209
|
-
it("taskPassRate 排除 errored,只能作为带限定名称的诊断指标:同 fixture 得 5/6", async () => {
|
|
210
|
-
const table = await measureRowsData([discriminating], {
|
|
211
|
-
dimensions: ["agent"],
|
|
212
|
-
measures: [passRate, taskPassRate, executionReliability],
|
|
213
|
-
});
|
|
214
|
-
const row = table.rows[0]!;
|
|
215
|
-
expect(metricCell(row, taskPassRate.name).value).toBeCloseTo(5 / 6);
|
|
216
|
-
expect(metricCell(row, passRate.name).value).toBeCloseTo(5 / 9);
|
|
217
|
-
// executionReliability:e1=1、e2=1、e3=0 → 2/3
|
|
218
|
-
expect(metricCell(row, executionReliability.name).value).toBeCloseTo(2 / 3);
|
|
219
|
-
});
|
|
220
|
-
|
|
221
|
-
it("2 passed + 5 errored 的默认通过率是 2/7,不是 100%", async () => {
|
|
222
|
-
const s = snap({
|
|
223
|
-
experimentId: "exp/err",
|
|
224
|
-
results: [
|
|
225
|
-
res("q1", "passed"),
|
|
226
|
-
res("q2", "passed"),
|
|
227
|
-
...[3, 4, 5, 6, 7].map((n) => res(`q${n}`, "errored", { error: erroredWith("x") })),
|
|
228
|
-
],
|
|
229
|
-
});
|
|
230
|
-
const data = await sampleSummary([s]);
|
|
231
|
-
expect(data.endToEndPassRate.value).toBeCloseTo(2 / 7);
|
|
232
|
-
});
|
|
233
|
-
|
|
234
|
-
it("skipped 对内置指标返回 null:不进有效样本但保留在 total,value 不受影响", async () => {
|
|
235
|
-
const s = snap({
|
|
236
|
-
experimentId: "exp/skip",
|
|
237
|
-
results: [res("a", "passed"), res("b", "skipped"), res("c", "failed")],
|
|
238
|
-
});
|
|
239
|
-
const data = await sampleSummary([s]);
|
|
240
|
-
expect(data.endToEndPassRate.value).toBeCloseTo(0.5); // (1+0)/2,skipped 不稀释
|
|
241
|
-
expect(data.endToEndPassRate.samples).toBe(2);
|
|
242
|
-
expect(data.endToEndPassRate.total).toBe(3);
|
|
243
|
-
});
|
|
244
|
-
|
|
245
|
-
it("null 表示测不了不参与聚合,0 正常参与:[null, 0, 1] 的 mean 是 0.5 而非 1/3", async () => {
|
|
246
|
-
const values = new Map([
|
|
247
|
-
["a", null],
|
|
248
|
-
["b", 0],
|
|
249
|
-
["c", 1],
|
|
250
|
-
]);
|
|
251
|
-
const metric: AttemptMetric = {
|
|
252
|
-
name: "tri",
|
|
253
|
-
value: (attempt) => values.get(attempt.evalId) ?? null,
|
|
254
|
-
};
|
|
255
|
-
const s = snap({ experimentId: "exp/tri", results: [res("a", "passed"), res("b", "passed"), res("c", "passed")] });
|
|
256
|
-
const table = await measureRowsData([s], { dimensions: ["agent"], measures: [metric] });
|
|
257
|
-
expect(metricCell(table.rows[0]!, "tri").value).toBeCloseTo(0.5);
|
|
258
|
-
});
|
|
259
|
-
|
|
260
|
-
it("跨快照计算先按身份键去重:局部补跑重叠快照下 samples 不虚增", async () => {
|
|
261
|
-
const carried = res("dup/a", "passed", { startedAt: "2026-07-01T09:00:00.000Z" });
|
|
262
|
-
const s1 = snap({ experimentId: "exp/dup", results: [carried], runStartedAt: "2026-07-01T09:00:00.000Z" });
|
|
263
|
-
// 携带合入:同一条结果(同身份键)原样出现在更新的快照里
|
|
264
|
-
const s2 = snap({ experimentId: "exp/dup", results: [{ ...carried }], runStartedAt: "2026-07-02T09:00:00.000Z" });
|
|
265
|
-
const data = await sampleSummary([s1, s2]);
|
|
266
|
-
expect(data.attempts).toBe(1);
|
|
267
|
-
expect(data.endToEndPassRate.samples).toBe(1);
|
|
268
|
-
});
|
|
269
|
-
|
|
270
|
-
it("自定义指标:where 是进入计算前的过滤;perEval + acrossEvals 两级分别生效", async () => {
|
|
271
|
-
const s = snap({
|
|
272
|
-
experimentId: "exp/custom",
|
|
273
|
-
results: [
|
|
274
|
-
res("a", "passed", { durationMs: 100, attempt: 0 }),
|
|
275
|
-
res("a", "passed", { durationMs: 300, attempt: 1 }),
|
|
276
|
-
res("b", "failed", { durationMs: 900 }),
|
|
277
|
-
res("c", "passed", { durationMs: 500 }),
|
|
278
|
-
],
|
|
279
|
-
});
|
|
280
|
-
const fastest: AttemptMetric = {
|
|
281
|
-
name: "fastest-pass",
|
|
282
|
-
where: (attempt) => attempt.result.verdict === "passed",
|
|
283
|
-
value: (attempt) => attempt.result.durationMs,
|
|
284
|
-
perEval: "min",
|
|
285
|
-
acrossEvals: "mean",
|
|
286
|
-
};
|
|
287
|
-
const table = await measureRowsData([s], { dimensions: ["agent"], measures: [fastest] });
|
|
288
|
-
const cell = metricCell(table.rows[0]!, "fastest-pass");
|
|
289
|
-
// b 被 where 排除;a 题内 min = 100,c = 500 → mean 300(双 mean 会是 (200+500)/2=350,可区分)
|
|
290
|
-
expect(cell.value).toBe(300);
|
|
291
|
-
expect(cell.samples).toBe(3);
|
|
292
|
-
expect(cell.total).toBe(4);
|
|
293
|
-
|
|
294
|
-
const allExcluded: AttemptMetric = {
|
|
295
|
-
name: "none",
|
|
296
|
-
where: () => false,
|
|
297
|
-
value: () => 1,
|
|
298
|
-
};
|
|
299
|
-
const empty = await measureRowsData([s], { dimensions: ["agent"], measures: [allExcluded] });
|
|
300
|
-
expect(metricCell(empty.rows[0]!, "none").value).toBeNull();
|
|
301
|
-
});
|
|
302
|
-
|
|
303
|
-
it("evalGroup 按完整父路径分组(无 / 取完整 id),与可比组同一条派生规则", async () => {
|
|
304
|
-
expect(evalGroupOf("a/b/c")).toBe("a/b");
|
|
305
|
-
expect(evalGroupOf("security/sql-injection")).toBe("security");
|
|
306
|
-
expect(evalGroupOf("standalone")).toBe("standalone");
|
|
307
|
-
const s = snap({
|
|
308
|
-
experimentId: "exp/g",
|
|
309
|
-
results: [res("a/b/c", "passed"), res("a/b/d", "failed"), res("solo", "passed")],
|
|
310
|
-
});
|
|
311
|
-
const matrix = await metricMatrixData([s], { rows: "evalGroup", columns: "agent", cell: passRate });
|
|
312
|
-
const rowKeys = [...new Set(matrix.cells.map((c) => c.row))];
|
|
313
|
-
expect(rowKeys.sort()).toEqual(["a/b", "solo"]);
|
|
314
|
-
});
|
|
315
|
-
|
|
316
|
-
it("报告消费落盘 verdict,不重新判卷:断言与 verdict 矛盾时以 verdict 为准", async () => {
|
|
317
|
-
const s = snap({
|
|
318
|
-
experimentId: "exp/v",
|
|
319
|
-
results: [
|
|
320
|
-
res("a", "failed", {
|
|
321
|
-
// 断言看起来全过,但 verdict 是 failed(如 --strict 翻案):按 verdict 记 0
|
|
322
|
-
assertions: [softAssertion("s", 1)],
|
|
323
|
-
}),
|
|
324
|
-
],
|
|
325
|
-
});
|
|
326
|
-
const data = await sampleSummary([s]);
|
|
327
|
-
expect(data.endToEndPassRate.value).toBe(0);
|
|
328
|
-
});
|
|
329
|
-
});
|
|
330
|
-
|
|
331
|
-
// ───────────────────────── MetricValue 与缺数据行为 ─────────────────────────
|
|
332
|
-
|
|
333
|
-
describe("MetricValue 诚实契约", () => {
|
|
334
|
-
it("measuredZero / partial / missing 三种格子互不混淆;refs 序列化后不丢", async () => {
|
|
335
|
-
const zero = snap({ experimentId: "exp/zero", results: [res("a", "failed")] });
|
|
336
|
-
const partial = snap({ experimentId: "exp/partial", results: [res("a", "passed"), res("b", "skipped")] });
|
|
337
|
-
const missing = snap({ experimentId: "exp/missing", results: [res("a", "skipped")] });
|
|
338
|
-
const table = await measureRowsData([zero, partial, missing], {
|
|
339
|
-
dimensions: ["experiment"],
|
|
340
|
-
measures: [passRate],
|
|
341
|
-
});
|
|
342
|
-
const cellOf = (key: string) => metricCell(table.rows.find((r) => r.key === key)!, passRate.name);
|
|
343
|
-
expect(cellOf("exp/zero")).toMatchObject({ value: 0, samples: 1, total: 1 });
|
|
344
|
-
expect(cellOf("exp/partial")).toMatchObject({ value: 1, samples: 1, total: 2 });
|
|
345
|
-
expect(cellOf("exp/missing")).toMatchObject({ value: null, samples: 0, total: 1 });
|
|
346
|
-
// 覆盖率与 refs 不因 JSON 序列化丢失;refs 跟随覆盖范围(含 null 值的 attempt)
|
|
347
|
-
const roundTrip = JSON.parse(JSON.stringify(table)) as typeof table;
|
|
348
|
-
expect(metricCell(roundTrip.rows.find((r) => r.key === "exp/partial")!, passRate.name).refs).toHaveLength(2);
|
|
349
|
-
expect(cellOf("exp/missing").refs).toHaveLength(1);
|
|
350
|
-
});
|
|
351
|
-
|
|
352
|
-
it("缺 o11y.json 时 assistantTurns / repeatedFailedCommands 为 missing;result.json 指标不受影响", async () => {
|
|
353
|
-
const s = snap({ experimentId: "exp/noo11y", results: [res("a", "passed", { durationMs: 1234 })] });
|
|
354
|
-
const table = await measureRowsData([s], {
|
|
355
|
-
dimensions: ["agent"],
|
|
356
|
-
measures: [assistantTurns, repeatedFailedCommands, durationMs],
|
|
357
|
-
});
|
|
358
|
-
const row = table.rows[0]!;
|
|
359
|
-
expect(metricCell(row, assistantTurns.name).value).toBeNull();
|
|
360
|
-
expect(metricCell(row, repeatedFailedCommands.name).value).toBeNull();
|
|
361
|
-
expect(metricCell(row, durationMs.name).value).toBe(1234);
|
|
362
|
-
});
|
|
363
|
-
|
|
364
|
-
it("durationMs 对 timeout attempt 删失:线值不进均值,cell.samples < cell.total 如实呈现覆盖率缺口", async () => {
|
|
365
|
-
const s = snap({
|
|
366
|
-
experimentId: "exp/censor",
|
|
367
|
-
results: [
|
|
368
|
-
res("a", "passed", { durationMs: 100 }),
|
|
369
|
-
res("b", "passed", { durationMs: 300 }),
|
|
370
|
-
// 撞 1200000ms(20m)硬线被砍断:durationMs 是右删失点,不是「跑了这么久」
|
|
371
|
-
res("c", "errored", { durationMs: 1200000, error: { code: "timeout", message: "timed out", origin: { scope: "attempt" as const, phase: "eval.run" } } }),
|
|
372
|
-
],
|
|
373
|
-
});
|
|
374
|
-
const table = await measureRowsData([s], { dimensions: ["experiment"], measures: [durationMs] });
|
|
375
|
-
const cell = metricCell(table.rows[0]!, durationMs.name);
|
|
376
|
-
// 均值只在 a/b 上算:(100+300)/2 = 200;若线值 1200000 混进来均值会失真到 400100
|
|
377
|
-
expect(cell.value).toBe(200);
|
|
378
|
-
expect(cell.samples).toBe(2);
|
|
379
|
-
expect(cell.total).toBe(3);
|
|
380
|
-
// 覆盖范围仍含被删失的 attempt(可下钻到「为什么测不了」),不是从格子里静默消失
|
|
381
|
-
expect(cell.refs).toHaveLength(3);
|
|
382
|
-
});
|
|
383
|
-
|
|
384
|
-
it("durationMs 只删失 timeout:其它 error.code 的 errored attempt 仍实测计入", async () => {
|
|
385
|
-
const s = snap({
|
|
386
|
-
experimentId: "exp/other-error",
|
|
387
|
-
results: [res("a", "errored", { durationMs: 500, error: erroredWith("boom") })],
|
|
388
|
-
});
|
|
389
|
-
const table = await measureRowsData([s], { dimensions: ["experiment"], measures: [durationMs] });
|
|
390
|
-
const cell = metricCell(table.rows[0]!, durationMs.name);
|
|
391
|
-
expect(cell.value).toBe(500);
|
|
392
|
-
expect(cell.samples).toBe(1);
|
|
393
|
-
expect(cell.total).toBe(1);
|
|
394
|
-
});
|
|
395
|
-
|
|
396
|
-
it("repeatedFailedCommands:同命令失败 3 次记 2;两条不同命令各失败 1 次记 0", async () => {
|
|
397
|
-
const repeat = snap({
|
|
398
|
-
experimentId: "exp/repeat",
|
|
399
|
-
results: [
|
|
400
|
-
res("a", "failed", {
|
|
401
|
-
o11y: o11ySummary({
|
|
402
|
-
shellCommands: [
|
|
403
|
-
{ command: "pnpm test", success: false },
|
|
404
|
-
{ command: "pnpm test", success: false },
|
|
405
|
-
{ command: "pnpm test", success: false },
|
|
406
|
-
{ command: "ls", success: true },
|
|
407
|
-
],
|
|
408
|
-
}),
|
|
409
|
-
}),
|
|
410
|
-
],
|
|
411
|
-
});
|
|
412
|
-
const distinct = snap({
|
|
413
|
-
experimentId: "exp/distinct",
|
|
414
|
-
results: [
|
|
415
|
-
res("a", "failed", {
|
|
416
|
-
o11y: o11ySummary({
|
|
417
|
-
shellCommands: [
|
|
418
|
-
{ command: "pnpm test", success: false },
|
|
419
|
-
{ command: "pnpm build", success: false },
|
|
420
|
-
],
|
|
421
|
-
}),
|
|
422
|
-
}),
|
|
423
|
-
],
|
|
424
|
-
});
|
|
425
|
-
const table = await measureRowsData([repeat, distinct], {
|
|
426
|
-
dimensions: ["experiment"],
|
|
427
|
-
measures: [repeatedFailedCommands],
|
|
428
|
-
});
|
|
429
|
-
const cellOf = (key: string) => metricCell(table.rows.find((r) => r.key === key)!, repeatedFailedCommands.name);
|
|
430
|
-
expect(cellOf("exp/repeat").value).toBe(2);
|
|
431
|
-
expect(cellOf("exp/distinct").value).toBe(0);
|
|
432
|
-
});
|
|
433
|
-
|
|
434
|
-
it("value 与 display 分别可断言;display 由 unit 或自定义 display(value, locale) 驱动", async () => {
|
|
435
|
-
const s = snap({
|
|
436
|
-
experimentId: "exp/display",
|
|
437
|
-
results: [
|
|
438
|
-
...[1, 1, 1, 1, 1].map((_, i) => res(`q${i}`, "passed")),
|
|
439
|
-
res("q5", "failed"),
|
|
440
|
-
],
|
|
441
|
-
});
|
|
442
|
-
const data = await sampleSummary([s]);
|
|
443
|
-
expect(data.endToEndPassRate.value).toBeCloseTo(5 / 6);
|
|
444
|
-
expect(data.endToEndPassRate.unit).toBe("%");
|
|
445
|
-
|
|
446
|
-
const localized: AttemptMetric = {
|
|
447
|
-
name: "loc",
|
|
448
|
-
value: () => 1,
|
|
449
|
-
display: (value, locale) => (locale === "zh-CN" ? `${value} 个` : `${value} item`),
|
|
450
|
-
};
|
|
451
|
-
const table = await measureRowsData([s], { dimensions: ["agent"], measures: [localized] });
|
|
452
|
-
const loc = metricCell(table.rows[0]!, "loc");
|
|
453
|
-
expect(loc.format && typeof loc.format === "object" && loc.format.kind === "custom").toBe(true);
|
|
454
|
-
if (loc.format && typeof loc.format === "object" && loc.format.kind === "custom") {
|
|
455
|
-
expect(loc.format.format(1, "en")).toBe("1 item");
|
|
456
|
-
expect(loc.format.format(1, "zh-CN")).toBe("1 个");
|
|
457
|
-
}
|
|
458
|
-
});
|
|
459
|
-
|
|
460
|
-
it("value() 抛错时整个计算失败,错误带 metric name 与 attempt locator,不伪装成测不了", async () => {
|
|
461
|
-
const bad: AttemptMetric = {
|
|
462
|
-
name: "explode",
|
|
463
|
-
value: () => {
|
|
464
|
-
throw new Error("boom");
|
|
465
|
-
},
|
|
466
|
-
};
|
|
467
|
-
const s = snap({ experimentId: "exp/bad", results: [res("a", "passed")] });
|
|
468
|
-
await expect(measureRowsData([s], { dimensions: ["agent"], measures: [bad] })).rejects.toThrow(/explode.*boom/s);
|
|
469
|
-
});
|
|
470
|
-
});
|
|
471
|
-
|
|
472
|
-
// ───────────────────────── Scoreboard ─────────────────────────
|
|
473
|
-
|
|
474
|
-
describe("scoreboardData", () => {
|
|
475
|
-
it("固定题集分母:未跑题按 0 分计入 notRun;跑了但 null 的题计入 unscorable,两个计数不合并", async () => {
|
|
476
|
-
const nullScore: AttemptMetric = {
|
|
477
|
-
name: "maybe-score",
|
|
478
|
-
value: (attempt) => (attempt.evalId === "s/unscorable" ? null : attempt.result.verdict === "passed" ? 1 : 0),
|
|
479
|
-
};
|
|
480
|
-
const s = snap({
|
|
481
|
-
experimentId: "exp/board",
|
|
482
|
-
results: [res("s/ran", "passed"), res("s/unscorable", "passed")],
|
|
483
|
-
});
|
|
484
|
-
const data = await scoreboardData([s], {
|
|
485
|
-
rows: "agent",
|
|
486
|
-
questions: ["s/ran", "s/unscorable", "s/never-1", "s/never-2"],
|
|
487
|
-
score: nullScore,
|
|
488
|
-
});
|
|
489
|
-
const row = data.rows[0]!;
|
|
490
|
-
// 分母恒 4:1 分挣到 1(s/ran),其余 0 → 100 × 1/4 = 25
|
|
491
|
-
expect(row.total.value).toBe(25);
|
|
492
|
-
expect(row.total.notRun).toBe(2);
|
|
493
|
-
expect(row.total.unscorable).toBe(1);
|
|
494
|
-
expect(row.subjects[0]!.questions).toBe(4);
|
|
495
|
-
});
|
|
496
|
-
|
|
497
|
-
it("权重按最长前缀命中;无命中默认 1;总分 fullMarks × earned / possible", async () => {
|
|
498
|
-
const s = snap({
|
|
499
|
-
experimentId: "exp/w",
|
|
500
|
-
results: [res("security/auth/a", "passed"), res("security/b", "failed"), res("misc", "passed")],
|
|
501
|
-
});
|
|
502
|
-
const data = await scoreboardData([s], {
|
|
503
|
-
rows: "agent",
|
|
504
|
-
questions: ["security/auth/a", "security/b", "misc"],
|
|
505
|
-
weights: { "security/": 2, "security/auth/": 4 },
|
|
506
|
-
fullMarks: 100,
|
|
507
|
-
});
|
|
508
|
-
const row = data.rows[0]!;
|
|
509
|
-
// earned = 4(auth/a) + 0 + 1(misc) = 5;possible = 4 + 2 + 1 = 7
|
|
510
|
-
expect(row.total.value).toBeCloseTo((100 * 5) / 7);
|
|
511
|
-
expect(data.weights[0]).toEqual({ prefix: "security/auth/", weight: 4 }); // 最长前缀在前
|
|
512
|
-
});
|
|
513
|
-
|
|
514
|
-
it("subject 缺省与 evalGroup 同一条规则(完整父路径);题集外的 eval 忽略并计入 ignoredEvals", async () => {
|
|
515
|
-
const s = snap({
|
|
516
|
-
experimentId: "exp/subject",
|
|
517
|
-
results: [res("a/b/c", "passed"), res("outside", "passed")],
|
|
518
|
-
});
|
|
519
|
-
const data = await scoreboardData([s], { rows: "agent", questions: ["a/b/c"] });
|
|
520
|
-
expect(data.rows[0]!.subjects[0]!.key).toBe("a/b");
|
|
521
|
-
expect(data.ignoredEvals).toBe(1);
|
|
522
|
-
});
|
|
523
|
-
|
|
524
|
-
it("questions 空数组 / 重复、非法权重、fullMarks<=0、score 出界、subject 空串都按完整用户反馈失败", async () => {
|
|
525
|
-
const s = snap({ experimentId: "exp/e", results: [res("a", "passed")] });
|
|
526
|
-
await expect(scoreboardData([s], { rows: "agent", questions: [] })).rejects.toThrow(/non-empty/);
|
|
527
|
-
await expect(scoreboardData([s], { rows: "agent", questions: ["a", "a"] })).rejects.toThrow(/twice/);
|
|
528
|
-
await expect(
|
|
529
|
-
scoreboardData([s], { rows: "agent", questions: ["a"], weights: { a: 0 } }),
|
|
530
|
-
).rejects.toThrow(/positive finite/);
|
|
531
|
-
await expect(scoreboardData([s], { rows: "agent", questions: ["a"], fullMarks: 0 })).rejects.toThrow(/fullMarks/);
|
|
532
|
-
await expect(
|
|
533
|
-
scoreboardData([s], { rows: "agent", questions: ["a"], score: { name: "big", value: () => 2 } }),
|
|
534
|
-
).rejects.toThrow(/\[0, 1\]/);
|
|
535
|
-
await expect(
|
|
536
|
-
scoreboardData([s], { rows: "agent", questions: ["a"], subject: () => "" }),
|
|
537
|
-
).rejects.toThrow(/empty/);
|
|
538
|
-
});
|
|
539
|
-
});
|
|
540
|
-
|
|
541
|
-
// ───────────────────────── 实体列表 ─────────────────────────
|
|
542
|
-
|
|
543
|
-
describe("实体列表 data", () => {
|
|
544
|
-
const failed = res("list/failed", "failed", {
|
|
545
|
-
assertions: [
|
|
546
|
-
{
|
|
547
|
-
name: "equals",
|
|
548
|
-
severity: "gate",
|
|
549
|
-
outcome: "failed" as const,
|
|
550
|
-
score: 0,
|
|
551
|
-
detail: "equals(42)",
|
|
552
|
-
expected: "42",
|
|
553
|
-
received: "41",
|
|
554
|
-
},
|
|
555
|
-
{ name: "second", severity: "gate", outcome: "failed" as const, score: 0, detail: "second-check" },
|
|
556
|
-
] as AssertionResult[],
|
|
557
|
-
usage: { inputTokens: 10, outputTokens: 5, costUSD: 0.1 },
|
|
558
|
-
});
|
|
559
|
-
const errored = res("list/errored", "errored", {
|
|
560
|
-
error: {
|
|
561
|
-
code: "sandbox-create-failed",
|
|
562
|
-
message: "docker daemon unreachable",
|
|
563
|
-
origin: { scope: "attempt" as const, phase: "sandbox.create" },
|
|
564
|
-
stack: "Error: docker daemon unreachable\n at boot (sandbox.ts:10:3)",
|
|
565
|
-
},
|
|
566
|
-
});
|
|
567
|
-
const passed = res("list/passed", "passed");
|
|
568
|
-
const skipped = res("list/skipped", "skipped");
|
|
569
|
-
const listSnap = () => snap({ experimentId: "exp/list", results: [failed, errored, passed, skipped] });
|
|
570
|
-
|
|
571
|
-
it("failureSummary 三态:failed 取主失败断言摘要、errored 取 error 一层摘要(phase · code · message)、passed/skipped 为 null", async () => {
|
|
572
|
-
const items = await attemptListData([listSnap()]);
|
|
573
|
-
const byEval = new Map(items.map((item) => [item.evalId, item]));
|
|
574
|
-
expect(byEval.get("list/failed")!.failureSummary).toContain("equals(42)");
|
|
575
|
-
expect(byEval.get("list/failed")!.failureSummary).toContain("received 41");
|
|
576
|
-
expect(byEval.get("list/failed")!.moreFailures).toBe(1);
|
|
577
|
-
expect(byEval.get("list/errored")!.failureSummary).toBe(
|
|
578
|
-
"sandbox.create · sandbox-create-failed · docker daemon unreachable",
|
|
579
|
-
);
|
|
580
|
-
expect(byEval.get("list/passed")!.failureSummary).toBeNull();
|
|
581
|
-
expect(byEval.get("list/skipped")!.failureSummary).toBeNull();
|
|
582
|
-
});
|
|
583
|
-
|
|
584
|
-
it("failureSummary 计分制口径(规则 6):passed 全部得分点挣满为 null,存在丢分得分点时取记录顺序第一条(含挣分尾缀),其余计入 moreFailures;中止 attempt 仍由既有规则 1 选中前置,不受规则 6 影响", async () => {
|
|
585
|
-
// 挣满:唯一得分点满分(pointsAssertion 默认 outcome passed、score 1),不是「没有得分点」的
|
|
586
|
-
// null,是「挣满」的 null——两种 null 的根因不同,fixture 用真实挣满而非省略得分点来区分。
|
|
587
|
-
const earnedInFull = res("exam/earned", "passed", {
|
|
588
|
-
evaluationKind: "points",
|
|
589
|
-
assertions: [pointsAssertion("installed", 1), pointsAssertion("configured", 1)],
|
|
590
|
-
});
|
|
591
|
-
// 丢分:两个得分点失败(严重度必须是 soft——points Eval 下失败的 gate 会中止,不可能与 passed
|
|
592
|
-
// 并存),取记录顺序第一条(healthy)为主摘要,second-loss 计入 moreFailures。
|
|
593
|
-
const lostPoints = res("exam/lost", "passed", {
|
|
594
|
-
evaluationKind: "points",
|
|
595
|
-
assertions: [
|
|
596
|
-
pointsAssertion("installed", 1), // 挣满,不是候选
|
|
597
|
-
pointsAssertion("healthy", 16, { severity: "soft", outcome: "failed", score: 0.8 }),
|
|
598
|
-
pointsAssertion("second-loss", 0, { severity: "soft", outcome: "failed", score: 0 }),
|
|
599
|
-
],
|
|
600
|
-
});
|
|
601
|
-
// 中止:记录顺序最后一条、唯一 failed 的 gate——既有规则 1 自然选中,计分制不改选择逻辑
|
|
602
|
-
// (docs/feature/assertions/library/display.md「主失败断言怎样选」规则 5)。
|
|
603
|
-
const aborted = res("exam/aborted", "failed", {
|
|
604
|
-
evaluationKind: "points",
|
|
605
|
-
assertions: [
|
|
606
|
-
pointsAssertion("cloned repo", 1),
|
|
607
|
-
{ name: "installed deps", severity: "gate", outcome: "failed" as const, score: 0 },
|
|
608
|
-
] as AssertionResult[],
|
|
609
|
-
});
|
|
610
|
-
const s = snap({ experimentId: "exam/rule6", results: [earnedInFull, lostPoints, aborted] });
|
|
611
|
-
const items = await attemptListData([s]);
|
|
612
|
-
const byEval = new Map(items.map((item) => [item.evalId, item]));
|
|
613
|
-
|
|
614
|
-
expect(byEval.get("exam/earned")!.failureSummary).toBeNull();
|
|
615
|
-
expect(byEval.get("exam/earned")!.moreFailures).toBe(0);
|
|
616
|
-
|
|
617
|
-
const lostSummary = byEval.get("exam/lost")!;
|
|
618
|
-
expect(lostSummary.failureSummary).toContain("healthy");
|
|
619
|
-
expect(lostSummary.failureSummary).not.toContain("second-loss"); // 只有首条丢分进正文
|
|
620
|
-
expect(lostSummary.failureSummary).toMatch(/\+16 pts/); // 首条丢分得分点的挣分尾缀
|
|
621
|
-
expect(lostSummary.moreFailures).toBe(1); // second-loss 单独计数,不进 failureSummary 正文
|
|
622
|
-
|
|
623
|
-
expect(byEval.get("exam/aborted")!.failureSummary).toContain("installed deps");
|
|
624
|
-
expect(byEval.get("exam/aborted")!.moreFailures).toBe(0);
|
|
625
|
-
});
|
|
626
|
-
|
|
627
|
-
it("序列化 JSON 不含第二条断言文本、stack、evidence 或 diagnostics;costUSD 缺失一律 null", async () => {
|
|
628
|
-
const items = await attemptListData([listSnap()]);
|
|
629
|
-
const json = JSON.stringify(items);
|
|
630
|
-
expect(json).not.toContain("second-check");
|
|
631
|
-
expect(json).not.toContain("sandbox.ts:10:3");
|
|
632
|
-
expect(json).not.toContain('"assertions"');
|
|
633
|
-
expect(json).not.toContain('"diagnostics"');
|
|
634
|
-
const byEval = new Map(items.map((item) => [item.evalId, item]));
|
|
635
|
-
expect(byEval.get("list/failed")!.costUSD).toBe(0.1);
|
|
636
|
-
expect(byEval.get("list/errored")!.costUSD).toBeNull();
|
|
637
|
-
});
|
|
638
|
-
|
|
639
|
-
it("experimentListData:evalVerdicts / passRate / costUSD / durationMs / tokens 齐全,默认按端到端通过率降序", async () => {
|
|
640
|
-
const winner = snap({ experimentId: "exp/win", results: [res("a", "passed"), res("b", "passed")] });
|
|
641
|
-
const loser = snap({ experimentId: "exp/lose", results: [res("a", "failed"), res("b", "passed")] });
|
|
642
|
-
const items = await experimentListData([loser, winner]);
|
|
643
|
-
expect(items.map((item) => item.experimentId)).toEqual(["exp/win", "exp/lose"]);
|
|
644
|
-
expect(items[0]!.evalVerdicts).toEqual({ passed: 2, failed: 0, errored: 0, skipped: 0 });
|
|
645
|
-
expect(items[0]!.endToEndPassRate.value).toBe(1);
|
|
646
|
-
expect(items[1]!.evals).toBe(2);
|
|
647
|
-
});
|
|
648
|
-
|
|
649
|
-
it("同一 experiment 的输入含不一致可比性配置时按完整用户反馈失败,指引 run 维度 / MetricLine", async () => {
|
|
650
|
-
const a = snap({ experimentId: "exp/mixed", model: "gpt-a", results: [res("x", "passed")] });
|
|
651
|
-
const b = snap({ experimentId: "exp/mixed", model: "gpt-b", results: [res("y", "passed")] });
|
|
652
|
-
await expect(experimentListData([a, b])).rejects.toThrow(/run.*MetricLine|MetricLine/s);
|
|
653
|
-
// current() 口径的 Sample(一实验一配置)照常计算
|
|
654
|
-
const clean = scopeOf([a]);
|
|
655
|
-
await expect(experimentListData(clean)).resolves.toHaveLength(1);
|
|
656
|
-
});
|
|
657
|
-
|
|
658
|
-
it("时效字段:attemptListData.historical 是 carried 的投影;experimentListData.historicalAttempts 计入携带的 attempt", async () => {
|
|
659
|
-
const carriedB = res("b", "passed", { artifactBase: "exp/hist/old-snap/b/a0" });
|
|
660
|
-
const s = snap({ experimentId: "exp/hist", results: [res("a", "passed"), carriedB] });
|
|
661
|
-
const attempts = await attemptListData([s]);
|
|
662
|
-
expect(attempts.find((item) => item.evalId === "a")!.historical).toBe(false);
|
|
663
|
-
expect(attempts.find((item) => item.evalId === "b")!.historical).toBe(true); // 携带条目
|
|
664
|
-
|
|
665
|
-
const items = await experimentListData([s]);
|
|
666
|
-
expect(items[0]!.historicalAttempts).toBe(1);
|
|
667
|
-
expect(items[0]!.attempts).toBe(2); // historicalAttempts 是子集,不改变 attempts 总数口径
|
|
668
|
-
});
|
|
669
|
-
|
|
670
|
-
it("current() 下跨快照拼入的历史执行(非携带)以水位基准比较标 historical;run 维度按真实来源分组、显示真实 startedAt", async () => {
|
|
671
|
-
// 两个真实贡献 Run:周一(旧,贡献 q2)与周二(新,贡献 q1)——不用 artifactBase/carried,
|
|
672
|
-
// 单纯是「所属快照早于该 experiment 的水位基准」这条 historicalOf 的第二个分支。
|
|
673
|
-
const monday = snap({ experimentId: "exp/multi", results: [res("q2", "passed")], runStartedAt: "2026-07-01T08:00:00.000Z" });
|
|
674
|
-
const tuesday = snap({ experimentId: "exp/multi", results: [res("q1", "passed")], runStartedAt: "2026-07-02T08:00:00.000Z" });
|
|
675
|
-
const scope = scopeOf([monday, tuesday]);
|
|
676
|
-
|
|
677
|
-
const attempts = await attemptListData(scope);
|
|
678
|
-
expect(attempts.find((item) => item.evalId === "q2")!.historical).toBe(true); // 来自旧快照,非携带
|
|
679
|
-
expect(attempts.find((item) => item.evalId === "q1")!.historical).toBe(false); // 来自水位基准本身
|
|
680
|
-
|
|
681
|
-
const items = await experimentListData(scope);
|
|
682
|
-
expect(items[0]!.historicalAttempts).toBe(1);
|
|
683
|
-
expect(items[0]!.attempts).toBe(2);
|
|
684
|
-
|
|
685
|
-
// run 维度按真实来源分组:两条 attempt 各自的真实 startedAt 不同,不被合并成一个键。
|
|
686
|
-
const table = await measureRowsData(scope, { dimensions: ["run"], measures: [passRate] });
|
|
687
|
-
expect(table.rows.map((r) => r.key).sort()).toEqual([
|
|
688
|
-
"exp/multi @ 2026-07-01T08:00:00.000Z",
|
|
689
|
-
"exp/multi @ 2026-07-02T08:00:00.000Z",
|
|
690
|
-
]);
|
|
691
|
-
});
|
|
692
|
-
|
|
693
|
-
it("占位行数据:missingEvalIds 来自 scope.coverage,不参与 evals/attempts 计数或任何指标聚合", async () => {
|
|
694
|
-
const s = snap({ experimentId: "exp/gap", results: [res("a", "passed"), res("b", "failed")] });
|
|
695
|
-
const scope = scopeOf([s], [], [{ experimentId: "exp/gap", run: s, knownEvalIds: ["a", "b", "c"], missingEvalIds: ["c"] }]);
|
|
696
|
-
const items = await experimentListData(scope);
|
|
697
|
-
expect(items[0]!.missingEvalIds).toEqual(["c"]);
|
|
698
|
-
// 占位行(c)不冒充有 attempt 的题:evals/attempts 分母仍是 2 道有 attempt 的题,不是 3。
|
|
699
|
-
expect(items[0]!.evals).toBe(2);
|
|
700
|
-
expect(items[0]!.attempts).toBe(2);
|
|
701
|
-
expect(items[0]!.evalRows.map((row) => row.evalId)).toEqual(["a", "b"]); // 占位行不在 evalRows 里(占位行只在渲染面合成)
|
|
702
|
-
});
|
|
703
|
-
|
|
704
|
-
it("coverage-only 实验也产生覆盖占位行;--fresh 清空全部 attempt 时缺口不再静默消失", async () => {
|
|
705
|
-
const anchor = snap({ experimentId: "exp/fresh", results: [] });
|
|
706
|
-
const scope = scopeOf([], [], [{ experimentId: "exp/fresh", run: anchor, knownEvalIds: ["a", "b"], missingEvalIds: ["a", "b"] }]);
|
|
707
|
-
const items = await experimentListData(scope);
|
|
708
|
-
expect(items).toHaveLength(1);
|
|
709
|
-
expect(items[0]).toMatchObject({
|
|
710
|
-
experimentId: "exp/fresh",
|
|
711
|
-
agent: "agent-x",
|
|
712
|
-
missingEvalIds: ["a", "b"],
|
|
713
|
-
evals: 0,
|
|
714
|
-
attempts: 0,
|
|
715
|
-
evalRows: [],
|
|
716
|
-
});
|
|
717
|
-
});
|
|
718
|
-
|
|
719
|
-
// ─────── 覆盖缺口的两档占位与过期结论参考(experiment-table.md「覆盖缺口的两档占位行」) ───────
|
|
720
|
-
|
|
721
|
-
describe("覆盖缺口的两档占位与过期结论参考", () => {
|
|
722
|
-
/**
|
|
723
|
-
* 区分力场景:evalId "b" 只在两份旧 configHash 快照下跑过(与当前基准不可比,过期结论候选
|
|
724
|
-
* 多于一条),"c" 从未在任何快照出现过(未跑到)。`current` 是当前基准,只贡献 "a"。
|
|
725
|
-
*/
|
|
726
|
-
function staleScope(opts: { fresh?: boolean } = {}): { scope: Sample; older: Run; newerStale: Run } {
|
|
727
|
-
const current = snap({ experimentId: "exp/stale", results: [res("a", "passed")] });
|
|
728
|
-
current.configHash = "cfg-2";
|
|
729
|
-
const older = snap({
|
|
730
|
-
experimentId: "exp/stale",
|
|
731
|
-
results: [res("b", "failed")],
|
|
732
|
-
runStartedAt: "2026-01-01T00:00:00.000Z",
|
|
733
|
-
});
|
|
734
|
-
older.configHash = "cfg-1";
|
|
735
|
-
const newerStale = snap({
|
|
736
|
-
experimentId: "exp/stale",
|
|
737
|
-
results: [res("b", "passed")],
|
|
738
|
-
runStartedAt: "2026-02-01T00:00:00.000Z",
|
|
739
|
-
});
|
|
740
|
-
newerStale.configHash = "cfg-1"; // 同样不可比,但比 older 晚出现——参考要取这一条
|
|
741
|
-
const coverage = [
|
|
742
|
-
{ experimentId: "exp/stale", run: current, knownEvalIds: ["a", "b", "c"], missingEvalIds: ["b", "c"] },
|
|
743
|
-
];
|
|
744
|
-
const scope = makeSample(
|
|
745
|
-
"current",
|
|
746
|
-
[current],
|
|
747
|
-
[...current.attempts],
|
|
748
|
-
[],
|
|
749
|
-
coverage,
|
|
750
|
-
opts.fresh ?? false,
|
|
751
|
-
[...current.attempts, ...older.attempts, ...newerStale.attempts],
|
|
752
|
-
);
|
|
753
|
-
return { scope, older, newerStale };
|
|
754
|
-
}
|
|
755
|
-
|
|
756
|
-
function recordCellOf(row: { cells: Readonly<globalThis.Record<string, Cell>> }): Cell {
|
|
757
|
-
return row.cells.record!;
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
it("两档占位行的结果格都是 missing 且都带补跑命令;只有过期结论的那档带 reference,取候选中最新的一条", async () => {
|
|
761
|
-
const { scope, newerStale } = staleScope();
|
|
762
|
-
const items = await experimentListData(scope);
|
|
763
|
-
const content = experimentListContent(items);
|
|
764
|
-
const sub = content.rows[0]!.subRows!;
|
|
765
|
-
const staleRow = sub.find((row) => row.key.endsWith("b:missing"))!;
|
|
766
|
-
const notRunRow = sub.find((row) => row.key.endsWith("c:missing"))!;
|
|
767
|
-
expect(staleRow.variant).toBe("placeholder");
|
|
768
|
-
expect(notRunRow.variant).toBe("placeholder");
|
|
769
|
-
|
|
770
|
-
const staleCell = recordCellOf(staleRow);
|
|
771
|
-
const notRunCell = recordCellOf(notRunRow);
|
|
772
|
-
if (staleCell.kind !== "missing" || notRunCell.kind !== "missing") throw new Error("expected missing cells");
|
|
773
|
-
expect(staleCell.code).toBe("noCurrentResult");
|
|
774
|
-
expect(notRunCell.code).toBe("noCurrentResult");
|
|
775
|
-
expect(staleCell.detail).toBe("niceeval exp exp/stale");
|
|
776
|
-
expect(notRunCell.detail).toBe("niceeval exp exp/stale");
|
|
777
|
-
expect(notRunCell.reference).toBeUndefined();
|
|
778
|
-
expect(staleCell.reference).toBeDefined();
|
|
779
|
-
// 两档必须给出不同的格——把两档折成同一种占位的实现在这里失败。
|
|
780
|
-
expect(staleCell).not.toEqual(notRunCell);
|
|
781
|
-
// 候选多于一条(older 与 newerStale 都不可比)时取最新那条:newerStale 的判定是 "passed"。
|
|
782
|
-
expect(staleCell.reference!.verdict).toBe("passed");
|
|
783
|
-
expect(staleCell.reference!.locator).toBe(
|
|
784
|
-
encodeAttemptLocator({ runId: newerStale.runId, evalId: "b", attempt: 0 }),
|
|
785
|
-
);
|
|
786
|
-
|
|
787
|
-
// text 面:带参考时用参考替代原因文案,不把「当前配置下无结果」再说一遍。
|
|
788
|
-
expect(formatCellText(staleCell, "en")).toMatch(/^✓ /);
|
|
789
|
-
expect(formatCellText(notRunCell, "en")).toBe("no result for current config · niceeval exp exp/stale");
|
|
790
|
-
expect(formatCellText(notRunCell, "zh-CN")).toBe("当前配置下无结果 · niceeval exp exp/stale");
|
|
791
|
-
});
|
|
792
|
-
|
|
793
|
-
it("reference 不进任何计数:带参考前后,该 experiment 的判定计票、通过率与覆盖分母逐字不变", async () => {
|
|
794
|
-
const { scope } = staleScope();
|
|
795
|
-
const [item] = await experimentListData(scope);
|
|
796
|
-
// 分母只数 "a" 这一道有 attempt 的题,"b" 的参考不进 evals/attempts/passRate 分母。
|
|
797
|
-
expect(item!.evals).toBe(1);
|
|
798
|
-
expect(item!.attempts).toBe(1);
|
|
799
|
-
expect(item!.endToEndPassRate).toMatchObject({ value: 1, samples: 1, total: 1 });
|
|
800
|
-
expect(item!.evalVerdicts).toEqual({ passed: 1, failed: 0, errored: 0, skipped: 0 });
|
|
801
|
-
});
|
|
802
|
-
|
|
803
|
-
it("sample.fresh 为 true 时两档占位行都不带 reference", async () => {
|
|
804
|
-
const nonFresh = experimentListContent(await experimentListData(staleScope({ fresh: false }).scope));
|
|
805
|
-
const fresh = experimentListContent(await experimentListData(staleScope({ fresh: true }).scope));
|
|
806
|
-
const staleCellNonFresh = recordCellOf(nonFresh.rows[0]!.subRows!.find((row) => row.key.endsWith("b:missing"))!);
|
|
807
|
-
const staleCellFresh = recordCellOf(fresh.rows[0]!.subRows!.find((row) => row.key.endsWith("b:missing"))!);
|
|
808
|
-
if (staleCellNonFresh.kind !== "missing" || staleCellFresh.kind !== "missing") throw new Error("expected missing cells");
|
|
809
|
-
expect(staleCellNonFresh.reference).toBeDefined();
|
|
810
|
-
expect(staleCellFresh.reference).toBeUndefined();
|
|
811
|
-
});
|
|
812
|
-
});
|
|
813
|
-
|
|
814
|
-
// ─────── 覆盖构成的四段与两面(experiment-table.md「覆盖构成」) ───────
|
|
815
|
-
|
|
816
|
-
describe("覆盖构成的四段与两面", () => {
|
|
817
|
-
/**
|
|
818
|
-
* 区分力场景:"retry" 重试两次,第一次携带(历史执行)、第二次新执行——按 attempt 计数的
|
|
819
|
-
* 错误实现会让合计超过题数;"solo" 只有历史执行;"stale" 只有过期结论;"gone" 未跑到。
|
|
820
|
-
*/
|
|
821
|
-
function compositionScope(): Sample {
|
|
822
|
-
const current = snap({
|
|
823
|
-
experimentId: "exp/composition",
|
|
824
|
-
results: [
|
|
825
|
-
res("retry", "passed", { artifactBase: "exp/composition/old/retry/a0" }), // 携带,attempt 0
|
|
826
|
-
res("retry", "passed", { attempt: 1 }), // 新执行,attempt 1
|
|
827
|
-
res("solo", "passed", { artifactBase: "exp/composition/old/solo/a0" }),
|
|
828
|
-
],
|
|
829
|
-
});
|
|
830
|
-
current.configHash = "cfg-current";
|
|
831
|
-
const staleRun = snap({ experimentId: "exp/composition", results: [res("stale", "failed")] });
|
|
832
|
-
staleRun.configHash = "cfg-old";
|
|
833
|
-
const coverage = [
|
|
834
|
-
{
|
|
835
|
-
experimentId: "exp/composition",
|
|
836
|
-
run: current,
|
|
837
|
-
knownEvalIds: ["retry", "solo", "stale", "gone"],
|
|
838
|
-
missingEvalIds: ["stale", "gone"],
|
|
839
|
-
},
|
|
840
|
-
];
|
|
841
|
-
return makeSample("current", [current], [...current.attempts], [], coverage, false, [
|
|
842
|
-
...current.attempts,
|
|
843
|
-
...staleRun.attempts,
|
|
844
|
-
]);
|
|
845
|
-
}
|
|
846
|
-
|
|
847
|
-
/** experiment 行的覆盖构成副行(subRows 末尾,key 以 `coverage:` 起头)的 record 格。 */
|
|
848
|
-
function coverageCellOf(content: ReturnType<typeof experimentListContent>, experimentId: string): Cell {
|
|
849
|
-
const row = content.rows[0]!.subRows!.find((r) => r.key === `${COVERAGE_ROW_PREFIX}${experimentId}`)!;
|
|
850
|
-
return row.cells.record!;
|
|
851
|
-
}
|
|
852
|
-
|
|
853
|
-
it("四段互斥且合计等于已知题数;一题同时有新执行与携带 attempt 时只落新执行段", async () => {
|
|
854
|
-
const [item] = await experimentListData(compositionScope());
|
|
855
|
-
const content = experimentListContent([item!]);
|
|
856
|
-
const coverageCell = coverageCellOf(content, "exp/composition");
|
|
857
|
-
if (coverageCell.kind !== "composition") throw new Error("expected composition cell");
|
|
858
|
-
const byLabel = new Map(
|
|
859
|
-
coverageCell.segments.map((segment) => [
|
|
860
|
-
typeof segment.label === "string" ? segment.label : segment.label.en,
|
|
861
|
-
segment.count,
|
|
862
|
-
]),
|
|
863
|
-
);
|
|
864
|
-
expect(byLabel.get("fresh")).toBe(1); // retry 只落新执行段,不因为它也有携带 attempt 而重复计
|
|
865
|
-
expect(byLabel.get("historical")).toBe(1); // solo
|
|
866
|
-
expect(byLabel.get("stale")).toBe(1); // stale
|
|
867
|
-
expect(byLabel.get("not run")).toBe(1); // gone
|
|
868
|
-
const total = [...coverageCell.segments].reduce((sum, s) => sum + s.count, 0);
|
|
869
|
-
expect(total).toBe(4); // 合计 = knownEvalIds 数,不是 attempts 数
|
|
870
|
-
});
|
|
871
|
-
|
|
872
|
-
it("计数为零的段不出现在两面输出里,与判定计票同一条规则", async () => {
|
|
873
|
-
const s = snap({ experimentId: "exp/all-fresh", results: [res("q", "passed")] });
|
|
874
|
-
const [item] = await experimentListData([s]);
|
|
875
|
-
const content = experimentListContent([item!]);
|
|
876
|
-
const coverageCell = coverageCellOf(content, "exp/all-fresh");
|
|
877
|
-
if (coverageCell.kind !== "composition") throw new Error("expected composition cell");
|
|
878
|
-
expect(coverageCell.segments.filter((s) => s.count > 0)).toHaveLength(1);
|
|
879
|
-
expect(formatCellText(coverageCell, "en")).toBe("1 fresh");
|
|
880
|
-
expect(formatCellText(coverageCell, "zh-CN")).toBe("1 新执行");
|
|
881
|
-
});
|
|
882
|
-
|
|
883
|
-
it("段名走 LocalizedText:zh-CN 与 en 取同一份 segments,只有文案不同,计数与段序逐字相同", async () => {
|
|
884
|
-
const [item] = await experimentListData(compositionScope());
|
|
885
|
-
const content = experimentListContent([item!]);
|
|
886
|
-
const coverageCell = coverageCellOf(content, "exp/composition");
|
|
887
|
-
if (coverageCell.kind !== "composition") throw new Error("expected composition cell");
|
|
888
|
-
const en = formatCellText(coverageCell, "en");
|
|
889
|
-
const zh = formatCellText(coverageCell, "zh-CN");
|
|
890
|
-
expect(en).toBe("1 fresh · 1 historical · 1 stale · 1 not run");
|
|
891
|
-
expect(zh).toBe("1 新执行 · 1 历史执行 · 1 过期结论 · 1 未跑到");
|
|
892
|
-
expect(coverageCell.segments.map((s) => s.count)).toEqual([1, 1, 1, 1]);
|
|
893
|
-
});
|
|
894
|
-
|
|
895
|
-
it("构成格不携带业务语义:它是 experiment 副行独有的格,换一套无关段名照常渲染,渲染面不认识段含义", async () => {
|
|
896
|
-
const [item] = await experimentListData(compositionScope());
|
|
897
|
-
const content = experimentListContent([item!]);
|
|
898
|
-
// 副行紧跟在 Eval / 组行之后,key 前缀把它与题目行区分开——它不是又一道 "retry" 之类的题。
|
|
899
|
-
const rowKeys = content.rows[0]!.subRows!.map((r) => r.key);
|
|
900
|
-
expect(rowKeys.at(-1)).toBe(`${COVERAGE_ROW_PREFIX}exp/composition`);
|
|
901
|
-
expect(rowKeys).toContain("retry");
|
|
902
|
-
|
|
903
|
-
const arbitrarySegments: Cell = {
|
|
904
|
-
kind: "composition",
|
|
905
|
-
segments: [{ label: "x", count: 2 }, { label: "y", count: 0 }],
|
|
906
|
-
};
|
|
907
|
-
expect(formatCellText(arbitrarySegments)).toBe("2 x");
|
|
908
|
-
});
|
|
909
|
-
});
|
|
910
|
-
|
|
911
|
-
// ─────── 计分制字段:evaluationKind 定义期投影 / totalScore 的两级聚合方向 / 预排 ───────
|
|
912
|
-
|
|
913
|
-
it('ExperimentListItem.evaluationKind 是定义期事实投影,不从 attempt 判定推断:failed 的计分制 eval 仍读 "points",passed 的通过制 eval 仍读 "pass"', async () => {
|
|
914
|
-
const pointsFailed = snap({ experimentId: "exam/points-fail", results: [res("a", "failed", { evaluationKind: "points" })] });
|
|
915
|
-
const plainPassed = snap({ experimentId: "exam/plain-pass", results: [res("a", "passed")] });
|
|
916
|
-
const items = await experimentListData([pointsFailed, plainPassed]);
|
|
917
|
-
const byId = new Map(items.map((item) => [item.experimentId, item]));
|
|
918
|
-
expect(byId.get("exam/points-fail")!.evaluationKind).toBe("points");
|
|
919
|
-
expect(byId.get("exam/plain-pass")!.evaluationKind).toBe("pass");
|
|
920
|
-
});
|
|
921
|
-
|
|
922
|
-
it("单一 Experiment 混型:题型为 mixed,通过率只读 pass Eval,总分只读 points Eval", async () => {
|
|
923
|
-
const mixed = snap({
|
|
924
|
-
experimentId: "exam/mixed-one",
|
|
925
|
-
results: [
|
|
926
|
-
res("plain", "failed"),
|
|
927
|
-
res("score", "passed", { evaluationKind: "points", assertions: [pointsAssertion("x", 5)] }),
|
|
928
|
-
],
|
|
929
|
-
});
|
|
930
|
-
const [item] = await experimentListData([mixed]);
|
|
931
|
-
expect(item!.evaluationKind).toBe("mixed");
|
|
932
|
-
expect(item!.endToEndPassRate.value).toBe(0);
|
|
933
|
-
expect(item!.endToEndPassRate.total).toBe(1);
|
|
934
|
-
expect(item!.totalScore.value).toBe(5);
|
|
935
|
-
expect(item!.evalRows.map((row) => [row.evalId, row.evaluationKind])).toEqual([
|
|
936
|
-
["plain", "pass"],
|
|
937
|
-
["score", "points"],
|
|
938
|
-
]);
|
|
939
|
-
});
|
|
940
|
-
|
|
941
|
-
it("ExperimentListItem.totalScore 是跨题 sum,不是 mean:3 分 + 5 分 = 8,不是 4", async () => {
|
|
942
|
-
const s = snap({
|
|
943
|
-
experimentId: "exam/multi",
|
|
944
|
-
results: [
|
|
945
|
-
res("q1", "passed", { evaluationKind: "points", assertions: [pointsAssertion("x", 3)] }),
|
|
946
|
-
res("q2", "passed", { evaluationKind: "points", assertions: [pointsAssertion("x", 5)] }),
|
|
947
|
-
],
|
|
948
|
-
});
|
|
949
|
-
const items = await experimentListData([s]);
|
|
950
|
-
expect(items[0]!.totalScore.value).toBe(8);
|
|
951
|
-
expect(items[0]!.totalScore.value).not.toBe(4); // 区分力:sum ≠ mean,同分 fixture 证明不了聚合方向
|
|
952
|
-
});
|
|
953
|
-
|
|
954
|
-
it("EvalListItem.totalScore / ExperimentListEvalRow.totalScore 是题内多轮的 mean,不是 sum:4 分与 2 分平均 3,不是 6", async () => {
|
|
955
|
-
const s = snap({
|
|
956
|
-
experimentId: "exam/retry",
|
|
957
|
-
results: [
|
|
958
|
-
res("q1", "passed", { attempt: 0, evaluationKind: "points", assertions: [pointsAssertion("x", 4)] }),
|
|
959
|
-
res("q1", "passed", { attempt: 1, evaluationKind: "points", assertions: [pointsAssertion("x", 2)] }),
|
|
960
|
-
],
|
|
961
|
-
});
|
|
962
|
-
const evalItems = await evalListData([s]);
|
|
963
|
-
expect(evalItems[0]!.totalScore.value).toBe(3);
|
|
964
|
-
expect(evalItems[0]!.totalScore.value).not.toBe(6); // 区分力:mean ≠ sum
|
|
965
|
-
|
|
966
|
-
const expItems = await experimentListData([s]);
|
|
967
|
-
expect(expItems[0]!.evalRows[0]!.totalScore.value).toBe(3);
|
|
968
|
-
});
|
|
969
|
-
|
|
970
|
-
it("通过制 experiment:ExperimentListItem.totalScore 为 null cell,同一行 passRate 仍是良态数字,二者并存不互斥", async () => {
|
|
971
|
-
const s = snap({ experimentId: "exam/passonly", results: [res("a", "passed"), res("b", "failed")] });
|
|
972
|
-
const items = await experimentListData([s]);
|
|
973
|
-
expect(items[0]!.totalScore.value).toBeNull();
|
|
974
|
-
expect(items[0]!.endToEndPassRate.value).toBe(0.5);
|
|
975
|
-
});
|
|
976
|
-
|
|
977
|
-
it("默认预排:纯计分制列表按 totalScore 降序,不是 passRate(两个 experiment 端到端通过率同为 1,只有总分能分出高低)", async () => {
|
|
978
|
-
const low = snap({
|
|
979
|
-
experimentId: "exam/a-lowscore",
|
|
980
|
-
results: [res("q1", "passed", { evaluationKind: "points", assertions: [pointsAssertion("x", 3)] })],
|
|
981
|
-
});
|
|
982
|
-
const high = snap({
|
|
983
|
-
experimentId: "exam/z-highscore",
|
|
984
|
-
results: [res("q1", "passed", { evaluationKind: "points", assertions: [pointsAssertion("x", 9)] })],
|
|
985
|
-
});
|
|
986
|
-
const items = await experimentListData([low, high]);
|
|
987
|
-
expect(items.map((item) => item.experimentId)).toEqual(["exam/z-highscore", "exam/a-lowscore"]);
|
|
988
|
-
});
|
|
989
|
-
|
|
990
|
-
it("默认预排:pass 与 points 混型退回 experiment id 字典序,两种读数不能互相排名", async () => {
|
|
991
|
-
const pointsExp = snap({
|
|
992
|
-
experimentId: "exam/z-points",
|
|
993
|
-
results: [res("q1", "passed", { evaluationKind: "points", assertions: [pointsAssertion("x", 100)] })],
|
|
994
|
-
});
|
|
995
|
-
const passExp = snap({ experimentId: "exam/a-pass", results: [res("q1", "passed")] });
|
|
996
|
-
const items = await experimentListData([pointsExp, passExp]);
|
|
997
|
-
// 若混型误走「只要有 points 行就按总分排」的分支,总分良态数字的 points 行会排到
|
|
998
|
-
// totalScore=null(沉底)的 pass 行前面,顺序会是 [z-points, a-pass];正确的混型退回
|
|
999
|
-
// 字典序应是 [a-pass, z-points]。
|
|
1000
|
-
expect(items.map((item) => item.experimentId)).toEqual(["exam/a-pass", "exam/z-points"]);
|
|
1001
|
-
});
|
|
1002
|
-
});
|
|
1003
|
-
|
|
1004
|
-
// ───────────────────────── sampleSummary ─────────────────────────
|
|
1005
|
-
|
|
1006
|
-
describe("sampleSummary", () => {
|
|
1007
|
-
it("evals 按 experimentId + evalId 计数(2 实验 × 6 题 = 12),与 evalVerdicts 同分母;两级计票在含重试时不同", async () => {
|
|
1008
|
-
const mk = (experimentId: string) =>
|
|
1009
|
-
snap({
|
|
1010
|
-
experimentId,
|
|
1011
|
-
results: [
|
|
1012
|
-
res("q1", "passed"),
|
|
1013
|
-
res("q2", "failed", { attempt: 0 }),
|
|
1014
|
-
res("q2", "passed", { attempt: 1 }), // 重试后过:eval 级 passed,attempt 级 1 failed + 1 passed
|
|
1015
|
-
res("q3", "passed"),
|
|
1016
|
-
res("q4", "failed"),
|
|
1017
|
-
res("q5", "errored", { error: erroredWith("x") }),
|
|
1018
|
-
res("q6", "skipped"),
|
|
1019
|
-
],
|
|
1020
|
-
});
|
|
1021
|
-
const data = await sampleSummary([mk("cmp/a"), mk("cmp/b")]);
|
|
1022
|
-
expect(data.experiments).toBe(2);
|
|
1023
|
-
expect(data.evals).toBe(12);
|
|
1024
|
-
expect(data.evalVerdicts).toEqual({ passed: 6, failed: 2, errored: 2, skipped: 2 });
|
|
1025
|
-
expect(
|
|
1026
|
-
data.evalVerdicts.passed + data.evalVerdicts.failed + data.evalVerdicts.errored + data.evalVerdicts.skipped,
|
|
1027
|
-
).toBe(data.evals);
|
|
1028
|
-
expect(data.attemptVerdicts).toEqual({ passed: 6, failed: 4, errored: 2, skipped: 2 });
|
|
1029
|
-
expect(data.attemptVerdicts).not.toEqual(data.evalVerdicts);
|
|
1030
|
-
expect(data.range.earliestStartedAt).not.toBeNull();
|
|
1031
|
-
expect(data.range.latestStartedAt).not.toBeNull();
|
|
1032
|
-
});
|
|
1033
|
-
|
|
1034
|
-
it("totalCostUSD 按 attempt 求和;一次成本都没报时 value 为 null,不伪造 0", async () => {
|
|
1035
|
-
const withCost = snap({
|
|
1036
|
-
experimentId: "exp/cost",
|
|
1037
|
-
results: [
|
|
1038
|
-
res("a", "passed", { usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.25 } }),
|
|
1039
|
-
res("b", "failed", { estimatedCostUSD: 0.05 }),
|
|
1040
|
-
res("c", "passed"),
|
|
1041
|
-
],
|
|
1042
|
-
});
|
|
1043
|
-
const data = await sampleSummary([withCost]);
|
|
1044
|
-
expect(data.totalCostUSD.value).toBeCloseTo(0.3);
|
|
1045
|
-
const none = await sampleSummary([snap({ experimentId: "exp/free", results: [res("a", "passed")] })]);
|
|
1046
|
-
expect(none.totalCostUSD.value).toBeNull();
|
|
1047
|
-
});
|
|
1048
|
-
|
|
1049
|
-
it("空范围的 range 为 null,不编造当前时间", async () => {
|
|
1050
|
-
const data = await sampleSummary([]);
|
|
1051
|
-
expect(data.range).toEqual({ earliestStartedAt: null, latestStartedAt: null });
|
|
1052
|
-
expect(data.evals).toBe(0);
|
|
1053
|
-
});
|
|
1054
|
-
|
|
1055
|
-
it("evaluationKindComposition 与 totalScore:纯通过制 Sample 省略 totalScore(不摆空列)", async () => {
|
|
1056
|
-
const data = await sampleSummary([snap({ experimentId: "exp/pass", results: [res("a", "passed")] })]);
|
|
1057
|
-
expect(data.evaluationKindComposition).toBe("pass");
|
|
1058
|
-
expect(data.totalScore).toBeUndefined();
|
|
1059
|
-
});
|
|
1060
|
-
|
|
1061
|
-
it("evaluationKindComposition 与 totalScore:纯计分制 Sample 携带 totalScore", async () => {
|
|
1062
|
-
const s = snap({
|
|
1063
|
-
experimentId: "exp/points",
|
|
1064
|
-
results: [res("a", "passed", { evaluationKind: "points", assertions: [pointsAssertion("x", 3)] })],
|
|
1065
|
-
});
|
|
1066
|
-
const data = await sampleSummary([s]);
|
|
1067
|
-
expect(data.evaluationKindComposition).toBe("points");
|
|
1068
|
-
expect(data.totalScore?.value).toBe(3);
|
|
1069
|
-
});
|
|
1070
|
-
|
|
1071
|
-
it("evaluationKindComposition:一个 Sample 里并排通过制与计分制两个 experiment → \"mixed\"", async () => {
|
|
1072
|
-
const passExp = snap({ experimentId: "exp/a-pass", results: [res("a", "passed")] });
|
|
1073
|
-
const pointsExp = snap({
|
|
1074
|
-
experimentId: "exp/b-points",
|
|
1075
|
-
results: [res("b", "passed", { evaluationKind: "points", assertions: [pointsAssertion("x", 5)] })],
|
|
1076
|
-
});
|
|
1077
|
-
const data = await sampleSummary([passExp, pointsExp]);
|
|
1078
|
-
expect(data.evaluationKindComposition).toBe("mixed");
|
|
1079
|
-
expect(data.totalScore).toBeDefined();
|
|
1080
|
-
});
|
|
1081
|
-
|
|
1082
|
-
it("evaluationKindComposition:同一 experiment 混型时通过率不被 points Eval 稀释", async () => {
|
|
1083
|
-
const mixed = snap({
|
|
1084
|
-
experimentId: "exp/mixed-one",
|
|
1085
|
-
results: [
|
|
1086
|
-
res("plain", "failed"),
|
|
1087
|
-
res("score", "passed", { evaluationKind: "points", assertions: [pointsAssertion("x", 5)] }),
|
|
1088
|
-
],
|
|
1089
|
-
});
|
|
1090
|
-
const data = await sampleSummary([mixed]);
|
|
1091
|
-
expect(data.evaluationKindComposition).toBe("mixed");
|
|
1092
|
-
expect(data.endToEndPassRate.value).toBe(0);
|
|
1093
|
-
expect(data.endToEndPassRate.total).toBe(1);
|
|
1094
|
-
expect(data.totalScore?.value).toBe(5);
|
|
1095
|
-
});
|
|
1096
|
-
|
|
1097
|
-
it("evaluationKindComposition() 与 SampleSummaryContent.evaluationKindComposition 同一 fixture 下一致(同规则同值,不各自重复判据)", async () => {
|
|
1098
|
-
const passInput = [snap({ experimentId: "exp/agree-pass", results: [res("a", "passed")] })];
|
|
1099
|
-
expect(await evaluationKindComposition(passInput)).toBe((await sampleSummary(passInput)).evaluationKindComposition);
|
|
1100
|
-
|
|
1101
|
-
const pointsInput = [
|
|
1102
|
-
snap({
|
|
1103
|
-
experimentId: "exp/agree-points",
|
|
1104
|
-
results: [res("a", "passed", { evaluationKind: "points", assertions: [pointsAssertion("x", 3)] })],
|
|
1105
|
-
}),
|
|
1106
|
-
];
|
|
1107
|
-
expect(await evaluationKindComposition(pointsInput)).toBe((await sampleSummary(pointsInput)).evaluationKindComposition);
|
|
1108
|
-
|
|
1109
|
-
// 混型:一个通过制 experiment 快照 + 一个计分制 experiment 快照并排——最可能出现
|
|
1110
|
-
// 复制粘贴分叉的形态。
|
|
1111
|
-
const mixedInput = [
|
|
1112
|
-
snap({ experimentId: "exp/agree-a-pass", results: [res("a", "passed")] }),
|
|
1113
|
-
snap({
|
|
1114
|
-
experimentId: "exp/agree-b-points",
|
|
1115
|
-
results: [res("b", "passed", { evaluationKind: "points", assertions: [pointsAssertion("x", 5)] })],
|
|
1116
|
-
}),
|
|
1117
|
-
];
|
|
1118
|
-
expect(await evaluationKindComposition(mixedInput)).toBe((await sampleSummary(mixedInput)).evaluationKindComposition);
|
|
1119
|
-
});
|
|
1120
|
-
});
|
|
1121
|
-
|
|
1122
|
-
// ───────────────────────── experimentListData / sampleSummary 的 selectedEvalIds 投影 ─────────────────────────
|
|
1123
|
-
// SampleOverview 是普通组合组件(把同一个 input 原样透传给 SampleSummary /
|
|
1124
|
-
// ExperimentScatter / ExperimentTable,不再有自己的 data 形态);经它展开后与直接调用这三个
|
|
1125
|
-
// 函数深等的验证挪到 dual-render.test.tsx(需要 resolve 管线)。这里只测三个函数自己的
|
|
1126
|
-
// selectedEvalIds 投影契约。
|
|
1127
|
-
|
|
1128
|
-
describe("experimentListData / sampleSummary 的 selectedEvalIds 投影", () => {
|
|
1129
|
-
it("不同深度目录的 experiments 一律进同一份 data,不按父路径分组比较", async () => {
|
|
1130
|
-
const g1a = snap({ experimentId: "compare/a", agent: "bub", results: [res("q", "passed")] });
|
|
1131
|
-
const g1b = snap({ experimentId: "compare/b", agent: "codex", results: [res("q", "failed")] });
|
|
1132
|
-
const g2 = snap({ experimentId: "bench/long/x", results: [res("q", "passed")] });
|
|
1133
|
-
const solo = snap({ experimentId: "standalone", results: [res("q", "errored", { error: erroredWith("x") })] });
|
|
1134
|
-
const items = await experimentListData([g1a, g1b, g2, solo]);
|
|
1135
|
-
expect(items.map((e) => e.experimentId).sort()).toEqual([
|
|
1136
|
-
"bench/long/x",
|
|
1137
|
-
"compare/a",
|
|
1138
|
-
"compare/b",
|
|
1139
|
-
"standalone",
|
|
1140
|
-
]);
|
|
1141
|
-
});
|
|
1142
|
-
|
|
1143
|
-
it("每个 experiment 只保留自己选择的 eval:A 声明 selectedEvalIds:[q1] 但夹带 q2 attempt,B 只选 q2;q2 不污染 A", async () => {
|
|
1144
|
-
const a = snap({
|
|
1145
|
-
experimentId: "exp/a",
|
|
1146
|
-
results: [res("q1", "passed"), res("q2", "failed")],
|
|
1147
|
-
experiment: { attempts: 1, earlyExit: false, selectedEvalIds: ["q1"] },
|
|
1148
|
-
});
|
|
1149
|
-
const b = snap({
|
|
1150
|
-
experimentId: "exp/b",
|
|
1151
|
-
results: [res("q2", "passed")],
|
|
1152
|
-
experiment: { attempts: 1, earlyExit: false, selectedEvalIds: ["q2"] },
|
|
1153
|
-
});
|
|
1154
|
-
|
|
1155
|
-
const items = await experimentListData([a, b]);
|
|
1156
|
-
const rowA = items.find((e) => e.experimentId === "exp/a")!;
|
|
1157
|
-
const rowB = items.find((e) => e.experimentId === "exp/b")!;
|
|
1158
|
-
expect(rowA.evals).toBe(1); // 只统计 q1,夹带的 q2 attempt 不计入
|
|
1159
|
-
expect(rowA.evalRows.map((r) => r.evalId)).toEqual(["q1"]);
|
|
1160
|
-
expect(rowB.evals).toBe(1);
|
|
1161
|
-
expect(rowB.evalRows.map((r) => r.evalId)).toEqual(["q2"]);
|
|
1162
|
-
|
|
1163
|
-
// 汇总口径同样不被污染:全 Sample 只有 2 个 eval(exp/a·q1、exp/b·q2),不是 3 个。
|
|
1164
|
-
const summary = await sampleSummary([a, b]);
|
|
1165
|
-
expect(summary.evals).toBe(2);
|
|
1166
|
-
});
|
|
1167
|
-
|
|
1168
|
-
it("第三方快照缺 experiment 信息时仍可见,按其实际 evals 参与(selectedEvalIdsOf 退化)", async () => {
|
|
1169
|
-
const thirdParty = snap({ experimentId: "third-party/exp", results: [res("q", "passed")] });
|
|
1170
|
-
expect(thirdParty.experiment).toBeUndefined();
|
|
1171
|
-
|
|
1172
|
-
const items = await experimentListData([thirdParty]);
|
|
1173
|
-
expect(items.map((e) => e.experimentId)).toEqual(["third-party/exp"]);
|
|
1174
|
-
expect(items[0]!.evals).toBe(1);
|
|
1175
|
-
});
|
|
1176
|
-
});
|
|
1177
|
-
|
|
1178
|
-
// ───────────────────────── metricMatrixData / measureRowsData ─────────────────────────
|
|
1179
|
-
|
|
1180
|
-
describe("metricMatrixData / measureRowsData", () => {
|
|
1181
|
-
it("矩阵稀疏:无 attempt 的组合不生成格子", async () => {
|
|
1182
|
-
const withCost = snap({
|
|
1183
|
-
experimentId: "cmp/priced",
|
|
1184
|
-
agent: "bub",
|
|
1185
|
-
results: [res("a", "passed", { usage: { inputTokens: 1, outputTokens: 1, costUSD: 0.2 } })],
|
|
1186
|
-
});
|
|
1187
|
-
const noCost = snap({ experimentId: "cmp/free", agent: "codex", results: [res("b", "passed")] });
|
|
1188
|
-
const matrix = await metricMatrixData([withCost, noCost], {
|
|
1189
|
-
rows: "eval",
|
|
1190
|
-
columns: "agent",
|
|
1191
|
-
cell: passRate,
|
|
1192
|
-
});
|
|
1193
|
-
// a×codex、b×bub 没有样本 → 不出现(不是 value: 0)
|
|
1194
|
-
expect(matrix.cells).toHaveLength(2);
|
|
1195
|
-
expect(matrix.rowDimension).toBe("eval");
|
|
1196
|
-
expect(matrix.columnDimension).toBe("agent");
|
|
1197
|
-
});
|
|
1198
|
-
|
|
1199
|
-
it("分组维度上未声明的 flag 归 (missing) 组,不丢行", async () => {
|
|
1200
|
-
const withFlag = snap({
|
|
1201
|
-
experimentId: "f/on",
|
|
1202
|
-
results: [res("a", "passed")],
|
|
1203
|
-
experiment: { attempts: 1, earlyExit: false, selectedEvalIds: [], flags: { memory: "mempal" } },
|
|
1204
|
-
});
|
|
1205
|
-
const withoutFlag = snap({ experimentId: "f/off", results: [res("a", "failed")] });
|
|
1206
|
-
const table = await measureRowsData([withFlag, withoutFlag], {
|
|
1207
|
-
dimensions: [flag("memory")],
|
|
1208
|
-
measures: [passRate],
|
|
1209
|
-
});
|
|
1210
|
-
expect(table.rows.map((r) => r.key).sort()).toEqual(["(missing)", "mempal"]);
|
|
1211
|
-
});
|
|
1212
|
-
|
|
1213
|
-
it("measureRowsData sort:必须是 columns 中同一实例且声明 better;方向随 better,缺数据沉底", async () => {
|
|
1214
|
-
const hi = snap({ experimentId: "s/hi", results: [res("a", "passed")] });
|
|
1215
|
-
const lo = snap({ experimentId: "s/lo", results: [res("a", "failed")] });
|
|
1216
|
-
const na = snap({ experimentId: "s/na", results: [res("a", "skipped")] });
|
|
1217
|
-
const byPass = await measureRowsData([lo, hi, na], {
|
|
1218
|
-
dimensions: ["experiment"],
|
|
1219
|
-
measures: [passRate],
|
|
1220
|
-
sort: passRate,
|
|
1221
|
-
});
|
|
1222
|
-
expect(byPass.rows.map((r) => r.key)).toEqual(["s/hi", "s/lo", "s/na"]);
|
|
1223
|
-
|
|
1224
|
-
const fast = snap({ experimentId: "d/fast", results: [res("a", "passed", { durationMs: 10 })] });
|
|
1225
|
-
const slow = snap({ experimentId: "d/slow", results: [res("a", "passed", { durationMs: 99 })] });
|
|
1226
|
-
const byDuration = await measureRowsData([slow, fast], {
|
|
1227
|
-
dimensions: ["experiment"],
|
|
1228
|
-
measures: [durationMs],
|
|
1229
|
-
sort: durationMs,
|
|
1230
|
-
});
|
|
1231
|
-
expect(byDuration.rows.map((r) => r.key)).toEqual(["d/fast", "d/slow"]); // lower better:低在前
|
|
1232
|
-
|
|
1233
|
-
await expect(
|
|
1234
|
-
measureRowsData([hi], { dimensions: ["experiment"], measures: [passRate], sort: durationMs }),
|
|
1235
|
-
).rejects.toThrow(/measures/);
|
|
1236
|
-
const noBetter: AttemptMetric = { name: "plain", value: () => 1 };
|
|
1237
|
-
await expect(
|
|
1238
|
-
measureRowsData([hi], { dimensions: ["experiment"], measures: [noBetter], sort: noBetter }),
|
|
1239
|
-
).rejects.toThrow(/better/);
|
|
1240
|
-
});
|
|
1241
|
-
|
|
1242
|
-
it("省略 sort 时按行 key 字典序(维度 domain 稳定序,不随文件扫描顺序)", async () => {
|
|
1243
|
-
const b = snap({ experimentId: "o/bbb", results: [res("a", "passed")] });
|
|
1244
|
-
const a = snap({ experimentId: "o/aaa", results: [res("a", "failed")] });
|
|
1245
|
-
const table = await measureRowsData([b, a], { dimensions: ["experiment"], measures: [passRate] });
|
|
1246
|
-
expect(table.rows.map((r) => r.key)).toEqual(["o/aaa", "o/bbb"]);
|
|
1247
|
-
});
|
|
1248
|
-
});
|
|
1249
|
-
|
|
1250
|
-
// ───────────────────────── metricLineData ─────────────────────────
|
|
1251
|
-
|
|
1252
|
-
describe("metricLineData", () => {
|
|
1253
|
-
const flaggedSnap = (experimentId: string, budget: number | undefined, verdicts: Verdict[]) =>
|
|
1254
|
-
snap({
|
|
1255
|
-
experimentId,
|
|
1256
|
-
results: verdicts.map((v, i) => res(`q${i}`, v)),
|
|
1257
|
-
experiment: {
|
|
1258
|
-
attempts: 1,
|
|
1259
|
-
earlyExit: false,
|
|
1260
|
-
selectedEvalIds: [],
|
|
1261
|
-
...(budget !== undefined ? { flags: { budget } } : {}),
|
|
1262
|
-
},
|
|
1263
|
-
});
|
|
1264
|
-
|
|
1265
|
-
it("未声明数值 flag 的 experiment 不伪造 x 值(不落到 x=0)并可数", async () => {
|
|
1266
|
-
const data = await metricLineData(
|
|
1267
|
-
[flaggedSnap("l/100", 100, ["passed"]), flaggedSnap("l/none", undefined, ["passed"])],
|
|
1268
|
-
{ x: numericFlag("budget"), y: passRate },
|
|
1269
|
-
);
|
|
1270
|
-
const missing = data.rows.filter((r) => r.x === null);
|
|
1271
|
-
expect(missing).toHaveLength(1);
|
|
1272
|
-
expect(data.rows.some((r) => r.x === 0)).toBe(false);
|
|
1273
|
-
});
|
|
1274
|
-
|
|
1275
|
-
it("点身份 = (series, x):同桶多 experiment 合成一个点,y 按 (series, x, experiment, eval) 顺序聚合", async () => {
|
|
1276
|
-
// 两个 experiment 同 x=100:各 1 题,一个 passed 一个 failed → 合成一点 y = (1+0)/2
|
|
1277
|
-
const data = await metricLineData(
|
|
1278
|
-
[flaggedSnap("m/one", 100, ["passed"]), flaggedSnap("m/two", 100, ["failed"])],
|
|
1279
|
-
{ x: numericFlag("budget"), y: passRate },
|
|
1280
|
-
);
|
|
1281
|
-
expect(data.rows).toHaveLength(1);
|
|
1282
|
-
expect(data.rows[0]!.key).toBe("100"); // x 的稳定十进制字符串
|
|
1283
|
-
expect(data.rows[0]!.x).toBe(100);
|
|
1284
|
-
expect(data.rows[0]!.y.value).toBeCloseTo(0.5);
|
|
1285
|
-
});
|
|
1286
|
-
|
|
1287
|
-
it("自定义 NumericAxis.of 在同一 experiment × eval 内不恒定时报完整用户反馈,不静默取首值", async () => {
|
|
1288
|
-
const s = snap({
|
|
1289
|
-
experimentId: "l/vary",
|
|
1290
|
-
results: [res("q", "passed", { attempt: 0, durationMs: 10 }), res("q", "passed", { attempt: 1, durationMs: 20 })],
|
|
1291
|
-
});
|
|
1292
|
-
const perAttempt = {
|
|
1293
|
-
name: "per-attempt",
|
|
1294
|
-
of: (attempt: AttemptHandle) => attempt.result.durationMs,
|
|
1295
|
-
};
|
|
1296
|
-
await expect(metricLineData([s], { x: perAttempt, y: passRate })).rejects.toThrow(/not constant/);
|
|
1297
|
-
});
|
|
1298
|
-
});
|
|
1299
|
-
|
|
1300
|
-
// deltaTableData(对照矩阵)与 stabilityMatrixData(稳定性矩阵)的测试住
|
|
1301
|
-
// src/report/slices/delta-table.test.ts 与 stability-matrix.test.ts
|
|
1302
|
-
// (专用组件测试文件,不挤共享 compute.test.ts)。
|
|
1303
|
-
|
|
1304
|
-
// ───────────────────────── examScore ─────────────────────────
|
|
1305
|
-
|
|
1306
|
-
describe("examScore", () => {
|
|
1307
|
-
it("gate 决定能否得分,soft 给质量分;errored 交白卷是 0 分不是缺数据", async () => {
|
|
1308
|
-
const s = snap({
|
|
1309
|
-
experimentId: "exam/x",
|
|
1310
|
-
results: [
|
|
1311
|
-
res("soft", "passed", { assertions: [softAssertion("a", 0.5), softAssertion("b", 1)] }),
|
|
1312
|
-
res("allgate", "passed"),
|
|
1313
|
-
res("crashed", "errored", { error: erroredWith("boom") }),
|
|
1314
|
-
],
|
|
1315
|
-
});
|
|
1316
|
-
const table = await measureRowsData([s], { dimensions: ["eval"], measures: [examScore] });
|
|
1317
|
-
const cellOf = (key: string) => metricCell(table.rows.find((r) => r.key === key)!, examScore.name);
|
|
1318
|
-
expect(cellOf("soft").value).toBeCloseTo(0.75);
|
|
1319
|
-
expect(cellOf("allgate").value).toBe(1);
|
|
1320
|
-
expect(cellOf("crashed").value).toBe(0);
|
|
1321
|
-
expect(cellOf("crashed").samples).toBe(1); // 0 分是测得的事实,不是缺数据
|
|
1322
|
-
});
|
|
1323
|
-
});
|
|
1324
|
-
|
|
1325
|
-
// ───────────────────────── totalScore ─────────────────────────
|
|
1326
|
-
|
|
1327
|
-
describe("totalScore", () => {
|
|
1328
|
-
it("计分制 eval:assertions[].points 之和 + scoreEntries[].points 之和,纯累加", async () => {
|
|
1329
|
-
const s = snap({
|
|
1330
|
-
experimentId: "score/x",
|
|
1331
|
-
results: [
|
|
1332
|
-
res("checkpoints", "passed", {
|
|
1333
|
-
evaluationKind: "points",
|
|
1334
|
-
assertions: [pointsAssertion("a", 1), pointsAssertion("b", 1), pointsAssertion("c", 0)],
|
|
1335
|
-
scoreEntries: [{ label: "代码精简", points: 15 }],
|
|
1336
|
-
}),
|
|
1337
|
-
],
|
|
1338
|
-
});
|
|
1339
|
-
const table = await measureRowsData([s], { dimensions: ["eval"], measures: [totalScore] });
|
|
1340
|
-
const cellOf = (key: string) => metricCell(table.rows.find((r) => r.key === key)!, totalScore.name);
|
|
1341
|
-
expect(cellOf("checkpoints").value).toBe(17); // 1 + 1 + 0 + 15
|
|
1342
|
-
});
|
|
1343
|
-
|
|
1344
|
-
it("errored 记 null(基础设施得 null,不折成 0),不进分母", async () => {
|
|
1345
|
-
const s = snap({
|
|
1346
|
-
experimentId: "score/errored",
|
|
1347
|
-
results: [res("crashed", "errored", { evaluationKind: "points", error: erroredWith("boom") })],
|
|
1348
|
-
});
|
|
1349
|
-
const table = await measureRowsData([s], { dimensions: ["eval"], measures: [totalScore] });
|
|
1350
|
-
const cell = metricCell(table.rows.find((r) => r.key === "crashed")!, totalScore.name);
|
|
1351
|
-
expect(cell.value).toBeNull();
|
|
1352
|
-
expect(cell.samples).toBe(0);
|
|
1353
|
-
});
|
|
1354
|
-
|
|
1355
|
-
it("skipped 记 null", async () => {
|
|
1356
|
-
const s = snap({
|
|
1357
|
-
experimentId: "score/skipped",
|
|
1358
|
-
results: [res("skip", "skipped", { evaluationKind: "points", skipReason: "not applicable" })],
|
|
1359
|
-
});
|
|
1360
|
-
const table = await measureRowsData([s], { dimensions: ["eval"], measures: [totalScore] });
|
|
1361
|
-
expect(metricCell(table.rows.find((r) => r.key === "skip")!, totalScore.name).value).toBeNull();
|
|
1362
|
-
});
|
|
1363
|
-
|
|
1364
|
-
it("failed(gate 挂了)仍照实求和已挣到的分——中止挣 0 靠 test() 控制流,不靠这个指标折成 0", async () => {
|
|
1365
|
-
const s = snap({
|
|
1366
|
-
experimentId: "score/failed",
|
|
1367
|
-
results: [
|
|
1368
|
-
res("partial", "failed", {
|
|
1369
|
-
evaluationKind: "points",
|
|
1370
|
-
assertions: [
|
|
1371
|
-
pointsAssertion("step1", 1),
|
|
1372
|
-
pointsAssertion("step2", 0, { outcome: "failed" as const, severity: "gate" }),
|
|
1373
|
-
],
|
|
1374
|
-
}),
|
|
1375
|
-
],
|
|
1376
|
-
});
|
|
1377
|
-
const table = await measureRowsData([s], { dimensions: ["eval"], measures: [totalScore] });
|
|
1378
|
-
expect(metricCell(table.rows.find((r) => r.key === "partial")!, totalScore.name).value).toBe(1);
|
|
1379
|
-
});
|
|
1380
|
-
|
|
1381
|
-
it("通过制 eval(evaluationKind 省略或 \"pass\"):恒 null,不参与聚合——两种题型天然不互相污染", async () => {
|
|
1382
|
-
const s = snap({
|
|
1383
|
-
experimentId: "score/mixed-scope",
|
|
1384
|
-
results: [res("pass-eval", "passed"), res("also-pass-eval", "passed", { evaluationKind: "pass" })],
|
|
1385
|
-
});
|
|
1386
|
-
const table = await measureRowsData([s], { dimensions: ["eval"], measures: [totalScore] });
|
|
1387
|
-
expect(metricCell(table.rows.find((r) => r.key === "pass-eval")!, totalScore.name).value).toBeNull();
|
|
1388
|
-
expect(metricCell(table.rows.find((r) => r.key === "also-pass-eval")!, totalScore.name).value).toBeNull();
|
|
1389
|
-
});
|
|
1390
|
-
|
|
1391
|
-
it("聚合口径:同一 eval 的多个 attempt 取均值(perEval mean),跨 eval 求和(acrossEvals sum)", async () => {
|
|
1392
|
-
const s = snap({
|
|
1393
|
-
experimentId: "score/aggregate",
|
|
1394
|
-
results: [
|
|
1395
|
-
// 同一题 "q1" 跑两轮:4 分与 2 分 → perEval mean = 3
|
|
1396
|
-
res("q1", "passed", { attempt: 0, evaluationKind: "points", assertions: [pointsAssertion("a", 4)] }),
|
|
1397
|
-
res("q1", "passed", { attempt: 1, evaluationKind: "points", assertions: [pointsAssertion("a", 2)] }),
|
|
1398
|
-
// 另一题 "q2" 跑一轮:5 分
|
|
1399
|
-
res("q2", "passed", { evaluationKind: "points", assertions: [pointsAssertion("a", 5)] }),
|
|
1400
|
-
],
|
|
1401
|
-
});
|
|
1402
|
-
const table = await measureRowsData([s], { dimensions: ["experiment"], measures: [totalScore] });
|
|
1403
|
-
// acrossEvals sum:q1 的 perEval 均值(3)+ q2 的值(5)= 8,不是全部 attempt 直接相加(11)。
|
|
1404
|
-
expect(metricCell(table.rows[0]!, totalScore.name).value).toBe(8);
|
|
1405
|
-
});
|
|
1406
|
-
|
|
1407
|
-
it("从公开导出面(niceeval/report 顶层)可以拿到同一个 totalScore Calculation,构建自定义报告不需要下钻内部模块", async () => {
|
|
1408
|
-
const { totalScore: totalScoreFromBarrel, aggregate, experiment } = await import("../index.ts");
|
|
1409
|
-
const { totalScore: totalScoreCalc } = await import("../model/calculation.ts");
|
|
1410
|
-
expect(totalScoreFromBarrel).toBe(totalScoreCalc); // 同引用:barrel 不复制一份新计算
|
|
1411
|
-
const s = snap({
|
|
1412
|
-
experimentId: "score/barrel",
|
|
1413
|
-
results: [res("checkpoints", "passed", { evaluationKind: "points", assertions: [pointsAssertion("a", 4)] })],
|
|
1414
|
-
});
|
|
1415
|
-
const rows = await aggregate(scopeOf([s]), {
|
|
1416
|
-
by: { experiment },
|
|
1417
|
-
values: { totalScore: totalScoreFromBarrel },
|
|
1418
|
-
});
|
|
1419
|
-
expect(rows[0]!.totalScore.value).toBe(4);
|
|
1420
|
-
});
|
|
1421
|
-
});
|
|
1422
|
-
|
|
1423
|
-
// ───────────────────────── labels 维度、series 归类与 connect ─────────────────────────
|
|
1424
|
-
|
|
1425
|
-
describe("labels 维度、series 归类与 connect", () => {
|
|
1426
|
-
const labeled = (
|
|
1427
|
-
experimentId: string,
|
|
1428
|
-
labels: globalThis.Record<string, string | number> | undefined,
|
|
1429
|
-
verdicts: Verdict[],
|
|
1430
|
-
agent = "agent-x",
|
|
1431
|
-
) =>
|
|
1432
|
-
snap({
|
|
1433
|
-
experimentId,
|
|
1434
|
-
agent,
|
|
1435
|
-
results: verdicts.map((v, i) => res(`q${i}`, v, { agent })),
|
|
1436
|
-
experiment: {
|
|
1437
|
-
attempts: 1,
|
|
1438
|
-
earlyExit: false,
|
|
1439
|
-
selectedEvalIds: verdicts.map((_, i) => `q${i}`),
|
|
1440
|
-
...(labels ? { labels } : {}),
|
|
1441
|
-
},
|
|
1442
|
-
});
|
|
1443
|
-
|
|
1444
|
-
it("label() 按声明值分组、未声明归 (missing);numericLabel 对字符串值返回 null 不猜序", async () => {
|
|
1445
|
-
const table = await measureRowsData(
|
|
1446
|
-
[labeled("m/one", { memory: "mempal" }, ["passed"]), labeled("m/two", undefined, ["failed"])],
|
|
1447
|
-
{ dimensions: [label("memory")], measures: [passRate] },
|
|
1448
|
-
);
|
|
1449
|
-
expect(table.rows.map((r) => r.key).sort()).toEqual(["(missing)", "mempal"]);
|
|
1450
|
-
|
|
1451
|
-
const line = await metricLineData(
|
|
1452
|
-
[labeled("m/num", { contextK: 32 }, ["passed"]), labeled("m/str", { contextK: "big" }, ["passed"])],
|
|
1453
|
-
{ x: numericLabel("contextK"), y: passRate },
|
|
1454
|
-
);
|
|
1455
|
-
expect(line.rows.filter((r) => r.x === null)).toHaveLength(1);
|
|
1456
|
-
expect(line.rows.some((r) => r.x === 32)).toBe(true);
|
|
1457
|
-
expect(line.rows.some((r) => r.x === 0)).toBe(false);
|
|
1458
|
-
});
|
|
1459
|
-
|
|
1460
|
-
it("同图撞色按图例顺序线性探测空格;无冲突键仍取散列格;超过色板才复用", () => {
|
|
1461
|
-
// 真实回归现场:bub / claude-code / codex 在同一张图里必须三色互异
|
|
1462
|
-
const real = colorIndicesForKeys(["bub", "claude-code", "codex"]);
|
|
1463
|
-
expect(new Set(real.values()).size).toBe(3);
|
|
1464
|
-
// 构造性冲突:找两个散列同格的键,后到者让位,先到者保持散列格(跨图稳定)
|
|
1465
|
-
const pool = Array.from({ length: 40 }, (_, i) => `k${i}`);
|
|
1466
|
-
const byIdx = new Map<number, string[]>();
|
|
1467
|
-
for (const k of pool) {
|
|
1468
|
-
const idx = colorIndexForKey(k);
|
|
1469
|
-
byIdx.set(idx, [...(byIdx.get(idx) ?? []), k]);
|
|
1470
|
-
}
|
|
1471
|
-
const [first, second] = [...byIdx.values()].find((keys) => keys.length >= 2)!;
|
|
1472
|
-
const resolved = colorIndicesForKeys([first, second]);
|
|
1473
|
-
expect(resolved.get(first)).toBe(colorIndexForKey(first));
|
|
1474
|
-
expect(resolved.get(second)).not.toBe(resolved.get(first));
|
|
1475
|
-
// 前 6 个键占满 6 色;第 7 个开始只能复用
|
|
1476
|
-
const seven = colorIndicesForKeys(["s0", "s1", "s2", "s3", "s4", "s5", "s6"]);
|
|
1477
|
-
expect(new Set([...seven.values()].slice(0, 6)).size).toBe(6);
|
|
1478
|
-
});
|
|
1479
|
-
});
|
|
1480
|
-
|
|
1481
|
-
describe("validate*Data 接受真实计算产物(不是只接受手写 literal)", () => {
|
|
1482
|
-
// 手写 literal 通过校验只证明 validator 认得自己设想的形状;真正的把关是喂真实计算产物——
|
|
1483
|
-
// 尤其是省略可选字段(如自定义指标不声明 label)的真实路径,不是校验作者假想的边界。
|
|
1484
|
-
const a = snap({ experimentId: "compare/a", results: [res("q1", "passed"), res("q2", "failed")] });
|
|
1485
|
-
const b = snap({ experimentId: "compare/b", results: [res("q1", "failed"), res("q2", "passed")] });
|
|
1486
|
-
const scope = scopeOf([a, b]);
|
|
1487
|
-
const noLabelMetric: AttemptMetric = { name: "custom-no-label", value: (attempt) => (attempt.result.verdict === "passed" ? 1 : 0) };
|
|
1488
|
-
|
|
1489
|
-
it("measureRowsData:含未声明 label 的自定义指标", async () => {
|
|
1490
|
-
const table = await measureRowsData(scope, { dimensions: ["agent"], measures: [costUSD, noLabelMetric] });
|
|
1491
|
-
expect(isDataset(table)).toBe(true);
|
|
1492
|
-
});
|
|
1493
|
-
|
|
1494
|
-
it("metricMatrixData", async () => {
|
|
1495
|
-
const matrix = await metricMatrixData(scope, { rows: "experiment", columns: "eval", cell: passRate });
|
|
1496
|
-
expect(validateMatrixData(matrix)).toBeNull();
|
|
1497
|
-
});
|
|
1498
|
-
|
|
1499
|
-
it("metricLineData", async () => {
|
|
1500
|
-
const line = await metricLineData(scope, { x: numericFlag("budget"), y: passRate });
|
|
1501
|
-
expect(validateLineData(line)).toBeNull();
|
|
1502
|
-
});
|
|
1503
|
-
|
|
1504
|
-
it("scoreboardData", async () => {
|
|
1505
|
-
const board = await scoreboardData(scope, { rows: "agent", questions: ["q1", "q2"] });
|
|
1506
|
-
expect(validateScoreboardData(board)).toBeNull();
|
|
1507
|
-
});
|
|
1508
|
-
|
|
1509
|
-
it("deltaTableData(含 conditionsByFlag 派生条件)", async () => {
|
|
1510
|
-
const withFlag = snap({
|
|
1511
|
-
experimentId: "compare/withflag",
|
|
1512
|
-
results: [res("q1", "passed")],
|
|
1513
|
-
experiment: { attempts: 1, earlyExit: false, selectedEvalIds: [], flags: { memory: "on" } },
|
|
1514
|
-
});
|
|
1515
|
-
const withoutFlag = snap({
|
|
1516
|
-
experimentId: "compare/noflag",
|
|
1517
|
-
results: [res("q1", "failed")],
|
|
1518
|
-
experiment: { attempts: 1, earlyExit: false, selectedEvalIds: [], flags: {} },
|
|
1519
|
-
});
|
|
1520
|
-
const deltaScope = scopeOf([withFlag, withoutFlag]);
|
|
1521
|
-
const delta = await deltaTableData(deltaScope, { by: "experiment", conditions: conditionsByFlag("memory") });
|
|
1522
|
-
expect(validateDeltaData(delta)).toBeNull();
|
|
1523
|
-
});
|
|
1524
|
-
|
|
1525
|
-
it("stabilityMatrixData", async () => {
|
|
1526
|
-
const stability = await stabilityMatrixData(scope, { by: "experiment" });
|
|
1527
|
-
expect(validateStabilityMatrixData(stability)).toBeNull();
|
|
1528
|
-
});
|
|
1529
|
-
|
|
1530
|
-
it("sampleSummary", async () => {
|
|
1531
|
-
const summary = await sampleSummary(scope);
|
|
1532
|
-
expect(validateSampleSummaryContent(summary)).toBeNull();
|
|
1533
|
-
});
|
|
1534
|
-
|
|
1535
|
-
it("experimentListData / evalListData / attemptListData(同一 Sample 三种粒度)", async () => {
|
|
1536
|
-
expect(validateExperimentListData(await experimentListData(scope))).toBeNull();
|
|
1537
|
-
expect(validateEvalListData(await evalListData(scope))).toBeNull();
|
|
1538
|
-
expect(validateAttemptListData(await attemptListData(scope))).toBeNull();
|
|
1539
|
-
});
|
|
1540
|
-
});
|