niceeval 0.17.1 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +5 -2
- package/dist/adapter-usage.cjs.map +1 -1
- package/dist/adapter-usage.d.cts +1 -1
- package/dist/adapter-usage.d.mts +1 -1
- package/dist/adapter-usage.d.ts +1 -1
- package/dist/adapter.cjs +8 -3
- package/dist/adapter.cjs.map +1 -1
- package/dist/adapter.d.cts +26 -15
- package/dist/adapter.d.mts +26 -15
- package/dist/adapter.d.ts +26 -15
- package/dist/agents/{effect-runtime.cjs → agent-effects.cjs} +12 -18
- package/dist/agents/agent-effects.cjs.map +1 -0
- package/dist/agents/agent-effects.d.cts +23 -0
- package/dist/agents/agent-effects.d.mts +23 -0
- package/dist/agents/agent-effects.d.ts +23 -0
- package/dist/agents/agent-effects.js +16 -0
- package/dist/agents/agent-effects.js.map +1 -0
- package/dist/agents/ai-sdk.cjs +1 -1
- package/dist/agents/ai-sdk.cjs.map +1 -1
- package/dist/agents/bub.cjs +31 -18
- package/dist/agents/bub.cjs.map +1 -1
- package/dist/agents/bub.d.cts +9 -12
- package/dist/agents/bub.d.mts +9 -12
- package/dist/agents/bub.d.ts +9 -12
- package/dist/agents/claude-code-native.cjs +2 -2
- package/dist/agents/claude-code-native.cjs.map +1 -1
- package/dist/agents/claude-code.cjs +17 -11
- package/dist/agents/claude-code.cjs.map +1 -1
- package/dist/agents/claude-code.d.cts +11 -11
- package/dist/agents/claude-code.d.mts +11 -11
- package/dist/agents/claude-code.d.ts +11 -11
- package/dist/agents/codex-app-server.cjs +2 -2
- package/dist/agents/codex-app-server.cjs.map +1 -1
- package/dist/agents/codex.cjs +18 -12
- package/dist/agents/codex.cjs.map +1 -1
- package/dist/agents/codex.d.cts +12 -12
- package/dist/agents/codex.d.mts +12 -12
- package/dist/agents/codex.d.ts +12 -12
- package/dist/agents/deepseek-harness.cjs +2 -2
- package/dist/agents/deepseek-harness.cjs.map +1 -1
- package/dist/agents/hermes.cjs +3 -3
- package/dist/agents/hermes.cjs.map +1 -1
- package/dist/agents/index.cjs +3 -3
- package/dist/agents/index.cjs.map +1 -1
- package/dist/agents/index.d.cts +3 -3
- package/dist/agents/index.d.mts +3 -3
- package/dist/agents/index.d.ts +3 -3
- package/dist/agents/index.js +2 -2
- package/dist/agents/index.mjs +2 -2
- package/dist/agents/mcp.cjs.map +1 -1
- package/dist/agents/mcp.d.cts +2 -2
- package/dist/agents/mcp.d.mts +2 -2
- package/dist/agents/mcp.d.ts +2 -2
- package/dist/agents/omp.cjs +2 -2
- package/dist/agents/omp.cjs.map +1 -1
- package/dist/agents/openai-compat.cjs +2 -2
- package/dist/agents/openai-compat.cjs.map +1 -1
- package/dist/agents/openclaw.cjs +3 -3
- package/dist/agents/openclaw.cjs.map +1 -1
- package/dist/agents/opencode.cjs +16 -4
- package/dist/agents/opencode.cjs.map +1 -1
- package/dist/agents/plugin-extensions.cjs +71 -0
- package/dist/agents/plugin-extensions.cjs.map +1 -0
- package/dist/agents/plugin-extensions.d.cts +26 -0
- package/dist/agents/plugin-extensions.d.mts +26 -0
- package/dist/agents/plugin-extensions.d.ts +26 -0
- package/dist/agents/plugin-extensions.js +8 -0
- package/dist/agents/plugin-extensions.js.map +1 -0
- package/dist/agents/post-setup.cjs +14 -3
- package/dist/agents/post-setup.cjs.map +1 -1
- package/dist/agents/post-setup.d.cts +4 -2
- package/dist/agents/post-setup.d.mts +4 -2
- package/dist/agents/post-setup.d.ts +4 -2
- package/dist/agents/post-setup.js +8 -6
- package/dist/agents/sdk-streams.cjs +3 -3
- package/dist/agents/sdk-streams.cjs.map +1 -1
- package/dist/agents/{session-slot.cjs → session-key.cjs} +4 -4
- package/dist/agents/session-key.cjs.map +1 -0
- package/dist/agents/session-key.d.cts +12 -0
- package/dist/agents/session-key.d.mts +12 -0
- package/dist/agents/session-key.d.ts +12 -0
- package/dist/agents/session-key.js +6 -0
- package/dist/agents/session-key.js.map +1 -0
- package/dist/agents/shared.cjs +2 -2
- package/dist/agents/shared.cjs.map +1 -1
- package/dist/agents/shared.d.cts +4 -4
- package/dist/agents/shared.d.mts +4 -4
- package/dist/agents/shared.d.ts +4 -4
- package/dist/agents/skills.cjs.map +1 -1
- package/dist/agents/skills.d.cts +3 -3
- package/dist/agents/skills.d.mts +3 -3
- package/dist/agents/skills.d.ts +3 -3
- package/dist/agents/types.cjs.map +1 -1
- package/dist/agents/types.d.cts +55 -54
- package/dist/agents/types.d.mts +55 -54
- package/dist/agents/types.d.ts +55 -54
- package/dist/agents/ui-message-stream.cjs +4 -4
- package/dist/agents/ui-message-stream.cjs.map +1 -1
- package/dist/agents/unrecognized-events.cjs +32 -0
- package/dist/agents/unrecognized-events.cjs.map +1 -0
- package/dist/agents/unrecognized-events.d.cts +8 -0
- package/dist/agents/unrecognized-events.d.mts +8 -0
- package/dist/agents/unrecognized-events.d.ts +8 -0
- package/dist/agents/unrecognized-events.js +6 -0
- package/dist/agents/unrecognized-events.js.map +1 -0
- package/dist/assertions/api.cjs.map +1 -1
- package/dist/assertions/api.d.cts +3 -0
- package/dist/assertions/api.d.mts +3 -0
- package/dist/assertions/api.d.ts +3 -0
- package/dist/assertions/collection.cjs +1 -1
- package/dist/assertions/collection.cjs.map +1 -1
- package/dist/assertions/judge.cjs +1 -1
- package/dist/assertions/judge.cjs.map +1 -1
- package/dist/assertions/match.cjs +52 -21
- package/dist/assertions/match.cjs.map +1 -1
- package/dist/assertions/match.d.cts +16 -5
- package/dist/assertions/match.d.mts +16 -5
- package/dist/assertions/match.d.ts +16 -5
- package/dist/assertions/match.js +96 -94
- package/dist/assertions/measurements.cjs +61 -0
- package/dist/assertions/measurements.cjs.map +1 -0
- package/dist/assertions/measurements.d.cts +24 -0
- package/dist/assertions/measurements.d.mts +24 -0
- package/dist/assertions/measurements.d.ts +24 -0
- package/dist/assertions/measurements.js +8 -0
- package/dist/assertions/measurements.js.map +1 -0
- package/dist/assertions/numeric.cjs +2 -0
- package/dist/assertions/numeric.cjs.map +1 -1
- package/dist/assertions/numeric.d.cts +1 -1
- package/dist/assertions/numeric.d.mts +1 -1
- package/dist/assertions/numeric.d.ts +1 -1
- package/dist/assertions/quantity.cjs +42 -0
- package/dist/assertions/quantity.cjs.map +1 -0
- package/dist/assertions/quantity.d.cts +18 -0
- package/dist/assertions/quantity.d.mts +18 -0
- package/dist/assertions/quantity.d.ts +18 -0
- package/dist/assertions/quantity.js +22 -0
- package/dist/assertions/quantity.js.map +1 -0
- package/dist/assertions/record/attachment.cjs +2 -0
- package/dist/assertions/record/attachment.cjs.map +1 -1
- package/dist/assertions/record/codec.cjs +5 -1
- package/dist/assertions/record/codec.cjs.map +1 -1
- package/dist/assertions/record/model.cjs.map +1 -1
- package/dist/assertions/record/model.d.cts +3 -1
- package/dist/assertions/record/model.d.mts +3 -1
- package/dist/assertions/record/model.d.ts +3 -1
- package/dist/assertions/runtime.cjs +11 -6
- package/dist/assertions/runtime.cjs.map +1 -1
- package/dist/cli/program.cjs +3 -0
- package/dist/cli/program.cjs.map +1 -1
- package/dist/context/assert-first.cjs +6 -4
- package/dist/context/assert-first.cjs.map +1 -1
- package/dist/context/assert-first.d.cts +9 -3
- package/dist/context/assert-first.d.mts +9 -3
- package/dist/context/assert-first.d.ts +9 -3
- package/dist/context/notes.cjs +47 -0
- package/dist/context/notes.cjs.map +1 -0
- package/dist/context/notes.d.cts +10 -0
- package/dist/context/notes.d.mts +10 -0
- package/dist/context/notes.d.ts +10 -0
- package/dist/context/notes.js +6 -0
- package/dist/context/notes.js.map +1 -0
- package/dist/context/send-failures.cjs +1 -7
- package/dist/context/send-failures.cjs.map +1 -1
- package/dist/context/send-failures.d.cts +2 -3
- package/dist/context/send-failures.d.mts +2 -3
- package/dist/context/send-failures.d.ts +2 -3
- package/dist/context/send-retry.cjs +0 -1
- package/dist/context/send-retry.cjs.map +1 -1
- package/dist/context/send-retry.d.cts +1 -2
- package/dist/context/send-retry.d.mts +1 -2
- package/dist/context/send-retry.d.ts +1 -2
- package/dist/context/session.cjs +10 -17
- package/dist/context/session.cjs.map +1 -1
- package/dist/context/session.d.cts +4 -10
- package/dist/context/session.d.mts +4 -10
- package/dist/context/session.d.ts +4 -10
- package/dist/define.cjs +89 -64
- package/dist/define.cjs.map +1 -1
- package/dist/define.d.cts +3 -3
- package/dist/define.d.mts +3 -3
- package/dist/define.d.ts +3 -3
- package/dist/eval/cli/contribution.cjs +1 -1
- package/dist/eval/cli/contribution.cjs.map +1 -1
- package/dist/eval/host/index.cjs +1 -1
- package/dist/eval/host/index.cjs.map +1 -1
- package/dist/eval/record/evaluation.cjs +1 -1
- package/dist/eval/record/evaluation.cjs.map +1 -1
- package/dist/eval/record/membership-provenance.d.cts +3 -3
- package/dist/eval/record/membership-provenance.d.mts +3 -3
- package/dist/eval/record/membership-provenance.d.ts +3 -3
- package/dist/eval/record/score.cjs +2 -2
- package/dist/eval/record/score.cjs.map +1 -1
- package/dist/eval/record/sealed-assertion.cjs +1 -1
- package/dist/eval/record/sealed-assertion.cjs.map +1 -1
- package/dist/eval/record/verdict.cjs +2 -2
- package/dist/eval/record/verdict.cjs.map +1 -1
- package/dist/expect/index.cjs +13 -1
- package/dist/expect/index.cjs.map +1 -1
- package/dist/expect/index.d.cts +5 -1
- package/dist/expect/index.d.mts +5 -1
- package/dist/expect/index.d.ts +5 -1
- package/dist/expect/index.js +92 -72
- package/dist/expect/index.mjs +92 -72
- package/dist/experiment/host/cli/contribution.cjs +52 -39
- package/dist/experiment/host/cli/contribution.cjs.map +1 -1
- package/dist/experiment/host/cli/contribution.d.cts +18 -4
- package/dist/experiment/host/cli/contribution.d.mts +18 -4
- package/dist/experiment/host/cli/contribution.d.ts +18 -4
- package/dist/experiment/host/cli/contribution.js +2 -2
- package/dist/experiment/host/cli/debug-protocol.cjs +2 -2
- package/dist/experiment/host/cli/debug-protocol.cjs.map +1 -1
- package/dist/experiment/host/cli/debug-protocol.d.cts +2 -2
- package/dist/experiment/host/cli/debug-protocol.d.mts +2 -2
- package/dist/experiment/host/cli/debug-protocol.d.ts +2 -2
- package/dist/experiment/host/cli/output-protocol.cjs +1 -1
- package/dist/experiment/host/cli/output-protocol.cjs.map +1 -1
- package/dist/experiment/host/cli/output-protocol.d.cts +2 -2
- package/dist/experiment/host/cli/output-protocol.d.mts +2 -2
- package/dist/experiment/host/cli/output-protocol.d.ts +2 -2
- package/dist/experiment/host/current.cjs +2 -2
- package/dist/experiment/host/current.cjs.map +1 -1
- package/dist/experiment/host/index.cjs +1 -1
- package/dist/experiment/host/index.cjs.map +1 -1
- package/dist/experiment/host/operations.cjs +24 -23
- package/dist/experiment/host/operations.cjs.map +1 -1
- package/dist/experiment/host/runtime.cjs +11 -10
- package/dist/experiment/host/runtime.cjs.map +1 -1
- package/dist/experiment/host/runtime.d.cts +3 -3
- package/dist/experiment/host/runtime.d.mts +3 -3
- package/dist/experiment/host/runtime.d.ts +3 -3
- package/dist/experiment/host/teardown.cjs +4 -3
- package/dist/experiment/host/teardown.cjs.map +1 -1
- package/dist/experiment/host/teardown.d.cts +3 -3
- package/dist/experiment/host/teardown.d.mts +3 -3
- package/dist/experiment/host/teardown.d.ts +3 -3
- package/dist/experiment/host/types.cjs.map +1 -1
- package/dist/experiment/host/types.d.cts +19 -19
- package/dist/experiment/host/types.d.mts +19 -19
- package/dist/experiment/host/types.d.ts +19 -19
- package/dist/index.cjs +3 -19
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +4 -8
- package/dist/index.d.mts +4 -8
- package/dist/index.d.ts +4 -8
- package/dist/index.js +12 -34
- package/dist/index.mjs +12 -34
- package/dist/inspection/assertions.cjs +1 -1
- package/dist/inspection/assertions.cjs.map +1 -1
- package/dist/inspection/cli/contribution.cjs +8 -7
- package/dist/inspection/cli/contribution.cjs.map +1 -1
- package/dist/inspection/facts.cjs +4 -4
- package/dist/inspection/facts.cjs.map +1 -1
- package/dist/inspection/index.cjs +1 -5
- package/dist/inspection/index.cjs.map +1 -1
- package/dist/inspection/index.d.cts +1 -2
- package/dist/inspection/index.d.mts +1 -2
- package/dist/inspection/index.d.ts +1 -2
- package/dist/inspection/index.js +20 -26
- package/dist/inspection/lifecycle.cjs +7 -0
- package/dist/inspection/lifecycle.cjs.map +1 -0
- package/dist/inspection/lifecycle.d.cts +4 -0
- package/dist/inspection/lifecycle.d.mts +4 -0
- package/dist/inspection/lifecycle.d.ts +4 -0
- package/dist/inspection/lifecycle.js +6 -0
- package/dist/inspection/lifecycle.js.map +1 -0
- package/dist/inspection/notes.cjs +40 -0
- package/dist/inspection/notes.cjs.map +1 -0
- package/dist/inspection/notes.d.cts +12 -0
- package/dist/inspection/notes.d.mts +12 -0
- package/dist/inspection/notes.d.ts +12 -0
- package/dist/inspection/notes.js +8 -0
- package/dist/inspection/notes.js.map +1 -0
- package/dist/inspection/overview.cjs +12 -15
- package/dist/inspection/overview.cjs.map +1 -1
- package/dist/inspection/overview.d.cts +8 -6
- package/dist/inspection/overview.d.mts +8 -6
- package/dist/inspection/overview.d.ts +8 -6
- package/dist/inspection/planned-attempt.cjs +45 -0
- package/dist/inspection/planned-attempt.cjs.map +1 -0
- package/dist/inspection/planned-attempt.d.cts +109 -0
- package/dist/inspection/planned-attempt.d.mts +109 -0
- package/dist/inspection/planned-attempt.d.ts +109 -0
- package/dist/inspection/planned-attempt.js +24 -0
- package/dist/inspection/planned-attempt.js.map +1 -0
- package/dist/inspection/project-result.cjs +5 -5
- package/dist/inspection/project-result.cjs.map +1 -1
- package/dist/inspection/project-result.d.cts +58 -57
- package/dist/inspection/project-result.d.mts +58 -57
- package/dist/inspection/project-result.d.ts +58 -57
- package/dist/inspection/project-result.js +4 -4
- package/dist/inspection/project.cjs +3 -3
- package/dist/inspection/project.cjs.map +1 -1
- package/dist/inspection/protocol-values.cjs +1 -1
- package/dist/inspection/protocol-values.cjs.map +1 -1
- package/dist/inspection/protocol-values.d.cts +1 -1
- package/dist/inspection/protocol-values.d.mts +1 -1
- package/dist/inspection/protocol-values.d.ts +1 -1
- package/dist/inspection/protocol.cjs +8 -8
- package/dist/inspection/protocol.cjs.map +1 -1
- package/dist/inspection/protocol.d.cts +237 -219
- package/dist/inspection/protocol.d.mts +237 -219
- package/dist/inspection/protocol.d.ts +237 -219
- package/dist/inspection/public.cjs +1 -5
- package/dist/inspection/public.cjs.map +1 -1
- package/dist/inspection/public.d.cts +1 -2
- package/dist/inspection/public.d.mts +1 -2
- package/dist/inspection/public.d.ts +1 -2
- package/dist/inspection/public.js +14 -20
- package/dist/inspection/public.mjs +166 -129
- package/dist/inspection/query.cjs +1 -1
- package/dist/inspection/query.cjs.map +1 -1
- package/dist/inspection/results.cjs +61 -52
- package/dist/inspection/results.cjs.map +1 -1
- package/dist/inspection/results.d.cts +157 -140
- package/dist/inspection/results.d.mts +157 -140
- package/dist/inspection/results.d.ts +157 -140
- package/dist/inspection/select.cjs +33 -28
- package/dist/inspection/select.cjs.map +1 -1
- package/dist/inspection/source.cjs +1 -1
- package/dist/inspection/source.cjs.map +1 -1
- package/dist/inspection/sources.cjs +2 -2
- package/dist/inspection/sources.cjs.map +1 -1
- package/dist/inspection/trace.cjs +9 -8
- package/dist/inspection/trace.cjs.map +1 -1
- package/dist/inspection/trace.d.cts +4 -4
- package/dist/inspection/trace.d.mts +4 -4
- package/dist/inspection/trace.d.ts +4 -4
- package/dist/loaders/index.cjs +7 -279
- package/dist/loaders/index.cjs.map +1 -1
- package/dist/loaders/index.d.cts +2 -95
- package/dist/loaders/index.d.mts +2 -95
- package/dist/loaders/index.d.ts +2 -95
- package/dist/loaders/index.js +10 -12
- package/dist/loaders/index.mjs +10 -12
- package/dist/loaders/runtime.cjs +283 -0
- package/dist/loaders/runtime.cjs.map +1 -0
- package/dist/loaders/runtime.d.cts +95 -0
- package/dist/loaders/runtime.d.mts +95 -0
- package/dist/loaders/runtime.d.ts +95 -0
- package/dist/loaders/runtime.js +16 -0
- package/dist/loaders/runtime.js.map +1 -0
- package/dist/migrations/judge-provider.md +6 -6
- package/dist/{model-slots.cjs → model-roles.cjs} +26 -7
- package/dist/model-roles.cjs.map +1 -0
- package/dist/model-roles.d.cts +26 -0
- package/dist/model-roles.d.mts +26 -0
- package/dist/model-roles.d.ts +26 -0
- package/dist/model-roles.js +14 -0
- package/dist/model-roles.js.map +1 -0
- package/dist/o11y/adapter-usage-projection.cjs +9 -8
- package/dist/o11y/adapter-usage-projection.cjs.map +1 -1
- package/dist/o11y/adapter-usage-projection.d.cts +23 -15
- package/dist/o11y/adapter-usage-projection.d.mts +23 -15
- package/dist/o11y/adapter-usage-projection.d.ts +23 -15
- package/dist/o11y/cache-read.cjs +14 -14
- package/dist/o11y/cache-read.cjs.map +1 -1
- package/dist/o11y/cache-read.d.cts +5 -5
- package/dist/o11y/cache-read.d.mts +5 -5
- package/dist/o11y/cache-read.d.ts +5 -5
- package/dist/o11y/derive.cjs.map +1 -1
- package/dist/o11y/derive.d.cts +2 -2
- package/dist/o11y/derive.d.mts +2 -2
- package/dist/o11y/derive.d.ts +2 -2
- package/dist/o11y/eval-usage.cjs +9 -9
- package/dist/o11y/eval-usage.cjs.map +1 -1
- package/dist/o11y/eval-usage.d.cts +7 -7
- package/dist/o11y/eval-usage.d.mts +7 -7
- package/dist/o11y/eval-usage.d.ts +7 -7
- package/dist/o11y/parsers/bub.cjs +57 -81
- package/dist/o11y/parsers/bub.cjs.map +1 -1
- package/dist/o11y/parsers/index.cjs.map +1 -1
- package/dist/o11y/parsers/index.d.cts +2 -0
- package/dist/o11y/parsers/index.d.mts +2 -0
- package/dist/o11y/parsers/index.d.ts +2 -0
- package/dist/o11y/parsers/opencode.cjs +129 -217
- package/dist/o11y/parsers/opencode.cjs.map +1 -1
- package/dist/o11y/parsers/opencode.d.cts +2 -5
- package/dist/o11y/parsers/opencode.d.mts +2 -5
- package/dist/o11y/parsers/opencode.d.ts +2 -5
- package/dist/o11y/types.cjs.map +1 -1
- package/dist/o11y/types.d.cts +1 -1
- package/dist/o11y/types.d.mts +1 -1
- package/dist/o11y/types.d.ts +1 -1
- package/dist/plugin/contracts.cjs +170 -120
- package/dist/plugin/contracts.cjs.map +1 -1
- package/dist/plugin/contracts.d.cts +102 -90
- package/dist/plugin/contracts.d.mts +102 -90
- package/dist/plugin/contracts.d.ts +102 -90
- package/dist/plugin/contracts.js +16 -12
- package/dist/plugin/index.cjs.map +1 -1
- package/dist/plugin/index.d.cts +1 -1
- package/dist/plugin/index.d.mts +1 -1
- package/dist/plugin/index.d.ts +1 -1
- package/dist/plugin/link.cjs +69 -64
- package/dist/plugin/link.cjs.map +1 -1
- package/dist/plugin/link.d.cts +5 -3
- package/dist/plugin/link.d.mts +5 -3
- package/dist/plugin/link.d.ts +5 -3
- package/dist/plugin/link.js +12 -8
- package/dist/record/codec/seal-manifest.d.cts +3 -3
- package/dist/record/codec/seal-manifest.d.mts +3 -3
- package/dist/record/codec/seal-manifest.d.ts +3 -3
- package/dist/record/family/attempt-note/definition.cjs +12 -0
- package/dist/record/family/attempt-note/definition.cjs.map +1 -0
- package/dist/record/family/attempt-note/definition.d.cts +6 -0
- package/dist/record/family/attempt-note/definition.d.mts +6 -0
- package/dist/record/family/attempt-note/definition.d.ts +6 -0
- package/dist/record/family/attempt-note/definition.js +6 -0
- package/dist/record/family/attempt-note/definition.js.map +1 -0
- package/dist/record/family/attempt-note/persistence.cjs +10 -0
- package/dist/record/family/attempt-note/persistence.cjs.map +1 -0
- package/dist/record/family/attempt-note/persistence.d.cts +6 -0
- package/dist/record/family/attempt-note/persistence.d.mts +6 -0
- package/dist/record/family/attempt-note/persistence.d.ts +6 -0
- package/dist/record/family/attempt-note/persistence.js +6 -0
- package/dist/record/family/attempt-note/persistence.js.map +1 -0
- package/dist/record/family/attempt-note/schema.cjs +20 -0
- package/dist/record/family/attempt-note/schema.cjs.map +1 -0
- package/dist/record/family/attempt-note/schema.d.cts +21 -0
- package/dist/record/family/attempt-note/schema.d.mts +21 -0
- package/dist/record/family/attempt-note/schema.d.ts +21 -0
- package/dist/record/family/attempt-note/schema.js +10 -0
- package/dist/record/family/attempt-note/schema.js.map +1 -0
- package/dist/record/family/catalog.cjs +32 -29
- package/dist/record/family/catalog.cjs.map +1 -1
- package/dist/record/family/catalog.d.cts +7 -2
- package/dist/record/family/catalog.d.mts +7 -2
- package/dist/record/family/catalog.d.ts +7 -2
- package/dist/record/family/current.cjs +48 -45
- package/dist/record/family/current.cjs.map +1 -1
- package/dist/record/family/current.d.cts +15 -0
- package/dist/record/family/current.d.mts +15 -0
- package/dist/record/family/current.d.ts +15 -0
- package/dist/record/model/index.js +66 -64
- package/dist/record/model/run-context.cjs +9 -3
- package/dist/record/model/run-context.cjs.map +1 -1
- package/dist/record/model/run-context.d.cts +9 -2
- package/dist/record/model/run-context.d.mts +9 -2
- package/dist/record/model/run-context.d.ts +9 -2
- package/dist/record/model/run-context.js +8 -6
- package/dist/record/model/seal-manifest.d.cts +1 -1
- package/dist/record/model/seal-manifest.d.mts +1 -1
- package/dist/record/model/seal-manifest.d.ts +1 -1
- package/dist/run/cli/contribution.cjs +6 -32
- package/dist/run/cli/contribution.cjs.map +1 -1
- package/dist/run/host/runtime.cjs +1 -1
- package/dist/run/host/runtime.cjs.map +1 -1
- package/dist/run/storage/sqlite.cjs +7 -7
- package/dist/run/storage/sqlite.cjs.map +1 -1
- package/dist/runner/adapter-usage.cjs +5 -3
- package/dist/runner/adapter-usage.cjs.map +1 -1
- package/dist/runner/adapter-usage.d.cts +2 -2
- package/dist/runner/adapter-usage.d.mts +2 -2
- package/dist/runner/adapter-usage.d.ts +2 -2
- package/dist/runner/adoption.cjs +10 -12
- package/dist/runner/adoption.cjs.map +1 -1
- package/dist/runner/attempt-notes.cjs +8 -0
- package/dist/runner/attempt-notes.cjs.map +1 -0
- package/dist/runner/attempt-notes.d.cts +4 -0
- package/dist/runner/attempt-notes.d.mts +4 -0
- package/dist/runner/attempt-notes.d.ts +4 -0
- package/dist/runner/attempt-notes.js +8 -0
- package/dist/runner/attempt-notes.js.map +1 -0
- package/dist/runner/attempt.cjs +61 -73
- package/dist/runner/attempt.cjs.map +1 -1
- package/dist/runner/attempt.d.cts +3 -3
- package/dist/runner/attempt.d.mts +3 -3
- package/dist/runner/attempt.d.ts +3 -3
- package/dist/runner/command-plan.cjs +21 -26
- package/dist/runner/command-plan.cjs.map +1 -1
- package/dist/runner/command-plan.d.cts +4 -4
- package/dist/runner/command-plan.d.mts +4 -4
- package/dist/runner/command-plan.d.ts +4 -4
- package/dist/runner/command-plan.js +4 -4
- package/dist/runner/config-identity.cjs.map +1 -1
- package/dist/runner/config-identity.d.cts +2 -2
- package/dist/runner/config-identity.d.mts +2 -2
- package/dist/runner/config-identity.d.ts +2 -2
- package/dist/runner/discover.cjs +6 -6
- package/dist/runner/discover.cjs.map +1 -1
- package/dist/runner/eval-selection.cjs +15 -1
- package/dist/runner/eval-selection.cjs.map +1 -1
- package/dist/runner/eval-selection.d.cts +4 -0
- package/dist/runner/eval-selection.d.mts +4 -0
- package/dist/runner/eval-selection.d.ts +4 -0
- package/dist/runner/feedback/human.cjs +1 -1
- package/dist/runner/feedback/human.cjs.map +1 -1
- package/dist/runner/feedback/json.cjs +4 -2
- package/dist/runner/feedback/json.cjs.map +1 -1
- package/dist/runner/feedback/json.d.cts +1 -1
- package/dist/runner/feedback/json.d.mts +1 -1
- package/dist/runner/feedback/json.d.ts +1 -1
- package/dist/runner/feedback/sink.cjs +1 -1
- package/dist/runner/feedback/sink.cjs.map +1 -1
- package/dist/runner/feedback/sink.d.cts +1 -1
- package/dist/runner/feedback/sink.d.mts +1 -1
- package/dist/runner/feedback/sink.d.ts +1 -1
- package/dist/runner/fingerprint.cjs +1 -1
- package/dist/runner/fingerprint.cjs.map +1 -1
- package/dist/runner/lock.cjs +1 -3
- package/dist/runner/lock.cjs.map +1 -1
- package/dist/runner/lock.d.cts +0 -2
- package/dist/runner/lock.d.mts +0 -2
- package/dist/runner/lock.d.ts +0 -2
- package/dist/runner/lock.js +14 -16
- package/dist/runner/project-target.cjs.map +1 -1
- package/dist/runner/project-target.d.cts +2 -2
- package/dist/runner/project-target.d.mts +2 -2
- package/dist/runner/project-target.d.ts +2 -2
- package/dist/runner/record/attachments.cjs +2 -2
- package/dist/runner/record/attachments.cjs.map +1 -1
- package/dist/runner/record/planning.cjs +8 -2
- package/dist/runner/record/planning.cjs.map +1 -1
- package/dist/runner/record.cjs +2 -0
- package/dist/runner/record.cjs.map +1 -1
- package/dist/runner/report.cjs +3 -3
- package/dist/runner/report.cjs.map +1 -1
- package/dist/runner/report.d.cts +1 -1
- package/dist/runner/report.d.mts +1 -1
- package/dist/runner/report.d.ts +1 -1
- package/dist/runner/reuse-plan.cjs +2 -2
- package/dist/runner/reuse-plan.cjs.map +1 -1
- package/dist/runner/run.cjs +10 -45
- package/dist/runner/run.cjs.map +1 -1
- package/dist/runner/sandbox-pool.cjs +1 -2
- package/dist/runner/sandbox-pool.cjs.map +1 -1
- package/dist/runner/setup-prefix-preparation.cjs +1 -1
- package/dist/runner/setup-prefix-preparation.cjs.map +1 -1
- package/dist/runner/teardown-registry.cjs +2 -2
- package/dist/runner/teardown-registry.cjs.map +1 -1
- package/dist/runner/teardown-registry.d.cts +1 -1
- package/dist/runner/teardown-registry.d.mts +1 -1
- package/dist/runner/teardown-registry.d.ts +1 -1
- package/dist/runner/types.cjs +4 -4
- package/dist/runner/types.cjs.map +1 -1
- package/dist/runner/types.d.cts +64 -92
- package/dist/runner/types.d.mts +64 -92
- package/dist/runner/types.d.ts +64 -92
- package/dist/runner/types.js +2 -2
- package/dist/sandbox/action.cjs +1 -1
- package/dist/sandbox/action.cjs.map +1 -1
- package/dist/sandbox/cli/contribution.cjs +17 -14
- package/dist/sandbox/cli/contribution.cjs.map +1 -1
- package/dist/sandbox/cli/contribution.d.cts +1 -1
- package/dist/sandbox/cli/contribution.d.mts +1 -1
- package/dist/sandbox/cli/contribution.d.ts +1 -1
- package/dist/sandbox/commands.cjs +1 -1
- package/dist/sandbox/commands.cjs.map +1 -1
- package/dist/sandbox/commands.d.cts +1 -1
- package/dist/sandbox/commands.d.mts +1 -1
- package/dist/sandbox/commands.d.ts +1 -1
- package/dist/sandbox/compose.cjs +2 -2
- package/dist/sandbox/compose.cjs.map +1 -1
- package/dist/sandbox/docker-profile/registry.cjs +5 -10
- package/dist/sandbox/docker-profile/registry.cjs.map +1 -1
- package/dist/sandbox/docker-profile/registry.d.cts +0 -4
- package/dist/sandbox/docker-profile/registry.d.mts +0 -4
- package/dist/sandbox/docker-profile/registry.d.ts +0 -4
- package/dist/sandbox/docker-profile/registry.js +6 -14
- package/dist/sandbox/docker-profile/runtime.cjs +1 -1
- package/dist/sandbox/docker-profile/runtime.cjs.map +1 -1
- package/dist/sandbox/docker-profile/schema.cjs +12 -16
- package/dist/sandbox/docker-profile/schema.cjs.map +1 -1
- package/dist/sandbox/docker-profile/schema.d.cts +2 -6
- package/dist/sandbox/docker-profile/schema.d.mts +2 -6
- package/dist/sandbox/docker-profile/schema.d.ts +2 -6
- package/dist/sandbox/docker-profile/schema.js +24 -32
- package/dist/sandbox/incus.cjs +2 -2
- package/dist/sandbox/incus.cjs.map +1 -1
- package/dist/sandbox/index.cjs +4 -36
- package/dist/sandbox/index.cjs.map +1 -1
- package/dist/sandbox/index.d.cts +4 -8
- package/dist/sandbox/index.d.mts +4 -8
- package/dist/sandbox/index.d.ts +4 -8
- package/dist/sandbox/index.js +116 -174
- package/dist/sandbox/index.mjs +116 -174
- package/dist/sandbox/layer.cjs +3 -3
- package/dist/sandbox/layer.cjs.map +1 -1
- package/dist/sandbox/link.cjs +5 -0
- package/dist/sandbox/link.cjs.map +1 -1
- package/dist/sandbox/link.d.cts +3 -0
- package/dist/sandbox/link.d.mts +3 -0
- package/dist/sandbox/link.d.ts +3 -0
- package/dist/sandbox/runtime.cjs +9 -4
- package/dist/sandbox/runtime.cjs.map +1 -1
- package/dist/shared/failure-class.cjs +5 -5
- package/dist/shared/failure-class.cjs.map +1 -1
- package/dist/shared/failure-class.d.cts +4 -10
- package/dist/shared/failure-class.d.mts +4 -10
- package/dist/shared/failure-class.d.ts +4 -10
- package/dist/shared/types.cjs.map +1 -1
- package/dist/shared/types.d.cts +1 -1
- package/dist/shared/types.d.mts +1 -1
- package/dist/shared/types.d.ts +1 -1
- package/dist/show/contribution.cjs +17 -2
- package/dist/show/contribution.cjs.map +1 -1
- package/dist/show/contribution.d.cts +14 -0
- package/dist/show/contribution.d.mts +14 -0
- package/dist/show/contribution.d.ts +14 -0
- package/dist/show/model.cjs +8 -5
- package/dist/show/model.cjs.map +1 -1
- package/dist/show/model.d.cts +11 -8
- package/dist/show/model.d.mts +11 -8
- package/dist/show/model.d.ts +11 -8
- package/dist/show/render.cjs +34 -22
- package/dist/show/render.cjs.map +1 -1
- package/dist/sources/projector.cjs +1 -1
- package/dist/sources/projector.cjs.map +1 -1
- package/dist/types.cjs +1 -1
- package/dist/types.cjs.map +1 -1
- package/dist/types.d.cts +1 -1
- package/dist/types.d.mts +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.js +2 -2
- package/dist/view/app-dist/.vite/manifest.json +2 -2
- package/dist/view/app-dist/assets/{index-B3ijZRph.js → index-CxrOUvHC.js} +17 -17
- package/dist/view/app-dist/assets/{quality-cost-scatter-B--DfJ9s.js → quality-cost-scatter-B6Aiss7x.js} +1 -1
- package/dist/view/app-dist/index.html +1 -1
- package/dist/view/cli/contribution.cjs +1 -1
- package/dist/view/cli/contribution.cjs.map +1 -1
- package/dist/view/function-runtime.mjs +318 -121
- package/dist/view/http-protocol.d.cts +1 -1
- package/dist/view/http-protocol.d.mts +1 -1
- package/dist/view/http-protocol.d.ts +1 -1
- package/dist/view/inspection-host.cjs +2 -2
- package/dist/view/inspection-host.cjs.map +1 -1
- package/dist/view/inspection-host.d.cts +349 -331
- package/dist/view/inspection-host.d.mts +349 -331
- package/dist/view/inspection-host.d.ts +349 -331
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +9 -9
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +8 -8
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +4 -4
- package/docs-site/zh/examples/integrations/langgraph.mdx +6 -6
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +8 -8
- package/docs-site/zh/explanation/adapter.mdx +6 -6
- package/docs-site/zh/explanation/assert.mdx +7 -7
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/hitl.mdx +2 -2
- package/docs-site/zh/explanation/judge.mdx +3 -3
- package/docs-site/zh/index.mdx +18 -0
- package/docs-site/zh/introduction.mdx +4 -3
- package/docs-site/zh/reference/builtin-agents.mdx +46 -49
- package/docs-site/zh/reference/cli.mdx +16 -14
- package/docs-site/zh/reference/define-agent.mdx +19 -19
- package/docs-site/zh/reference/define-config.mdx +7 -15
- package/docs-site/zh/reference/define-eval.mdx +20 -14
- package/docs-site/zh/reference/define-experiment.mdx +18 -39
- package/docs-site/zh/reference/events.mdx +1 -1
- package/docs-site/zh/reference/expect.mdx +21 -0
- package/docs-site/zh/reference/official-adapters.mdx +12 -5
- package/docs-site/zh/troubleshooting/debugging.mdx +2 -2
- package/docs-site/zh/troubleshooting/recover-after-kill.mdx +1 -1
- package/docs-site/zh/tutorials/agent-onboarding.mdx +2 -2
- package/docs-site/zh/tutorials/authoring.mdx +4 -3
- package/docs-site/zh/tutorials/compare-runs.mdx +1 -1
- package/docs-site/zh/tutorials/concurrency.mdx +8 -8
- package/docs-site/zh/tutorials/configuration.mdx +5 -5
- package/docs-site/zh/tutorials/control-run-cost.mdx +49 -9
- package/docs-site/zh/tutorials/eval-groups.mdx +1 -1
- package/docs-site/zh/tutorials/evaluation-kinds.mdx +5 -5
- package/docs-site/zh/tutorials/experiment-services.mdx +32 -32
- package/docs-site/zh/tutorials/handle-execution-failures.mdx +2 -28
- package/docs-site/zh/tutorials/plugins.mdx +49 -143
- package/docs-site/zh/tutorials/sandbox-agent.mdx +1 -1
- package/docs-site/zh/tutorials/sandbox-reuse.mdx +7 -7
- package/docs-site/zh/tutorials/upgrade-0-18.mdx +64 -0
- package/docs-site/zh/tutorials/upgrade-0-19.mdx +121 -0
- package/docs-site/zh/tutorials/upgrade-0-20.mdx +12 -0
- package/docs-site/zh/tutorials/verify-judge.mdx +4 -3
- package/docs-site/zh/tutorials/viewing-results.mdx +2 -2
- package/docs-site/zh/tutorials/write-experiment.mdx +1 -1
- package/docs-site/zh/tutorials/write-send.mdx +13 -13
- package/package.json +1 -1
- package/dist/agents/effect-runtime.cjs.map +0 -1
- package/dist/agents/effect-runtime.d.cts +0 -29
- package/dist/agents/effect-runtime.d.mts +0 -29
- package/dist/agents/effect-runtime.d.ts +0 -29
- package/dist/agents/effect-runtime.js +0 -16
- package/dist/agents/effect-runtime.js.map +0 -1
- package/dist/agents/session-slot.cjs.map +0 -1
- package/dist/agents/session-slot.d.cts +0 -12
- package/dist/agents/session-slot.d.mts +0 -12
- package/dist/agents/session-slot.d.ts +0 -12
- package/dist/agents/session-slot.js +0 -6
- package/dist/agents/session-slot.js.map +0 -1
- package/dist/model-slots.cjs.map +0 -1
- package/dist/model-slots.d.cts +0 -19
- package/dist/model-slots.d.mts +0 -19
- package/dist/model-slots.d.ts +0 -19
- package/dist/model-slots.js +0 -8
- package/dist/model-slots.js.map +0 -1
package/dist/show/model.cjs.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model.js","sourceRoot":"","sources":["model.ts"],"names":[],"mappings":";;AAqSA,0CAWC;AACD,8CAQC;AAGD,gCA8BC;AAGD,wCA4BC;AACD,wCAqBC;AACD,oCAuCC;AACD,gDAgBC;AACD,sCAEC;AACD,oCAEC;AACD,kCAEC;AA5MD,MAAM,MAAM,GAAG,CAAC,KAGf,EAAU,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;AAC3D,MAAM,SAAS,GAAG,CAAC,KAAyC,EAAa,EAAE,CAAC,CAAC;IACzE,cAAc,EAAE,KAAK,CAAC,cAAc;IACpC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,QAAQ;IACpC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,QAAQ;IACpC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM;IAClC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM;IAClC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO;IACpC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO;IACpC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;IACxC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;IACpC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;CAC/B,CAAC,CAAC;AACH,MAAM,KAAK,GAAG,CAAC,MAAyC,EAAyB,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACtG,YAAY,EAAE,IAAI,CAAC,YAAY;IAC/B,MAAM,EAAE,IAAI,CAAC,MAAM;IACnB,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC;IAC1B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACnC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,KAAK,WAAW;YACjD,CAAC,CAAC;gBACE,GAAG,MAAM,CAAC,WAAW;gBACrB,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC;gBACvC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC;gBACjD,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC;aAC5C;YACD,CAAC,CAAC,MAAM,CAAC,WAAW;KAC3B,CAAC,CAAC;CACN,CAAC,CAAC,CAAC;AACJ,SAAgB,eAAe,CAAC,QAAsD;IAClF,OAAO;QACH,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC3C,WAAW,EAAE,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAC5D,YAAY,EAAE,UAAU,CAAC,YAAY;YACrC,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC;SACnC,CAAC,CAAC;QACH,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;KACxC,CAAC;AACN,CAAC;AACD,SAAgB,iBAAiB,CAAC,QAAwD;IACtF,OAAO;QACH,YAAY,EAAE,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,YAAY;QACzD,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;QACpD,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC;QACvC,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,WAAW;QAC5C,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,UAAU;KAC7C,CAAC;AACN,CAAC;AACD,4CAA4C;AAC5C,wFAAwF;AACxF,SAAgB,UAAU,CAAC,QAAiD;IACxE,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,OAAO;QACH,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,OAAO,IAAI,IAAI;QACjD,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;QAC9E,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ;QACrC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,SAAS;QACtC,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACnC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,WAAW,EAAE,MAAM,CAAC,WAAW;SAClC,CAAC,CAAC;KACN,CAAC;AACN,CAAC;AACD,gDAAgD;AAChD,qDAAqD;AACrD,SAAgB,cAAc,CAAC,QAAqD,EAAE,KAAqD;IACvI,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC;IAC/B,OAAO;QACH,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,UAAU;QACnC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS;QAC/B,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM;QACzB,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM;QACzB,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO;QAC3B,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW;QACnC,YAAY,EAAE,KAAK,CAAC,SAAS,CAAC,YAAY;QAC1C,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,QAAQ,EAAE;YACN,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK;YAC3C,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK;YACrC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK;YACjC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK;YACnC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK;YACjC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK;SAClC;QACD,UAAU,EAAE;YACR,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK;YAC7B,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SACnG;QACD,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC9E,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;KACvE,CAAC;AACN,CAAC;AACD,SAAgB,cAAc,CAAC,QAAyD,EAAE,OAAe;IACrG,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC;IAC/B,OAAO;QACH,OAAO;QACP,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5I,UAAU,EAAE;YACR,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK;YAC7B,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,MAAM,EAAE,IAAI,CAAC,MAAM;aACtB,CAAC,CAAC;YACH,kBAAkB,EAAE,KAAK,CAAC,UAAU,CAAC,kBAAkB;YACvD,sBAAsB,EAAE,KAAK,CAAC,UAAU,CAAC,sBAAsB;SAClE;QACD,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;KAC3C,CAAC;AACN,CAAC;AACD,SAAgB,YAAY,CAAC,QAAuD,EAAE,OAAe;IACjG,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;IAC7B,OAAO;QACH,OAAO;QACP,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,YAAY,EAAE;YACV,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK;YAC/B,WAAW,EAAE,KAAK,CAAC,YAAY,CAAC,WAAW;YAC3C,oBAAoB,EAAE,KAAK,CAAC,YAAY,CAAC,oBAAoB;YAC7D,sBAAsB,EAAE,KAAK,CAAC,YAAY,CAAC,sBAAsB;YACjE,cAAc,EAAE,KAAK,CAAC,YAAY,CAAC,cAAc;YACjD,gBAAgB,EAAE,KAAK,CAAC,YAAY,CAAC,gBAAgB;YACrD,SAAS,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM;YAC1C,cAAc,EAAE,KAAK,CAAC,YAAY,CAAC,cAAc;YACjD,gBAAgB,EAAE,KAAK,CAAC,YAAY,CAAC,gBAAgB;YACrD,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC3C,GAAG,IAAI;gBACP,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC;aAC5H,CAAC,CAAC;SACN;QACD,QAAQ,EAAE;YACN,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK;YAC3B,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW;YACvC,oBAAoB,EAAE,KAAK,CAAC,QAAQ,CAAC,oBAAoB;YACzD,sBAAsB,EAAE,KAAK,CAAC,QAAQ,CAAC,sBAAsB;YAC7D,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;YAC/B,mBAAmB,EAAE,KAAK,CAAC,QAAQ,CAAC,mBAAmB;YACvD,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;SACxI;QACD,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE;YACR,OAAO,EAAE,KAAK,CAAC,aAAa,CAAC,OAAO;YACpC,iBAAiB,EAAE,KAAK,CAAC,aAAa,CAAC,iBAAiB,CAAC,GAAG;YAC5D,UAAU,EAAE,KAAK,CAAC,aAAa,CAAC,UAAU;YAC1C,kBAAkB,EAAE,KAAK,CAAC,aAAa,CAAC,kBAAkB;YAC1D,4BAA4B,EAAE,KAAK,CAAC,aAAa,CAAC,iBAAiB,CAAC,cAAc;YAClF,qBAAqB,EAAE,KAAK,CAAC,aAAa,CAAC,qBAAqB;SACnE;KACJ,CAAC;AACN,CAAC;AACD,SAAgB,kBAAkB,CAAC,QAA8D,EAAE,OAAe;IAC9G,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC7B,OAAO;QACH,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,QAAQ,EAAE,IAAI,CAAC,IAAI,KAAK,MAAM;YAC1B,CAAC,CAAC,IAAI,CAAC,MAAM;YACb,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,iBAAiB;gBAC7B,CAAC,CAAC,IAAI,CAAC,gBAAgB;gBACvB,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS;oBACrB,CAAC,CAAC,IAAI,CAAC,SAAS;oBAChB,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,iBAAiB;wBAC7B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO;wBACpB,CAAC,CAAC,IAAI,CAAC,UAAU;QACjC,IAAI;KACP,CAAC;AACN,CAAC;AACD,SAAgB,aAAa,CAAC,QAAwD,EAAE,OAAe;IACnG,OAAO,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;AAC3C,CAAC;AACD,SAAgB,YAAY,CAAC,QAAuD,EAAE,OAAe;IACjG,OAAO,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;AAC1C,CAAC;AACD,SAAgB,WAAW,CAAC,QAAsD,EAAE,OAAe;IAC/F,OAAO,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC","sourcesContent":["// @concord-file ne-surface-show-model\n// @concord-implements docs/feature/inspection/cli.md\n// @concord-implements docs/feature/run-inspection/README.md\nimport type { InspectionOverviewResult, InspectionScoredValue, InspectionSuccessDocumentFor, InspectionTraceDetailResult, } from \"../inspection/index.ts\";\nimport type { AttemptOutcome as RecordAttemptOutcome, MembershipAction as RecordMembershipAction } from \"../record/model/core.ts\";\nimport type { AdapterIdentity } from \"../record/model/run-context.ts\";\nimport type { Verdict as PublicVerdict } from \"../shared/types.ts\";\nimport type { AgentTurnOutcome as ReceiptAgentTurnOutcome, SANDBOX_COMMAND_PHASES } from \"../record/family/protocol-values.ts\";\nexport type MetricState = \"available\" | \"partial\" | \"unavailable\" | \"empty\" | \"unsupported\" | \"failed\";\nexport type Verdict = PublicVerdict;\nexport type MembershipAction = RecordMembershipAction;\nexport type AttemptOutcome = RecordAttemptOutcome;\nexport type AgentTurnOutcome = ReceiptAgentTurnOutcome;\nexport type SectionState = \"available\" | \"not-recorded\" | \"partial\" | \"unavailable\";\nexport type ProjectionState = \"complete\" | \"partial\" | \"not-recorded\" | \"invalid\";\nexport type SourceState = \"available\" | \"not-recorded\" | \"invalid\";\nexport type ScoredValue = InspectionScoredValue;\nexport type SourceContent = {\n readonly state: \"available\";\n readonly text: string;\n} | {\n readonly state: \"omitted\";\n readonly reason: \"inspection-result-byte-limit\";\n readonly byteLength: number;\n readonly byteLimit: number;\n};\nexport type AssertionSource = {\n readonly state: \"mapped\";\n readonly sourceItemId: string;\n readonly sha256: string;\n} | {\n readonly state: \"unmapped\";\n readonly reason: \"source-snapshot-not-recorded\" | \"position-unrepresentable\";\n};\nexport type CommandPhase = (typeof SANDBOX_COMMAND_PHASES)[number];\nexport type CommandOutcome = \"exited\" | \"terminated\" | \"not-started\";\nexport type Metric = {\n readonly state: MetricState;\n readonly value: number | null;\n};\nexport type ExecutionValue = {\n readonly state: \"available\";\n readonly value: string;\n} | {\n readonly state: \"mixed\";\n} | {\n readonly state: \"unavailable\";\n};\nexport type AdapterValue = {\n readonly state: \"available\";\n readonly value: AdapterIdentity;\n} | {\n readonly state: \"mixed\";\n};\nexport type Aggregate = {\n readonly evaluationKind: \"pass\" | \"points\" | \"mixed\";\n readonly expected: number;\n readonly observed: number;\n readonly passed: number;\n readonly failed: number;\n readonly errored: number;\n readonly skipped: number;\n readonly passRate: Metric;\n readonly score: Metric;\n readonly durationMs: Metric;\n readonly tokens: Metric;\n};\nexport interface OverviewView {\n readonly totals: Aggregate;\n readonly experiments: readonly {\n readonly experimentId: string;\n readonly adapter: AdapterValue;\n readonly model: ExecutionValue;\n readonly aggregate: Aggregate;\n }[];\n readonly cells: readonly {\n readonly experimentId: string;\n readonly evalId: string;\n readonly aggregate: Aggregate;\n readonly members: readonly {\n readonly publication: {\n readonly state: \"pending\";\n } | {\n readonly state: \"absent\";\n readonly reason: string;\n } | {\n readonly state: \"published\";\n readonly action: MembershipAction;\n readonly attemptId: string;\n readonly attemptLocator: string;\n readonly originRunId: string;\n readonly originSlotId: string;\n readonly verdict: Verdict | null;\n readonly score: Metric;\n readonly durationMs: Metric;\n readonly tokens: Metric;\n };\n }[];\n }[];\n}\nexport interface ExperimentView {\n readonly experimentId: string;\n readonly aggregate: Aggregate;\n readonly cells: OverviewView[\"cells\"];\n readonly costSummary: InspectionSuccessDocumentFor<\"experiment.get\">[\"experiment\"][\"costSummary\"];\n readonly modelUsage: InspectionSuccessDocumentFor<\"experiment.get\">[\"experiment\"][\"modelUsage\"];\n}\nexport interface RunView {\n readonly runId: string;\n readonly experimentId: string;\n readonly adapter: AdapterIdentity | null;\n readonly state: \"active\" | \"completed\" | \"interrupted\" | \"failed\";\n readonly startedAt: number;\n readonly completedAt?: number;\n readonly expected: number;\n readonly observed: number;\n readonly coverage: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"coverage\"];\n readonly usage: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"usage\"];\n readonly limitations: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"limitations\"];\n readonly members: readonly {\n readonly slotId: string;\n readonly evalId: string;\n readonly attemptOrdinal: number;\n readonly locator: string | null;\n readonly state: MembershipAction | \"pending\";\n readonly relation: \"origin\" | \"reference\" | null;\n readonly outcome: AttemptOutcome | null;\n readonly verdict: Verdict | null;\n readonly score: ScoredValue | null;\n readonly coverage: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"slots\"][number][\"coverage\"];\n readonly usage: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"slots\"][number][\"usage\"];\n readonly limitations: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"slots\"][number][\"limitations\"];\n }[];\n}\nexport interface AttemptView {\n readonly totalCosts?: InspectionSuccessDocumentFor<\"attempt.usage\">[\"usage\"][\"totalCosts\"];\n readonly locator: string;\n readonly verdict: Verdict | null;\n readonly attemptId: string;\n readonly evalId: string;\n readonly slotId: string;\n readonly outcome: AttemptOutcome;\n readonly originRunId: string;\n readonly experimentId: string;\n readonly score: ScoredValue;\n readonly sections: {\n readonly assertions: SectionState;\n readonly sources: SectionState;\n readonly trace: SectionState;\n readonly timing: SectionState;\n readonly usage: SectionState;\n readonly diff: SectionState;\n };\n readonly assertions: {\n readonly state: SourceState;\n readonly entries: readonly {\n readonly entryId: string;\n readonly label?: string;\n readonly key?: string;\n readonly groupPath: readonly string[];\n }[];\n };\n readonly evidenceCoverage: readonly string[];\n readonly limitations: readonly string[];\n}\nexport interface SourcesView {\n readonly locator: string;\n readonly state: SourceState;\n readonly items: readonly {\n readonly path: string;\n readonly sourceItemId: string;\n readonly byteLength: number;\n readonly content: SourceContent;\n }[];\n readonly assertions: {\n readonly state: SourceState;\n readonly sites: readonly {\n readonly entryId: string;\n readonly role: \"declaration\" | \"threshold\" | \"score\" | \"gate\" | \"optional\" | \"stop\";\n readonly start: {\n readonly line: number;\n readonly column: number;\n };\n readonly end: {\n readonly line: number;\n readonly column: number;\n };\n readonly source: AssertionSource;\n }[];\n readonly hasMoreSourceSites: boolean;\n readonly omittedSourceSiteCount: number;\n };\n readonly hasMore: boolean;\n readonly omittedItemCount: number;\n}\ntype WithoutTurnId<Value> = Value extends unknown ? Omit<Value, \"turnId\"> : never;\nexport type TraceItem = WithoutTurnId<InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"conversation\"][\"items\"][number]>;\nexport interface TraceView {\n readonly locator: string;\n readonly execution: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"execution\"];\n readonly conversation: {\n readonly state: ProjectionState;\n readonly limitations: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"conversation\"][\"limitations\"];\n readonly limitationsTruncated: boolean;\n readonly omittedLimitationCount: number;\n readonly turnsTruncated: boolean;\n readonly omittedTurnCount: number;\n readonly itemCount: number;\n readonly itemsTruncated: boolean;\n readonly omittedItemCount: number;\n readonly turns: readonly {\n readonly turnId: string;\n readonly sequence: number;\n readonly outcome: AgentTurnOutcome;\n readonly items: readonly TraceItem[];\n }[];\n };\n readonly commands: {\n readonly state: ProjectionState;\n readonly limitations: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"commands\"][\"limitations\"];\n readonly limitationsTruncated: boolean;\n readonly omittedLimitationCount: number;\n readonly hasMore: boolean;\n readonly omittedCommandCount: number;\n readonly items: readonly {\n readonly commandId: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"commands\"][\"items\"][number][\"commandId\"];\n readonly phase: CommandPhase;\n readonly outcome: CommandOutcome;\n }[];\n };\n readonly diagnostics: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"diagnostics\"];\n readonly identities: {\n readonly itemIds: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"identityIndex\"][\"itemIds\"];\n readonly toolOccurrenceIds: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"identityIndex\"][\"toolOccurrenceIds\"][\"ids\"];\n readonly commandIds: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"identityIndex\"][\"commandIds\"];\n readonly omittedItemIdCount: number;\n readonly omittedToolOccurrenceIdCount: number;\n readonly omittedCommandIdCount: number;\n };\n}\nexport interface TraceDetailView {\n readonly locator: string;\n readonly kind: InspectionTraceDetailResult[\"kind\"];\n readonly stableId: string;\n readonly body: InspectionTraceDetailResult;\n}\nexport type ExecutionDisplayPreviewBlock = Extract<TraceView[\"execution\"][\"events\"][number][\"display\"], {\n state: \"present\";\n}>[\"blocks\"][number];\nexport type ExecutionDisplayDetailBlock = NonNullable<Extract<TraceDetailView[\"body\"], {\n kind: \"execution-event\";\n}>[\"event\"][\"display\"]>[number];\nexport type TimingView = {\n readonly locator: string;\n} & InspectionSuccessDocumentFor<\"attempt.timing\">[\"timing\"];\nexport type UsageView = {\n readonly locator: string;\n} & InspectionSuccessDocumentFor<\"attempt.usage\">[\"usage\"];\nexport type DiffView = {\n readonly locator: string;\n} & InspectionSuccessDocumentFor<\"attempt.diff\">[\"diff\"];\nconst metric = (value: {\n readonly state: MetricState;\n readonly value: number | null;\n}): Metric => ({ state: value.state, value: value.value });\nconst aggregate = (value: InspectionOverviewResult[\"totals\"]): Aggregate => ({\n evaluationKind: value.evaluationKind,\n expected: value.denominator.expected,\n observed: value.denominator.observed,\n passed: value.verdict.tally.passed,\n failed: value.verdict.tally.failed,\n errored: value.verdict.tally.errored,\n skipped: value.verdict.tally.skipped,\n passRate: metric(value.verdict.passRate),\n score: metric(value.score),\n durationMs: metric(value.durationMs),\n tokens: metric(value.tokens),\n});\nconst cells = (values: InspectionOverviewResult[\"cells\"]): OverviewView[\"cells\"] => values.map((cell) => ({\n experimentId: cell.experimentId,\n evalId: cell.evalId,\n aggregate: aggregate(cell),\n members: cell.members.map((member) => ({\n publication: member.publication.state === \"published\"\n ? {\n ...member.publication,\n score: metric(member.publication.score),\n durationMs: metric(member.publication.durationMs),\n tokens: metric(member.publication.tokens),\n }\n : member.publication,\n })),\n}));\nexport function projectOverview(document: InspectionSuccessDocumentFor<\"overview.get\">): OverviewView {\n return {\n totals: aggregate(document.overview.totals),\n experiments: document.overview.experiments.map((experiment) => ({\n experimentId: experiment.experimentId,\n adapter: experiment.adapter,\n model: experiment.model,\n aggregate: aggregate(experiment),\n })),\n cells: cells(document.overview.cells),\n };\n}\nexport function projectExperiment(document: InspectionSuccessDocumentFor<\"experiment.get\">): ExperimentView {\n return {\n experimentId: document.experiment.experiment.experimentId,\n aggregate: aggregate(document.experiment.experiment),\n cells: cells(document.experiment.cells),\n costSummary: document.experiment.costSummary,\n modelUsage: document.experiment.modelUsage,\n };\n}\n// @concord-code ne-surface-project-run-view\n// @concord-implements docs/feature/inspection/use-case/inspection-check-completeness.md\nexport function projectRun(document: InspectionSuccessDocumentFor<\"run.get\">): RunView {\n const detail = document.run;\n const value = detail.value;\n return {\n runId: value.runId,\n experimentId: value.experimentId,\n adapter: value.context?.execution.adapter ?? null,\n state: value.state,\n startedAt: value.startedAt,\n ...(value.completedAt === undefined ? {} : { completedAt: value.completedAt }),\n expected: detail.denominator.expected,\n observed: detail.denominator.published,\n coverage: detail.coverage,\n usage: detail.usage,\n limitations: detail.limitations,\n members: detail.slots.map((member) => ({\n slotId: member.slotId,\n evalId: member.evalId,\n attemptOrdinal: member.attemptOrdinal,\n locator: member.locator,\n state: member.state,\n relation: member.relation,\n outcome: member.outcome,\n verdict: member.verdict,\n score: member.score,\n coverage: member.coverage,\n usage: member.usage,\n limitations: member.limitations,\n })),\n };\n}\n// @concord-code ne-surface-project-attempt-view\n// @concord-implements docs/feature/inspection/cli.md\nexport function projectAttempt(document: InspectionSuccessDocumentFor<\"attempt.get\">, usage?: InspectionSuccessDocumentFor<\"attempt.usage\">): AttemptView {\n const value = document.attempt;\n return {\n locator: value.locator,\n totalCosts: usage?.usage.totalCosts,\n verdict: value.verdict,\n attemptId: value.core.attemptId,\n evalId: value.core.evalId,\n slotId: value.core.slotId,\n outcome: value.core.outcome,\n originRunId: value.core.originRunId,\n experimentId: value.originRun.experimentId,\n score: value.score,\n sections: {\n assertions: value.sections.assertions.state,\n sources: value.sections.sources.state,\n trace: value.sections.trace.state,\n timing: value.sections.timing.state,\n usage: value.sections.usage.state,\n diff: value.sections.diff.state,\n },\n assertions: {\n state: value.assertions.state,\n entries: value.assertions.entries.map((entry) => ({ entryId: entry.entryId, ...entry.display })),\n },\n evidenceCoverage: value.evidenceCoverage.map((entry) => JSON.stringify(entry)),\n limitations: value.limitations.map((entry) => JSON.stringify(entry)),\n };\n}\nexport function projectSources(document: InspectionSuccessDocumentFor<\"attempt.sources\">, locator: string): SourcesView {\n const value = document.sources;\n return {\n locator,\n state: value.state,\n items: value.items.map((item) => ({ path: item.path, sourceItemId: item.sourceItemId, byteLength: item.byteLength, content: item.content })),\n assertions: {\n state: value.assertions.state,\n sites: value.assertions.sourceSites.map((site) => ({\n entryId: site.entryId,\n role: site.role,\n start: site.start,\n end: site.end,\n source: site.source,\n })),\n hasMoreSourceSites: value.assertions.hasMoreSourceSites,\n omittedSourceSiteCount: value.assertions.omittedSourceSiteCount,\n },\n hasMore: value.hasMore,\n omittedItemCount: value.omittedItemCount,\n };\n}\nexport function projectTrace(document: InspectionSuccessDocumentFor<\"attempt.trace\">, locator: string): TraceView {\n const value = document.trace;\n return {\n locator,\n execution: value.execution,\n conversation: {\n state: value.conversation.state,\n limitations: value.conversation.limitations,\n limitationsTruncated: value.conversation.limitationsTruncated,\n omittedLimitationCount: value.conversation.omittedLimitationCount,\n turnsTruncated: value.conversation.turnsTruncated,\n omittedTurnCount: value.conversation.omittedTurnCount,\n itemCount: value.conversation.items.length,\n itemsTruncated: value.conversation.itemsTruncated,\n omittedItemCount: value.conversation.omittedItemCount,\n turns: value.conversation.turns.map((turn) => ({\n ...turn,\n items: value.conversation.items.filter((item) => item.turnId === turn.turnId).map(({ turnId: _turnId, ...item }) => item),\n })),\n },\n commands: {\n state: value.commands.state,\n limitations: value.commands.limitations,\n limitationsTruncated: value.commands.limitationsTruncated,\n omittedLimitationCount: value.commands.omittedLimitationCount,\n hasMore: value.commands.hasMore,\n omittedCommandCount: value.commands.omittedCommandCount,\n items: value.commands.items.map((command) => ({ commandId: command.commandId, phase: command.phase, outcome: command.outcome.kind })),\n },\n diagnostics: value.diagnostics,\n identities: {\n itemIds: value.identityIndex.itemIds,\n toolOccurrenceIds: value.identityIndex.toolOccurrenceIds.ids,\n commandIds: value.identityIndex.commandIds,\n omittedItemIdCount: value.identityIndex.omittedItemIdCount,\n omittedToolOccurrenceIdCount: value.identityIndex.toolOccurrenceIds.omittedIdCount,\n omittedCommandIdCount: value.identityIndex.omittedCommandIdCount,\n },\n };\n}\nexport function projectTraceDetail(document: InspectionSuccessDocumentFor<\"attempt.trace.detail\">, locator: string): TraceDetailView {\n const body = document.detail;\n return {\n locator,\n kind: body.kind,\n stableId: body.kind === \"item\"\n ? body.itemId\n : body.kind === \"tool-occurrence\"\n ? body.toolOccurrenceId\n : body.kind === \"command\"\n ? body.commandId\n : body.kind === \"execution-event\"\n ? body.event.eventId\n : body.evidenceId,\n body,\n };\n}\nexport function projectTiming(document: InspectionSuccessDocumentFor<\"attempt.timing\">, locator: string): TimingView {\n return { locator, ...document.timing };\n}\nexport function projectUsage(document: InspectionSuccessDocumentFor<\"attempt.usage\">, locator: string): UsageView {\n return { locator, ...document.usage };\n}\nexport function projectDiff(document: InspectionSuccessDocumentFor<\"attempt.diff\">, locator: string): DiffView {\n return { locator, ...document.diff };\n}\n"]}
|
|
1
|
+
{"version":3,"file":"model.js","sourceRoot":"","sources":["model.ts"],"names":[],"mappings":";;AAwSA,0CAWC;AACD,8CAUC;AAGD,gCA8BC;AAGD,wCA6BC;AACD,wCAqBC;AACD,oCAuCC;AACD,gDAgBC;AACD,sCAEC;AACD,oCAEC;AACD,kCAEC;AA/MD,MAAM,MAAM,GAAG,CAAC,KAGf,EAAU,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;AAC3D,MAAM,SAAS,GAAG,CAAC,KAAyC,EAAa,EAAE,CAAC,CAAC;IACzE,cAAc,EAAE,KAAK,CAAC,cAAc;IACpC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,QAAQ;IACpC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,QAAQ;IACpC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM;IAClC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM;IAClC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO;IACpC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO;IACpC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;IACxC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;IACpC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;CAC/B,CAAC,CAAC;AACH,MAAM,KAAK,GAAG,CAAC,MAAyC,EAAyB,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACtG,YAAY,EAAE,IAAI,CAAC,YAAY;IAC/B,MAAM,EAAE,IAAI,CAAC,MAAM;IACnB,SAAS,EAAE,SAAS,CAAC,IAAI,CAAC;IAC1B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACnC,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,KAAK,WAAW;YACjD,CAAC,CAAC;gBACE,GAAG,MAAM,CAAC,WAAW;gBACrB,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC;gBACvC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC;gBACjD,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC;aAC5C;YACD,CAAC,CAAC,MAAM,CAAC,WAAW;KAC3B,CAAC,CAAC;CACN,CAAC,CAAC,CAAC;AACJ,SAAgB,eAAe,CAAC,QAAsD;IAClF,OAAO;QACH,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC3C,WAAW,EAAE,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YAC5D,YAAY,EAAE,UAAU,CAAC,YAAY;YACrC,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,KAAK,EAAE,UAAU,CAAC,KAAK;YACvB,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC;SACnC,CAAC,CAAC;QACH,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;KACxC,CAAC;AACN,CAAC;AACD,SAAgB,iBAAiB,CAAC,QAAwD;IACtF,OAAO;QACH,YAAY,EAAE,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,YAAY;QACzD,KAAK,EAAE,QAAQ,CAAC,UAAU,CAAC,KAAK;QAChC,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,OAAO;QAC/C,SAAS,EAAE,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC;QACpD,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC;QACvC,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,WAAW;QAC5C,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,UAAU;KAC7C,CAAC;AACN,CAAC;AACD,gBAAgB;AAChB,wFAAwF;AACxF,SAAgB,UAAU,CAAC,QAAiD;IACxE,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC;IAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,OAAO;QACH,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,OAAO,IAAI,IAAI;QACjD,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;QAC9E,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ;QACrC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,SAAS;QACtC,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,OAAO,EAAE,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAC7C,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;YACzC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,WAAW,EAAE,MAAM,CAAC,WAAW;SAClC,CAAC,CAAC;KACN,CAAC;AACN,CAAC;AACD,gBAAgB;AAChB,qDAAqD;AACrD,SAAgB,cAAc,CAAC,QAAqD,EAAE,KAAqD;IACvI,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC;IAC/B,OAAO;QACH,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,UAAU;QACnC,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS;QAC/B,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM;QACzB,gBAAgB,EAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB;QAC7C,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,OAAO;QAC3B,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW;QACnC,YAAY,EAAE,KAAK,CAAC,SAAS,CAAC,YAAY;QAC1C,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,QAAQ,EAAE;YACN,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK;YAC3C,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK;YACrC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK;YACjC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK;YACnC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK;YACjC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK;SAClC;QACD,UAAU,EAAE;YACR,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK;YAC7B,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;SACnG;QACD,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC9E,WAAW,EAAE,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;KACvE,CAAC;AACN,CAAC;AACD,SAAgB,cAAc,CAAC,QAAyD,EAAE,OAAe;IACrG,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC;IAC/B,OAAO;QACH,OAAO;QACP,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC5I,UAAU,EAAE;YACR,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,KAAK;YAC7B,KAAK,EAAE,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC/C,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,MAAM,EAAE,IAAI,CAAC,MAAM;aACtB,CAAC,CAAC;YACH,kBAAkB,EAAE,KAAK,CAAC,UAAU,CAAC,kBAAkB;YACvD,sBAAsB,EAAE,KAAK,CAAC,UAAU,CAAC,sBAAsB;SAClE;QACD,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;KAC3C,CAAC;AACN,CAAC;AACD,SAAgB,YAAY,CAAC,QAAuD,EAAE,OAAe;IACjG,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;IAC7B,OAAO;QACH,OAAO;QACP,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,YAAY,EAAE;YACV,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK;YAC/B,WAAW,EAAE,KAAK,CAAC,YAAY,CAAC,WAAW;YAC3C,oBAAoB,EAAE,KAAK,CAAC,YAAY,CAAC,oBAAoB;YAC7D,sBAAsB,EAAE,KAAK,CAAC,YAAY,CAAC,sBAAsB;YACjE,cAAc,EAAE,KAAK,CAAC,YAAY,CAAC,cAAc;YACjD,gBAAgB,EAAE,KAAK,CAAC,YAAY,CAAC,gBAAgB;YACrD,SAAS,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM;YAC1C,cAAc,EAAE,KAAK,CAAC,YAAY,CAAC,cAAc;YACjD,gBAAgB,EAAE,KAAK,CAAC,YAAY,CAAC,gBAAgB;YACrD,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC3C,GAAG,IAAI;gBACP,KAAK,EAAE,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC;aAC5H,CAAC,CAAC;SACN;QACD,QAAQ,EAAE;YACN,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK;YAC3B,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,WAAW;YACvC,oBAAoB,EAAE,KAAK,CAAC,QAAQ,CAAC,oBAAoB;YACzD,sBAAsB,EAAE,KAAK,CAAC,QAAQ,CAAC,sBAAsB;YAC7D,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO;YAC/B,mBAAmB,EAAE,KAAK,CAAC,QAAQ,CAAC,mBAAmB;YACvD,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;SACxI;QACD,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE;YACR,OAAO,EAAE,KAAK,CAAC,aAAa,CAAC,OAAO;YACpC,iBAAiB,EAAE,KAAK,CAAC,aAAa,CAAC,iBAAiB,CAAC,GAAG;YAC5D,UAAU,EAAE,KAAK,CAAC,aAAa,CAAC,UAAU;YAC1C,kBAAkB,EAAE,KAAK,CAAC,aAAa,CAAC,kBAAkB;YAC1D,4BAA4B,EAAE,KAAK,CAAC,aAAa,CAAC,iBAAiB,CAAC,cAAc;YAClF,qBAAqB,EAAE,KAAK,CAAC,aAAa,CAAC,qBAAqB;SACnE;KACJ,CAAC;AACN,CAAC;AACD,SAAgB,kBAAkB,CAAC,QAA8D,EAAE,OAAe;IAC9G,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC7B,OAAO;QACH,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,QAAQ,EAAE,IAAI,CAAC,IAAI,KAAK,MAAM;YAC1B,CAAC,CAAC,IAAI,CAAC,MAAM;YACb,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,iBAAiB;gBAC7B,CAAC,CAAC,IAAI,CAAC,gBAAgB;gBACvB,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS;oBACrB,CAAC,CAAC,IAAI,CAAC,SAAS;oBAChB,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,iBAAiB;wBAC7B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO;wBACpB,CAAC,CAAC,IAAI,CAAC,UAAU;QACjC,IAAI;KACP,CAAC;AACN,CAAC;AACD,SAAgB,aAAa,CAAC,QAAwD,EAAE,OAAe;IACnG,OAAO,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC;AAC3C,CAAC;AACD,SAAgB,YAAY,CAAC,QAAuD,EAAE,OAAe;IACjG,OAAO,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;AAC1C,CAAC;AACD,SAAgB,WAAW,CAAC,QAAsD,EAAE,OAAe;IAC/F,OAAO,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC","sourcesContent":["// @concord-file\n// @concord-implements docs/feature/inspection/cli.md\n// @concord-implements docs/feature/run-inspection/README.md\nimport type { InspectionOverviewResult, InspectionScoredValue, InspectionSuccessDocumentFor, InspectionTraceDetailResult, } from \"../inspection/index.ts\";\nimport type { AttemptOutcome as RecordAttemptOutcome, MembershipAction as RecordMembershipAction } from \"../record/model/core.ts\";\nimport type { AdapterIdentity } from \"../record/model/run-context.ts\";\nimport type { Verdict as PublicVerdict } from \"../shared/types.ts\";\nimport type { AgentTurnOutcome as ReceiptAgentTurnOutcome } from \"../record/family/protocol-values.ts\";\nexport type MetricState = \"available\" | \"partial\" | \"unavailable\" | \"empty\" | \"unsupported\" | \"failed\";\nexport type Verdict = PublicVerdict;\nexport type MembershipAction = RecordMembershipAction;\nexport type AttemptOutcome = RecordAttemptOutcome;\nexport type AgentTurnOutcome = ReceiptAgentTurnOutcome;\nexport type SectionState = \"available\" | \"not-recorded\" | \"partial\" | \"unavailable\";\nexport type ProjectionState = \"complete\" | \"partial\" | \"not-recorded\" | \"invalid\";\nexport type SourceState = \"available\" | \"not-recorded\" | \"invalid\";\nexport type ScoredValue = InspectionScoredValue;\nexport type SourceContent = {\n readonly state: \"available\";\n readonly text: string;\n} | {\n readonly state: \"omitted\";\n readonly reason: \"inspection-result-byte-limit\";\n readonly byteLength: number;\n readonly byteLimit: number;\n};\nexport type AssertionSource = {\n readonly state: \"mapped\";\n readonly sourceItemId: string;\n readonly sha256: string;\n} | {\n readonly state: \"unmapped\";\n readonly reason: \"source-snapshot-not-recorded\" | \"position-unrepresentable\";\n};\nexport type CommandPhase = InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"commands\"][\"items\"][number][\"phase\"];\nexport type CommandOutcome = \"exited\" | \"terminated\" | \"not-started\";\nexport type Metric = {\n readonly state: MetricState;\n readonly value: number | null;\n};\nexport type ExecutionValue = {\n readonly state: \"available\";\n readonly value: string;\n} | {\n readonly state: \"mixed\";\n} | {\n readonly state: \"unavailable\";\n};\nexport type AdapterValue = {\n readonly state: \"available\";\n readonly value: AdapterIdentity;\n} | {\n readonly state: \"mixed\";\n};\nexport type Aggregate = {\n readonly evaluationKind: \"pass\" | \"points\" | \"mixed\";\n readonly expected: number;\n readonly observed: number;\n readonly passed: number;\n readonly failed: number;\n readonly errored: number;\n readonly skipped: number;\n readonly passRate: Metric;\n readonly score: Metric;\n readonly durationMs: Metric;\n readonly tokens: Metric;\n};\nexport interface OverviewView {\n readonly totals: Aggregate;\n readonly experiments: readonly {\n readonly experimentId: string;\n readonly adapter: AdapterValue;\n readonly model: ExecutionValue;\n readonly aggregate: Aggregate;\n }[];\n readonly cells: readonly {\n readonly experimentId: string;\n readonly evalId: string;\n readonly aggregate: Aggregate;\n readonly members: readonly {\n readonly publication: {\n readonly state: \"pending\";\n } | {\n readonly state: \"absent\";\n readonly reason: string;\n } | {\n readonly state: \"published\";\n readonly action: MembershipAction;\n readonly attemptId: string;\n readonly attemptLocator: string;\n readonly originRunId: string;\n readonly originPlannedAttemptId: string;\n readonly verdict: Verdict | null;\n readonly score: Metric;\n readonly durationMs: Metric;\n readonly tokens: Metric;\n };\n }[];\n }[];\n}\nexport interface ExperimentView {\n readonly plugins: InspectionSuccessDocumentFor<\"experiment.get\">[\"experiment\"][\"experiment\"][\"plugins\"];\n readonly notes: InspectionSuccessDocumentFor<\"experiment.get\">[\"experiment\"][\"notes\"];\n readonly experimentId: string;\n readonly aggregate: Aggregate;\n readonly cells: OverviewView[\"cells\"];\n readonly costSummary: InspectionSuccessDocumentFor<\"experiment.get\">[\"experiment\"][\"costSummary\"];\n readonly modelUsage: InspectionSuccessDocumentFor<\"experiment.get\">[\"experiment\"][\"modelUsage\"];\n}\nexport interface RunView {\n readonly runId: string;\n readonly experimentId: string;\n readonly adapter: AdapterIdentity | null;\n readonly state: \"active\" | \"completed\" | \"interrupted\" | \"failed\";\n readonly startedAt: number;\n readonly completedAt?: number;\n readonly expected: number;\n readonly observed: number;\n readonly coverage: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"coverage\"];\n readonly usage: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"usage\"];\n readonly limitations: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"limitations\"];\n readonly members: readonly {\n readonly plannedAttemptId: string;\n readonly evalId: string;\n readonly attemptOrdinal: number;\n readonly locator: string | null;\n readonly state: MembershipAction | \"pending\";\n readonly relation: \"origin\" | \"reference\" | null;\n readonly outcome: AttemptOutcome | null;\n readonly verdict: Verdict | null;\n readonly score: ScoredValue | null;\n readonly coverage: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"plannedAttempts\"][number][\"coverage\"];\n readonly usage: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"plannedAttempts\"][number][\"usage\"];\n readonly limitations: InspectionSuccessDocumentFor<\"run.get\">[\"run\"][\"plannedAttempts\"][number][\"limitations\"];\n }[];\n}\nexport interface AttemptView {\n readonly notes: InspectionSuccessDocumentFor<\"attempt.get\">[\"attempt\"][\"notes\"];\n readonly totalCosts?: InspectionSuccessDocumentFor<\"attempt.usage\">[\"usage\"][\"totalCosts\"];\n readonly locator: string;\n readonly verdict: Verdict | null;\n readonly attemptId: string;\n readonly evalId: string;\n readonly plannedAttemptId: string;\n readonly outcome: AttemptOutcome;\n readonly originRunId: string;\n readonly experimentId: string;\n readonly score: ScoredValue;\n readonly sections: {\n readonly assertions: SectionState;\n readonly sources: SectionState;\n readonly trace: SectionState;\n readonly timing: SectionState;\n readonly usage: SectionState;\n readonly diff: SectionState;\n };\n readonly assertions: {\n readonly state: SourceState;\n readonly entries: readonly {\n readonly entryId: string;\n readonly label?: string;\n readonly key?: string;\n readonly groupPath: readonly string[];\n }[];\n };\n readonly evidenceCoverage: readonly string[];\n readonly limitations: readonly string[];\n}\nexport interface SourcesView {\n readonly locator: string;\n readonly state: SourceState;\n readonly items: readonly {\n readonly path: string;\n readonly sourceItemId: string;\n readonly byteLength: number;\n readonly content: SourceContent;\n }[];\n readonly assertions: {\n readonly state: SourceState;\n readonly sites: readonly {\n readonly entryId: string;\n readonly role: \"declaration\" | \"threshold\" | \"score\" | \"gate\" | \"optional\" | \"stop\";\n readonly start: {\n readonly line: number;\n readonly column: number;\n };\n readonly end: {\n readonly line: number;\n readonly column: number;\n };\n readonly source: AssertionSource;\n }[];\n readonly hasMoreSourceSites: boolean;\n readonly omittedSourceSiteCount: number;\n };\n readonly hasMore: boolean;\n readonly omittedItemCount: number;\n}\ntype WithoutTurnId<Value> = Value extends unknown ? Omit<Value, \"turnId\"> : never;\nexport type TraceItem = WithoutTurnId<InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"conversation\"][\"items\"][number]>;\nexport interface TraceView {\n readonly locator: string;\n readonly execution: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"execution\"];\n readonly conversation: {\n readonly state: ProjectionState;\n readonly limitations: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"conversation\"][\"limitations\"];\n readonly limitationsTruncated: boolean;\n readonly omittedLimitationCount: number;\n readonly turnsTruncated: boolean;\n readonly omittedTurnCount: number;\n readonly itemCount: number;\n readonly itemsTruncated: boolean;\n readonly omittedItemCount: number;\n readonly turns: readonly {\n readonly turnId: string;\n readonly sequence: number;\n readonly outcome: AgentTurnOutcome;\n readonly items: readonly TraceItem[];\n }[];\n };\n readonly commands: {\n readonly state: ProjectionState;\n readonly limitations: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"commands\"][\"limitations\"];\n readonly limitationsTruncated: boolean;\n readonly omittedLimitationCount: number;\n readonly hasMore: boolean;\n readonly omittedCommandCount: number;\n readonly items: readonly {\n readonly commandId: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"commands\"][\"items\"][number][\"commandId\"];\n readonly phase: CommandPhase;\n readonly outcome: CommandOutcome;\n }[];\n };\n readonly diagnostics: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"diagnostics\"];\n readonly identities: {\n readonly itemIds: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"identityIndex\"][\"itemIds\"];\n readonly toolOccurrenceIds: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"identityIndex\"][\"toolOccurrenceIds\"][\"ids\"];\n readonly commandIds: InspectionSuccessDocumentFor<\"attempt.trace\">[\"trace\"][\"identityIndex\"][\"commandIds\"];\n readonly omittedItemIdCount: number;\n readonly omittedToolOccurrenceIdCount: number;\n readonly omittedCommandIdCount: number;\n };\n}\nexport interface TraceDetailView {\n readonly locator: string;\n readonly kind: InspectionTraceDetailResult[\"kind\"];\n readonly stableId: string;\n readonly body: InspectionTraceDetailResult;\n}\nexport type ExecutionDisplayPreviewBlock = Extract<TraceView[\"execution\"][\"events\"][number][\"display\"], {\n state: \"present\";\n}>[\"blocks\"][number];\nexport type ExecutionDisplayDetailBlock = NonNullable<Extract<TraceDetailView[\"body\"], {\n kind: \"execution-event\";\n}>[\"event\"][\"display\"]>[number];\nexport type TimingView = {\n readonly locator: string;\n} & InspectionSuccessDocumentFor<\"attempt.timing\">[\"timing\"];\nexport type UsageView = {\n readonly locator: string;\n} & InspectionSuccessDocumentFor<\"attempt.usage\">[\"usage\"];\nexport type DiffView = {\n readonly locator: string;\n} & InspectionSuccessDocumentFor<\"attempt.diff\">[\"diff\"];\nconst metric = (value: {\n readonly state: MetricState;\n readonly value: number | null;\n}): Metric => ({ state: value.state, value: value.value });\nconst aggregate = (value: InspectionOverviewResult[\"totals\"]): Aggregate => ({\n evaluationKind: value.evaluationKind,\n expected: value.denominator.expected,\n observed: value.denominator.observed,\n passed: value.verdict.tally.passed,\n failed: value.verdict.tally.failed,\n errored: value.verdict.tally.errored,\n skipped: value.verdict.tally.skipped,\n passRate: metric(value.verdict.passRate),\n score: metric(value.score),\n durationMs: metric(value.durationMs),\n tokens: metric(value.tokens),\n});\nconst cells = (values: InspectionOverviewResult[\"cells\"]): OverviewView[\"cells\"] => values.map((cell) => ({\n experimentId: cell.experimentId,\n evalId: cell.evalId,\n aggregate: aggregate(cell),\n members: cell.members.map((member) => ({\n publication: member.publication.state === \"published\"\n ? {\n ...member.publication,\n score: metric(member.publication.score),\n durationMs: metric(member.publication.durationMs),\n tokens: metric(member.publication.tokens),\n }\n : member.publication,\n })),\n}));\nexport function projectOverview(document: InspectionSuccessDocumentFor<\"overview.get\">): OverviewView {\n return {\n totals: aggregate(document.overview.totals),\n experiments: document.overview.experiments.map((experiment) => ({\n experimentId: experiment.experimentId,\n adapter: experiment.adapter,\n model: experiment.model,\n aggregate: aggregate(experiment),\n })),\n cells: cells(document.overview.cells),\n };\n}\nexport function projectExperiment(document: InspectionSuccessDocumentFor<\"experiment.get\">): ExperimentView {\n return {\n experimentId: document.experiment.experiment.experimentId,\n notes: document.experiment.notes,\n plugins: document.experiment.experiment.plugins,\n aggregate: aggregate(document.experiment.experiment),\n cells: cells(document.experiment.cells),\n costSummary: document.experiment.costSummary,\n modelUsage: document.experiment.modelUsage,\n };\n}\n// @concord-code\n// @concord-implements docs/feature/inspection/use-case/inspection-check-completeness.md\nexport function projectRun(document: InspectionSuccessDocumentFor<\"run.get\">): RunView {\n const detail = document.run;\n const value = detail.value;\n return {\n runId: value.runId,\n experimentId: value.experimentId,\n adapter: value.context?.execution.adapter ?? null,\n state: value.state,\n startedAt: value.startedAt,\n ...(value.completedAt === undefined ? {} : { completedAt: value.completedAt }),\n expected: detail.denominator.expected,\n observed: detail.denominator.published,\n coverage: detail.coverage,\n usage: detail.usage,\n limitations: detail.limitations,\n members: detail.plannedAttempts.map((member) => ({\n plannedAttemptId: member.plannedAttemptId,\n evalId: member.evalId,\n attemptOrdinal: member.attemptOrdinal,\n locator: member.locator,\n state: member.state,\n relation: member.relation,\n outcome: member.outcome,\n verdict: member.verdict,\n score: member.score,\n coverage: member.coverage,\n usage: member.usage,\n limitations: member.limitations,\n })),\n };\n}\n// @concord-code\n// @concord-implements docs/feature/inspection/cli.md\nexport function projectAttempt(document: InspectionSuccessDocumentFor<\"attempt.get\">, usage?: InspectionSuccessDocumentFor<\"attempt.usage\">): AttemptView {\n const value = document.attempt;\n return {\n locator: value.locator,\n notes: value.notes,\n totalCosts: usage?.usage.totalCosts,\n verdict: value.verdict,\n attemptId: value.core.attemptId,\n evalId: value.core.evalId,\n plannedAttemptId: value.core.plannedAttemptId,\n outcome: value.core.outcome,\n originRunId: value.core.originRunId,\n experimentId: value.originRun.experimentId,\n score: value.score,\n sections: {\n assertions: value.sections.assertions.state,\n sources: value.sections.sources.state,\n trace: value.sections.trace.state,\n timing: value.sections.timing.state,\n usage: value.sections.usage.state,\n diff: value.sections.diff.state,\n },\n assertions: {\n state: value.assertions.state,\n entries: value.assertions.entries.map((entry) => ({ entryId: entry.entryId, ...entry.display })),\n },\n evidenceCoverage: value.evidenceCoverage.map((entry) => JSON.stringify(entry)),\n limitations: value.limitations.map((entry) => JSON.stringify(entry)),\n };\n}\nexport function projectSources(document: InspectionSuccessDocumentFor<\"attempt.sources\">, locator: string): SourcesView {\n const value = document.sources;\n return {\n locator,\n state: value.state,\n items: value.items.map((item) => ({ path: item.path, sourceItemId: item.sourceItemId, byteLength: item.byteLength, content: item.content })),\n assertions: {\n state: value.assertions.state,\n sites: value.assertions.sourceSites.map((site) => ({\n entryId: site.entryId,\n role: site.role,\n start: site.start,\n end: site.end,\n source: site.source,\n })),\n hasMoreSourceSites: value.assertions.hasMoreSourceSites,\n omittedSourceSiteCount: value.assertions.omittedSourceSiteCount,\n },\n hasMore: value.hasMore,\n omittedItemCount: value.omittedItemCount,\n };\n}\nexport function projectTrace(document: InspectionSuccessDocumentFor<\"attempt.trace\">, locator: string): TraceView {\n const value = document.trace;\n return {\n locator,\n execution: value.execution,\n conversation: {\n state: value.conversation.state,\n limitations: value.conversation.limitations,\n limitationsTruncated: value.conversation.limitationsTruncated,\n omittedLimitationCount: value.conversation.omittedLimitationCount,\n turnsTruncated: value.conversation.turnsTruncated,\n omittedTurnCount: value.conversation.omittedTurnCount,\n itemCount: value.conversation.items.length,\n itemsTruncated: value.conversation.itemsTruncated,\n omittedItemCount: value.conversation.omittedItemCount,\n turns: value.conversation.turns.map((turn) => ({\n ...turn,\n items: value.conversation.items.filter((item) => item.turnId === turn.turnId).map(({ turnId: _turnId, ...item }) => item),\n })),\n },\n commands: {\n state: value.commands.state,\n limitations: value.commands.limitations,\n limitationsTruncated: value.commands.limitationsTruncated,\n omittedLimitationCount: value.commands.omittedLimitationCount,\n hasMore: value.commands.hasMore,\n omittedCommandCount: value.commands.omittedCommandCount,\n items: value.commands.items.map((command) => ({ commandId: command.commandId, phase: command.phase, outcome: command.outcome.kind })),\n },\n diagnostics: value.diagnostics,\n identities: {\n itemIds: value.identityIndex.itemIds,\n toolOccurrenceIds: value.identityIndex.toolOccurrenceIds.ids,\n commandIds: value.identityIndex.commandIds,\n omittedItemIdCount: value.identityIndex.omittedItemIdCount,\n omittedToolOccurrenceIdCount: value.identityIndex.toolOccurrenceIds.omittedIdCount,\n omittedCommandIdCount: value.identityIndex.omittedCommandIdCount,\n },\n };\n}\nexport function projectTraceDetail(document: InspectionSuccessDocumentFor<\"attempt.trace.detail\">, locator: string): TraceDetailView {\n const body = document.detail;\n return {\n locator,\n kind: body.kind,\n stableId: body.kind === \"item\"\n ? body.itemId\n : body.kind === \"tool-occurrence\"\n ? body.toolOccurrenceId\n : body.kind === \"command\"\n ? body.commandId\n : body.kind === \"execution-event\"\n ? body.event.eventId\n : body.evidenceId,\n body,\n };\n}\nexport function projectTiming(document: InspectionSuccessDocumentFor<\"attempt.timing\">, locator: string): TimingView {\n return { locator, ...document.timing };\n}\nexport function projectUsage(document: InspectionSuccessDocumentFor<\"attempt.usage\">, locator: string): UsageView {\n return { locator, ...document.usage };\n}\nexport function projectDiff(document: InspectionSuccessDocumentFor<\"attempt.diff\">, locator: string): DiffView {\n return { locator, ...document.diff };\n}\n"]}
|
package/dist/show/model.d.cts
CHANGED
|
@@ -2,7 +2,7 @@ import type { InspectionScoredValue, InspectionSuccessDocumentFor, InspectionTra
|
|
|
2
2
|
import type { AttemptOutcome as RecordAttemptOutcome, MembershipAction as RecordMembershipAction } from "../record/model/core.cjs";
|
|
3
3
|
import type { AdapterIdentity } from "../record/model/run-context.cjs";
|
|
4
4
|
import type { Verdict as PublicVerdict } from "../shared/types.cjs";
|
|
5
|
-
import type { AgentTurnOutcome as ReceiptAgentTurnOutcome
|
|
5
|
+
import type { AgentTurnOutcome as ReceiptAgentTurnOutcome } from "../record/family/protocol-values.cjs";
|
|
6
6
|
export type MetricState = "available" | "partial" | "unavailable" | "empty" | "unsupported" | "failed";
|
|
7
7
|
export type Verdict = PublicVerdict;
|
|
8
8
|
export type MembershipAction = RecordMembershipAction;
|
|
@@ -29,7 +29,7 @@ export type AssertionSource = {
|
|
|
29
29
|
readonly state: "unmapped";
|
|
30
30
|
readonly reason: "source-snapshot-not-recorded" | "position-unrepresentable";
|
|
31
31
|
};
|
|
32
|
-
export type CommandPhase =
|
|
32
|
+
export type CommandPhase = InspectionSuccessDocumentFor<"attempt.trace">["trace"]["commands"]["items"][number]["phase"];
|
|
33
33
|
export type CommandOutcome = "exited" | "terminated" | "not-started";
|
|
34
34
|
export type Metric = {
|
|
35
35
|
readonly state: MetricState;
|
|
@@ -86,7 +86,7 @@ export interface OverviewView {
|
|
|
86
86
|
readonly attemptId: string;
|
|
87
87
|
readonly attemptLocator: string;
|
|
88
88
|
readonly originRunId: string;
|
|
89
|
-
readonly
|
|
89
|
+
readonly originPlannedAttemptId: string;
|
|
90
90
|
readonly verdict: Verdict | null;
|
|
91
91
|
readonly score: Metric;
|
|
92
92
|
readonly durationMs: Metric;
|
|
@@ -96,6 +96,8 @@ export interface OverviewView {
|
|
|
96
96
|
}[];
|
|
97
97
|
}
|
|
98
98
|
export interface ExperimentView {
|
|
99
|
+
readonly plugins: InspectionSuccessDocumentFor<"experiment.get">["experiment"]["experiment"]["plugins"];
|
|
100
|
+
readonly notes: InspectionSuccessDocumentFor<"experiment.get">["experiment"]["notes"];
|
|
99
101
|
readonly experimentId: string;
|
|
100
102
|
readonly aggregate: Aggregate;
|
|
101
103
|
readonly cells: OverviewView["cells"];
|
|
@@ -115,7 +117,7 @@ export interface RunView {
|
|
|
115
117
|
readonly usage: InspectionSuccessDocumentFor<"run.get">["run"]["usage"];
|
|
116
118
|
readonly limitations: InspectionSuccessDocumentFor<"run.get">["run"]["limitations"];
|
|
117
119
|
readonly members: readonly {
|
|
118
|
-
readonly
|
|
120
|
+
readonly plannedAttemptId: string;
|
|
119
121
|
readonly evalId: string;
|
|
120
122
|
readonly attemptOrdinal: number;
|
|
121
123
|
readonly locator: string | null;
|
|
@@ -124,18 +126,19 @@ export interface RunView {
|
|
|
124
126
|
readonly outcome: AttemptOutcome | null;
|
|
125
127
|
readonly verdict: Verdict | null;
|
|
126
128
|
readonly score: ScoredValue | null;
|
|
127
|
-
readonly coverage: InspectionSuccessDocumentFor<"run.get">["run"]["
|
|
128
|
-
readonly usage: InspectionSuccessDocumentFor<"run.get">["run"]["
|
|
129
|
-
readonly limitations: InspectionSuccessDocumentFor<"run.get">["run"]["
|
|
129
|
+
readonly coverage: InspectionSuccessDocumentFor<"run.get">["run"]["plannedAttempts"][number]["coverage"];
|
|
130
|
+
readonly usage: InspectionSuccessDocumentFor<"run.get">["run"]["plannedAttempts"][number]["usage"];
|
|
131
|
+
readonly limitations: InspectionSuccessDocumentFor<"run.get">["run"]["plannedAttempts"][number]["limitations"];
|
|
130
132
|
}[];
|
|
131
133
|
}
|
|
132
134
|
export interface AttemptView {
|
|
135
|
+
readonly notes: InspectionSuccessDocumentFor<"attempt.get">["attempt"]["notes"];
|
|
133
136
|
readonly totalCosts?: InspectionSuccessDocumentFor<"attempt.usage">["usage"]["totalCosts"];
|
|
134
137
|
readonly locator: string;
|
|
135
138
|
readonly verdict: Verdict | null;
|
|
136
139
|
readonly attemptId: string;
|
|
137
140
|
readonly evalId: string;
|
|
138
|
-
readonly
|
|
141
|
+
readonly plannedAttemptId: string;
|
|
139
142
|
readonly outcome: AttemptOutcome;
|
|
140
143
|
readonly originRunId: string;
|
|
141
144
|
readonly experimentId: string;
|
package/dist/show/model.d.mts
CHANGED
|
@@ -2,7 +2,7 @@ import type { InspectionScoredValue, InspectionSuccessDocumentFor, InspectionTra
|
|
|
2
2
|
import type { AttemptOutcome as RecordAttemptOutcome, MembershipAction as RecordMembershipAction } from "../record/model/core.mjs";
|
|
3
3
|
import type { AdapterIdentity } from "../record/model/run-context.mjs";
|
|
4
4
|
import type { Verdict as PublicVerdict } from "../shared/types.mjs";
|
|
5
|
-
import type { AgentTurnOutcome as ReceiptAgentTurnOutcome
|
|
5
|
+
import type { AgentTurnOutcome as ReceiptAgentTurnOutcome } from "../record/family/protocol-values.mjs";
|
|
6
6
|
export type MetricState = "available" | "partial" | "unavailable" | "empty" | "unsupported" | "failed";
|
|
7
7
|
export type Verdict = PublicVerdict;
|
|
8
8
|
export type MembershipAction = RecordMembershipAction;
|
|
@@ -29,7 +29,7 @@ export type AssertionSource = {
|
|
|
29
29
|
readonly state: "unmapped";
|
|
30
30
|
readonly reason: "source-snapshot-not-recorded" | "position-unrepresentable";
|
|
31
31
|
};
|
|
32
|
-
export type CommandPhase =
|
|
32
|
+
export type CommandPhase = InspectionSuccessDocumentFor<"attempt.trace">["trace"]["commands"]["items"][number]["phase"];
|
|
33
33
|
export type CommandOutcome = "exited" | "terminated" | "not-started";
|
|
34
34
|
export type Metric = {
|
|
35
35
|
readonly state: MetricState;
|
|
@@ -86,7 +86,7 @@ export interface OverviewView {
|
|
|
86
86
|
readonly attemptId: string;
|
|
87
87
|
readonly attemptLocator: string;
|
|
88
88
|
readonly originRunId: string;
|
|
89
|
-
readonly
|
|
89
|
+
readonly originPlannedAttemptId: string;
|
|
90
90
|
readonly verdict: Verdict | null;
|
|
91
91
|
readonly score: Metric;
|
|
92
92
|
readonly durationMs: Metric;
|
|
@@ -96,6 +96,8 @@ export interface OverviewView {
|
|
|
96
96
|
}[];
|
|
97
97
|
}
|
|
98
98
|
export interface ExperimentView {
|
|
99
|
+
readonly plugins: InspectionSuccessDocumentFor<"experiment.get">["experiment"]["experiment"]["plugins"];
|
|
100
|
+
readonly notes: InspectionSuccessDocumentFor<"experiment.get">["experiment"]["notes"];
|
|
99
101
|
readonly experimentId: string;
|
|
100
102
|
readonly aggregate: Aggregate;
|
|
101
103
|
readonly cells: OverviewView["cells"];
|
|
@@ -115,7 +117,7 @@ export interface RunView {
|
|
|
115
117
|
readonly usage: InspectionSuccessDocumentFor<"run.get">["run"]["usage"];
|
|
116
118
|
readonly limitations: InspectionSuccessDocumentFor<"run.get">["run"]["limitations"];
|
|
117
119
|
readonly members: readonly {
|
|
118
|
-
readonly
|
|
120
|
+
readonly plannedAttemptId: string;
|
|
119
121
|
readonly evalId: string;
|
|
120
122
|
readonly attemptOrdinal: number;
|
|
121
123
|
readonly locator: string | null;
|
|
@@ -124,18 +126,19 @@ export interface RunView {
|
|
|
124
126
|
readonly outcome: AttemptOutcome | null;
|
|
125
127
|
readonly verdict: Verdict | null;
|
|
126
128
|
readonly score: ScoredValue | null;
|
|
127
|
-
readonly coverage: InspectionSuccessDocumentFor<"run.get">["run"]["
|
|
128
|
-
readonly usage: InspectionSuccessDocumentFor<"run.get">["run"]["
|
|
129
|
-
readonly limitations: InspectionSuccessDocumentFor<"run.get">["run"]["
|
|
129
|
+
readonly coverage: InspectionSuccessDocumentFor<"run.get">["run"]["plannedAttempts"][number]["coverage"];
|
|
130
|
+
readonly usage: InspectionSuccessDocumentFor<"run.get">["run"]["plannedAttempts"][number]["usage"];
|
|
131
|
+
readonly limitations: InspectionSuccessDocumentFor<"run.get">["run"]["plannedAttempts"][number]["limitations"];
|
|
130
132
|
}[];
|
|
131
133
|
}
|
|
132
134
|
export interface AttemptView {
|
|
135
|
+
readonly notes: InspectionSuccessDocumentFor<"attempt.get">["attempt"]["notes"];
|
|
133
136
|
readonly totalCosts?: InspectionSuccessDocumentFor<"attempt.usage">["usage"]["totalCosts"];
|
|
134
137
|
readonly locator: string;
|
|
135
138
|
readonly verdict: Verdict | null;
|
|
136
139
|
readonly attemptId: string;
|
|
137
140
|
readonly evalId: string;
|
|
138
|
-
readonly
|
|
141
|
+
readonly plannedAttemptId: string;
|
|
139
142
|
readonly outcome: AttemptOutcome;
|
|
140
143
|
readonly originRunId: string;
|
|
141
144
|
readonly experimentId: string;
|
package/dist/show/model.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { InspectionScoredValue, InspectionSuccessDocumentFor, InspectionTra
|
|
|
2
2
|
import type { AttemptOutcome as RecordAttemptOutcome, MembershipAction as RecordMembershipAction } from "../record/model/core.js";
|
|
3
3
|
import type { AdapterIdentity } from "../record/model/run-context.js";
|
|
4
4
|
import type { Verdict as PublicVerdict } from "../shared/types.js";
|
|
5
|
-
import type { AgentTurnOutcome as ReceiptAgentTurnOutcome
|
|
5
|
+
import type { AgentTurnOutcome as ReceiptAgentTurnOutcome } from "../record/family/protocol-values.js";
|
|
6
6
|
export type MetricState = "available" | "partial" | "unavailable" | "empty" | "unsupported" | "failed";
|
|
7
7
|
export type Verdict = PublicVerdict;
|
|
8
8
|
export type MembershipAction = RecordMembershipAction;
|
|
@@ -29,7 +29,7 @@ export type AssertionSource = {
|
|
|
29
29
|
readonly state: "unmapped";
|
|
30
30
|
readonly reason: "source-snapshot-not-recorded" | "position-unrepresentable";
|
|
31
31
|
};
|
|
32
|
-
export type CommandPhase =
|
|
32
|
+
export type CommandPhase = InspectionSuccessDocumentFor<"attempt.trace">["trace"]["commands"]["items"][number]["phase"];
|
|
33
33
|
export type CommandOutcome = "exited" | "terminated" | "not-started";
|
|
34
34
|
export type Metric = {
|
|
35
35
|
readonly state: MetricState;
|
|
@@ -86,7 +86,7 @@ export interface OverviewView {
|
|
|
86
86
|
readonly attemptId: string;
|
|
87
87
|
readonly attemptLocator: string;
|
|
88
88
|
readonly originRunId: string;
|
|
89
|
-
readonly
|
|
89
|
+
readonly originPlannedAttemptId: string;
|
|
90
90
|
readonly verdict: Verdict | null;
|
|
91
91
|
readonly score: Metric;
|
|
92
92
|
readonly durationMs: Metric;
|
|
@@ -96,6 +96,8 @@ export interface OverviewView {
|
|
|
96
96
|
}[];
|
|
97
97
|
}
|
|
98
98
|
export interface ExperimentView {
|
|
99
|
+
readonly plugins: InspectionSuccessDocumentFor<"experiment.get">["experiment"]["experiment"]["plugins"];
|
|
100
|
+
readonly notes: InspectionSuccessDocumentFor<"experiment.get">["experiment"]["notes"];
|
|
99
101
|
readonly experimentId: string;
|
|
100
102
|
readonly aggregate: Aggregate;
|
|
101
103
|
readonly cells: OverviewView["cells"];
|
|
@@ -115,7 +117,7 @@ export interface RunView {
|
|
|
115
117
|
readonly usage: InspectionSuccessDocumentFor<"run.get">["run"]["usage"];
|
|
116
118
|
readonly limitations: InspectionSuccessDocumentFor<"run.get">["run"]["limitations"];
|
|
117
119
|
readonly members: readonly {
|
|
118
|
-
readonly
|
|
120
|
+
readonly plannedAttemptId: string;
|
|
119
121
|
readonly evalId: string;
|
|
120
122
|
readonly attemptOrdinal: number;
|
|
121
123
|
readonly locator: string | null;
|
|
@@ -124,18 +126,19 @@ export interface RunView {
|
|
|
124
126
|
readonly outcome: AttemptOutcome | null;
|
|
125
127
|
readonly verdict: Verdict | null;
|
|
126
128
|
readonly score: ScoredValue | null;
|
|
127
|
-
readonly coverage: InspectionSuccessDocumentFor<"run.get">["run"]["
|
|
128
|
-
readonly usage: InspectionSuccessDocumentFor<"run.get">["run"]["
|
|
129
|
-
readonly limitations: InspectionSuccessDocumentFor<"run.get">["run"]["
|
|
129
|
+
readonly coverage: InspectionSuccessDocumentFor<"run.get">["run"]["plannedAttempts"][number]["coverage"];
|
|
130
|
+
readonly usage: InspectionSuccessDocumentFor<"run.get">["run"]["plannedAttempts"][number]["usage"];
|
|
131
|
+
readonly limitations: InspectionSuccessDocumentFor<"run.get">["run"]["plannedAttempts"][number]["limitations"];
|
|
130
132
|
}[];
|
|
131
133
|
}
|
|
132
134
|
export interface AttemptView {
|
|
135
|
+
readonly notes: InspectionSuccessDocumentFor<"attempt.get">["attempt"]["notes"];
|
|
133
136
|
readonly totalCosts?: InspectionSuccessDocumentFor<"attempt.usage">["usage"]["totalCosts"];
|
|
134
137
|
readonly locator: string;
|
|
135
138
|
readonly verdict: Verdict | null;
|
|
136
139
|
readonly attemptId: string;
|
|
137
140
|
readonly evalId: string;
|
|
138
|
-
readonly
|
|
141
|
+
readonly plannedAttemptId: string;
|
|
139
142
|
readonly outcome: AttemptOutcome;
|
|
140
143
|
readonly originRunId: string;
|
|
141
144
|
readonly experimentId: string;
|
package/dist/show/render.cjs
CHANGED
|
@@ -11,9 +11,10 @@ exports.renderTiming = renderTiming;
|
|
|
11
11
|
exports.renderUsage = renderUsage;
|
|
12
12
|
exports.renderDiff = renderDiff;
|
|
13
13
|
exports.traceSelector = traceSelector;
|
|
14
|
-
// @concord-file
|
|
14
|
+
// @concord-file
|
|
15
15
|
// @concord-implements docs/feature/inspection/cli.md
|
|
16
16
|
// @concord-implements docs/feature/run-inspection/README.md
|
|
17
|
+
const quantity_ts_1 = require("../assertions/quantity.cjs");
|
|
17
18
|
const index_ts_1 = require("../terminal/index.cjs");
|
|
18
19
|
const bytes_ts_1 = require("../inspection/bytes.cjs");
|
|
19
20
|
const schema_ts_1 = require("../record/family/execution-traces/schema.cjs");
|
|
@@ -260,7 +261,7 @@ function renderOverview(value, options = {}) {
|
|
|
260
261
|
title: `Attempts · ${group.name ?? group.experiments[0].experimentId}`,
|
|
261
262
|
blocks: group.experiments.flatMap((experiment) => {
|
|
262
263
|
const experimentCells = value.cells.filter((cell) => cell.experimentId === experiment.experimentId && cell.members.length > 0);
|
|
263
|
-
const gaps = current.
|
|
264
|
+
const gaps = current.plannedAttempts.filter((slot) => slot.experimentId === experiment.experimentId && slot.state === "gap");
|
|
264
265
|
return [
|
|
265
266
|
...(group.name === null ? [] : [{ kind: "divider", title: `Experiment ${experiment.experimentId}`, attachNext: true }]),
|
|
266
267
|
...gaps.flatMap((slot) => slot.state !== "gap" ? [] : [
|
|
@@ -326,17 +327,23 @@ function renderExperiment(value) {
|
|
|
326
327
|
kind: "panel",
|
|
327
328
|
title: `Experiment ${value.experimentId}`,
|
|
328
329
|
blocks: [
|
|
330
|
+
{ kind: "divider", title: "Plugins" },
|
|
331
|
+
{ kind: "table", columns: ["Name", "Key", "Identity"].map((header) => ({ header })),
|
|
332
|
+
rows: value.plugins.map((plugin) => [plugin.name, plugin.key, JSON.stringify(plugin.identity)]) },
|
|
333
|
+
{ kind: "divider", title: "Notes" },
|
|
334
|
+
{ kind: "table", columns: ["Label", "Count", "Min", "Median", "P90", "Max"].map((header) => ({ header })),
|
|
335
|
+
rows: value.notes.map((note) => [note.label, String(note.count), ...[note.min, note.median, note.p90, note.max].map((n) => (0, quantity_ts_1.formatNumericUnit)(n, note.unit))]) },
|
|
329
336
|
{ kind: "divider", title: totalCosts.state === "complete" ? "Experiment total costs" : "Experiment known subtotal" },
|
|
330
337
|
...totalCostBlocks(totalCosts).slice(1),
|
|
331
338
|
{ kind: "keyValue", entries: [
|
|
332
|
-
{ key: "Scope", value: "Latest recorded
|
|
333
|
-
{ key: "
|
|
334
|
-
{ key: "
|
|
339
|
+
{ key: "Scope", value: "Latest recorded attempts; replaced executions excluded" },
|
|
340
|
+
{ key: "Planned attempts", value: String(coverage.plannedAttemptCount) },
|
|
341
|
+
{ key: "Recorded attempts", value: String(coverage.recordedAttemptCount) },
|
|
335
342
|
{ key: "Unique origin Attempts", value: String(coverage.originAttemptCount) },
|
|
336
343
|
{ key: "Attempts with complete costs", value: String(coverage.completeAttemptCount) },
|
|
337
344
|
{ key: "Attempts with partial costs", value: String(coverage.partialAttemptCount) },
|
|
338
345
|
{ key: "Attempts with unavailable costs", value: String(coverage.unavailableAttemptCount) },
|
|
339
|
-
{ key: "
|
|
346
|
+
{ key: "Missing attempts", value: String(coverage.missingAttemptCount) },
|
|
340
347
|
] },
|
|
341
348
|
{ kind: "divider", title: "Summary" },
|
|
342
349
|
aggregateEntries(value.aggregate),
|
|
@@ -365,7 +372,7 @@ function renderExperiment(value) {
|
|
|
365
372
|
function executionValue(value) {
|
|
366
373
|
return value.state === "available" ? value.value : value.state;
|
|
367
374
|
}
|
|
368
|
-
// @concord-code
|
|
375
|
+
// @concord-code
|
|
369
376
|
// @concord-implements docs/feature/inspection/use-case/inspection-check-completeness.md
|
|
370
377
|
function renderRun(value) {
|
|
371
378
|
const blocks = [
|
|
@@ -420,7 +427,7 @@ function renderRun(value) {
|
|
|
420
427
|
for (const member of value.members) {
|
|
421
428
|
blocks.push({
|
|
422
429
|
kind: "panel",
|
|
423
|
-
title: `Member ${member.
|
|
430
|
+
title: `Member ${member.plannedAttemptId}`,
|
|
424
431
|
meta: member.state,
|
|
425
432
|
blocks: [
|
|
426
433
|
{
|
|
@@ -504,12 +511,12 @@ function runLocatedLimitationTable(values) {
|
|
|
504
511
|
return {
|
|
505
512
|
kind: "table",
|
|
506
513
|
columns: [
|
|
507
|
-
{ header: "
|
|
514
|
+
{ header: "Planned attempt" },
|
|
508
515
|
{ header: "Attempt" },
|
|
509
516
|
{ header: "Limitation" },
|
|
510
517
|
],
|
|
511
518
|
rows: values.map((value) => [
|
|
512
|
-
value.
|
|
519
|
+
value.plannedAttemptId,
|
|
513
520
|
value.locator ?? "not-recorded",
|
|
514
521
|
formatRunMemberLimitation(value.limitation),
|
|
515
522
|
]),
|
|
@@ -534,7 +541,7 @@ function formatAttemptLimitation(value) {
|
|
|
534
541
|
}
|
|
535
542
|
return `assertion-material; ${value.state}; ${value.reason}; ${value.limitations.length} material limitations`;
|
|
536
543
|
}
|
|
537
|
-
// @concord-code
|
|
544
|
+
// @concord-code
|
|
538
545
|
// @concord-implements docs/feature/inspection/cli.md
|
|
539
546
|
// @concord-implements docs/feature/error-classification/use-case/reading-errored.md
|
|
540
547
|
function renderAttempt(value) {
|
|
@@ -554,7 +561,7 @@ function renderAttempt(value) {
|
|
|
554
561
|
{ key: "Eval", value: value.evalId },
|
|
555
562
|
{ key: "Run", value: value.originRunId },
|
|
556
563
|
{ key: "Attempt", value: value.attemptId },
|
|
557
|
-
{ key: "
|
|
564
|
+
{ key: "Planned attempt", value: value.plannedAttemptId },
|
|
558
565
|
{ key: "Outcome", value: value.outcome },
|
|
559
566
|
{ key: "Verdict", value: value.verdict ?? "not-available" },
|
|
560
567
|
{ key: "Score", value: score(value.score, value.verdict) },
|
|
@@ -586,6 +593,11 @@ function renderAttempt(value) {
|
|
|
586
593
|
entry.groupPath.length === 0 ? "root" : entry.groupPath.join(" / "),
|
|
587
594
|
]),
|
|
588
595
|
},
|
|
596
|
+
{ kind: "divider", title: "Notes" },
|
|
597
|
+
{ kind: "keyValue", entries: value.notes.map((note) => ({
|
|
598
|
+
key: note.label,
|
|
599
|
+
value: note.value === null ? "unavailable" : typeof note.value === "number" ? (0, quantity_ts_1.formatNumericUnit)(note.value, note.unit) : JSON.stringify(note.value),
|
|
600
|
+
})) },
|
|
589
601
|
{ kind: "divider", title: "Evidence" },
|
|
590
602
|
{
|
|
591
603
|
kind: "keyValue",
|
|
@@ -768,7 +780,7 @@ function displayDetailBlocks(block, locator, eventId) {
|
|
|
768
780
|
const lines = displayLines(block, eventId, TERMINAL_OPTIONS.width);
|
|
769
781
|
const blocks = [{ kind: "code", text: lines.join("\n") }];
|
|
770
782
|
if (block.kind === "image") {
|
|
771
|
-
const request = JSON.stringify({ protocol: "niceeval.query/
|
|
783
|
+
const request = JSON.stringify({ protocol: "niceeval.query/v2", operation: { kind: "attempt.artifact", locator, artifactId: block.artifactId } });
|
|
772
784
|
blocks.push({ kind: "code", text: `niceeval query run --request - <<'NICEEVAL_REQUEST'\n${request}\nNICEEVAL_REQUEST` });
|
|
773
785
|
}
|
|
774
786
|
return blocks;
|
|
@@ -1430,14 +1442,14 @@ function cacheReadBlocks(report) {
|
|
|
1430
1442
|
];
|
|
1431
1443
|
return [{ kind: "divider", title: "Cache read rate" }, { kind: "table",
|
|
1432
1444
|
columns: [{ header: "Scope" }, { header: "Rate" }, { header: "Covered calls" }, { header: "Collection" }, { header: "Reasons" }],
|
|
1433
|
-
rows: [row("Total", report.total), ...report.
|
|
1434
|
-
...report.models.map((group) => row(`${group.
|
|
1445
|
+
rows: [row("Total", report.total), ...report.roles.map((group) => row(`Role: ${group.modelRole ?? "not-recorded"}`, group)),
|
|
1446
|
+
...report.models.map((group) => row(`${group.modelRole ?? "not-recorded"} / ${group.provider ?? "not-recorded"} / ${group.model ?? "not-recorded"}`, group))], overflow: "wrap" }];
|
|
1435
1447
|
}
|
|
1436
1448
|
function modelUsageBlocks(value) {
|
|
1437
1449
|
const configured = value.configuredModels;
|
|
1438
1450
|
const groups = value.modelGroups;
|
|
1439
1451
|
const recordedCalls = (count) => count === null ? "—" : count === 0 ? "0 (no recorded calls)" : String(count);
|
|
1440
|
-
const defaultBinding = configured.state === "available" && configured.bindings.length === 1 && configured.bindings[0].
|
|
1452
|
+
const defaultBinding = configured.state === "available" && configured.bindings.length === 1 && configured.bindings[0].modelRole === "default" ? configured.bindings[0] : null;
|
|
1441
1453
|
const configurationBlocks = configured.state !== "available"
|
|
1442
1454
|
? [{ kind: "keyValue", entries: [{ key: "Configuration", value: configured.state }] }]
|
|
1443
1455
|
: defaultBinding !== null
|
|
@@ -1447,12 +1459,12 @@ function modelUsageBlocks(value) {
|
|
|
1447
1459
|
{ key: "Recorded calls", value: recordedCalls(defaultBinding.recordedCalls) },
|
|
1448
1460
|
] }]
|
|
1449
1461
|
: configured.bindings.length === 0
|
|
1450
|
-
? [{ kind: "keyValue", entries: [{ key: "Configured
|
|
1451
|
-
: [{ kind: "table", columns: [{ header: "
|
|
1452
|
-
rows: configured.bindings.map((binding) => [binding.
|
|
1462
|
+
? [{ kind: "keyValue", entries: [{ key: "Configured roles", value: "0" }] }]
|
|
1463
|
+
: [{ kind: "table", columns: [{ header: "Role" }, { header: "Configured model" }, { header: "Effort" }, { header: "Recorded calls" }],
|
|
1464
|
+
rows: configured.bindings.map((binding) => [binding.modelRole, binding.model ?? "—", binding.reasoningEffort ?? "—", recordedCalls(binding.recordedCalls)]), overflow: "wrap" }];
|
|
1453
1465
|
return [
|
|
1454
1466
|
...cacheReadBlocks(value.cacheRead),
|
|
1455
|
-
{ kind: "divider", title: "Configured model
|
|
1467
|
+
{ kind: "divider", title: "Configured model roles" },
|
|
1456
1468
|
...configurationBlocks,
|
|
1457
1469
|
{ kind: "divider", title: "Actual model usage" },
|
|
1458
1470
|
{ kind: "keyValue", entries: [
|
|
@@ -1462,10 +1474,10 @@ function modelUsageBlocks(value) {
|
|
|
1462
1474
|
...("reason" in groups && groups.reason !== undefined ? [{ key: "Reason", value: groups.reason }] : []),
|
|
1463
1475
|
] },
|
|
1464
1476
|
{ kind: "table", columns: [
|
|
1465
|
-
{ header: "
|
|
1477
|
+
{ header: "Role / provider / actual model", maxWidth: 30 },
|
|
1466
1478
|
{ header: "Calls / tokens", maxWidth: 18 }, { header: "Application cost", maxWidth: 24 },
|
|
1467
1479
|
], rows: groups.groups.map((group) => [
|
|
1468
|
-
`
|
|
1480
|
+
`Role: ${group.modelRole ?? "not-recorded"}\nProvider: ${group.provider ?? "not-recorded"}\n${group.model ?? "not-recorded"}`,
|
|
1469
1481
|
`${group.recordedCalls} calls\nInput incl. cache: ${metric(group.tokens.inputTotalTokens)}\nOutput: ${metric(group.tokens.outputTokens)}\nTotal: ${metric(group.tokens.totalTokens)}`,
|
|
1470
1482
|
group.costs.values.length === 0 ? group.costs.state : group.costs.values.map((cost) => `${cost.value} ${cost.currency} (${group.costs.state}${group.costs.state === "partial" ? "; known subtotal" : ""}; ${cost.coveredCalls}/${group.costs.totalCalls} covered; ${cost.source})`).join("; "),
|
|
1471
1483
|
]), overflow: "wrap" },
|