niceeval 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +4 -2
- package/dist/adapter-usage.cjs.map +1 -1
- package/dist/adapter-usage.d.cts +1 -1
- package/dist/adapter-usage.d.mts +1 -1
- package/dist/adapter-usage.d.ts +1 -1
- package/dist/adapter.cjs +8 -7
- package/dist/adapter.cjs.map +1 -1
- package/dist/adapter.d.cts +21 -21
- package/dist/adapter.d.mts +21 -21
- package/dist/adapter.d.ts +21 -21
- package/dist/agents/{effect-runtime.cjs → agent-effects.cjs} +12 -18
- package/dist/agents/agent-effects.cjs.map +1 -0
- package/dist/agents/agent-effects.d.cts +23 -0
- package/dist/agents/agent-effects.d.mts +23 -0
- package/dist/agents/agent-effects.d.ts +23 -0
- package/dist/agents/agent-effects.js +16 -0
- package/dist/agents/agent-effects.js.map +1 -0
- package/dist/agents/ai-sdk.cjs +1 -1
- package/dist/agents/ai-sdk.cjs.map +1 -1
- package/dist/agents/bub.cjs +31 -18
- package/dist/agents/bub.cjs.map +1 -1
- package/dist/agents/bub.d.cts +9 -12
- package/dist/agents/bub.d.mts +9 -12
- package/dist/agents/bub.d.ts +9 -12
- package/dist/agents/claude-code-native.cjs +2 -2
- package/dist/agents/claude-code-native.cjs.map +1 -1
- package/dist/agents/claude-code.cjs +17 -11
- package/dist/agents/claude-code.cjs.map +1 -1
- package/dist/agents/claude-code.d.cts +11 -11
- package/dist/agents/claude-code.d.mts +11 -11
- package/dist/agents/claude-code.d.ts +11 -11
- package/dist/agents/codex-app-server.cjs +2 -2
- package/dist/agents/codex-app-server.cjs.map +1 -1
- package/dist/agents/codex.cjs +18 -12
- package/dist/agents/codex.cjs.map +1 -1
- package/dist/agents/codex.d.cts +12 -12
- package/dist/agents/codex.d.mts +12 -12
- package/dist/agents/codex.d.ts +12 -12
- package/dist/agents/deepseek-harness.cjs +2 -2
- package/dist/agents/deepseek-harness.cjs.map +1 -1
- package/dist/agents/hermes.cjs +3 -3
- package/dist/agents/hermes.cjs.map +1 -1
- package/dist/agents/index.cjs +3 -3
- package/dist/agents/index.cjs.map +1 -1
- package/dist/agents/index.d.cts +3 -3
- package/dist/agents/index.d.mts +3 -3
- package/dist/agents/index.d.ts +3 -3
- package/dist/agents/index.js +2 -2
- package/dist/agents/index.mjs +2 -2
- package/dist/agents/mcp.cjs.map +1 -1
- package/dist/agents/mcp.d.cts +2 -2
- package/dist/agents/mcp.d.mts +2 -2
- package/dist/agents/mcp.d.ts +2 -2
- package/dist/agents/omp.cjs +2 -2
- package/dist/agents/omp.cjs.map +1 -1
- package/dist/agents/openai-compat.cjs +2 -2
- package/dist/agents/openai-compat.cjs.map +1 -1
- package/dist/agents/openclaw.cjs +3 -3
- package/dist/agents/openclaw.cjs.map +1 -1
- package/dist/agents/opencode.cjs +16 -4
- package/dist/agents/opencode.cjs.map +1 -1
- package/dist/agents/plugin-extensions.cjs +71 -0
- package/dist/agents/plugin-extensions.cjs.map +1 -0
- package/dist/agents/plugin-extensions.d.cts +26 -0
- package/dist/agents/plugin-extensions.d.mts +26 -0
- package/dist/agents/plugin-extensions.d.ts +26 -0
- package/dist/agents/plugin-extensions.js +8 -0
- package/dist/agents/plugin-extensions.js.map +1 -0
- package/dist/agents/post-setup.cjs +14 -3
- package/dist/agents/post-setup.cjs.map +1 -1
- package/dist/agents/post-setup.d.cts +4 -2
- package/dist/agents/post-setup.d.mts +4 -2
- package/dist/agents/post-setup.d.ts +4 -2
- package/dist/agents/post-setup.js +8 -6
- package/dist/agents/sdk-streams.cjs +3 -3
- package/dist/agents/sdk-streams.cjs.map +1 -1
- package/dist/agents/{session-slot.cjs → session-key.cjs} +4 -4
- package/dist/agents/session-key.cjs.map +1 -0
- package/dist/agents/session-key.d.cts +12 -0
- package/dist/agents/session-key.d.mts +12 -0
- package/dist/agents/session-key.d.ts +12 -0
- package/dist/agents/session-key.js +6 -0
- package/dist/agents/session-key.js.map +1 -0
- package/dist/agents/shared.cjs +2 -2
- package/dist/agents/shared.cjs.map +1 -1
- package/dist/agents/shared.d.cts +4 -4
- package/dist/agents/shared.d.mts +4 -4
- package/dist/agents/shared.d.ts +4 -4
- package/dist/agents/skills.cjs.map +1 -1
- package/dist/agents/skills.d.cts +3 -3
- package/dist/agents/skills.d.mts +3 -3
- package/dist/agents/skills.d.ts +3 -3
- package/dist/agents/types.cjs.map +1 -1
- package/dist/agents/types.d.cts +55 -54
- package/dist/agents/types.d.mts +55 -54
- package/dist/agents/types.d.ts +55 -54
- package/dist/agents/ui-message-stream.cjs +4 -4
- package/dist/agents/ui-message-stream.cjs.map +1 -1
- package/dist/agents/unrecognized-events.cjs +32 -0
- package/dist/agents/unrecognized-events.cjs.map +1 -0
- package/dist/agents/unrecognized-events.d.cts +8 -0
- package/dist/agents/unrecognized-events.d.mts +8 -0
- package/dist/agents/unrecognized-events.d.ts +8 -0
- package/dist/agents/unrecognized-events.js +6 -0
- package/dist/agents/unrecognized-events.js.map +1 -0
- package/dist/assertions/api.cjs.map +1 -1
- package/dist/assertions/api.d.cts +3 -0
- package/dist/assertions/api.d.mts +3 -0
- package/dist/assertions/api.d.ts +3 -0
- package/dist/assertions/collection.cjs +1 -1
- package/dist/assertions/collection.cjs.map +1 -1
- package/dist/assertions/judge.cjs +1 -1
- package/dist/assertions/judge.cjs.map +1 -1
- package/dist/assertions/match.cjs +52 -21
- package/dist/assertions/match.cjs.map +1 -1
- package/dist/assertions/match.d.cts +16 -5
- package/dist/assertions/match.d.mts +16 -5
- package/dist/assertions/match.d.ts +16 -5
- package/dist/assertions/match.js +96 -94
- package/dist/assertions/measurements.cjs +22 -6
- package/dist/assertions/measurements.cjs.map +1 -1
- package/dist/assertions/measurements.d.cts +8 -1
- package/dist/assertions/measurements.d.mts +8 -1
- package/dist/assertions/measurements.d.ts +8 -1
- package/dist/assertions/numeric.cjs +2 -0
- package/dist/assertions/numeric.cjs.map +1 -1
- package/dist/assertions/numeric.d.cts +1 -1
- package/dist/assertions/numeric.d.mts +1 -1
- package/dist/assertions/numeric.d.ts +1 -1
- package/dist/assertions/quantity.cjs +42 -0
- package/dist/assertions/quantity.cjs.map +1 -0
- package/dist/assertions/quantity.d.cts +18 -0
- package/dist/assertions/quantity.d.mts +18 -0
- package/dist/assertions/quantity.d.ts +18 -0
- package/dist/assertions/quantity.js +22 -0
- package/dist/assertions/quantity.js.map +1 -0
- package/dist/assertions/record/attachment.cjs +2 -0
- package/dist/assertions/record/attachment.cjs.map +1 -1
- package/dist/assertions/record/codec.cjs +5 -1
- package/dist/assertions/record/codec.cjs.map +1 -1
- package/dist/assertions/record/model.cjs.map +1 -1
- package/dist/assertions/record/model.d.cts +3 -1
- package/dist/assertions/record/model.d.mts +3 -1
- package/dist/assertions/record/model.d.ts +3 -1
- package/dist/assertions/runtime.cjs +11 -6
- package/dist/assertions/runtime.cjs.map +1 -1
- package/dist/cli/program.cjs +3 -0
- package/dist/cli/program.cjs.map +1 -1
- package/dist/context/assert-first.cjs +6 -4
- package/dist/context/assert-first.cjs.map +1 -1
- package/dist/context/assert-first.d.cts +9 -3
- package/dist/context/assert-first.d.mts +9 -3
- package/dist/context/assert-first.d.ts +9 -3
- package/dist/context/notes.cjs +47 -0
- package/dist/context/notes.cjs.map +1 -0
- package/dist/context/notes.d.cts +10 -0
- package/dist/context/notes.d.mts +10 -0
- package/dist/context/notes.d.ts +10 -0
- package/dist/context/notes.js +6 -0
- package/dist/context/notes.js.map +1 -0
- package/dist/context/send-failures.cjs +1 -7
- package/dist/context/send-failures.cjs.map +1 -1
- package/dist/context/send-failures.d.cts +2 -3
- package/dist/context/send-failures.d.mts +2 -3
- package/dist/context/send-failures.d.ts +2 -3
- package/dist/context/send-retry.cjs +0 -1
- package/dist/context/send-retry.cjs.map +1 -1
- package/dist/context/send-retry.d.cts +1 -2
- package/dist/context/send-retry.d.mts +1 -2
- package/dist/context/send-retry.d.ts +1 -2
- package/dist/context/session.cjs +10 -17
- package/dist/context/session.cjs.map +1 -1
- package/dist/context/session.d.cts +4 -10
- package/dist/context/session.d.mts +4 -10
- package/dist/context/session.d.ts +4 -10
- package/dist/define.cjs +88 -65
- package/dist/define.cjs.map +1 -1
- package/dist/define.d.cts +3 -3
- package/dist/define.d.mts +3 -3
- package/dist/define.d.ts +3 -3
- package/dist/eval/cli/contribution.cjs +1 -1
- package/dist/eval/cli/contribution.cjs.map +1 -1
- package/dist/eval/host/index.cjs +1 -1
- package/dist/eval/host/index.cjs.map +1 -1
- package/dist/eval/record/evaluation.cjs +1 -1
- package/dist/eval/record/evaluation.cjs.map +1 -1
- package/dist/eval/record/membership-provenance.d.cts +3 -3
- package/dist/eval/record/membership-provenance.d.mts +3 -3
- package/dist/eval/record/membership-provenance.d.ts +3 -3
- package/dist/eval/record/score.cjs +2 -2
- package/dist/eval/record/score.cjs.map +1 -1
- package/dist/eval/record/sealed-assertion.cjs +1 -1
- package/dist/eval/record/sealed-assertion.cjs.map +1 -1
- package/dist/eval/record/verdict.cjs +2 -2
- package/dist/eval/record/verdict.cjs.map +1 -1
- package/dist/expect/index.cjs +10 -1
- package/dist/expect/index.cjs.map +1 -1
- package/dist/expect/index.d.cts +3 -1
- package/dist/expect/index.d.mts +3 -1
- package/dist/expect/index.d.ts +3 -1
- package/dist/expect/index.js +92 -76
- package/dist/expect/index.mjs +92 -76
- package/dist/experiment/host/cli/contribution.cjs +52 -39
- package/dist/experiment/host/cli/contribution.cjs.map +1 -1
- package/dist/experiment/host/cli/contribution.d.cts +18 -4
- package/dist/experiment/host/cli/contribution.d.mts +18 -4
- package/dist/experiment/host/cli/contribution.d.ts +18 -4
- package/dist/experiment/host/cli/contribution.js +2 -2
- package/dist/experiment/host/cli/debug-protocol.cjs +2 -2
- package/dist/experiment/host/cli/debug-protocol.cjs.map +1 -1
- package/dist/experiment/host/cli/debug-protocol.d.cts +2 -2
- package/dist/experiment/host/cli/debug-protocol.d.mts +2 -2
- package/dist/experiment/host/cli/debug-protocol.d.ts +2 -2
- package/dist/experiment/host/cli/output-protocol.cjs +1 -1
- package/dist/experiment/host/cli/output-protocol.cjs.map +1 -1
- package/dist/experiment/host/cli/output-protocol.d.cts +2 -2
- package/dist/experiment/host/cli/output-protocol.d.mts +2 -2
- package/dist/experiment/host/cli/output-protocol.d.ts +2 -2
- package/dist/experiment/host/current.cjs +2 -2
- package/dist/experiment/host/current.cjs.map +1 -1
- package/dist/experiment/host/index.cjs +1 -1
- package/dist/experiment/host/index.cjs.map +1 -1
- package/dist/experiment/host/operations.cjs +21 -22
- package/dist/experiment/host/operations.cjs.map +1 -1
- package/dist/experiment/host/runtime.cjs +8 -9
- package/dist/experiment/host/runtime.cjs.map +1 -1
- package/dist/experiment/host/runtime.d.cts +3 -3
- package/dist/experiment/host/runtime.d.mts +3 -3
- package/dist/experiment/host/runtime.d.ts +3 -3
- package/dist/experiment/host/teardown.cjs +3 -3
- package/dist/experiment/host/teardown.cjs.map +1 -1
- package/dist/experiment/host/teardown.d.cts +3 -3
- package/dist/experiment/host/teardown.d.mts +3 -3
- package/dist/experiment/host/teardown.d.ts +3 -3
- package/dist/experiment/host/types.cjs.map +1 -1
- package/dist/experiment/host/types.d.cts +19 -19
- package/dist/experiment/host/types.d.mts +19 -19
- package/dist/experiment/host/types.d.ts +19 -19
- package/dist/index.cjs +3 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +3 -3
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/dist/inspection/assertions.cjs +1 -1
- package/dist/inspection/assertions.cjs.map +1 -1
- package/dist/inspection/cli/contribution.cjs +8 -7
- package/dist/inspection/cli/contribution.cjs.map +1 -1
- package/dist/inspection/facts.cjs +4 -4
- package/dist/inspection/facts.cjs.map +1 -1
- package/dist/inspection/index.cjs +1 -5
- package/dist/inspection/index.cjs.map +1 -1
- package/dist/inspection/index.d.cts +1 -2
- package/dist/inspection/index.d.mts +1 -2
- package/dist/inspection/index.d.ts +1 -2
- package/dist/inspection/index.js +20 -26
- package/dist/inspection/lifecycle.cjs +7 -0
- package/dist/inspection/lifecycle.cjs.map +1 -0
- package/dist/inspection/lifecycle.d.cts +4 -0
- package/dist/inspection/lifecycle.d.mts +4 -0
- package/dist/inspection/lifecycle.d.ts +4 -0
- package/dist/inspection/lifecycle.js +6 -0
- package/dist/inspection/lifecycle.js.map +1 -0
- package/dist/inspection/notes.cjs +40 -0
- package/dist/inspection/notes.cjs.map +1 -0
- package/dist/inspection/notes.d.cts +12 -0
- package/dist/inspection/notes.d.mts +12 -0
- package/dist/inspection/notes.d.ts +12 -0
- package/dist/inspection/notes.js +8 -0
- package/dist/inspection/notes.js.map +1 -0
- package/dist/inspection/overview.cjs +12 -15
- package/dist/inspection/overview.cjs.map +1 -1
- package/dist/inspection/overview.d.cts +8 -6
- package/dist/inspection/overview.d.mts +8 -6
- package/dist/inspection/overview.d.ts +8 -6
- package/dist/inspection/planned-attempt.cjs +45 -0
- package/dist/inspection/planned-attempt.cjs.map +1 -0
- package/dist/inspection/planned-attempt.d.cts +109 -0
- package/dist/inspection/planned-attempt.d.mts +109 -0
- package/dist/inspection/planned-attempt.d.ts +109 -0
- package/dist/inspection/planned-attempt.js +24 -0
- package/dist/inspection/planned-attempt.js.map +1 -0
- package/dist/inspection/project-result.cjs +5 -5
- package/dist/inspection/project-result.cjs.map +1 -1
- package/dist/inspection/project-result.d.cts +58 -57
- package/dist/inspection/project-result.d.mts +58 -57
- package/dist/inspection/project-result.d.ts +58 -57
- package/dist/inspection/project-result.js +4 -4
- package/dist/inspection/project.cjs +3 -3
- package/dist/inspection/project.cjs.map +1 -1
- package/dist/inspection/protocol-values.cjs +1 -1
- package/dist/inspection/protocol-values.cjs.map +1 -1
- package/dist/inspection/protocol-values.d.cts +1 -1
- package/dist/inspection/protocol-values.d.mts +1 -1
- package/dist/inspection/protocol-values.d.ts +1 -1
- package/dist/inspection/protocol.cjs +8 -8
- package/dist/inspection/protocol.cjs.map +1 -1
- package/dist/inspection/protocol.d.cts +237 -219
- package/dist/inspection/protocol.d.mts +237 -219
- package/dist/inspection/protocol.d.ts +237 -219
- package/dist/inspection/public.cjs +1 -5
- package/dist/inspection/public.cjs.map +1 -1
- package/dist/inspection/public.d.cts +1 -2
- package/dist/inspection/public.d.mts +1 -2
- package/dist/inspection/public.d.ts +1 -2
- package/dist/inspection/public.js +14 -20
- package/dist/inspection/public.mjs +166 -129
- package/dist/inspection/query.cjs +1 -1
- package/dist/inspection/query.cjs.map +1 -1
- package/dist/inspection/results.cjs +61 -52
- package/dist/inspection/results.cjs.map +1 -1
- package/dist/inspection/results.d.cts +157 -140
- package/dist/inspection/results.d.mts +157 -140
- package/dist/inspection/results.d.ts +157 -140
- package/dist/inspection/select.cjs +33 -28
- package/dist/inspection/select.cjs.map +1 -1
- package/dist/inspection/source.cjs +1 -1
- package/dist/inspection/source.cjs.map +1 -1
- package/dist/inspection/sources.cjs +2 -2
- package/dist/inspection/sources.cjs.map +1 -1
- package/dist/inspection/trace.cjs +9 -8
- package/dist/inspection/trace.cjs.map +1 -1
- package/dist/inspection/trace.d.cts +4 -4
- package/dist/inspection/trace.d.mts +4 -4
- package/dist/inspection/trace.d.ts +4 -4
- package/dist/loaders/index.cjs +7 -279
- package/dist/loaders/index.cjs.map +1 -1
- package/dist/loaders/index.d.cts +2 -95
- package/dist/loaders/index.d.mts +2 -95
- package/dist/loaders/index.d.ts +2 -95
- package/dist/loaders/index.js +10 -12
- package/dist/loaders/index.mjs +10 -12
- package/dist/loaders/runtime.cjs +283 -0
- package/dist/loaders/runtime.cjs.map +1 -0
- package/dist/loaders/runtime.d.cts +95 -0
- package/dist/loaders/runtime.d.mts +95 -0
- package/dist/loaders/runtime.d.ts +95 -0
- package/dist/loaders/runtime.js +16 -0
- package/dist/loaders/runtime.js.map +1 -0
- package/dist/migrations/judge-provider.md +6 -6
- package/dist/{model-slots.cjs → model-roles.cjs} +12 -12
- package/dist/{model-slots.cjs.map → model-roles.cjs.map} +1 -1
- package/dist/{model-slots.d.cts → model-roles.d.cts} +10 -10
- package/dist/{model-slots.d.mts → model-roles.d.mts} +10 -10
- package/dist/{model-slots.d.ts → model-roles.d.ts} +10 -10
- package/dist/model-roles.js +14 -0
- package/dist/model-roles.js.map +1 -0
- package/dist/o11y/adapter-usage-projection.cjs +9 -8
- package/dist/o11y/adapter-usage-projection.cjs.map +1 -1
- package/dist/o11y/adapter-usage-projection.d.cts +23 -15
- package/dist/o11y/adapter-usage-projection.d.mts +23 -15
- package/dist/o11y/adapter-usage-projection.d.ts +23 -15
- package/dist/o11y/cache-read.cjs +14 -14
- package/dist/o11y/cache-read.cjs.map +1 -1
- package/dist/o11y/cache-read.d.cts +5 -5
- package/dist/o11y/cache-read.d.mts +5 -5
- package/dist/o11y/cache-read.d.ts +5 -5
- package/dist/o11y/derive.cjs.map +1 -1
- package/dist/o11y/derive.d.cts +2 -2
- package/dist/o11y/derive.d.mts +2 -2
- package/dist/o11y/derive.d.ts +2 -2
- package/dist/o11y/eval-usage.cjs +9 -9
- package/dist/o11y/eval-usage.cjs.map +1 -1
- package/dist/o11y/eval-usage.d.cts +7 -7
- package/dist/o11y/eval-usage.d.mts +7 -7
- package/dist/o11y/eval-usage.d.ts +7 -7
- package/dist/o11y/parsers/bub.cjs +57 -81
- package/dist/o11y/parsers/bub.cjs.map +1 -1
- package/dist/o11y/parsers/index.cjs.map +1 -1
- package/dist/o11y/parsers/index.d.cts +2 -0
- package/dist/o11y/parsers/index.d.mts +2 -0
- package/dist/o11y/parsers/index.d.ts +2 -0
- package/dist/o11y/parsers/opencode.cjs +129 -217
- package/dist/o11y/parsers/opencode.cjs.map +1 -1
- package/dist/o11y/parsers/opencode.d.cts +2 -5
- package/dist/o11y/parsers/opencode.d.mts +2 -5
- package/dist/o11y/parsers/opencode.d.ts +2 -5
- package/dist/o11y/types.cjs.map +1 -1
- package/dist/o11y/types.d.cts +1 -1
- package/dist/o11y/types.d.mts +1 -1
- package/dist/o11y/types.d.ts +1 -1
- package/dist/plugin/contracts.cjs +170 -120
- package/dist/plugin/contracts.cjs.map +1 -1
- package/dist/plugin/contracts.d.cts +102 -90
- package/dist/plugin/contracts.d.mts +102 -90
- package/dist/plugin/contracts.d.ts +102 -90
- package/dist/plugin/contracts.js +16 -12
- package/dist/plugin/index.cjs.map +1 -1
- package/dist/plugin/index.d.cts +1 -1
- package/dist/plugin/index.d.mts +1 -1
- package/dist/plugin/index.d.ts +1 -1
- package/dist/plugin/link.cjs +69 -64
- package/dist/plugin/link.cjs.map +1 -1
- package/dist/plugin/link.d.cts +5 -3
- package/dist/plugin/link.d.mts +5 -3
- package/dist/plugin/link.d.ts +5 -3
- package/dist/plugin/link.js +12 -8
- package/dist/record/codec/seal-manifest.d.cts +3 -3
- package/dist/record/codec/seal-manifest.d.mts +3 -3
- package/dist/record/codec/seal-manifest.d.ts +3 -3
- package/dist/record/family/attempt-note/definition.cjs +12 -0
- package/dist/record/family/attempt-note/definition.cjs.map +1 -0
- package/dist/record/family/attempt-note/definition.d.cts +6 -0
- package/dist/record/family/attempt-note/definition.d.mts +6 -0
- package/dist/record/family/attempt-note/definition.d.ts +6 -0
- package/dist/record/family/attempt-note/definition.js +6 -0
- package/dist/record/family/attempt-note/definition.js.map +1 -0
- package/dist/record/family/attempt-note/persistence.cjs +10 -0
- package/dist/record/family/attempt-note/persistence.cjs.map +1 -0
- package/dist/record/family/attempt-note/persistence.d.cts +6 -0
- package/dist/record/family/attempt-note/persistence.d.mts +6 -0
- package/dist/record/family/attempt-note/persistence.d.ts +6 -0
- package/dist/record/family/attempt-note/persistence.js +6 -0
- package/dist/record/family/attempt-note/persistence.js.map +1 -0
- package/dist/record/family/attempt-note/schema.cjs +20 -0
- package/dist/record/family/attempt-note/schema.cjs.map +1 -0
- package/dist/record/family/attempt-note/schema.d.cts +21 -0
- package/dist/record/family/attempt-note/schema.d.mts +21 -0
- package/dist/record/family/attempt-note/schema.d.ts +21 -0
- package/dist/record/family/attempt-note/schema.js +10 -0
- package/dist/record/family/attempt-note/schema.js.map +1 -0
- package/dist/record/family/catalog.cjs +32 -29
- package/dist/record/family/catalog.cjs.map +1 -1
- package/dist/record/family/catalog.d.cts +7 -2
- package/dist/record/family/catalog.d.mts +7 -2
- package/dist/record/family/catalog.d.ts +7 -2
- package/dist/record/family/current.cjs +48 -45
- package/dist/record/family/current.cjs.map +1 -1
- package/dist/record/family/current.d.cts +15 -0
- package/dist/record/family/current.d.mts +15 -0
- package/dist/record/family/current.d.ts +15 -0
- package/dist/record/model/index.js +66 -64
- package/dist/record/model/run-context.cjs +9 -3
- package/dist/record/model/run-context.cjs.map +1 -1
- package/dist/record/model/run-context.d.cts +9 -2
- package/dist/record/model/run-context.d.mts +9 -2
- package/dist/record/model/run-context.d.ts +9 -2
- package/dist/record/model/run-context.js +8 -6
- package/dist/record/model/seal-manifest.d.cts +1 -1
- package/dist/record/model/seal-manifest.d.mts +1 -1
- package/dist/record/model/seal-manifest.d.ts +1 -1
- package/dist/run/cli/contribution.cjs +6 -32
- package/dist/run/cli/contribution.cjs.map +1 -1
- package/dist/run/host/runtime.cjs +1 -1
- package/dist/run/host/runtime.cjs.map +1 -1
- package/dist/run/storage/sqlite.cjs +7 -7
- package/dist/run/storage/sqlite.cjs.map +1 -1
- package/dist/runner/adapter-usage.cjs +5 -3
- package/dist/runner/adapter-usage.cjs.map +1 -1
- package/dist/runner/adapter-usage.d.cts +2 -2
- package/dist/runner/adapter-usage.d.mts +2 -2
- package/dist/runner/adapter-usage.d.ts +2 -2
- package/dist/runner/adoption.cjs +9 -12
- package/dist/runner/adoption.cjs.map +1 -1
- package/dist/runner/attempt-notes.cjs +8 -0
- package/dist/runner/attempt-notes.cjs.map +1 -0
- package/dist/runner/attempt-notes.d.cts +4 -0
- package/dist/runner/attempt-notes.d.mts +4 -0
- package/dist/runner/attempt-notes.d.ts +4 -0
- package/dist/runner/attempt-notes.js +8 -0
- package/dist/runner/attempt-notes.js.map +1 -0
- package/dist/runner/attempt.cjs +61 -73
- package/dist/runner/attempt.cjs.map +1 -1
- package/dist/runner/attempt.d.cts +3 -3
- package/dist/runner/attempt.d.mts +3 -3
- package/dist/runner/attempt.d.ts +3 -3
- package/dist/runner/command-plan.cjs +21 -26
- package/dist/runner/command-plan.cjs.map +1 -1
- package/dist/runner/command-plan.d.cts +4 -4
- package/dist/runner/command-plan.d.mts +4 -4
- package/dist/runner/command-plan.d.ts +4 -4
- package/dist/runner/command-plan.js +4 -4
- package/dist/runner/config-identity.cjs.map +1 -1
- package/dist/runner/config-identity.d.cts +2 -2
- package/dist/runner/config-identity.d.mts +2 -2
- package/dist/runner/config-identity.d.ts +2 -2
- package/dist/runner/discover.cjs +6 -6
- package/dist/runner/discover.cjs.map +1 -1
- package/dist/runner/eval-selection.cjs +2 -2
- package/dist/runner/eval-selection.cjs.map +1 -1
- package/dist/runner/eval-selection.d.cts +2 -2
- package/dist/runner/eval-selection.d.mts +2 -2
- package/dist/runner/eval-selection.d.ts +2 -2
- package/dist/runner/feedback/human.cjs +1 -1
- package/dist/runner/feedback/human.cjs.map +1 -1
- package/dist/runner/feedback/json.cjs +4 -2
- package/dist/runner/feedback/json.cjs.map +1 -1
- package/dist/runner/feedback/json.d.cts +1 -1
- package/dist/runner/feedback/json.d.mts +1 -1
- package/dist/runner/feedback/json.d.ts +1 -1
- package/dist/runner/feedback/sink.cjs +1 -1
- package/dist/runner/feedback/sink.cjs.map +1 -1
- package/dist/runner/feedback/sink.d.cts +1 -1
- package/dist/runner/feedback/sink.d.mts +1 -1
- package/dist/runner/feedback/sink.d.ts +1 -1
- package/dist/runner/fingerprint.cjs +1 -1
- package/dist/runner/fingerprint.cjs.map +1 -1
- package/dist/runner/project-target.cjs.map +1 -1
- package/dist/runner/project-target.d.cts +2 -2
- package/dist/runner/project-target.d.mts +2 -2
- package/dist/runner/project-target.d.ts +2 -2
- package/dist/runner/record/attachments.cjs +2 -2
- package/dist/runner/record/attachments.cjs.map +1 -1
- package/dist/runner/record/planning.cjs +8 -2
- package/dist/runner/record/planning.cjs.map +1 -1
- package/dist/runner/record.cjs +2 -0
- package/dist/runner/record.cjs.map +1 -1
- package/dist/runner/report.cjs +3 -3
- package/dist/runner/report.cjs.map +1 -1
- package/dist/runner/report.d.cts +1 -1
- package/dist/runner/report.d.mts +1 -1
- package/dist/runner/report.d.ts +1 -1
- package/dist/runner/reuse-plan.cjs +2 -2
- package/dist/runner/reuse-plan.cjs.map +1 -1
- package/dist/runner/run.cjs +10 -45
- package/dist/runner/run.cjs.map +1 -1
- package/dist/runner/sandbox-pool.cjs +1 -2
- package/dist/runner/sandbox-pool.cjs.map +1 -1
- package/dist/runner/setup-prefix-preparation.cjs +1 -1
- package/dist/runner/setup-prefix-preparation.cjs.map +1 -1
- package/dist/runner/teardown-registry.cjs +2 -2
- package/dist/runner/teardown-registry.cjs.map +1 -1
- package/dist/runner/teardown-registry.d.cts +1 -1
- package/dist/runner/teardown-registry.d.mts +1 -1
- package/dist/runner/teardown-registry.d.ts +1 -1
- package/dist/runner/types.cjs +4 -4
- package/dist/runner/types.cjs.map +1 -1
- package/dist/runner/types.d.cts +64 -93
- package/dist/runner/types.d.mts +64 -93
- package/dist/runner/types.d.ts +64 -93
- package/dist/runner/types.js +2 -2
- package/dist/sandbox/action.cjs +1 -1
- package/dist/sandbox/action.cjs.map +1 -1
- package/dist/sandbox/cli/contribution.cjs +17 -14
- package/dist/sandbox/cli/contribution.cjs.map +1 -1
- package/dist/sandbox/cli/contribution.d.cts +1 -1
- package/dist/sandbox/cli/contribution.d.mts +1 -1
- package/dist/sandbox/cli/contribution.d.ts +1 -1
- package/dist/sandbox/commands.cjs +1 -1
- package/dist/sandbox/commands.cjs.map +1 -1
- package/dist/sandbox/commands.d.cts +1 -1
- package/dist/sandbox/commands.d.mts +1 -1
- package/dist/sandbox/commands.d.ts +1 -1
- package/dist/sandbox/compose.cjs +2 -2
- package/dist/sandbox/compose.cjs.map +1 -1
- package/dist/sandbox/docker-profile/registry.cjs +5 -9
- package/dist/sandbox/docker-profile/registry.cjs.map +1 -1
- package/dist/sandbox/docker-profile/registry.d.cts +0 -3
- package/dist/sandbox/docker-profile/registry.d.mts +0 -3
- package/dist/sandbox/docker-profile/registry.d.ts +0 -3
- package/dist/sandbox/docker-profile/registry.js +6 -12
- package/dist/sandbox/docker-profile/runtime.cjs +1 -1
- package/dist/sandbox/docker-profile/runtime.cjs.map +1 -1
- package/dist/sandbox/docker-profile/schema.cjs +12 -16
- package/dist/sandbox/docker-profile/schema.cjs.map +1 -1
- package/dist/sandbox/docker-profile/schema.d.cts +2 -6
- package/dist/sandbox/docker-profile/schema.d.mts +2 -6
- package/dist/sandbox/docker-profile/schema.d.ts +2 -6
- package/dist/sandbox/docker-profile/schema.js +24 -32
- package/dist/sandbox/incus.cjs +2 -2
- package/dist/sandbox/incus.cjs.map +1 -1
- package/dist/sandbox/index.cjs +4 -35
- package/dist/sandbox/index.cjs.map +1 -1
- package/dist/sandbox/index.d.cts +4 -8
- package/dist/sandbox/index.d.mts +4 -8
- package/dist/sandbox/index.d.ts +4 -8
- package/dist/sandbox/index.js +116 -172
- package/dist/sandbox/index.mjs +116 -172
- package/dist/sandbox/layer.cjs +3 -3
- package/dist/sandbox/layer.cjs.map +1 -1
- package/dist/sandbox/link.cjs +5 -0
- package/dist/sandbox/link.cjs.map +1 -1
- package/dist/sandbox/link.d.cts +3 -0
- package/dist/sandbox/link.d.mts +3 -0
- package/dist/sandbox/link.d.ts +3 -0
- package/dist/sandbox/runtime.cjs +9 -4
- package/dist/sandbox/runtime.cjs.map +1 -1
- package/dist/shared/failure-class.cjs +5 -5
- package/dist/shared/failure-class.cjs.map +1 -1
- package/dist/shared/failure-class.d.cts +4 -10
- package/dist/shared/failure-class.d.mts +4 -10
- package/dist/shared/failure-class.d.ts +4 -10
- package/dist/shared/types.cjs.map +1 -1
- package/dist/shared/types.d.cts +1 -1
- package/dist/shared/types.d.mts +1 -1
- package/dist/shared/types.d.ts +1 -1
- package/dist/show/contribution.cjs +17 -2
- package/dist/show/contribution.cjs.map +1 -1
- package/dist/show/contribution.d.cts +14 -0
- package/dist/show/contribution.d.mts +14 -0
- package/dist/show/contribution.d.ts +14 -0
- package/dist/show/model.cjs +8 -5
- package/dist/show/model.cjs.map +1 -1
- package/dist/show/model.d.cts +11 -8
- package/dist/show/model.d.mts +11 -8
- package/dist/show/model.d.ts +11 -8
- package/dist/show/render.cjs +34 -22
- package/dist/show/render.cjs.map +1 -1
- package/dist/sources/projector.cjs +1 -1
- package/dist/sources/projector.cjs.map +1 -1
- package/dist/types.cjs +1 -1
- package/dist/types.cjs.map +1 -1
- package/dist/types.d.cts +1 -1
- package/dist/types.d.mts +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.js +2 -2
- package/dist/view/app-dist/.vite/manifest.json +2 -2
- package/dist/view/app-dist/assets/{index-B3ijZRph.js → index-CxrOUvHC.js} +17 -17
- package/dist/view/app-dist/assets/{quality-cost-scatter-B--DfJ9s.js → quality-cost-scatter-B6Aiss7x.js} +1 -1
- package/dist/view/app-dist/index.html +1 -1
- package/dist/view/cli/contribution.cjs +1 -1
- package/dist/view/cli/contribution.cjs.map +1 -1
- package/dist/view/function-runtime.mjs +318 -121
- package/dist/view/http-protocol.d.cts +1 -1
- package/dist/view/http-protocol.d.mts +1 -1
- package/dist/view/http-protocol.d.ts +1 -1
- package/dist/view/inspection-host.cjs +2 -2
- package/dist/view/inspection-host.cjs.map +1 -1
- package/dist/view/inspection-host.d.cts +349 -331
- package/dist/view/inspection-host.d.mts +349 -331
- package/dist/view/inspection-host.d.ts +349 -331
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +5 -5
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +4 -4
- package/docs-site/zh/examples/integrations/langgraph.mdx +4 -4
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +5 -5
- package/docs-site/zh/explanation/adapter.mdx +6 -6
- package/docs-site/zh/explanation/assert.mdx +7 -7
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/hitl.mdx +2 -2
- package/docs-site/zh/explanation/judge.mdx +2 -2
- package/docs-site/zh/introduction.mdx +2 -2
- package/docs-site/zh/reference/builtin-agents.mdx +46 -49
- package/docs-site/zh/reference/cli.mdx +16 -14
- package/docs-site/zh/reference/define-agent.mdx +19 -19
- package/docs-site/zh/reference/define-config.mdx +7 -15
- package/docs-site/zh/reference/define-eval.mdx +20 -14
- package/docs-site/zh/reference/define-experiment.mdx +18 -39
- package/docs-site/zh/reference/events.mdx +1 -1
- package/docs-site/zh/reference/expect.mdx +21 -0
- package/docs-site/zh/reference/official-adapters.mdx +12 -5
- package/docs-site/zh/troubleshooting/debugging.mdx +2 -2
- package/docs-site/zh/troubleshooting/recover-after-kill.mdx +1 -1
- package/docs-site/zh/tutorials/agent-onboarding.mdx +2 -2
- package/docs-site/zh/tutorials/authoring.mdx +2 -2
- package/docs-site/zh/tutorials/compare-runs.mdx +1 -1
- package/docs-site/zh/tutorials/concurrency.mdx +8 -8
- package/docs-site/zh/tutorials/configuration.mdx +5 -5
- package/docs-site/zh/tutorials/control-run-cost.mdx +19 -19
- package/docs-site/zh/tutorials/eval-groups.mdx +1 -1
- package/docs-site/zh/tutorials/evaluation-kinds.mdx +5 -5
- package/docs-site/zh/tutorials/experiment-services.mdx +32 -32
- package/docs-site/zh/tutorials/handle-execution-failures.mdx +2 -28
- package/docs-site/zh/tutorials/plugins.mdx +49 -143
- package/docs-site/zh/tutorials/sandbox-agent.mdx +1 -1
- package/docs-site/zh/tutorials/sandbox-reuse.mdx +7 -7
- package/docs-site/zh/tutorials/upgrade-0-18.mdx +1 -1
- package/docs-site/zh/tutorials/upgrade-0-19.mdx +121 -0
- package/docs-site/zh/tutorials/upgrade-0-20.mdx +12 -0
- package/docs-site/zh/tutorials/verify-judge.mdx +2 -2
- package/docs-site/zh/tutorials/viewing-results.mdx +2 -2
- package/docs-site/zh/tutorials/write-experiment.mdx +1 -1
- package/docs-site/zh/tutorials/write-send.mdx +13 -13
- package/package.json +1 -1
- package/dist/agents/effect-runtime.cjs.map +0 -1
- package/dist/agents/effect-runtime.d.cts +0 -29
- package/dist/agents/effect-runtime.d.mts +0 -29
- package/dist/agents/effect-runtime.d.ts +0 -29
- package/dist/agents/effect-runtime.js +0 -16
- package/dist/agents/effect-runtime.js.map +0 -1
- package/dist/agents/session-slot.cjs.map +0 -1
- package/dist/agents/session-slot.d.cts +0 -12
- package/dist/agents/session-slot.d.mts +0 -12
- package/dist/agents/session-slot.d.ts +0 -12
- package/dist/agents/session-slot.js +0 -6
- package/dist/agents/session-slot.js.map +0 -1
- package/dist/model-slots.js +0 -14
- package/dist/model-slots.js.map +0 -1
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
import type { ExperimentHookContext } from "../runner/types.cjs";
|
|
2
2
|
import type { JsonValue, ScopedFeedback } from "../shared/types.cjs";
|
|
3
3
|
import { type SandboxLayer } from "../sandbox/layer.cjs";
|
|
4
|
+
import type { SandboxAction, SandboxAfterAction } from "../sandbox/action.cjs";
|
|
5
|
+
import type { SandboxCommandContext, SandboxCommandTarget } from "../sandbox/commands.cjs";
|
|
6
|
+
import type { CodexConfig } from "../agents/codex.cjs";
|
|
7
|
+
import type { ClaudeCodeConfig } from "../agents/claude-code.cjs";
|
|
8
|
+
import type { BubConfig } from "../agents/bub.cjs";
|
|
9
|
+
import type { SkillSpec, McpServer } from "../agents/types.cjs";
|
|
4
10
|
export type PluginOwner = "experiment" | "group" | "eval";
|
|
5
|
-
export type PluginScope =
|
|
11
|
+
export type PluginScope = "experiment" | "group" | "sandbox" | "attempt";
|
|
6
12
|
export type PluginOnUnavailable = "stop-group" | "replace-sandbox";
|
|
7
13
|
declare const PLUGIN_INSTANCE: unique symbol;
|
|
8
14
|
export interface PluginInstance<Owners extends PluginOwner = PluginOwner> {
|
|
@@ -14,124 +20,130 @@ export interface GroupPluginContext extends ScopedFeedback {
|
|
|
14
20
|
readonly evalGroupId: string;
|
|
15
21
|
readonly signal: AbortSignal;
|
|
16
22
|
}
|
|
17
|
-
|
|
23
|
+
type EvalGroupRef = NonNullable<SandboxCommandContext["evalGroup"]>;
|
|
24
|
+
export interface AttemptPluginContext extends ScopedFeedback {
|
|
18
25
|
readonly experimentId: string;
|
|
19
26
|
readonly evalId: string;
|
|
20
27
|
readonly attempt: number;
|
|
21
|
-
readonly
|
|
28
|
+
readonly evalGroup?: EvalGroupRef;
|
|
22
29
|
readonly signal: AbortSignal;
|
|
23
30
|
}
|
|
24
31
|
export type ExperimentPluginContext = ExperimentHookContext;
|
|
32
|
+
export type PluginElements<Element> = Element | readonly Element[];
|
|
25
33
|
export type PluginLifecycleFragment<Hook> = Readonly<{
|
|
26
|
-
readonly
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
readonly teardown?: Hook;
|
|
30
|
-
} | {
|
|
31
|
-
readonly setup?: Hook;
|
|
32
|
-
readonly teardown: Hook;
|
|
33
|
-
})>;
|
|
34
|
+
readonly before?: PluginElements<Hook>;
|
|
35
|
+
readonly after?: PluginElements<Hook>;
|
|
36
|
+
}>;
|
|
34
37
|
export type ExperimentPluginFragment = PluginLifecycleFragment<PluginHook<ExperimentHookContext>>;
|
|
35
38
|
export type GroupPluginFragment = PluginLifecycleFragment<PluginHook<GroupPluginContext>>;
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
readonly
|
|
42
|
-
readonly
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
readonly
|
|
39
|
+
type GroupContext<Context, Required extends boolean> = Required extends true ? Omit<Context, "evalGroup"> & {
|
|
40
|
+
readonly evalGroup: EvalGroupRef;
|
|
41
|
+
} : Context;
|
|
42
|
+
export type AttemptPluginFragment<Required extends boolean = false> = PluginLifecycleFragment<PluginHook<GroupContext<AttemptPluginContext, Required>>>;
|
|
43
|
+
export interface SandboxPluginFragment<Required extends boolean = false> {
|
|
44
|
+
readonly before?: PluginElements<SandboxAction | ((sandbox: SandboxCommandTarget, context: GroupContext<SandboxCommandContext, Required>) => void | Promise<void>)>;
|
|
45
|
+
readonly after?: PluginElements<SandboxAfterAction | ((sandbox: SandboxCommandTarget, context: GroupContext<Omit<SandboxCommandContext, "onCleanup">, Required>) => void | Promise<void>)>;
|
|
46
|
+
}
|
|
47
|
+
export interface PluginAgentExtensions {
|
|
48
|
+
readonly skills?: readonly SkillSpec[];
|
|
49
|
+
readonly mcpServers?: readonly McpServer[];
|
|
50
|
+
readonly codex?: Pick<CodexConfig, "plugins" | "configFile" | "before" | "after">;
|
|
51
|
+
readonly claudeCode?: Pick<ClaudeCodeConfig, "plugins" | "settingsFile" | "before" | "after">;
|
|
52
|
+
readonly bub?: Pick<BubConfig, "pythonPlugins" | "before" | "after">;
|
|
46
53
|
}
|
|
47
|
-
|
|
48
|
-
readonly
|
|
54
|
+
interface PluginFields<Required extends boolean> {
|
|
55
|
+
readonly agent?: PluginAgentExtensions;
|
|
56
|
+
readonly experiment?: ExperimentPluginFragment;
|
|
57
|
+
readonly group?: GroupPluginFragment;
|
|
58
|
+
readonly sandbox?: SandboxPluginFragment<Required>;
|
|
59
|
+
readonly attempt?: AttemptPluginFragment<Required>;
|
|
60
|
+
}
|
|
61
|
+
type NonEmptyFields<Required extends boolean> = PluginFields<Required> & ({
|
|
62
|
+
readonly agent: PluginAgentExtensions;
|
|
49
63
|
} | {
|
|
50
|
-
readonly
|
|
64
|
+
readonly experiment: ExperimentPluginFragment;
|
|
51
65
|
} | {
|
|
52
|
-
readonly
|
|
66
|
+
readonly group: GroupPluginFragment;
|
|
53
67
|
} | {
|
|
54
|
-
readonly
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
readonly
|
|
63
|
-
|
|
64
|
-
readonly [Owner in Exclude<PluginOwner, Owners>]?: never;
|
|
65
|
-
};
|
|
66
|
-
type OptionedPluginDefinition<Options, Owners extends PluginOwner> = Omit<PluginDefinition<Options>, PluginOwner | "instanceKey"> & ExactHostFragments<Options, Owners> & {
|
|
67
|
-
readonly instanceKey: (options: Options) => string;
|
|
68
|
-
};
|
|
69
|
-
type OptionedSandboxOnlyPluginDefinition<Options> = Omit<PluginDefinition<Options>, PluginOwner | "sandbox" | "instanceKey"> & {
|
|
70
|
-
readonly [Owner in PluginOwner]?: never;
|
|
71
|
-
} & {
|
|
72
|
-
readonly sandbox: (options: Options) => SandboxPluginFragment;
|
|
73
|
-
readonly instanceKey: (options: Options) => string;
|
|
74
|
-
};
|
|
75
|
-
type HostScopesOf<Definition> = (Definition extends {
|
|
76
|
-
readonly experiment: (...args: any[]) => unknown;
|
|
77
|
-
} ? "experiment" : never) | (Definition extends {
|
|
78
|
-
readonly group: (...args: any[]) => unknown;
|
|
79
|
-
} ? "group" : never) | (Definition extends {
|
|
80
|
-
readonly eval: (...args: any[]) => unknown;
|
|
81
|
-
} ? "eval" : never);
|
|
82
|
-
type ScopesOf<Definition> = [HostScopesOf<Definition>] extends [never] ? Definition extends {
|
|
83
|
-
readonly sandbox: (...args: any[]) => unknown;
|
|
84
|
-
} ? PluginOwner : never : HostScopesOf<Definition>;
|
|
85
|
-
type NormalizedPluginLifecycleFragment<Hook> = Readonly<{
|
|
68
|
+
readonly sandbox: SandboxPluginFragment<Required>;
|
|
69
|
+
} | {
|
|
70
|
+
readonly attempt: AttemptPluginFragment<Required>;
|
|
71
|
+
});
|
|
72
|
+
interface PluginIdentity {
|
|
73
|
+
readonly behaviorRevision?: never;
|
|
74
|
+
readonly instanceKey?: never;
|
|
75
|
+
readonly eval?: never;
|
|
76
|
+
readonly name: string;
|
|
77
|
+
readonly key?: string;
|
|
86
78
|
readonly identity: Readonly<globalThis.Record<string, JsonValue>>;
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
readonly
|
|
90
|
-
|
|
79
|
+
}
|
|
80
|
+
type GroupRequiredPluginDefinition = PluginIdentity & {
|
|
81
|
+
readonly requires: {
|
|
82
|
+
readonly evalGroup: true;
|
|
83
|
+
};
|
|
84
|
+
} & NonEmptyFields<true>;
|
|
85
|
+
type OptionalGroupPluginDefinition = PluginIdentity & {
|
|
86
|
+
readonly requires?: {
|
|
87
|
+
readonly evalGroup?: false;
|
|
88
|
+
};
|
|
89
|
+
} & NonEmptyFields<false>;
|
|
90
|
+
export type PluginDefinition = GroupRequiredPluginDefinition | OptionalGroupPluginDefinition;
|
|
91
|
+
type OwnersOf<Definition> = Definition extends {
|
|
92
|
+
readonly agent: unknown;
|
|
93
|
+
} | {
|
|
94
|
+
readonly experiment: unknown;
|
|
95
|
+
} ? "experiment" : Definition extends {
|
|
96
|
+
readonly group: unknown;
|
|
97
|
+
} ? "experiment" | "group" : PluginOwner;
|
|
98
|
+
interface NormalizedLifecycle<Context> {
|
|
99
|
+
readonly before: readonly PluginHook<Context>[];
|
|
100
|
+
readonly after: readonly PluginHook<Context>[];
|
|
101
|
+
}
|
|
91
102
|
export interface PluginInstanceData {
|
|
92
103
|
readonly name: string;
|
|
93
|
-
readonly
|
|
94
|
-
readonly
|
|
95
|
-
readonly
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
readonly
|
|
104
|
+
readonly key: string;
|
|
105
|
+
readonly identity: Readonly<globalThis.Record<string, JsonValue>>;
|
|
106
|
+
readonly requires: Readonly<{
|
|
107
|
+
evalGroup?: true;
|
|
108
|
+
}>;
|
|
109
|
+
readonly agent?: PluginAgentExtensions;
|
|
110
|
+
readonly experiment?: NormalizedLifecycle<ExperimentHookContext>;
|
|
111
|
+
readonly group?: NormalizedLifecycle<GroupPluginContext>;
|
|
112
|
+
readonly attempt?: NormalizedLifecycle<AttemptPluginContext>;
|
|
113
|
+
readonly sandbox?: SandboxLayer<"command-only">;
|
|
114
|
+
readonly shape: JsonValue;
|
|
115
|
+
readonly agentProjection?: JsonValue;
|
|
99
116
|
}
|
|
100
117
|
export interface LinkedPluginLifecycle {
|
|
101
118
|
readonly scope: PluginOwner;
|
|
102
119
|
readonly name: string;
|
|
103
|
-
readonly
|
|
104
|
-
readonly instanceKey: string;
|
|
120
|
+
readonly key: string;
|
|
105
121
|
readonly identity: Readonly<globalThis.Record<string, JsonValue>>;
|
|
122
|
+
readonly requires: PluginInstanceData["requires"];
|
|
123
|
+
readonly shape: JsonValue;
|
|
124
|
+
readonly agentProjection?: JsonValue;
|
|
106
125
|
readonly arrayPosition: number;
|
|
126
|
+
readonly beforeCount: number;
|
|
127
|
+
readonly afterCount: number;
|
|
107
128
|
readonly hasSetup: boolean;
|
|
108
129
|
readonly hasTeardown: boolean;
|
|
109
130
|
readonly hostDeclared: boolean;
|
|
110
|
-
readonly sandboxLayer?:
|
|
131
|
+
readonly sandboxLayer?: SandboxLayer<"command-only">;
|
|
111
132
|
readonly setup?: PluginHook<any>;
|
|
112
133
|
readonly teardown?: PluginHook<any>;
|
|
113
134
|
}
|
|
114
|
-
export declare function
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
export declare function definePlugin<Options
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
export declare function definePlugin<Options>(definition: OptionedPluginDefinition<Options, "group">): (options: Options) => PluginInstance<"group">;
|
|
123
|
-
export declare function definePlugin<Options>(definition: OptionedPluginDefinition<Options, "eval">): (options: Options) => PluginInstance<"eval">;
|
|
124
|
-
export declare function definePlugin<Options>(definition: OptionedSandboxOnlyPluginDefinition<Options>): (options: Options) => PluginInstance<PluginOwner>;
|
|
125
|
-
/** Broadly annotated dynamic definitions retain runtime owner validation. */
|
|
126
|
-
export declare function definePlugin<Options>(definition: PluginDefinition<Options> & AtLeastOneScope<Options> & {
|
|
127
|
-
readonly instanceKey: (options: Options) => string;
|
|
128
|
-
}): (options: Options) => PluginInstance<PluginOwner>;
|
|
129
|
-
export declare function definePlugin<const Definition extends PluginDefinition<void> & AtLeastOneScope<void>>(definition: Definition & {
|
|
130
|
-
readonly instanceKey?: never;
|
|
131
|
-
}): () => PluginInstance<ScopesOf<Definition>>;
|
|
135
|
+
export declare function pluginSourceOf(declaration: object): string | undefined;
|
|
136
|
+
export declare function definePlugin<const Definition extends GroupRequiredPluginDefinition>(spec: Definition): PluginInstance<OwnersOf<Definition>>;
|
|
137
|
+
export declare function definePlugin<const Definition extends OptionalGroupPluginDefinition>(spec: Definition): PluginInstance<OwnersOf<Definition>>;
|
|
138
|
+
export declare function definePlugin<Options, const Required extends boolean = false, const Definition extends PluginIdentity = PluginIdentity>(factory: (options: Options) => Definition & {
|
|
139
|
+
readonly requires?: {
|
|
140
|
+
readonly evalGroup: Required;
|
|
141
|
+
};
|
|
142
|
+
} & NonEmptyFields<Required>): (options: Options) => PluginInstance<OwnersOf<Definition>>;
|
|
132
143
|
export declare function isPluginInstance(value: unknown): value is PluginInstance;
|
|
133
144
|
export declare function pluginInstanceDataOf(value: PluginInstance<any>): PluginInstanceData;
|
|
134
|
-
export declare function
|
|
135
|
-
export declare function
|
|
145
|
+
export declare function assertPluginOwner(plugin: PluginInstance<any>, scope: PluginOwner, label: string): void;
|
|
146
|
+
export declare function linkPluginLifecycles<Scope extends PluginOwner>(plugins: readonly PluginInstance<Scope>[], scope: Scope, field?: "experiment" | "group" | "attempt"): readonly LinkedPluginLifecycle[];
|
|
147
|
+
export declare function pluginLifecycleIdentity({ scope, name, key, identity, requires, shape, agentProjection, arrayPosition }: LinkedPluginLifecycle): JsonValue;
|
|
136
148
|
export declare function pluginLifecycleProjection(lifecycles: readonly LinkedPluginLifecycle[]): JsonValue;
|
|
137
149
|
export {};
|
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
import type { ExperimentHookContext } from "../runner/types.mjs";
|
|
2
2
|
import type { JsonValue, ScopedFeedback } from "../shared/types.mjs";
|
|
3
3
|
import { type SandboxLayer } from "../sandbox/layer.mjs";
|
|
4
|
+
import type { SandboxAction, SandboxAfterAction } from "../sandbox/action.mjs";
|
|
5
|
+
import type { SandboxCommandContext, SandboxCommandTarget } from "../sandbox/commands.mjs";
|
|
6
|
+
import type { CodexConfig } from "../agents/codex.mjs";
|
|
7
|
+
import type { ClaudeCodeConfig } from "../agents/claude-code.mjs";
|
|
8
|
+
import type { BubConfig } from "../agents/bub.mjs";
|
|
9
|
+
import type { SkillSpec, McpServer } from "../agents/types.mjs";
|
|
4
10
|
export type PluginOwner = "experiment" | "group" | "eval";
|
|
5
|
-
export type PluginScope =
|
|
11
|
+
export type PluginScope = "experiment" | "group" | "sandbox" | "attempt";
|
|
6
12
|
export type PluginOnUnavailable = "stop-group" | "replace-sandbox";
|
|
7
13
|
declare const PLUGIN_INSTANCE: unique symbol;
|
|
8
14
|
export interface PluginInstance<Owners extends PluginOwner = PluginOwner> {
|
|
@@ -14,124 +20,130 @@ export interface GroupPluginContext extends ScopedFeedback {
|
|
|
14
20
|
readonly evalGroupId: string;
|
|
15
21
|
readonly signal: AbortSignal;
|
|
16
22
|
}
|
|
17
|
-
|
|
23
|
+
type EvalGroupRef = NonNullable<SandboxCommandContext["evalGroup"]>;
|
|
24
|
+
export interface AttemptPluginContext extends ScopedFeedback {
|
|
18
25
|
readonly experimentId: string;
|
|
19
26
|
readonly evalId: string;
|
|
20
27
|
readonly attempt: number;
|
|
21
|
-
readonly
|
|
28
|
+
readonly evalGroup?: EvalGroupRef;
|
|
22
29
|
readonly signal: AbortSignal;
|
|
23
30
|
}
|
|
24
31
|
export type ExperimentPluginContext = ExperimentHookContext;
|
|
32
|
+
export type PluginElements<Element> = Element | readonly Element[];
|
|
25
33
|
export type PluginLifecycleFragment<Hook> = Readonly<{
|
|
26
|
-
readonly
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
readonly teardown?: Hook;
|
|
30
|
-
} | {
|
|
31
|
-
readonly setup?: Hook;
|
|
32
|
-
readonly teardown: Hook;
|
|
33
|
-
})>;
|
|
34
|
+
readonly before?: PluginElements<Hook>;
|
|
35
|
+
readonly after?: PluginElements<Hook>;
|
|
36
|
+
}>;
|
|
34
37
|
export type ExperimentPluginFragment = PluginLifecycleFragment<PluginHook<ExperimentHookContext>>;
|
|
35
38
|
export type GroupPluginFragment = PluginLifecycleFragment<PluginHook<GroupPluginContext>>;
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
readonly
|
|
42
|
-
readonly
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
readonly
|
|
39
|
+
type GroupContext<Context, Required extends boolean> = Required extends true ? Omit<Context, "evalGroup"> & {
|
|
40
|
+
readonly evalGroup: EvalGroupRef;
|
|
41
|
+
} : Context;
|
|
42
|
+
export type AttemptPluginFragment<Required extends boolean = false> = PluginLifecycleFragment<PluginHook<GroupContext<AttemptPluginContext, Required>>>;
|
|
43
|
+
export interface SandboxPluginFragment<Required extends boolean = false> {
|
|
44
|
+
readonly before?: PluginElements<SandboxAction | ((sandbox: SandboxCommandTarget, context: GroupContext<SandboxCommandContext, Required>) => void | Promise<void>)>;
|
|
45
|
+
readonly after?: PluginElements<SandboxAfterAction | ((sandbox: SandboxCommandTarget, context: GroupContext<Omit<SandboxCommandContext, "onCleanup">, Required>) => void | Promise<void>)>;
|
|
46
|
+
}
|
|
47
|
+
export interface PluginAgentExtensions {
|
|
48
|
+
readonly skills?: readonly SkillSpec[];
|
|
49
|
+
readonly mcpServers?: readonly McpServer[];
|
|
50
|
+
readonly codex?: Pick<CodexConfig, "plugins" | "configFile" | "before" | "after">;
|
|
51
|
+
readonly claudeCode?: Pick<ClaudeCodeConfig, "plugins" | "settingsFile" | "before" | "after">;
|
|
52
|
+
readonly bub?: Pick<BubConfig, "pythonPlugins" | "before" | "after">;
|
|
46
53
|
}
|
|
47
|
-
|
|
48
|
-
readonly
|
|
54
|
+
interface PluginFields<Required extends boolean> {
|
|
55
|
+
readonly agent?: PluginAgentExtensions;
|
|
56
|
+
readonly experiment?: ExperimentPluginFragment;
|
|
57
|
+
readonly group?: GroupPluginFragment;
|
|
58
|
+
readonly sandbox?: SandboxPluginFragment<Required>;
|
|
59
|
+
readonly attempt?: AttemptPluginFragment<Required>;
|
|
60
|
+
}
|
|
61
|
+
type NonEmptyFields<Required extends boolean> = PluginFields<Required> & ({
|
|
62
|
+
readonly agent: PluginAgentExtensions;
|
|
49
63
|
} | {
|
|
50
|
-
readonly
|
|
64
|
+
readonly experiment: ExperimentPluginFragment;
|
|
51
65
|
} | {
|
|
52
|
-
readonly
|
|
66
|
+
readonly group: GroupPluginFragment;
|
|
53
67
|
} | {
|
|
54
|
-
readonly
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
readonly
|
|
63
|
-
|
|
64
|
-
readonly [Owner in Exclude<PluginOwner, Owners>]?: never;
|
|
65
|
-
};
|
|
66
|
-
type OptionedPluginDefinition<Options, Owners extends PluginOwner> = Omit<PluginDefinition<Options>, PluginOwner | "instanceKey"> & ExactHostFragments<Options, Owners> & {
|
|
67
|
-
readonly instanceKey: (options: Options) => string;
|
|
68
|
-
};
|
|
69
|
-
type OptionedSandboxOnlyPluginDefinition<Options> = Omit<PluginDefinition<Options>, PluginOwner | "sandbox" | "instanceKey"> & {
|
|
70
|
-
readonly [Owner in PluginOwner]?: never;
|
|
71
|
-
} & {
|
|
72
|
-
readonly sandbox: (options: Options) => SandboxPluginFragment;
|
|
73
|
-
readonly instanceKey: (options: Options) => string;
|
|
74
|
-
};
|
|
75
|
-
type HostScopesOf<Definition> = (Definition extends {
|
|
76
|
-
readonly experiment: (...args: any[]) => unknown;
|
|
77
|
-
} ? "experiment" : never) | (Definition extends {
|
|
78
|
-
readonly group: (...args: any[]) => unknown;
|
|
79
|
-
} ? "group" : never) | (Definition extends {
|
|
80
|
-
readonly eval: (...args: any[]) => unknown;
|
|
81
|
-
} ? "eval" : never);
|
|
82
|
-
type ScopesOf<Definition> = [HostScopesOf<Definition>] extends [never] ? Definition extends {
|
|
83
|
-
readonly sandbox: (...args: any[]) => unknown;
|
|
84
|
-
} ? PluginOwner : never : HostScopesOf<Definition>;
|
|
85
|
-
type NormalizedPluginLifecycleFragment<Hook> = Readonly<{
|
|
68
|
+
readonly sandbox: SandboxPluginFragment<Required>;
|
|
69
|
+
} | {
|
|
70
|
+
readonly attempt: AttemptPluginFragment<Required>;
|
|
71
|
+
});
|
|
72
|
+
interface PluginIdentity {
|
|
73
|
+
readonly behaviorRevision?: never;
|
|
74
|
+
readonly instanceKey?: never;
|
|
75
|
+
readonly eval?: never;
|
|
76
|
+
readonly name: string;
|
|
77
|
+
readonly key?: string;
|
|
86
78
|
readonly identity: Readonly<globalThis.Record<string, JsonValue>>;
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
readonly
|
|
90
|
-
|
|
79
|
+
}
|
|
80
|
+
type GroupRequiredPluginDefinition = PluginIdentity & {
|
|
81
|
+
readonly requires: {
|
|
82
|
+
readonly evalGroup: true;
|
|
83
|
+
};
|
|
84
|
+
} & NonEmptyFields<true>;
|
|
85
|
+
type OptionalGroupPluginDefinition = PluginIdentity & {
|
|
86
|
+
readonly requires?: {
|
|
87
|
+
readonly evalGroup?: false;
|
|
88
|
+
};
|
|
89
|
+
} & NonEmptyFields<false>;
|
|
90
|
+
export type PluginDefinition = GroupRequiredPluginDefinition | OptionalGroupPluginDefinition;
|
|
91
|
+
type OwnersOf<Definition> = Definition extends {
|
|
92
|
+
readonly agent: unknown;
|
|
93
|
+
} | {
|
|
94
|
+
readonly experiment: unknown;
|
|
95
|
+
} ? "experiment" : Definition extends {
|
|
96
|
+
readonly group: unknown;
|
|
97
|
+
} ? "experiment" | "group" : PluginOwner;
|
|
98
|
+
interface NormalizedLifecycle<Context> {
|
|
99
|
+
readonly before: readonly PluginHook<Context>[];
|
|
100
|
+
readonly after: readonly PluginHook<Context>[];
|
|
101
|
+
}
|
|
91
102
|
export interface PluginInstanceData {
|
|
92
103
|
readonly name: string;
|
|
93
|
-
readonly
|
|
94
|
-
readonly
|
|
95
|
-
readonly
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
readonly
|
|
104
|
+
readonly key: string;
|
|
105
|
+
readonly identity: Readonly<globalThis.Record<string, JsonValue>>;
|
|
106
|
+
readonly requires: Readonly<{
|
|
107
|
+
evalGroup?: true;
|
|
108
|
+
}>;
|
|
109
|
+
readonly agent?: PluginAgentExtensions;
|
|
110
|
+
readonly experiment?: NormalizedLifecycle<ExperimentHookContext>;
|
|
111
|
+
readonly group?: NormalizedLifecycle<GroupPluginContext>;
|
|
112
|
+
readonly attempt?: NormalizedLifecycle<AttemptPluginContext>;
|
|
113
|
+
readonly sandbox?: SandboxLayer<"command-only">;
|
|
114
|
+
readonly shape: JsonValue;
|
|
115
|
+
readonly agentProjection?: JsonValue;
|
|
99
116
|
}
|
|
100
117
|
export interface LinkedPluginLifecycle {
|
|
101
118
|
readonly scope: PluginOwner;
|
|
102
119
|
readonly name: string;
|
|
103
|
-
readonly
|
|
104
|
-
readonly instanceKey: string;
|
|
120
|
+
readonly key: string;
|
|
105
121
|
readonly identity: Readonly<globalThis.Record<string, JsonValue>>;
|
|
122
|
+
readonly requires: PluginInstanceData["requires"];
|
|
123
|
+
readonly shape: JsonValue;
|
|
124
|
+
readonly agentProjection?: JsonValue;
|
|
106
125
|
readonly arrayPosition: number;
|
|
126
|
+
readonly beforeCount: number;
|
|
127
|
+
readonly afterCount: number;
|
|
107
128
|
readonly hasSetup: boolean;
|
|
108
129
|
readonly hasTeardown: boolean;
|
|
109
130
|
readonly hostDeclared: boolean;
|
|
110
|
-
readonly sandboxLayer?:
|
|
131
|
+
readonly sandboxLayer?: SandboxLayer<"command-only">;
|
|
111
132
|
readonly setup?: PluginHook<any>;
|
|
112
133
|
readonly teardown?: PluginHook<any>;
|
|
113
134
|
}
|
|
114
|
-
export declare function
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
export declare function definePlugin<Options
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
export declare function definePlugin<Options>(definition: OptionedPluginDefinition<Options, "group">): (options: Options) => PluginInstance<"group">;
|
|
123
|
-
export declare function definePlugin<Options>(definition: OptionedPluginDefinition<Options, "eval">): (options: Options) => PluginInstance<"eval">;
|
|
124
|
-
export declare function definePlugin<Options>(definition: OptionedSandboxOnlyPluginDefinition<Options>): (options: Options) => PluginInstance<PluginOwner>;
|
|
125
|
-
/** Broadly annotated dynamic definitions retain runtime owner validation. */
|
|
126
|
-
export declare function definePlugin<Options>(definition: PluginDefinition<Options> & AtLeastOneScope<Options> & {
|
|
127
|
-
readonly instanceKey: (options: Options) => string;
|
|
128
|
-
}): (options: Options) => PluginInstance<PluginOwner>;
|
|
129
|
-
export declare function definePlugin<const Definition extends PluginDefinition<void> & AtLeastOneScope<void>>(definition: Definition & {
|
|
130
|
-
readonly instanceKey?: never;
|
|
131
|
-
}): () => PluginInstance<ScopesOf<Definition>>;
|
|
135
|
+
export declare function pluginSourceOf(declaration: object): string | undefined;
|
|
136
|
+
export declare function definePlugin<const Definition extends GroupRequiredPluginDefinition>(spec: Definition): PluginInstance<OwnersOf<Definition>>;
|
|
137
|
+
export declare function definePlugin<const Definition extends OptionalGroupPluginDefinition>(spec: Definition): PluginInstance<OwnersOf<Definition>>;
|
|
138
|
+
export declare function definePlugin<Options, const Required extends boolean = false, const Definition extends PluginIdentity = PluginIdentity>(factory: (options: Options) => Definition & {
|
|
139
|
+
readonly requires?: {
|
|
140
|
+
readonly evalGroup: Required;
|
|
141
|
+
};
|
|
142
|
+
} & NonEmptyFields<Required>): (options: Options) => PluginInstance<OwnersOf<Definition>>;
|
|
132
143
|
export declare function isPluginInstance(value: unknown): value is PluginInstance;
|
|
133
144
|
export declare function pluginInstanceDataOf(value: PluginInstance<any>): PluginInstanceData;
|
|
134
|
-
export declare function
|
|
135
|
-
export declare function
|
|
145
|
+
export declare function assertPluginOwner(plugin: PluginInstance<any>, scope: PluginOwner, label: string): void;
|
|
146
|
+
export declare function linkPluginLifecycles<Scope extends PluginOwner>(plugins: readonly PluginInstance<Scope>[], scope: Scope, field?: "experiment" | "group" | "attempt"): readonly LinkedPluginLifecycle[];
|
|
147
|
+
export declare function pluginLifecycleIdentity({ scope, name, key, identity, requires, shape, agentProjection, arrayPosition }: LinkedPluginLifecycle): JsonValue;
|
|
136
148
|
export declare function pluginLifecycleProjection(lifecycles: readonly LinkedPluginLifecycle[]): JsonValue;
|
|
137
149
|
export {};
|