niceeval 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INDEX.md +4 -2
- package/dist/adapter-usage.cjs.map +1 -1
- package/dist/adapter-usage.d.cts +1 -1
- package/dist/adapter-usage.d.mts +1 -1
- package/dist/adapter-usage.d.ts +1 -1
- package/dist/adapter.cjs +8 -7
- package/dist/adapter.cjs.map +1 -1
- package/dist/adapter.d.cts +21 -21
- package/dist/adapter.d.mts +21 -21
- package/dist/adapter.d.ts +21 -21
- package/dist/agents/{effect-runtime.cjs → agent-effects.cjs} +12 -18
- package/dist/agents/agent-effects.cjs.map +1 -0
- package/dist/agents/agent-effects.d.cts +23 -0
- package/dist/agents/agent-effects.d.mts +23 -0
- package/dist/agents/agent-effects.d.ts +23 -0
- package/dist/agents/agent-effects.js +16 -0
- package/dist/agents/agent-effects.js.map +1 -0
- package/dist/agents/ai-sdk.cjs +1 -1
- package/dist/agents/ai-sdk.cjs.map +1 -1
- package/dist/agents/bub.cjs +31 -18
- package/dist/agents/bub.cjs.map +1 -1
- package/dist/agents/bub.d.cts +9 -12
- package/dist/agents/bub.d.mts +9 -12
- package/dist/agents/bub.d.ts +9 -12
- package/dist/agents/claude-code-native.cjs +2 -2
- package/dist/agents/claude-code-native.cjs.map +1 -1
- package/dist/agents/claude-code.cjs +17 -11
- package/dist/agents/claude-code.cjs.map +1 -1
- package/dist/agents/claude-code.d.cts +11 -11
- package/dist/agents/claude-code.d.mts +11 -11
- package/dist/agents/claude-code.d.ts +11 -11
- package/dist/agents/codex-app-server.cjs +2 -2
- package/dist/agents/codex-app-server.cjs.map +1 -1
- package/dist/agents/codex.cjs +18 -12
- package/dist/agents/codex.cjs.map +1 -1
- package/dist/agents/codex.d.cts +12 -12
- package/dist/agents/codex.d.mts +12 -12
- package/dist/agents/codex.d.ts +12 -12
- package/dist/agents/deepseek-harness.cjs +2 -2
- package/dist/agents/deepseek-harness.cjs.map +1 -1
- package/dist/agents/hermes.cjs +3 -3
- package/dist/agents/hermes.cjs.map +1 -1
- package/dist/agents/index.cjs +3 -3
- package/dist/agents/index.cjs.map +1 -1
- package/dist/agents/index.d.cts +3 -3
- package/dist/agents/index.d.mts +3 -3
- package/dist/agents/index.d.ts +3 -3
- package/dist/agents/index.js +2 -2
- package/dist/agents/index.mjs +2 -2
- package/dist/agents/mcp.cjs.map +1 -1
- package/dist/agents/mcp.d.cts +2 -2
- package/dist/agents/mcp.d.mts +2 -2
- package/dist/agents/mcp.d.ts +2 -2
- package/dist/agents/omp.cjs +2 -2
- package/dist/agents/omp.cjs.map +1 -1
- package/dist/agents/openai-compat.cjs +2 -2
- package/dist/agents/openai-compat.cjs.map +1 -1
- package/dist/agents/openclaw.cjs +3 -3
- package/dist/agents/openclaw.cjs.map +1 -1
- package/dist/agents/opencode.cjs +16 -4
- package/dist/agents/opencode.cjs.map +1 -1
- package/dist/agents/plugin-extensions.cjs +71 -0
- package/dist/agents/plugin-extensions.cjs.map +1 -0
- package/dist/agents/plugin-extensions.d.cts +26 -0
- package/dist/agents/plugin-extensions.d.mts +26 -0
- package/dist/agents/plugin-extensions.d.ts +26 -0
- package/dist/agents/plugin-extensions.js +8 -0
- package/dist/agents/plugin-extensions.js.map +1 -0
- package/dist/agents/post-setup.cjs +14 -3
- package/dist/agents/post-setup.cjs.map +1 -1
- package/dist/agents/post-setup.d.cts +4 -2
- package/dist/agents/post-setup.d.mts +4 -2
- package/dist/agents/post-setup.d.ts +4 -2
- package/dist/agents/post-setup.js +8 -6
- package/dist/agents/sdk-streams.cjs +3 -3
- package/dist/agents/sdk-streams.cjs.map +1 -1
- package/dist/agents/{session-slot.cjs → session-key.cjs} +4 -4
- package/dist/agents/session-key.cjs.map +1 -0
- package/dist/agents/session-key.d.cts +12 -0
- package/dist/agents/session-key.d.mts +12 -0
- package/dist/agents/session-key.d.ts +12 -0
- package/dist/agents/session-key.js +6 -0
- package/dist/agents/session-key.js.map +1 -0
- package/dist/agents/shared.cjs +2 -2
- package/dist/agents/shared.cjs.map +1 -1
- package/dist/agents/shared.d.cts +4 -4
- package/dist/agents/shared.d.mts +4 -4
- package/dist/agents/shared.d.ts +4 -4
- package/dist/agents/skills.cjs.map +1 -1
- package/dist/agents/skills.d.cts +3 -3
- package/dist/agents/skills.d.mts +3 -3
- package/dist/agents/skills.d.ts +3 -3
- package/dist/agents/types.cjs.map +1 -1
- package/dist/agents/types.d.cts +55 -54
- package/dist/agents/types.d.mts +55 -54
- package/dist/agents/types.d.ts +55 -54
- package/dist/agents/ui-message-stream.cjs +4 -4
- package/dist/agents/ui-message-stream.cjs.map +1 -1
- package/dist/agents/unrecognized-events.cjs +32 -0
- package/dist/agents/unrecognized-events.cjs.map +1 -0
- package/dist/agents/unrecognized-events.d.cts +8 -0
- package/dist/agents/unrecognized-events.d.mts +8 -0
- package/dist/agents/unrecognized-events.d.ts +8 -0
- package/dist/agents/unrecognized-events.js +6 -0
- package/dist/agents/unrecognized-events.js.map +1 -0
- package/dist/assertions/api.cjs.map +1 -1
- package/dist/assertions/api.d.cts +3 -0
- package/dist/assertions/api.d.mts +3 -0
- package/dist/assertions/api.d.ts +3 -0
- package/dist/assertions/collection.cjs +1 -1
- package/dist/assertions/collection.cjs.map +1 -1
- package/dist/assertions/judge.cjs +1 -1
- package/dist/assertions/judge.cjs.map +1 -1
- package/dist/assertions/match.cjs +52 -21
- package/dist/assertions/match.cjs.map +1 -1
- package/dist/assertions/match.d.cts +16 -5
- package/dist/assertions/match.d.mts +16 -5
- package/dist/assertions/match.d.ts +16 -5
- package/dist/assertions/match.js +96 -94
- package/dist/assertions/measurements.cjs +22 -6
- package/dist/assertions/measurements.cjs.map +1 -1
- package/dist/assertions/measurements.d.cts +8 -1
- package/dist/assertions/measurements.d.mts +8 -1
- package/dist/assertions/measurements.d.ts +8 -1
- package/dist/assertions/numeric.cjs +2 -0
- package/dist/assertions/numeric.cjs.map +1 -1
- package/dist/assertions/numeric.d.cts +1 -1
- package/dist/assertions/numeric.d.mts +1 -1
- package/dist/assertions/numeric.d.ts +1 -1
- package/dist/assertions/quantity.cjs +42 -0
- package/dist/assertions/quantity.cjs.map +1 -0
- package/dist/assertions/quantity.d.cts +18 -0
- package/dist/assertions/quantity.d.mts +18 -0
- package/dist/assertions/quantity.d.ts +18 -0
- package/dist/assertions/quantity.js +22 -0
- package/dist/assertions/quantity.js.map +1 -0
- package/dist/assertions/record/attachment.cjs +2 -0
- package/dist/assertions/record/attachment.cjs.map +1 -1
- package/dist/assertions/record/codec.cjs +5 -1
- package/dist/assertions/record/codec.cjs.map +1 -1
- package/dist/assertions/record/model.cjs.map +1 -1
- package/dist/assertions/record/model.d.cts +3 -1
- package/dist/assertions/record/model.d.mts +3 -1
- package/dist/assertions/record/model.d.ts +3 -1
- package/dist/assertions/runtime.cjs +11 -6
- package/dist/assertions/runtime.cjs.map +1 -1
- package/dist/cli/program.cjs +3 -0
- package/dist/cli/program.cjs.map +1 -1
- package/dist/context/assert-first.cjs +6 -4
- package/dist/context/assert-first.cjs.map +1 -1
- package/dist/context/assert-first.d.cts +9 -3
- package/dist/context/assert-first.d.mts +9 -3
- package/dist/context/assert-first.d.ts +9 -3
- package/dist/context/notes.cjs +47 -0
- package/dist/context/notes.cjs.map +1 -0
- package/dist/context/notes.d.cts +10 -0
- package/dist/context/notes.d.mts +10 -0
- package/dist/context/notes.d.ts +10 -0
- package/dist/context/notes.js +6 -0
- package/dist/context/notes.js.map +1 -0
- package/dist/context/send-failures.cjs +1 -7
- package/dist/context/send-failures.cjs.map +1 -1
- package/dist/context/send-failures.d.cts +2 -3
- package/dist/context/send-failures.d.mts +2 -3
- package/dist/context/send-failures.d.ts +2 -3
- package/dist/context/send-retry.cjs +0 -1
- package/dist/context/send-retry.cjs.map +1 -1
- package/dist/context/send-retry.d.cts +1 -2
- package/dist/context/send-retry.d.mts +1 -2
- package/dist/context/send-retry.d.ts +1 -2
- package/dist/context/session.cjs +10 -17
- package/dist/context/session.cjs.map +1 -1
- package/dist/context/session.d.cts +4 -10
- package/dist/context/session.d.mts +4 -10
- package/dist/context/session.d.ts +4 -10
- package/dist/define.cjs +88 -65
- package/dist/define.cjs.map +1 -1
- package/dist/define.d.cts +3 -3
- package/dist/define.d.mts +3 -3
- package/dist/define.d.ts +3 -3
- package/dist/eval/cli/contribution.cjs +1 -1
- package/dist/eval/cli/contribution.cjs.map +1 -1
- package/dist/eval/host/index.cjs +1 -1
- package/dist/eval/host/index.cjs.map +1 -1
- package/dist/eval/record/evaluation.cjs +1 -1
- package/dist/eval/record/evaluation.cjs.map +1 -1
- package/dist/eval/record/membership-provenance.d.cts +3 -3
- package/dist/eval/record/membership-provenance.d.mts +3 -3
- package/dist/eval/record/membership-provenance.d.ts +3 -3
- package/dist/eval/record/score.cjs +2 -2
- package/dist/eval/record/score.cjs.map +1 -1
- package/dist/eval/record/sealed-assertion.cjs +1 -1
- package/dist/eval/record/sealed-assertion.cjs.map +1 -1
- package/dist/eval/record/verdict.cjs +2 -2
- package/dist/eval/record/verdict.cjs.map +1 -1
- package/dist/expect/index.cjs +10 -1
- package/dist/expect/index.cjs.map +1 -1
- package/dist/expect/index.d.cts +3 -1
- package/dist/expect/index.d.mts +3 -1
- package/dist/expect/index.d.ts +3 -1
- package/dist/expect/index.js +92 -76
- package/dist/expect/index.mjs +92 -76
- package/dist/experiment/host/cli/contribution.cjs +52 -39
- package/dist/experiment/host/cli/contribution.cjs.map +1 -1
- package/dist/experiment/host/cli/contribution.d.cts +18 -4
- package/dist/experiment/host/cli/contribution.d.mts +18 -4
- package/dist/experiment/host/cli/contribution.d.ts +18 -4
- package/dist/experiment/host/cli/contribution.js +2 -2
- package/dist/experiment/host/cli/debug-protocol.cjs +2 -2
- package/dist/experiment/host/cli/debug-protocol.cjs.map +1 -1
- package/dist/experiment/host/cli/debug-protocol.d.cts +2 -2
- package/dist/experiment/host/cli/debug-protocol.d.mts +2 -2
- package/dist/experiment/host/cli/debug-protocol.d.ts +2 -2
- package/dist/experiment/host/cli/output-protocol.cjs +1 -1
- package/dist/experiment/host/cli/output-protocol.cjs.map +1 -1
- package/dist/experiment/host/cli/output-protocol.d.cts +2 -2
- package/dist/experiment/host/cli/output-protocol.d.mts +2 -2
- package/dist/experiment/host/cli/output-protocol.d.ts +2 -2
- package/dist/experiment/host/current.cjs +2 -2
- package/dist/experiment/host/current.cjs.map +1 -1
- package/dist/experiment/host/index.cjs +1 -1
- package/dist/experiment/host/index.cjs.map +1 -1
- package/dist/experiment/host/operations.cjs +21 -22
- package/dist/experiment/host/operations.cjs.map +1 -1
- package/dist/experiment/host/runtime.cjs +8 -9
- package/dist/experiment/host/runtime.cjs.map +1 -1
- package/dist/experiment/host/runtime.d.cts +3 -3
- package/dist/experiment/host/runtime.d.mts +3 -3
- package/dist/experiment/host/runtime.d.ts +3 -3
- package/dist/experiment/host/teardown.cjs +3 -3
- package/dist/experiment/host/teardown.cjs.map +1 -1
- package/dist/experiment/host/teardown.d.cts +3 -3
- package/dist/experiment/host/teardown.d.mts +3 -3
- package/dist/experiment/host/teardown.d.ts +3 -3
- package/dist/experiment/host/types.cjs.map +1 -1
- package/dist/experiment/host/types.d.cts +19 -19
- package/dist/experiment/host/types.d.mts +19 -19
- package/dist/experiment/host/types.d.ts +19 -19
- package/dist/index.cjs +3 -3
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +3 -3
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2 -2
- package/dist/index.mjs +2 -2
- package/dist/inspection/assertions.cjs +1 -1
- package/dist/inspection/assertions.cjs.map +1 -1
- package/dist/inspection/cli/contribution.cjs +8 -7
- package/dist/inspection/cli/contribution.cjs.map +1 -1
- package/dist/inspection/facts.cjs +4 -4
- package/dist/inspection/facts.cjs.map +1 -1
- package/dist/inspection/index.cjs +1 -5
- package/dist/inspection/index.cjs.map +1 -1
- package/dist/inspection/index.d.cts +1 -2
- package/dist/inspection/index.d.mts +1 -2
- package/dist/inspection/index.d.ts +1 -2
- package/dist/inspection/index.js +20 -26
- package/dist/inspection/lifecycle.cjs +7 -0
- package/dist/inspection/lifecycle.cjs.map +1 -0
- package/dist/inspection/lifecycle.d.cts +4 -0
- package/dist/inspection/lifecycle.d.mts +4 -0
- package/dist/inspection/lifecycle.d.ts +4 -0
- package/dist/inspection/lifecycle.js +6 -0
- package/dist/inspection/lifecycle.js.map +1 -0
- package/dist/inspection/notes.cjs +40 -0
- package/dist/inspection/notes.cjs.map +1 -0
- package/dist/inspection/notes.d.cts +12 -0
- package/dist/inspection/notes.d.mts +12 -0
- package/dist/inspection/notes.d.ts +12 -0
- package/dist/inspection/notes.js +8 -0
- package/dist/inspection/notes.js.map +1 -0
- package/dist/inspection/overview.cjs +12 -15
- package/dist/inspection/overview.cjs.map +1 -1
- package/dist/inspection/overview.d.cts +8 -6
- package/dist/inspection/overview.d.mts +8 -6
- package/dist/inspection/overview.d.ts +8 -6
- package/dist/inspection/planned-attempt.cjs +45 -0
- package/dist/inspection/planned-attempt.cjs.map +1 -0
- package/dist/inspection/planned-attempt.d.cts +109 -0
- package/dist/inspection/planned-attempt.d.mts +109 -0
- package/dist/inspection/planned-attempt.d.ts +109 -0
- package/dist/inspection/planned-attempt.js +24 -0
- package/dist/inspection/planned-attempt.js.map +1 -0
- package/dist/inspection/project-result.cjs +5 -5
- package/dist/inspection/project-result.cjs.map +1 -1
- package/dist/inspection/project-result.d.cts +58 -57
- package/dist/inspection/project-result.d.mts +58 -57
- package/dist/inspection/project-result.d.ts +58 -57
- package/dist/inspection/project-result.js +4 -4
- package/dist/inspection/project.cjs +3 -3
- package/dist/inspection/project.cjs.map +1 -1
- package/dist/inspection/protocol-values.cjs +1 -1
- package/dist/inspection/protocol-values.cjs.map +1 -1
- package/dist/inspection/protocol-values.d.cts +1 -1
- package/dist/inspection/protocol-values.d.mts +1 -1
- package/dist/inspection/protocol-values.d.ts +1 -1
- package/dist/inspection/protocol.cjs +8 -8
- package/dist/inspection/protocol.cjs.map +1 -1
- package/dist/inspection/protocol.d.cts +237 -219
- package/dist/inspection/protocol.d.mts +237 -219
- package/dist/inspection/protocol.d.ts +237 -219
- package/dist/inspection/public.cjs +1 -5
- package/dist/inspection/public.cjs.map +1 -1
- package/dist/inspection/public.d.cts +1 -2
- package/dist/inspection/public.d.mts +1 -2
- package/dist/inspection/public.d.ts +1 -2
- package/dist/inspection/public.js +14 -20
- package/dist/inspection/public.mjs +166 -129
- package/dist/inspection/query.cjs +1 -1
- package/dist/inspection/query.cjs.map +1 -1
- package/dist/inspection/results.cjs +61 -52
- package/dist/inspection/results.cjs.map +1 -1
- package/dist/inspection/results.d.cts +157 -140
- package/dist/inspection/results.d.mts +157 -140
- package/dist/inspection/results.d.ts +157 -140
- package/dist/inspection/select.cjs +33 -28
- package/dist/inspection/select.cjs.map +1 -1
- package/dist/inspection/source.cjs +1 -1
- package/dist/inspection/source.cjs.map +1 -1
- package/dist/inspection/sources.cjs +2 -2
- package/dist/inspection/sources.cjs.map +1 -1
- package/dist/inspection/trace.cjs +9 -8
- package/dist/inspection/trace.cjs.map +1 -1
- package/dist/inspection/trace.d.cts +4 -4
- package/dist/inspection/trace.d.mts +4 -4
- package/dist/inspection/trace.d.ts +4 -4
- package/dist/loaders/index.cjs +7 -279
- package/dist/loaders/index.cjs.map +1 -1
- package/dist/loaders/index.d.cts +2 -95
- package/dist/loaders/index.d.mts +2 -95
- package/dist/loaders/index.d.ts +2 -95
- package/dist/loaders/index.js +10 -12
- package/dist/loaders/index.mjs +10 -12
- package/dist/loaders/runtime.cjs +283 -0
- package/dist/loaders/runtime.cjs.map +1 -0
- package/dist/loaders/runtime.d.cts +95 -0
- package/dist/loaders/runtime.d.mts +95 -0
- package/dist/loaders/runtime.d.ts +95 -0
- package/dist/loaders/runtime.js +16 -0
- package/dist/loaders/runtime.js.map +1 -0
- package/dist/migrations/judge-provider.md +6 -6
- package/dist/{model-slots.cjs → model-roles.cjs} +12 -12
- package/dist/{model-slots.cjs.map → model-roles.cjs.map} +1 -1
- package/dist/{model-slots.d.cts → model-roles.d.cts} +10 -10
- package/dist/{model-slots.d.mts → model-roles.d.mts} +10 -10
- package/dist/{model-slots.d.ts → model-roles.d.ts} +10 -10
- package/dist/model-roles.js +14 -0
- package/dist/model-roles.js.map +1 -0
- package/dist/o11y/adapter-usage-projection.cjs +9 -8
- package/dist/o11y/adapter-usage-projection.cjs.map +1 -1
- package/dist/o11y/adapter-usage-projection.d.cts +23 -15
- package/dist/o11y/adapter-usage-projection.d.mts +23 -15
- package/dist/o11y/adapter-usage-projection.d.ts +23 -15
- package/dist/o11y/cache-read.cjs +14 -14
- package/dist/o11y/cache-read.cjs.map +1 -1
- package/dist/o11y/cache-read.d.cts +5 -5
- package/dist/o11y/cache-read.d.mts +5 -5
- package/dist/o11y/cache-read.d.ts +5 -5
- package/dist/o11y/derive.cjs.map +1 -1
- package/dist/o11y/derive.d.cts +2 -2
- package/dist/o11y/derive.d.mts +2 -2
- package/dist/o11y/derive.d.ts +2 -2
- package/dist/o11y/eval-usage.cjs +9 -9
- package/dist/o11y/eval-usage.cjs.map +1 -1
- package/dist/o11y/eval-usage.d.cts +7 -7
- package/dist/o11y/eval-usage.d.mts +7 -7
- package/dist/o11y/eval-usage.d.ts +7 -7
- package/dist/o11y/parsers/bub.cjs +57 -81
- package/dist/o11y/parsers/bub.cjs.map +1 -1
- package/dist/o11y/parsers/index.cjs.map +1 -1
- package/dist/o11y/parsers/index.d.cts +2 -0
- package/dist/o11y/parsers/index.d.mts +2 -0
- package/dist/o11y/parsers/index.d.ts +2 -0
- package/dist/o11y/parsers/opencode.cjs +129 -217
- package/dist/o11y/parsers/opencode.cjs.map +1 -1
- package/dist/o11y/parsers/opencode.d.cts +2 -5
- package/dist/o11y/parsers/opencode.d.mts +2 -5
- package/dist/o11y/parsers/opencode.d.ts +2 -5
- package/dist/o11y/types.cjs.map +1 -1
- package/dist/o11y/types.d.cts +1 -1
- package/dist/o11y/types.d.mts +1 -1
- package/dist/o11y/types.d.ts +1 -1
- package/dist/plugin/contracts.cjs +170 -120
- package/dist/plugin/contracts.cjs.map +1 -1
- package/dist/plugin/contracts.d.cts +102 -90
- package/dist/plugin/contracts.d.mts +102 -90
- package/dist/plugin/contracts.d.ts +102 -90
- package/dist/plugin/contracts.js +16 -12
- package/dist/plugin/index.cjs.map +1 -1
- package/dist/plugin/index.d.cts +1 -1
- package/dist/plugin/index.d.mts +1 -1
- package/dist/plugin/index.d.ts +1 -1
- package/dist/plugin/link.cjs +69 -64
- package/dist/plugin/link.cjs.map +1 -1
- package/dist/plugin/link.d.cts +5 -3
- package/dist/plugin/link.d.mts +5 -3
- package/dist/plugin/link.d.ts +5 -3
- package/dist/plugin/link.js +12 -8
- package/dist/record/codec/seal-manifest.d.cts +3 -3
- package/dist/record/codec/seal-manifest.d.mts +3 -3
- package/dist/record/codec/seal-manifest.d.ts +3 -3
- package/dist/record/family/attempt-note/definition.cjs +12 -0
- package/dist/record/family/attempt-note/definition.cjs.map +1 -0
- package/dist/record/family/attempt-note/definition.d.cts +6 -0
- package/dist/record/family/attempt-note/definition.d.mts +6 -0
- package/dist/record/family/attempt-note/definition.d.ts +6 -0
- package/dist/record/family/attempt-note/definition.js +6 -0
- package/dist/record/family/attempt-note/definition.js.map +1 -0
- package/dist/record/family/attempt-note/persistence.cjs +10 -0
- package/dist/record/family/attempt-note/persistence.cjs.map +1 -0
- package/dist/record/family/attempt-note/persistence.d.cts +6 -0
- package/dist/record/family/attempt-note/persistence.d.mts +6 -0
- package/dist/record/family/attempt-note/persistence.d.ts +6 -0
- package/dist/record/family/attempt-note/persistence.js +6 -0
- package/dist/record/family/attempt-note/persistence.js.map +1 -0
- package/dist/record/family/attempt-note/schema.cjs +20 -0
- package/dist/record/family/attempt-note/schema.cjs.map +1 -0
- package/dist/record/family/attempt-note/schema.d.cts +21 -0
- package/dist/record/family/attempt-note/schema.d.mts +21 -0
- package/dist/record/family/attempt-note/schema.d.ts +21 -0
- package/dist/record/family/attempt-note/schema.js +10 -0
- package/dist/record/family/attempt-note/schema.js.map +1 -0
- package/dist/record/family/catalog.cjs +32 -29
- package/dist/record/family/catalog.cjs.map +1 -1
- package/dist/record/family/catalog.d.cts +7 -2
- package/dist/record/family/catalog.d.mts +7 -2
- package/dist/record/family/catalog.d.ts +7 -2
- package/dist/record/family/current.cjs +48 -45
- package/dist/record/family/current.cjs.map +1 -1
- package/dist/record/family/current.d.cts +15 -0
- package/dist/record/family/current.d.mts +15 -0
- package/dist/record/family/current.d.ts +15 -0
- package/dist/record/model/index.js +66 -64
- package/dist/record/model/run-context.cjs +9 -3
- package/dist/record/model/run-context.cjs.map +1 -1
- package/dist/record/model/run-context.d.cts +9 -2
- package/dist/record/model/run-context.d.mts +9 -2
- package/dist/record/model/run-context.d.ts +9 -2
- package/dist/record/model/run-context.js +8 -6
- package/dist/record/model/seal-manifest.d.cts +1 -1
- package/dist/record/model/seal-manifest.d.mts +1 -1
- package/dist/record/model/seal-manifest.d.ts +1 -1
- package/dist/run/cli/contribution.cjs +6 -32
- package/dist/run/cli/contribution.cjs.map +1 -1
- package/dist/run/host/runtime.cjs +1 -1
- package/dist/run/host/runtime.cjs.map +1 -1
- package/dist/run/storage/sqlite.cjs +7 -7
- package/dist/run/storage/sqlite.cjs.map +1 -1
- package/dist/runner/adapter-usage.cjs +5 -3
- package/dist/runner/adapter-usage.cjs.map +1 -1
- package/dist/runner/adapter-usage.d.cts +2 -2
- package/dist/runner/adapter-usage.d.mts +2 -2
- package/dist/runner/adapter-usage.d.ts +2 -2
- package/dist/runner/adoption.cjs +9 -12
- package/dist/runner/adoption.cjs.map +1 -1
- package/dist/runner/attempt-notes.cjs +8 -0
- package/dist/runner/attempt-notes.cjs.map +1 -0
- package/dist/runner/attempt-notes.d.cts +4 -0
- package/dist/runner/attempt-notes.d.mts +4 -0
- package/dist/runner/attempt-notes.d.ts +4 -0
- package/dist/runner/attempt-notes.js +8 -0
- package/dist/runner/attempt-notes.js.map +1 -0
- package/dist/runner/attempt.cjs +61 -73
- package/dist/runner/attempt.cjs.map +1 -1
- package/dist/runner/attempt.d.cts +3 -3
- package/dist/runner/attempt.d.mts +3 -3
- package/dist/runner/attempt.d.ts +3 -3
- package/dist/runner/command-plan.cjs +21 -26
- package/dist/runner/command-plan.cjs.map +1 -1
- package/dist/runner/command-plan.d.cts +4 -4
- package/dist/runner/command-plan.d.mts +4 -4
- package/dist/runner/command-plan.d.ts +4 -4
- package/dist/runner/command-plan.js +4 -4
- package/dist/runner/config-identity.cjs.map +1 -1
- package/dist/runner/config-identity.d.cts +2 -2
- package/dist/runner/config-identity.d.mts +2 -2
- package/dist/runner/config-identity.d.ts +2 -2
- package/dist/runner/discover.cjs +6 -6
- package/dist/runner/discover.cjs.map +1 -1
- package/dist/runner/eval-selection.cjs +2 -2
- package/dist/runner/eval-selection.cjs.map +1 -1
- package/dist/runner/eval-selection.d.cts +2 -2
- package/dist/runner/eval-selection.d.mts +2 -2
- package/dist/runner/eval-selection.d.ts +2 -2
- package/dist/runner/feedback/human.cjs +1 -1
- package/dist/runner/feedback/human.cjs.map +1 -1
- package/dist/runner/feedback/json.cjs +4 -2
- package/dist/runner/feedback/json.cjs.map +1 -1
- package/dist/runner/feedback/json.d.cts +1 -1
- package/dist/runner/feedback/json.d.mts +1 -1
- package/dist/runner/feedback/json.d.ts +1 -1
- package/dist/runner/feedback/sink.cjs +1 -1
- package/dist/runner/feedback/sink.cjs.map +1 -1
- package/dist/runner/feedback/sink.d.cts +1 -1
- package/dist/runner/feedback/sink.d.mts +1 -1
- package/dist/runner/feedback/sink.d.ts +1 -1
- package/dist/runner/fingerprint.cjs +1 -1
- package/dist/runner/fingerprint.cjs.map +1 -1
- package/dist/runner/project-target.cjs.map +1 -1
- package/dist/runner/project-target.d.cts +2 -2
- package/dist/runner/project-target.d.mts +2 -2
- package/dist/runner/project-target.d.ts +2 -2
- package/dist/runner/record/attachments.cjs +2 -2
- package/dist/runner/record/attachments.cjs.map +1 -1
- package/dist/runner/record/planning.cjs +8 -2
- package/dist/runner/record/planning.cjs.map +1 -1
- package/dist/runner/record.cjs +2 -0
- package/dist/runner/record.cjs.map +1 -1
- package/dist/runner/report.cjs +3 -3
- package/dist/runner/report.cjs.map +1 -1
- package/dist/runner/report.d.cts +1 -1
- package/dist/runner/report.d.mts +1 -1
- package/dist/runner/report.d.ts +1 -1
- package/dist/runner/reuse-plan.cjs +2 -2
- package/dist/runner/reuse-plan.cjs.map +1 -1
- package/dist/runner/run.cjs +10 -45
- package/dist/runner/run.cjs.map +1 -1
- package/dist/runner/sandbox-pool.cjs +1 -2
- package/dist/runner/sandbox-pool.cjs.map +1 -1
- package/dist/runner/setup-prefix-preparation.cjs +1 -1
- package/dist/runner/setup-prefix-preparation.cjs.map +1 -1
- package/dist/runner/teardown-registry.cjs +2 -2
- package/dist/runner/teardown-registry.cjs.map +1 -1
- package/dist/runner/teardown-registry.d.cts +1 -1
- package/dist/runner/teardown-registry.d.mts +1 -1
- package/dist/runner/teardown-registry.d.ts +1 -1
- package/dist/runner/types.cjs +4 -4
- package/dist/runner/types.cjs.map +1 -1
- package/dist/runner/types.d.cts +64 -93
- package/dist/runner/types.d.mts +64 -93
- package/dist/runner/types.d.ts +64 -93
- package/dist/runner/types.js +2 -2
- package/dist/sandbox/action.cjs +1 -1
- package/dist/sandbox/action.cjs.map +1 -1
- package/dist/sandbox/cli/contribution.cjs +17 -14
- package/dist/sandbox/cli/contribution.cjs.map +1 -1
- package/dist/sandbox/cli/contribution.d.cts +1 -1
- package/dist/sandbox/cli/contribution.d.mts +1 -1
- package/dist/sandbox/cli/contribution.d.ts +1 -1
- package/dist/sandbox/commands.cjs +1 -1
- package/dist/sandbox/commands.cjs.map +1 -1
- package/dist/sandbox/commands.d.cts +1 -1
- package/dist/sandbox/commands.d.mts +1 -1
- package/dist/sandbox/commands.d.ts +1 -1
- package/dist/sandbox/compose.cjs +2 -2
- package/dist/sandbox/compose.cjs.map +1 -1
- package/dist/sandbox/docker-profile/registry.cjs +5 -9
- package/dist/sandbox/docker-profile/registry.cjs.map +1 -1
- package/dist/sandbox/docker-profile/registry.d.cts +0 -3
- package/dist/sandbox/docker-profile/registry.d.mts +0 -3
- package/dist/sandbox/docker-profile/registry.d.ts +0 -3
- package/dist/sandbox/docker-profile/registry.js +6 -12
- package/dist/sandbox/docker-profile/runtime.cjs +1 -1
- package/dist/sandbox/docker-profile/runtime.cjs.map +1 -1
- package/dist/sandbox/docker-profile/schema.cjs +12 -16
- package/dist/sandbox/docker-profile/schema.cjs.map +1 -1
- package/dist/sandbox/docker-profile/schema.d.cts +2 -6
- package/dist/sandbox/docker-profile/schema.d.mts +2 -6
- package/dist/sandbox/docker-profile/schema.d.ts +2 -6
- package/dist/sandbox/docker-profile/schema.js +24 -32
- package/dist/sandbox/incus.cjs +2 -2
- package/dist/sandbox/incus.cjs.map +1 -1
- package/dist/sandbox/index.cjs +4 -35
- package/dist/sandbox/index.cjs.map +1 -1
- package/dist/sandbox/index.d.cts +4 -8
- package/dist/sandbox/index.d.mts +4 -8
- package/dist/sandbox/index.d.ts +4 -8
- package/dist/sandbox/index.js +116 -172
- package/dist/sandbox/index.mjs +116 -172
- package/dist/sandbox/layer.cjs +3 -3
- package/dist/sandbox/layer.cjs.map +1 -1
- package/dist/sandbox/link.cjs +5 -0
- package/dist/sandbox/link.cjs.map +1 -1
- package/dist/sandbox/link.d.cts +3 -0
- package/dist/sandbox/link.d.mts +3 -0
- package/dist/sandbox/link.d.ts +3 -0
- package/dist/sandbox/runtime.cjs +9 -4
- package/dist/sandbox/runtime.cjs.map +1 -1
- package/dist/shared/failure-class.cjs +5 -5
- package/dist/shared/failure-class.cjs.map +1 -1
- package/dist/shared/failure-class.d.cts +4 -10
- package/dist/shared/failure-class.d.mts +4 -10
- package/dist/shared/failure-class.d.ts +4 -10
- package/dist/shared/types.cjs.map +1 -1
- package/dist/shared/types.d.cts +1 -1
- package/dist/shared/types.d.mts +1 -1
- package/dist/shared/types.d.ts +1 -1
- package/dist/show/contribution.cjs +17 -2
- package/dist/show/contribution.cjs.map +1 -1
- package/dist/show/contribution.d.cts +14 -0
- package/dist/show/contribution.d.mts +14 -0
- package/dist/show/contribution.d.ts +14 -0
- package/dist/show/model.cjs +8 -5
- package/dist/show/model.cjs.map +1 -1
- package/dist/show/model.d.cts +11 -8
- package/dist/show/model.d.mts +11 -8
- package/dist/show/model.d.ts +11 -8
- package/dist/show/render.cjs +34 -22
- package/dist/show/render.cjs.map +1 -1
- package/dist/sources/projector.cjs +1 -1
- package/dist/sources/projector.cjs.map +1 -1
- package/dist/types.cjs +1 -1
- package/dist/types.cjs.map +1 -1
- package/dist/types.d.cts +1 -1
- package/dist/types.d.mts +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/types.js +2 -2
- package/dist/view/app-dist/.vite/manifest.json +2 -2
- package/dist/view/app-dist/assets/{index-B3ijZRph.js → index-CxrOUvHC.js} +17 -17
- package/dist/view/app-dist/assets/{quality-cost-scatter-B--DfJ9s.js → quality-cost-scatter-B6Aiss7x.js} +1 -1
- package/dist/view/app-dist/index.html +1 -1
- package/dist/view/cli/contribution.cjs +1 -1
- package/dist/view/cli/contribution.cjs.map +1 -1
- package/dist/view/function-runtime.mjs +318 -121
- package/dist/view/http-protocol.d.cts +1 -1
- package/dist/view/http-protocol.d.mts +1 -1
- package/dist/view/http-protocol.d.ts +1 -1
- package/dist/view/inspection-host.cjs +2 -2
- package/dist/view/inspection-host.cjs.map +1 -1
- package/dist/view/inspection-host.d.cts +349 -331
- package/dist/view/inspection-host.d.mts +349 -331
- package/dist/view/inspection-host.d.ts +349 -331
- package/docs-site/zh/examples/integrations/ai-sdk-v7.mdx +6 -6
- package/docs-site/zh/examples/integrations/claude-sdk.mdx +5 -5
- package/docs-site/zh/examples/integrations/codex-sdk.mdx +4 -4
- package/docs-site/zh/examples/integrations/langgraph.mdx +4 -4
- package/docs-site/zh/examples/integrations/pi-sdk.mdx +5 -5
- package/docs-site/zh/explanation/adapter.mdx +6 -6
- package/docs-site/zh/explanation/assert.mdx +7 -7
- package/docs-site/zh/explanation/drive.mdx +4 -4
- package/docs-site/zh/explanation/hitl.mdx +2 -2
- package/docs-site/zh/explanation/judge.mdx +2 -2
- package/docs-site/zh/introduction.mdx +2 -2
- package/docs-site/zh/reference/builtin-agents.mdx +46 -49
- package/docs-site/zh/reference/cli.mdx +16 -14
- package/docs-site/zh/reference/define-agent.mdx +19 -19
- package/docs-site/zh/reference/define-config.mdx +7 -15
- package/docs-site/zh/reference/define-eval.mdx +20 -14
- package/docs-site/zh/reference/define-experiment.mdx +18 -39
- package/docs-site/zh/reference/events.mdx +1 -1
- package/docs-site/zh/reference/expect.mdx +21 -0
- package/docs-site/zh/reference/official-adapters.mdx +12 -5
- package/docs-site/zh/troubleshooting/debugging.mdx +2 -2
- package/docs-site/zh/troubleshooting/recover-after-kill.mdx +1 -1
- package/docs-site/zh/tutorials/agent-onboarding.mdx +2 -2
- package/docs-site/zh/tutorials/authoring.mdx +2 -2
- package/docs-site/zh/tutorials/compare-runs.mdx +1 -1
- package/docs-site/zh/tutorials/concurrency.mdx +8 -8
- package/docs-site/zh/tutorials/configuration.mdx +5 -5
- package/docs-site/zh/tutorials/control-run-cost.mdx +19 -19
- package/docs-site/zh/tutorials/eval-groups.mdx +1 -1
- package/docs-site/zh/tutorials/evaluation-kinds.mdx +5 -5
- package/docs-site/zh/tutorials/experiment-services.mdx +32 -32
- package/docs-site/zh/tutorials/handle-execution-failures.mdx +2 -28
- package/docs-site/zh/tutorials/plugins.mdx +49 -143
- package/docs-site/zh/tutorials/sandbox-agent.mdx +1 -1
- package/docs-site/zh/tutorials/sandbox-reuse.mdx +7 -7
- package/docs-site/zh/tutorials/upgrade-0-18.mdx +1 -1
- package/docs-site/zh/tutorials/upgrade-0-19.mdx +121 -0
- package/docs-site/zh/tutorials/upgrade-0-20.mdx +12 -0
- package/docs-site/zh/tutorials/verify-judge.mdx +2 -2
- package/docs-site/zh/tutorials/viewing-results.mdx +2 -2
- package/docs-site/zh/tutorials/write-experiment.mdx +1 -1
- package/docs-site/zh/tutorials/write-send.mdx +13 -13
- package/package.json +1 -1
- package/dist/agents/effect-runtime.cjs.map +0 -1
- package/dist/agents/effect-runtime.d.cts +0 -29
- package/dist/agents/effect-runtime.d.mts +0 -29
- package/dist/agents/effect-runtime.d.ts +0 -29
- package/dist/agents/effect-runtime.js +0 -16
- package/dist/agents/effect-runtime.js.map +0 -1
- package/dist/agents/session-slot.cjs.map +0 -1
- package/dist/agents/session-slot.d.cts +0 -12
- package/dist/agents/session-slot.d.mts +0 -12
- package/dist/agents/session-slot.d.ts +0 -12
- package/dist/agents/session-slot.js +0 -6
- package/dist/agents/session-slot.js.map +0 -1
- package/dist/model-slots.js +0 -14
- package/dist/model-slots.js.map +0 -1
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Internal discovery provenance. A shared helper is evaluated only once in a
|
|
3
|
+
* fresh module graph, so discovery uses its source module to copy these paths
|
|
4
|
+
* to every Eval entry that statically imports that helper.
|
|
5
|
+
*/
|
|
6
|
+
export interface LoaderCaptureOrigin {
|
|
7
|
+
readonly sourcePath: string;
|
|
8
|
+
readonly paths: readonly string[];
|
|
9
|
+
readonly criteriaPaths: readonly string[];
|
|
10
|
+
readonly privatePaths: readonly string[];
|
|
11
|
+
}
|
|
12
|
+
export interface LoaderCapturePaths {
|
|
13
|
+
readonly paths: readonly string[];
|
|
14
|
+
readonly criteriaPaths: readonly string[];
|
|
15
|
+
readonly privatePaths: readonly string[];
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* 发现期包住一个 eval 模块的求值,记录它经公开 loader 读取的数据文件(`paths`)、
|
|
19
|
+
* 经 `loadCriteria` 登记的判据树(`criteriaPaths`)、经 `loadPrivate` 登记的永不上传路径
|
|
20
|
+
* (`privatePaths`)。三格分开是因为指纹与泄题门口径不同:
|
|
21
|
+
* data 哈希已读进内存的内容;criteria / private 按内容流式哈希,且 private 另进泄题门。
|
|
22
|
+
*/
|
|
23
|
+
export declare function captureLoadedFiles<T>(load: () => Promise<T>): Promise<{
|
|
24
|
+
value: T;
|
|
25
|
+
paths: string[];
|
|
26
|
+
criteriaPaths: string[];
|
|
27
|
+
privatePaths: string[];
|
|
28
|
+
origins: readonly LoaderCaptureOrigin[];
|
|
29
|
+
unattributed: LoaderCapturePaths;
|
|
30
|
+
}>;
|
|
31
|
+
/**
|
|
32
|
+
* 读入一个 JSON 数据文件并解析。路径写项目根相对的字符串,或 eval 文件相对的
|
|
33
|
+
* `new URL(p, import.meta.url)`。内容哈希进读它的那条 eval 的指纹。`decode` 必须在
|
|
34
|
+
* 这个动态输入边界完成结构验证与归一;不提供「用泛型直接信任文件」的入口。
|
|
35
|
+
*/
|
|
36
|
+
type DecodedData<T> = unknown extends T ? never : T;
|
|
37
|
+
/**
|
|
38
|
+
* 动态文件值到领域值的唯一出口。`unknown`(以及会把它吞掉的 `any`)不能成为解码结果;
|
|
39
|
+
* decoder 必须返回一个比动态输入更窄、可供 Eval 定义直接使用的领域类型。
|
|
40
|
+
*/
|
|
41
|
+
export type DataDecoder<T> = (value: unknown) => DecodedData<T> | Promise<DecodedData<T>>;
|
|
42
|
+
export declare function loadJson<T>(path: string | URL, decode: DataDecoder<T>): Promise<T>;
|
|
43
|
+
/**
|
|
44
|
+
* 按 utf-8 读入一个文件的原文。判据不是结构化数据时用它:隐藏测试脚本、参考实现、
|
|
45
|
+
* shell 模板都是判据文件。读入即宣告「它参与判定」——文件内容哈希进读它的那条 eval
|
|
46
|
+
* 的指纹,改一字节就重跑那条 eval。用 `fs` 自行读入的文件不进指纹。
|
|
47
|
+
* 路径写项目根相对的字符串,或 eval 文件相对的 `new URL(p, import.meta.url)`。
|
|
48
|
+
*/
|
|
49
|
+
export declare function loadText(path: string | URL): Promise<string>;
|
|
50
|
+
/** eval 文件相对的 glob;pattern 与 URL 分开,避免 URL parser 改写 glob 字符。 */
|
|
51
|
+
export interface RelativeCriteriaPattern {
|
|
52
|
+
readonly pattern: string;
|
|
53
|
+
readonly relativeTo: URL;
|
|
54
|
+
}
|
|
55
|
+
export type CriteriaPattern = string | RelativeCriteriaPattern;
|
|
56
|
+
/**
|
|
57
|
+
* 登记一棵判据树进这条 eval 的指纹,不把内容读进内存。判分标准是一整棵树(隐藏测试目录、
|
|
58
|
+
* 参考实现、跑测脚本)、内容只经 `t.sandbox.uploadDirectory` 整体送进沙箱时用它:
|
|
59
|
+
* 发现期展开 glob、把每个匹配文件流式哈希进指纹,返回排序后的项目根相对路径清单。
|
|
60
|
+
* 树里改一字节、增一个文件、删一个文件,都作废引用这棵树的那条 eval;权限位与修改时间不进哈希,
|
|
61
|
+
* 重新 clone 一份工作树不作废。单个文件、且内容要写进沙箱时用 `loadText`。
|
|
62
|
+
*
|
|
63
|
+
* pattern 是项目根相对的 glob(`**` 跨层、`*` / `?` 段内、`[...]` 字符类、`{a,b}` 分支),
|
|
64
|
+
* `!` 前缀为排除;按声明顺序求值,后写的覆盖先写的。匹配集按文件系统枚举、不看 git:
|
|
65
|
+
* 新写的判据没 `git add` 也照样进指纹,代价是本地跑测冒出的生成物同样被盖到,用 `!` 排除。
|
|
66
|
+
*
|
|
67
|
+
* 只能在 eval 文件的模块顶层调用(发现期求值那一刻),`test(t)` 运行期调用直接报错——
|
|
68
|
+
* 携带决策发生在任何 attempt 执行之前,那时才登记已经来不及。
|
|
69
|
+
*
|
|
70
|
+
* 两种用法错误直接抛:某个 include pattern 一个文件都没匹配到(或命中的都被 `!` 排除了)——
|
|
71
|
+
* 多半是写错了或文件搬走了,别的 pattern 有命中也不放过,静默放过等于判据悄悄变窄;
|
|
72
|
+
* 以及匹配落到项目根外(符号链接穿出根)。`!` 排除 pattern 不受「必须有命中」这条约束。
|
|
73
|
+
*
|
|
74
|
+
* @param patterns 项目根相对 glob,或 `{ pattern, relativeTo: import.meta.url }`;`!` 排除只收字符串。
|
|
75
|
+
* @returns 匹配集的项目根相对路径,排序后(正斜杠分隔),不含内容。
|
|
76
|
+
*/
|
|
77
|
+
export declare function loadCriteria(...patterns: CriteriaPattern[]): Promise<string[]>;
|
|
78
|
+
/**
|
|
79
|
+
* 登记永不上传的 private 路径进这条 eval 的判据指纹与泄题门,不把内容读进内存。
|
|
80
|
+
* solution、生成器、参考答案必须与 eval 共址时用它:发现期与全部 build context /
|
|
81
|
+
* bind mount 交叉检查,任何阶段都不得进入 Agent 可见面(见 docs/feature/sandbox/case.md「泄题门」)。
|
|
82
|
+
*
|
|
83
|
+
* pattern 语法、发现期约束、空匹配与穿出项目根的报错与 `loadCriteria` 同形。
|
|
84
|
+
*
|
|
85
|
+
* @param patterns 一个或多个项目根相对 glob,`!` 前缀为排除。
|
|
86
|
+
* @returns 匹配集的项目根相对路径,排序后(正斜杠分隔),不含内容。
|
|
87
|
+
*/
|
|
88
|
+
export declare function loadPrivate(...patterns: string[]): Promise<string[]>;
|
|
89
|
+
/**
|
|
90
|
+
* 读入一个 YAML 数据文件并解析,需要项目里装了 `yaml`。路径写项目根相对的字符串,
|
|
91
|
+
* 或 eval 文件相对的 `new URL(p, import.meta.url)`。内容哈希进读它的那条 eval 的指纹。解析结果与
|
|
92
|
+
* JSON 一样必须立即经 `decode` 验证,不让未证明的动态值进入 Eval 定义。
|
|
93
|
+
*/
|
|
94
|
+
export declare function loadYaml<T>(path: string | URL, decode: DataDecoder<T>): Promise<T>;
|
|
95
|
+
export {};
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Internal discovery provenance. A shared helper is evaluated only once in a
|
|
3
|
+
* fresh module graph, so discovery uses its source module to copy these paths
|
|
4
|
+
* to every Eval entry that statically imports that helper.
|
|
5
|
+
*/
|
|
6
|
+
export interface LoaderCaptureOrigin {
|
|
7
|
+
readonly sourcePath: string;
|
|
8
|
+
readonly paths: readonly string[];
|
|
9
|
+
readonly criteriaPaths: readonly string[];
|
|
10
|
+
readonly privatePaths: readonly string[];
|
|
11
|
+
}
|
|
12
|
+
export interface LoaderCapturePaths {
|
|
13
|
+
readonly paths: readonly string[];
|
|
14
|
+
readonly criteriaPaths: readonly string[];
|
|
15
|
+
readonly privatePaths: readonly string[];
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* 发现期包住一个 eval 模块的求值,记录它经公开 loader 读取的数据文件(`paths`)、
|
|
19
|
+
* 经 `loadCriteria` 登记的判据树(`criteriaPaths`)、经 `loadPrivate` 登记的永不上传路径
|
|
20
|
+
* (`privatePaths`)。三格分开是因为指纹与泄题门口径不同:
|
|
21
|
+
* data 哈希已读进内存的内容;criteria / private 按内容流式哈希,且 private 另进泄题门。
|
|
22
|
+
*/
|
|
23
|
+
export declare function captureLoadedFiles<T>(load: () => Promise<T>): Promise<{
|
|
24
|
+
value: T;
|
|
25
|
+
paths: string[];
|
|
26
|
+
criteriaPaths: string[];
|
|
27
|
+
privatePaths: string[];
|
|
28
|
+
origins: readonly LoaderCaptureOrigin[];
|
|
29
|
+
unattributed: LoaderCapturePaths;
|
|
30
|
+
}>;
|
|
31
|
+
/**
|
|
32
|
+
* 读入一个 JSON 数据文件并解析。路径写项目根相对的字符串,或 eval 文件相对的
|
|
33
|
+
* `new URL(p, import.meta.url)`。内容哈希进读它的那条 eval 的指纹。`decode` 必须在
|
|
34
|
+
* 这个动态输入边界完成结构验证与归一;不提供「用泛型直接信任文件」的入口。
|
|
35
|
+
*/
|
|
36
|
+
type DecodedData<T> = unknown extends T ? never : T;
|
|
37
|
+
/**
|
|
38
|
+
* 动态文件值到领域值的唯一出口。`unknown`(以及会把它吞掉的 `any`)不能成为解码结果;
|
|
39
|
+
* decoder 必须返回一个比动态输入更窄、可供 Eval 定义直接使用的领域类型。
|
|
40
|
+
*/
|
|
41
|
+
export type DataDecoder<T> = (value: unknown) => DecodedData<T> | Promise<DecodedData<T>>;
|
|
42
|
+
export declare function loadJson<T>(path: string | URL, decode: DataDecoder<T>): Promise<T>;
|
|
43
|
+
/**
|
|
44
|
+
* 按 utf-8 读入一个文件的原文。判据不是结构化数据时用它:隐藏测试脚本、参考实现、
|
|
45
|
+
* shell 模板都是判据文件。读入即宣告「它参与判定」——文件内容哈希进读它的那条 eval
|
|
46
|
+
* 的指纹,改一字节就重跑那条 eval。用 `fs` 自行读入的文件不进指纹。
|
|
47
|
+
* 路径写项目根相对的字符串,或 eval 文件相对的 `new URL(p, import.meta.url)`。
|
|
48
|
+
*/
|
|
49
|
+
export declare function loadText(path: string | URL): Promise<string>;
|
|
50
|
+
/** eval 文件相对的 glob;pattern 与 URL 分开,避免 URL parser 改写 glob 字符。 */
|
|
51
|
+
export interface RelativeCriteriaPattern {
|
|
52
|
+
readonly pattern: string;
|
|
53
|
+
readonly relativeTo: URL;
|
|
54
|
+
}
|
|
55
|
+
export type CriteriaPattern = string | RelativeCriteriaPattern;
|
|
56
|
+
/**
|
|
57
|
+
* 登记一棵判据树进这条 eval 的指纹,不把内容读进内存。判分标准是一整棵树(隐藏测试目录、
|
|
58
|
+
* 参考实现、跑测脚本)、内容只经 `t.sandbox.uploadDirectory` 整体送进沙箱时用它:
|
|
59
|
+
* 发现期展开 glob、把每个匹配文件流式哈希进指纹,返回排序后的项目根相对路径清单。
|
|
60
|
+
* 树里改一字节、增一个文件、删一个文件,都作废引用这棵树的那条 eval;权限位与修改时间不进哈希,
|
|
61
|
+
* 重新 clone 一份工作树不作废。单个文件、且内容要写进沙箱时用 `loadText`。
|
|
62
|
+
*
|
|
63
|
+
* pattern 是项目根相对的 glob(`**` 跨层、`*` / `?` 段内、`[...]` 字符类、`{a,b}` 分支),
|
|
64
|
+
* `!` 前缀为排除;按声明顺序求值,后写的覆盖先写的。匹配集按文件系统枚举、不看 git:
|
|
65
|
+
* 新写的判据没 `git add` 也照样进指纹,代价是本地跑测冒出的生成物同样被盖到,用 `!` 排除。
|
|
66
|
+
*
|
|
67
|
+
* 只能在 eval 文件的模块顶层调用(发现期求值那一刻),`test(t)` 运行期调用直接报错——
|
|
68
|
+
* 携带决策发生在任何 attempt 执行之前,那时才登记已经来不及。
|
|
69
|
+
*
|
|
70
|
+
* 两种用法错误直接抛:某个 include pattern 一个文件都没匹配到(或命中的都被 `!` 排除了)——
|
|
71
|
+
* 多半是写错了或文件搬走了,别的 pattern 有命中也不放过,静默放过等于判据悄悄变窄;
|
|
72
|
+
* 以及匹配落到项目根外(符号链接穿出根)。`!` 排除 pattern 不受「必须有命中」这条约束。
|
|
73
|
+
*
|
|
74
|
+
* @param patterns 项目根相对 glob,或 `{ pattern, relativeTo: import.meta.url }`;`!` 排除只收字符串。
|
|
75
|
+
* @returns 匹配集的项目根相对路径,排序后(正斜杠分隔),不含内容。
|
|
76
|
+
*/
|
|
77
|
+
export declare function loadCriteria(...patterns: CriteriaPattern[]): Promise<string[]>;
|
|
78
|
+
/**
|
|
79
|
+
* 登记永不上传的 private 路径进这条 eval 的判据指纹与泄题门,不把内容读进内存。
|
|
80
|
+
* solution、生成器、参考答案必须与 eval 共址时用它:发现期与全部 build context /
|
|
81
|
+
* bind mount 交叉检查,任何阶段都不得进入 Agent 可见面(见 docs/feature/sandbox/case.md「泄题门」)。
|
|
82
|
+
*
|
|
83
|
+
* pattern 语法、发现期约束、空匹配与穿出项目根的报错与 `loadCriteria` 同形。
|
|
84
|
+
*
|
|
85
|
+
* @param patterns 一个或多个项目根相对 glob,`!` 前缀为排除。
|
|
86
|
+
* @returns 匹配集的项目根相对路径,排序后(正斜杠分隔),不含内容。
|
|
87
|
+
*/
|
|
88
|
+
export declare function loadPrivate(...patterns: string[]): Promise<string[]>;
|
|
89
|
+
/**
|
|
90
|
+
* 读入一个 YAML 数据文件并解析,需要项目里装了 `yaml`。路径写项目根相对的字符串,
|
|
91
|
+
* 或 eval 文件相对的 `new URL(p, import.meta.url)`。内容哈希进读它的那条 eval 的指纹。解析结果与
|
|
92
|
+
* JSON 一样必须立即经 `decode` 验证,不让未证明的动态值进入 Eval 定义。
|
|
93
|
+
*/
|
|
94
|
+
export declare function loadYaml<T>(path: string | URL, decode: DataDecoder<T>): Promise<T>;
|
|
95
|
+
export {};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { createRequire as __niceevalCreateRequire } from "node:module";
|
|
2
|
+
const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
|
|
3
|
+
const __niceevalCanonical = __niceevalRequire("./runtime.cjs");
|
|
4
|
+
const __niceevalExport0 = __niceevalCanonical["captureLoadedFiles"];
|
|
5
|
+
export { __niceevalExport0 as captureLoadedFiles };
|
|
6
|
+
const __niceevalExport1 = __niceevalCanonical["loadCriteria"];
|
|
7
|
+
export { __niceevalExport1 as loadCriteria };
|
|
8
|
+
const __niceevalExport2 = __niceevalCanonical["loadJson"];
|
|
9
|
+
export { __niceevalExport2 as loadJson };
|
|
10
|
+
const __niceevalExport3 = __niceevalCanonical["loadPrivate"];
|
|
11
|
+
export { __niceevalExport3 as loadPrivate };
|
|
12
|
+
const __niceevalExport4 = __niceevalCanonical["loadText"];
|
|
13
|
+
export { __niceevalExport4 as loadText };
|
|
14
|
+
const __niceevalExport5 = __niceevalCanonical["loadYaml"];
|
|
15
|
+
export { __niceevalExport5 as loadYaml };
|
|
16
|
+
//# sourceMappingURL=runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runtime.js","sources":[],"names":[],"mappings":""}
|
|
@@ -20,7 +20,7 @@ shape can contain any of these legacy fields:
|
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
Choose the provider that actually serves the model, construct it explicitly,
|
|
23
|
-
and assign that value to `
|
|
23
|
+
and assign that value to `judgeProvider`. This makes the service, protocol,
|
|
24
24
|
endpoint, credential source, and execution limits unambiguous.
|
|
25
25
|
|
|
26
26
|
## OpenAI and OpenAI-compatible endpoints
|
|
@@ -30,7 +30,7 @@ import { defineConfig } from "niceeval";
|
|
|
30
30
|
import { OpenAIProvider } from "niceeval/judge";
|
|
31
31
|
|
|
32
32
|
export default defineConfig({
|
|
33
|
-
|
|
33
|
+
judgeProvider: OpenAIProvider({
|
|
34
34
|
model: "judge-model",
|
|
35
35
|
// Keep this only for an OpenAI-compatible custom gateway:
|
|
36
36
|
// baseUrl: "https://gateway.example.test/v1",
|
|
@@ -52,7 +52,7 @@ import { defineConfig } from "niceeval";
|
|
|
52
52
|
import { VercelProvider } from "niceeval/judge";
|
|
53
53
|
|
|
54
54
|
export default defineConfig({
|
|
55
|
-
|
|
55
|
+
judgeProvider: VercelProvider({ model: "provider/model" }),
|
|
56
56
|
});
|
|
57
57
|
```
|
|
58
58
|
|
|
@@ -65,7 +65,7 @@ import { defineConfig } from "niceeval";
|
|
|
65
65
|
import { OpenRouterProvider } from "niceeval/judge";
|
|
66
66
|
|
|
67
67
|
export default defineConfig({
|
|
68
|
-
|
|
68
|
+
judgeProvider: OpenRouterProvider({ model: "provider/model" }),
|
|
69
69
|
});
|
|
70
70
|
```
|
|
71
71
|
|
|
@@ -78,7 +78,7 @@ import { defineConfig } from "niceeval";
|
|
|
78
78
|
import { TypesafeProvider } from "niceeval/judge";
|
|
79
79
|
|
|
80
80
|
export default defineConfig({
|
|
81
|
-
|
|
81
|
+
judgeProvider: TypesafeProvider({ model: "jev-1.13.0" }),
|
|
82
82
|
});
|
|
83
83
|
```
|
|
84
84
|
|
|
@@ -99,7 +99,7 @@ If an existing deployment must keep its current variable name temporarily,
|
|
|
99
99
|
select it explicitly without copying the secret into source code:
|
|
100
100
|
|
|
101
101
|
```ts
|
|
102
|
-
|
|
102
|
+
judgeProvider: OpenAIProvider({
|
|
103
103
|
model: "judge-model",
|
|
104
104
|
apiKeyEnv: "NICEEVAL_JUDGE_KEY",
|
|
105
105
|
})
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
4
|
-
exports.
|
|
3
|
+
exports.ResolvedModelRolesSchema = void 0;
|
|
4
|
+
exports.defineModelRoles = defineModelRoles;
|
|
5
5
|
exports.normalizeModelRequirements = normalizeModelRequirements;
|
|
6
|
-
exports.
|
|
7
|
-
exports.
|
|
6
|
+
exports.assertRequiredModelRoles = assertRequiredModelRoles;
|
|
7
|
+
exports.normalizeModelRoles = normalizeModelRoles;
|
|
8
8
|
const effect_1 = require("effect");
|
|
9
9
|
const modelText = effect_1.Schema.String.check(effect_1.Schema.isPattern(/\S/u));
|
|
10
|
-
const
|
|
11
|
-
const selections = effect_1.Schema.Record(
|
|
10
|
+
const roleKey = effect_1.Schema.String.check(effect_1.Schema.isPattern(/^[A-Za-z][A-Za-z0-9_-]{0,63}$(?![\s\S])/u));
|
|
11
|
+
const selections = effect_1.Schema.Record(roleKey, effect_1.Schema.Struct({
|
|
12
12
|
model: modelText,
|
|
13
13
|
reasoningEffort: effect_1.Schema.optionalKey(modelText),
|
|
14
14
|
})).check(effect_1.Schema.makeFilter((value) => Object.keys(value).length > 0 && Object.keys(value).length <= 64));
|
|
15
15
|
/** Share immutable model purposes without binding an Eval to one Experiment. */
|
|
16
|
-
function
|
|
16
|
+
function defineModelRoles(input) {
|
|
17
17
|
const decoded = effect_1.Schema.decodeUnknownSync(selections, { onExcessProperty: "error" })(input);
|
|
18
18
|
return Object.freeze(Object.fromEntries(Object.keys(decoded).sort().map((key) => [key, Object.freeze({ ...decoded[key] })])));
|
|
19
19
|
}
|
|
@@ -23,17 +23,17 @@ function normalizeModelRequirements(value) {
|
|
|
23
23
|
const decoded = effect_1.Schema.decodeUnknownSync(effect_1.Schema.Struct({ models: selections }), { onExcessProperty: "error" })(value);
|
|
24
24
|
return Object.freeze(Object.keys(decoded.models).sort());
|
|
25
25
|
}
|
|
26
|
-
function
|
|
26
|
+
function assertRequiredModelRoles(required, models, subject) {
|
|
27
27
|
const missing = required?.filter((key) => !Object.hasOwn(models ?? {}, key)) ?? [];
|
|
28
28
|
if (missing.length > 0)
|
|
29
|
-
throw new TypeError(`${subject} requires undeclared model
|
|
29
|
+
throw new TypeError(`${subject} requires undeclared model roles: ${missing.join(", ")}.`);
|
|
30
30
|
}
|
|
31
|
-
exports.
|
|
31
|
+
exports.ResolvedModelRolesSchema = effect_1.Schema.Record(roleKey, effect_1.Schema.Struct({
|
|
32
32
|
model: effect_1.Schema.NullOr(modelText),
|
|
33
33
|
reasoningEffort: effect_1.Schema.NullOr(modelText),
|
|
34
34
|
})).check(effect_1.Schema.makeFilter((value) => Object.keys(value).length <= 64));
|
|
35
35
|
/** Only the definition boundary interprets author syntax. Downstream owners pass this value through. */
|
|
36
|
-
function
|
|
36
|
+
function normalizeModelRoles(input) {
|
|
37
37
|
if (input.models !== undefined) {
|
|
38
38
|
if (input.model !== undefined || input.reasoningEffort !== undefined) {
|
|
39
39
|
throw new TypeError("Experiment models cannot be combined with model or reasoningEffort.");
|
|
@@ -51,4 +51,4 @@ function normalizeModelSlots(input) {
|
|
|
51
51
|
reasoningEffort: input.reasoningEffort === undefined ? null : effect_1.Schema.decodeUnknownSync(modelText)(input.reasoningEffort),
|
|
52
52
|
}) });
|
|
53
53
|
}
|
|
54
|
-
//# sourceMappingURL=model-
|
|
54
|
+
//# sourceMappingURL=model-roles.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-
|
|
1
|
+
{"version":3,"file":"model-roles.js","sourceRoot":"","sources":["model-roles.ts"],"names":[],"mappings":";;;AAsBA,4CAGC;AACD,gEAKC;AACD,4DAIC;AAMD,kDAqBC;AA/DD,mCAAgC;AAYhC,MAAM,SAAS,GAAG,eAAM,CAAC,MAAM,CAAC,KAAK,CAAC,eAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AAC/D,MAAM,OAAO,GAAG,eAAM,CAAC,MAAM,CAAC,KAAK,CAAC,eAAM,CAAC,SAAS,CAAC,0CAA0C,CAAC,CAAC,CAAC;AAClG,MAAM,UAAU,GAAG,eAAM,CAAC,MAAM,CAAC,OAAO,EAAE,eAAM,CAAC,MAAM,CAAC;IACpD,KAAK,EAAE,SAAS;IAChB,eAAe,EAAE,eAAM,CAAC,WAAW,CAAC,SAAS,CAAC;CACjD,CAAC,CAAC,CAAC,KAAK,CAAC,eAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;AAI1G,gFAAgF;AAChF,SAAgB,gBAAgB,CAA4B,KAAiD;IACzG,MAAM,OAAO,GAAG,eAAM,CAAC,iBAAiB,CAAC,UAAU,EAAE,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;IAC3F,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,CAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAA+C,CAAC;AACjL,CAAC;AACD,SAAgB,0BAA0B,CAAC,KAAc;IACrD,IAAI,KAAK,KAAK,SAAS;QACnB,OAAO,SAAS,CAAC;IACrB,MAAM,OAAO,GAAG,eAAM,CAAC,iBAAiB,CAAC,eAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;IACtH,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AAC7D,CAAC;AACD,SAAgB,wBAAwB,CAAC,QAAuC,EAAE,MAAsC,EAAE,OAAe;IACrI,MAAM,OAAO,GAAG,QAAQ,EAAE,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACnF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAClB,MAAM,IAAI,SAAS,CAAC,GAAG,OAAO,qCAAqC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClG,CAAC;AACY,QAAA,wBAAwB,GAAqC,eAAM,CAAC,MAAM,CAAC,OAAO,EAAE,eAAM,CAAC,MAAM,CAAC;IAC3G,KAAK,EAAE,eAAM,CAAC,MAAM,CAAC,SAAS,CAAC;IAC/B,eAAe,EAAE,eAAM,CAAC,MAAM,CAAC,SAAS,CAAC;CAC5C,CAAC,CAAC,CAAC,KAAK,CAAC,eAAM,CAAC,UAAU,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;AACzE,wGAAwG;AACxG,SAAgB,mBAAmB,CAAC,KAInC;IACG,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC7B,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;YACnE,MAAM,IAAI,SAAS,CAAC,qEAAqE,CAAC,CAAC;QAC/F,CAAC;QACD,MAAM,OAAO,GAAG,eAAM,CAAC,iBAAiB,CAAC,UAAU,EAAE,EAAE,gBAAgB,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAClG,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC5E,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAE,CAAC;YAC5B,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC,CAAC,CAAC;IACT,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS;QAChE,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7B,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC;YACtC,KAAK,EAAE,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,eAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC;YAC1F,eAAe,EAAE,KAAK,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,eAAM,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC;SAC3H,CAAC,EAAE,CAAC,CAAC;AACd,CAAC","sourcesContent":["import { Schema } from \"effect\";\n/** A model selected for one application-defined purpose. */\nexport interface ModelRoleSelection {\n readonly model: string;\n readonly reasoningEffort?: string;\n}\n/** Frozen selection; null preserves an unspecified native default. */\nexport interface ResolvedModelRole {\n readonly model: string | null;\n readonly reasoningEffort: string | null;\n}\nexport type ResolvedModelRoles = Readonly<Record<string, ResolvedModelRole>>;\nconst modelText = Schema.String.check(Schema.isPattern(/\\S/u));\nconst roleKey = Schema.String.check(Schema.isPattern(/^[A-Za-z][A-Za-z0-9_-]{0,63}$(?![\\s\\S])/u));\nconst selections = Schema.Record(roleKey, Schema.Struct({\n model: modelText,\n reasoningEffort: Schema.optionalKey(modelText),\n})).check(Schema.makeFilter((value) => Object.keys(value).length > 0 && Object.keys(value).length <= 64));\nexport interface ModelRequirements<Keys extends string = string> {\n readonly models: Readonly<Record<Keys, ModelRoleSelection>>;\n}\n/** Share immutable model purposes without binding an Eval to one Experiment. */\nexport function defineModelRoles<const Keys extends string>(input: Readonly<Record<Keys, ModelRoleSelection>>): Readonly<Record<Keys, ModelRoleSelection>> {\n const decoded = Schema.decodeUnknownSync(selections, { onExcessProperty: \"error\" })(input);\n return Object.freeze(Object.fromEntries(Object.keys(decoded).sort().map((key) => [key, Object.freeze({ ...decoded[key]! })]))) as Readonly<Record<Keys, ModelRoleSelection>>;\n}\nexport function normalizeModelRequirements(value: unknown): readonly string[] | undefined {\n if (value === undefined)\n return undefined;\n const decoded = Schema.decodeUnknownSync(Schema.Struct({ models: selections }), { onExcessProperty: \"error\" })(value);\n return Object.freeze(Object.keys(decoded.models).sort());\n}\nexport function assertRequiredModelRoles(required: readonly string[] | undefined, models: ResolvedModelRoles | undefined, subject: string): void {\n const missing = required?.filter((key) => !Object.hasOwn(models ?? {}, key)) ?? [];\n if (missing.length > 0)\n throw new TypeError(`${subject} requires undeclared model roles: ${missing.join(\", \")}.`);\n}\nexport const ResolvedModelRolesSchema: Schema.Codec<ResolvedModelRoles> = Schema.Record(roleKey, Schema.Struct({\n model: Schema.NullOr(modelText),\n reasoningEffort: Schema.NullOr(modelText),\n})).check(Schema.makeFilter((value) => Object.keys(value).length <= 64));\n/** Only the definition boundary interprets author syntax. Downstream owners pass this value through. */\nexport function normalizeModelRoles(input: {\n readonly model?: string;\n readonly reasoningEffort?: string;\n readonly models?: Readonly<Record<string, ModelRoleSelection>>;\n}): ResolvedModelRoles {\n if (input.models !== undefined) {\n if (input.model !== undefined || input.reasoningEffort !== undefined) {\n throw new TypeError(\"Experiment models cannot be combined with model or reasoningEffort.\");\n }\n const decoded = Schema.decodeUnknownSync(selections, { onExcessProperty: \"error\" })(input.models);\n return Object.freeze(Object.fromEntries(Object.keys(decoded).sort().map((key) => {\n const value = decoded[key]!;\n return [key, Object.freeze({ model: value.model, reasoningEffort: value.reasoningEffort ?? null })];\n })));\n }\n if (input.model === undefined && input.reasoningEffort === undefined)\n return Object.freeze({});\n return Object.freeze({ default: Object.freeze({\n model: input.model === undefined ? null : Schema.decodeUnknownSync(modelText)(input.model),\n reasoningEffort: input.reasoningEffort === undefined ? null : Schema.decodeUnknownSync(modelText)(input.reasoningEffort),\n }) });\n}\n"]}
|
|
@@ -1,26 +1,26 @@
|
|
|
1
1
|
import { Schema } from "effect";
|
|
2
2
|
/** A model selected for one application-defined purpose. */
|
|
3
|
-
export interface
|
|
3
|
+
export interface ModelRoleSelection {
|
|
4
4
|
readonly model: string;
|
|
5
5
|
readonly reasoningEffort?: string;
|
|
6
6
|
}
|
|
7
7
|
/** Frozen selection; null preserves an unspecified native default. */
|
|
8
|
-
export interface
|
|
8
|
+
export interface ResolvedModelRole {
|
|
9
9
|
readonly model: string | null;
|
|
10
10
|
readonly reasoningEffort: string | null;
|
|
11
11
|
}
|
|
12
|
-
export type
|
|
12
|
+
export type ResolvedModelRoles = Readonly<Record<string, ResolvedModelRole>>;
|
|
13
13
|
export interface ModelRequirements<Keys extends string = string> {
|
|
14
|
-
readonly models: Readonly<Record<Keys,
|
|
14
|
+
readonly models: Readonly<Record<Keys, ModelRoleSelection>>;
|
|
15
15
|
}
|
|
16
16
|
/** Share immutable model purposes without binding an Eval to one Experiment. */
|
|
17
|
-
export declare function
|
|
17
|
+
export declare function defineModelRoles<const Keys extends string>(input: Readonly<Record<Keys, ModelRoleSelection>>): Readonly<Record<Keys, ModelRoleSelection>>;
|
|
18
18
|
export declare function normalizeModelRequirements(value: unknown): readonly string[] | undefined;
|
|
19
|
-
export declare function
|
|
20
|
-
export declare const
|
|
19
|
+
export declare function assertRequiredModelRoles(required: readonly string[] | undefined, models: ResolvedModelRoles | undefined, subject: string): void;
|
|
20
|
+
export declare const ResolvedModelRolesSchema: Schema.Codec<ResolvedModelRoles>;
|
|
21
21
|
/** Only the definition boundary interprets author syntax. Downstream owners pass this value through. */
|
|
22
|
-
export declare function
|
|
22
|
+
export declare function normalizeModelRoles(input: {
|
|
23
23
|
readonly model?: string;
|
|
24
24
|
readonly reasoningEffort?: string;
|
|
25
|
-
readonly models?: Readonly<Record<string,
|
|
26
|
-
}):
|
|
25
|
+
readonly models?: Readonly<Record<string, ModelRoleSelection>>;
|
|
26
|
+
}): ResolvedModelRoles;
|
|
@@ -1,26 +1,26 @@
|
|
|
1
1
|
import { Schema } from "effect";
|
|
2
2
|
/** A model selected for one application-defined purpose. */
|
|
3
|
-
export interface
|
|
3
|
+
export interface ModelRoleSelection {
|
|
4
4
|
readonly model: string;
|
|
5
5
|
readonly reasoningEffort?: string;
|
|
6
6
|
}
|
|
7
7
|
/** Frozen selection; null preserves an unspecified native default. */
|
|
8
|
-
export interface
|
|
8
|
+
export interface ResolvedModelRole {
|
|
9
9
|
readonly model: string | null;
|
|
10
10
|
readonly reasoningEffort: string | null;
|
|
11
11
|
}
|
|
12
|
-
export type
|
|
12
|
+
export type ResolvedModelRoles = Readonly<Record<string, ResolvedModelRole>>;
|
|
13
13
|
export interface ModelRequirements<Keys extends string = string> {
|
|
14
|
-
readonly models: Readonly<Record<Keys,
|
|
14
|
+
readonly models: Readonly<Record<Keys, ModelRoleSelection>>;
|
|
15
15
|
}
|
|
16
16
|
/** Share immutable model purposes without binding an Eval to one Experiment. */
|
|
17
|
-
export declare function
|
|
17
|
+
export declare function defineModelRoles<const Keys extends string>(input: Readonly<Record<Keys, ModelRoleSelection>>): Readonly<Record<Keys, ModelRoleSelection>>;
|
|
18
18
|
export declare function normalizeModelRequirements(value: unknown): readonly string[] | undefined;
|
|
19
|
-
export declare function
|
|
20
|
-
export declare const
|
|
19
|
+
export declare function assertRequiredModelRoles(required: readonly string[] | undefined, models: ResolvedModelRoles | undefined, subject: string): void;
|
|
20
|
+
export declare const ResolvedModelRolesSchema: Schema.Codec<ResolvedModelRoles>;
|
|
21
21
|
/** Only the definition boundary interprets author syntax. Downstream owners pass this value through. */
|
|
22
|
-
export declare function
|
|
22
|
+
export declare function normalizeModelRoles(input: {
|
|
23
23
|
readonly model?: string;
|
|
24
24
|
readonly reasoningEffort?: string;
|
|
25
|
-
readonly models?: Readonly<Record<string,
|
|
26
|
-
}):
|
|
25
|
+
readonly models?: Readonly<Record<string, ModelRoleSelection>>;
|
|
26
|
+
}): ResolvedModelRoles;
|
|
@@ -1,26 +1,26 @@
|
|
|
1
1
|
import { Schema } from "effect";
|
|
2
2
|
/** A model selected for one application-defined purpose. */
|
|
3
|
-
export interface
|
|
3
|
+
export interface ModelRoleSelection {
|
|
4
4
|
readonly model: string;
|
|
5
5
|
readonly reasoningEffort?: string;
|
|
6
6
|
}
|
|
7
7
|
/** Frozen selection; null preserves an unspecified native default. */
|
|
8
|
-
export interface
|
|
8
|
+
export interface ResolvedModelRole {
|
|
9
9
|
readonly model: string | null;
|
|
10
10
|
readonly reasoningEffort: string | null;
|
|
11
11
|
}
|
|
12
|
-
export type
|
|
12
|
+
export type ResolvedModelRoles = Readonly<Record<string, ResolvedModelRole>>;
|
|
13
13
|
export interface ModelRequirements<Keys extends string = string> {
|
|
14
|
-
readonly models: Readonly<Record<Keys,
|
|
14
|
+
readonly models: Readonly<Record<Keys, ModelRoleSelection>>;
|
|
15
15
|
}
|
|
16
16
|
/** Share immutable model purposes without binding an Eval to one Experiment. */
|
|
17
|
-
export declare function
|
|
17
|
+
export declare function defineModelRoles<const Keys extends string>(input: Readonly<Record<Keys, ModelRoleSelection>>): Readonly<Record<Keys, ModelRoleSelection>>;
|
|
18
18
|
export declare function normalizeModelRequirements(value: unknown): readonly string[] | undefined;
|
|
19
|
-
export declare function
|
|
20
|
-
export declare const
|
|
19
|
+
export declare function assertRequiredModelRoles(required: readonly string[] | undefined, models: ResolvedModelRoles | undefined, subject: string): void;
|
|
20
|
+
export declare const ResolvedModelRolesSchema: Schema.Codec<ResolvedModelRoles>;
|
|
21
21
|
/** Only the definition boundary interprets author syntax. Downstream owners pass this value through. */
|
|
22
|
-
export declare function
|
|
22
|
+
export declare function normalizeModelRoles(input: {
|
|
23
23
|
readonly model?: string;
|
|
24
24
|
readonly reasoningEffort?: string;
|
|
25
|
-
readonly models?: Readonly<Record<string,
|
|
26
|
-
}):
|
|
25
|
+
readonly models?: Readonly<Record<string, ModelRoleSelection>>;
|
|
26
|
+
}): ResolvedModelRoles;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { createRequire as __niceevalCreateRequire } from "node:module";
|
|
2
|
+
const __niceevalRequire = __niceevalCreateRequire(import.meta.url);
|
|
3
|
+
const __niceevalCanonical = __niceevalRequire("./model-roles.cjs");
|
|
4
|
+
const __niceevalExport0 = __niceevalCanonical["ResolvedModelRolesSchema"];
|
|
5
|
+
export { __niceevalExport0 as ResolvedModelRolesSchema };
|
|
6
|
+
const __niceevalExport1 = __niceevalCanonical["assertRequiredModelRoles"];
|
|
7
|
+
export { __niceevalExport1 as assertRequiredModelRoles };
|
|
8
|
+
const __niceevalExport2 = __niceevalCanonical["defineModelRoles"];
|
|
9
|
+
export { __niceevalExport2 as defineModelRoles };
|
|
10
|
+
const __niceevalExport3 = __niceevalCanonical["normalizeModelRequirements"];
|
|
11
|
+
export { __niceevalExport3 as normalizeModelRequirements };
|
|
12
|
+
const __niceevalExport4 = __niceevalCanonical["normalizeModelRoles"];
|
|
13
|
+
export { __niceevalExport4 as normalizeModelRoles };
|
|
14
|
+
//# sourceMappingURL=model-roles.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-roles.js","sources":[],"names":[],"mappings":""}
|
|
@@ -14,7 +14,8 @@ function effectiveUsageCalls(usage) {
|
|
|
14
14
|
const receipts = new Map((usage.priceReceipts ?? []).map((receipt) => [receipt.callId, receipt]));
|
|
15
15
|
return Object.freeze(usage.calls.map((call) => {
|
|
16
16
|
const receipt = receipts.get(call.callId);
|
|
17
|
-
|
|
17
|
+
const { modelSlot, ...fields } = call;
|
|
18
|
+
return Object.freeze({ ...fields, modelRole: modelSlot, effectiveCost: call.cost !== null ? Object.freeze({ ...call.cost, state: "complete" }) : receipt === undefined ? null : Object.freeze({ amount: receipt.amount, currency: receipt.currency, source: receipt.source, state: receipt.state }) });
|
|
18
19
|
}));
|
|
19
20
|
}
|
|
20
21
|
function effectiveCostTotals(calls, collectionState) {
|
|
@@ -73,7 +74,7 @@ function adapterInspectionTokenTotal(usage, metric) {
|
|
|
73
74
|
}
|
|
74
75
|
/** Tuple order is stable across hosts, with missing identities before strings. */
|
|
75
76
|
function compareAdapterModelGroups(left, right) {
|
|
76
|
-
for (const key of ["
|
|
77
|
+
for (const key of ["modelRole", "provider", "model"]) {
|
|
77
78
|
const a = left[key];
|
|
78
79
|
const b = right[key];
|
|
79
80
|
if (a === b)
|
|
@@ -98,9 +99,9 @@ function projectAdapterModelUsage(usage, models, unavailable = { basis: "unavail
|
|
|
98
99
|
? Object.freeze({ state: "not-recorded" })
|
|
99
100
|
: Object.freeze({
|
|
100
101
|
state: "available",
|
|
101
|
-
bindings: Object.freeze(Object.keys(models).sort().map((
|
|
102
|
-
|
|
103
|
-
recordedCalls: usage === null ? null : counts.get(
|
|
102
|
+
bindings: Object.freeze(Object.keys(models).sort().map((modelRole) => Object.freeze({
|
|
103
|
+
modelRole, ...models[modelRole],
|
|
104
|
+
recordedCalls: usage === null ? null : counts.get(modelRole) ?? 0,
|
|
104
105
|
}))),
|
|
105
106
|
});
|
|
106
107
|
if (usage === null)
|
|
@@ -115,7 +116,7 @@ function projectAdapterModelUsage(usage, models, unavailable = { basis: "unavail
|
|
|
115
116
|
});
|
|
116
117
|
const grouped = new Map();
|
|
117
118
|
for (const call of effectiveUsageCalls(usage)) {
|
|
118
|
-
const identity = JSON.stringify([call.
|
|
119
|
+
const identity = JSON.stringify([call.modelRole, call.provider, call.model]);
|
|
119
120
|
const group = grouped.get(identity);
|
|
120
121
|
if (group === undefined)
|
|
121
122
|
grouped.set(identity, [call]);
|
|
@@ -124,10 +125,10 @@ function projectAdapterModelUsage(usage, models, unavailable = { basis: "unavail
|
|
|
124
125
|
}
|
|
125
126
|
const allGroups = [...grouped.values()].sort((left, right) => compareAdapterModelGroups(left[0], right[0]));
|
|
126
127
|
const groups = allGroups.slice(0, 64).map((calls) => {
|
|
127
|
-
const {
|
|
128
|
+
const { modelRole, provider, model } = calls[0];
|
|
128
129
|
const groupUsage = { ...usage, calls };
|
|
129
130
|
return Object.freeze({
|
|
130
|
-
|
|
131
|
+
modelRole, provider, model, recordedCalls: calls.length,
|
|
131
132
|
tokens: Object.freeze({
|
|
132
133
|
inputTotalTokens: adapterInspectionTokenTotal(groupUsage, "inputTotalTokens"),
|
|
133
134
|
outputTokens: adapterInspectionTokenTotal(groupUsage, "outputTokens"),
|