@prismatic-io/lux 0.0.2-preview.4 → 0.0.2-preview.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/answerers/claude-code/index.d.ts +15 -3
- package/lib/answerers/claude-code/index.d.ts.map +1 -1
- package/lib/answerers/claude-code/index.js +6 -5
- package/lib/answerers/claude-code/index.js.map +1 -1
- package/lib/answerers/persona/index.d.ts +44 -5
- package/lib/answerers/persona/index.d.ts.map +1 -1
- package/lib/answerers/persona/index.js +60 -34
- package/lib/answerers/persona/index.js.map +1 -1
- package/lib/answerers/terminal/index.d.ts +9 -3
- package/lib/answerers/terminal/index.d.ts.map +1 -1
- package/lib/answerers/terminal/index.js +6 -4
- package/lib/answerers/terminal/index.js.map +1 -1
- package/lib/assertions/authoring.d.ts +2 -2
- package/lib/assertions/authoring.d.ts.map +1 -1
- package/lib/assertions/authoring.js.map +1 -1
- package/lib/assertions/core/command-exits-zero.d.ts +5 -2
- package/lib/assertions/core/command-exits-zero.d.ts.map +1 -1
- package/lib/assertions/core/command-exits-zero.js +59 -14
- package/lib/assertions/core/command-exits-zero.js.map +1 -1
- package/lib/assertions/core/event-checks.d.ts +3 -0
- package/lib/assertions/core/event-checks.d.ts.map +1 -1
- package/lib/assertions/core/event-checks.js +49 -3
- package/lib/assertions/core/event-checks.js.map +1 -1
- package/lib/assertions/core/file-checks.d.ts +6 -0
- package/lib/assertions/core/file-checks.d.ts.map +1 -1
- package/lib/assertions/core/index.d.ts.map +1 -1
- package/lib/assertions/core/index.js +2 -1
- package/lib/assertions/core/index.js.map +1 -1
- package/lib/assertions/core/json-pointer-equals.d.ts +1 -0
- package/lib/assertions/core/json-pointer-equals.d.ts.map +1 -1
- package/lib/assertions/core/output-checks.d.ts +3 -0
- package/lib/assertions/core/output-checks.d.ts.map +1 -1
- package/lib/assertions/core/predicate.d.ts +1 -0
- package/lib/assertions/core/predicate.d.ts.map +1 -1
- package/lib/assertions/core/resource-checks.d.ts +25 -0
- package/lib/assertions/core/resource-checks.d.ts.map +1 -1
- package/lib/assertions/core/resource-checks.js +35 -5
- package/lib/assertions/core/resource-checks.js.map +1 -1
- package/lib/assertions/core/safe-regex.d.ts.map +1 -1
- package/lib/assertions/core/safe-regex.js +7 -1
- package/lib/assertions/core/safe-regex.js.map +1 -1
- package/lib/assertions/rubric/evidence.d.ts +88 -0
- package/lib/assertions/rubric/evidence.d.ts.map +1 -0
- package/lib/assertions/rubric/evidence.js +75 -0
- package/lib/assertions/rubric/evidence.js.map +1 -0
- package/lib/assertions/rubric/index.d.ts +88 -34
- package/lib/assertions/rubric/index.d.ts.map +1 -1
- package/lib/assertions/rubric/index.js +59 -43
- package/lib/assertions/rubric/index.js.map +1 -1
- package/lib/assertions/rubric/internal.d.ts +30 -5
- package/lib/assertions/rubric/internal.d.ts.map +1 -1
- package/lib/assertions/rubric/internal.js +203 -41
- package/lib/assertions/rubric/internal.js.map +1 -1
- package/lib/authoring.d.ts +37 -2
- package/lib/authoring.d.ts.map +1 -1
- package/lib/authoring.js +36 -1
- package/lib/authoring.js.map +1 -1
- package/lib/cli/bin.js +2 -13
- package/lib/cli/bin.js.map +1 -1
- package/lib/cli/command-runtime.d.ts +3 -3
- package/lib/cli/command-runtime.d.ts.map +1 -1
- package/lib/cli/command-runtime.js +17 -6
- package/lib/cli/command-runtime.js.map +1 -1
- package/lib/cli/doctor.d.ts +55 -0
- package/lib/cli/doctor.d.ts.map +1 -0
- package/lib/cli/doctor.js +14 -0
- package/lib/cli/doctor.js.map +1 -0
- package/lib/cli/harness-plan.d.ts +27 -0
- package/lib/cli/harness-plan.d.ts.map +1 -0
- package/lib/cli/harness-plan.js +75 -0
- package/lib/cli/harness-plan.js.map +1 -0
- package/lib/cli/init-templates.d.ts +11 -0
- package/lib/cli/init-templates.d.ts.map +1 -0
- package/lib/cli/init-templates.js +215 -0
- package/lib/cli/init-templates.js.map +1 -0
- package/lib/cli/init.d.ts +10 -0
- package/lib/cli/init.d.ts.map +1 -1
- package/lib/cli/init.js +34 -173
- package/lib/cli/init.js.map +1 -1
- package/lib/cli/output-schemas.d.ts +1412 -0
- package/lib/cli/output-schemas.d.ts.map +1 -0
- package/lib/cli/output-schemas.js +94 -0
- package/lib/cli/output-schemas.js.map +1 -0
- package/lib/cli/program.d.ts +140 -12
- package/lib/cli/program.d.ts.map +1 -1
- package/lib/cli/program.js +783 -650
- package/lib/cli/program.js.map +1 -1
- package/lib/cli/render/format.d.ts.map +1 -1
- package/lib/cli/render/format.js +1 -1
- package/lib/cli/render/format.js.map +1 -1
- package/lib/cli/render/reporter.d.ts.map +1 -1
- package/lib/cli/render/reporter.js +35 -8
- package/lib/cli/render/reporter.js.map +1 -1
- package/lib/cli/run-options.d.ts +19 -4
- package/lib/cli/run-options.d.ts.map +1 -1
- package/lib/cli/run-options.js +100 -19
- package/lib/cli/run-options.js.map +1 -1
- package/lib/cli/view.d.ts +15 -0
- package/lib/cli/view.d.ts.map +1 -0
- package/lib/cli/view.js +168 -0
- package/lib/cli/view.js.map +1 -0
- package/lib/container/driver.d.ts +27 -0
- package/lib/container/driver.d.ts.map +1 -0
- package/lib/container/driver.js +199 -0
- package/lib/container/driver.js.map +1 -0
- package/lib/container/paths.d.ts +3 -0
- package/lib/container/paths.d.ts.map +1 -0
- package/lib/container/paths.js +27 -0
- package/lib/container/paths.js.map +1 -0
- package/lib/container/protocol.d.ts +12 -0
- package/lib/container/protocol.d.ts.map +1 -0
- package/lib/container/protocol.js +38 -0
- package/lib/container/protocol.js.map +1 -0
- package/lib/container/resources.d.ts +14 -0
- package/lib/container/resources.d.ts.map +1 -0
- package/lib/container/resources.js +141 -0
- package/lib/container/resources.js.map +1 -0
- package/lib/container/worker.d.ts +2 -0
- package/lib/container/worker.d.ts.map +1 -0
- package/lib/container/worker.js +85 -0
- package/lib/container/worker.js.map +1 -0
- package/lib/core/action-events.d.ts +29 -0
- package/lib/core/action-events.d.ts.map +1 -0
- package/lib/core/action-events.js +51 -0
- package/lib/core/action-events.js.map +1 -0
- package/lib/core/agent-activity.d.ts +49 -0
- package/lib/core/agent-activity.d.ts.map +1 -0
- package/lib/core/agent-activity.js +206 -0
- package/lib/core/agent-activity.js.map +1 -0
- package/lib/core/agent-environment.d.ts +5 -0
- package/lib/core/agent-environment.d.ts.map +1 -0
- package/lib/core/agent-environment.js +47 -0
- package/lib/core/agent-environment.js.map +1 -0
- package/lib/core/agent-events.d.ts +26 -0
- package/lib/core/agent-events.d.ts.map +1 -0
- package/lib/core/agent-events.js +13 -0
- package/lib/core/agent-events.js.map +1 -0
- package/lib/core/annotation.d.ts +6 -2
- package/lib/core/annotation.d.ts.map +1 -1
- package/lib/core/annotation.js +2 -1
- package/lib/core/annotation.js.map +1 -1
- package/lib/core/answerer.d.ts +12 -0
- package/lib/core/answerer.d.ts.map +1 -1
- package/lib/core/answerer.js +2 -0
- package/lib/core/answerer.js.map +1 -1
- package/lib/core/artifact-evidence.d.ts +6 -1
- package/lib/core/artifact-evidence.d.ts.map +1 -1
- package/lib/core/artifact-evidence.js +27 -6
- package/lib/core/artifact-evidence.js.map +1 -1
- package/lib/core/assertion.d.ts +10 -4
- package/lib/core/assertion.d.ts.map +1 -1
- package/lib/core/assertion.js +8 -2
- package/lib/core/assertion.js.map +1 -1
- package/lib/core/case-defaults.d.ts +16 -0
- package/lib/core/case-defaults.d.ts.map +1 -0
- package/lib/core/case-defaults.js +42 -0
- package/lib/core/case-defaults.js.map +1 -0
- package/lib/core/case.d.ts +120 -6
- package/lib/core/case.d.ts.map +1 -1
- package/lib/core/case.js +51 -5
- package/lib/core/case.js.map +1 -1
- package/lib/core/command-verification.d.ts +10 -0
- package/lib/core/command-verification.d.ts.map +1 -0
- package/lib/core/command-verification.js +66 -0
- package/lib/core/command-verification.js.map +1 -0
- package/lib/core/driver-execution.d.ts +83 -0
- package/lib/core/driver-execution.d.ts.map +1 -0
- package/lib/core/driver-execution.js +72 -0
- package/lib/core/driver-execution.js.map +1 -0
- package/lib/core/driver-metrics.d.ts +98 -0
- package/lib/core/driver-metrics.d.ts.map +1 -0
- package/lib/core/driver-metrics.js +36 -0
- package/lib/core/driver-metrics.js.map +1 -0
- package/lib/core/driver.d.ts +64 -0
- package/lib/core/driver.d.ts.map +1 -1
- package/lib/core/driver.js +28 -0
- package/lib/core/driver.js.map +1 -1
- package/lib/core/duration.d.ts +5 -0
- package/lib/core/duration.d.ts.map +1 -0
- package/lib/core/duration.js +49 -0
- package/lib/core/duration.js.map +1 -0
- package/lib/core/environment.d.ts +18 -0
- package/lib/core/environment.d.ts.map +1 -0
- package/lib/core/environment.js +21 -0
- package/lib/core/environment.js.map +1 -0
- package/lib/core/evaluation-clusters.d.ts +17 -0
- package/lib/core/evaluation-clusters.d.ts.map +1 -0
- package/lib/core/evaluation-clusters.js +23 -0
- package/lib/core/evaluation-clusters.js.map +1 -0
- package/lib/core/events.d.ts +34 -0
- package/lib/core/events.d.ts.map +1 -1
- package/lib/core/events.js +11 -2
- package/lib/core/events.js.map +1 -1
- package/lib/core/experiment.d.ts +33 -2
- package/lib/core/experiment.d.ts.map +1 -1
- package/lib/core/experiment.js +40 -19
- package/lib/core/experiment.js.map +1 -1
- package/lib/core/file-tree.d.ts +7 -0
- package/lib/core/file-tree.d.ts.map +1 -0
- package/lib/core/file-tree.js +35 -0
- package/lib/core/file-tree.js.map +1 -0
- package/lib/core/global-id.d.ts +6 -0
- package/lib/core/global-id.d.ts.map +1 -0
- package/lib/core/global-id.js +17 -0
- package/lib/core/global-id.js.map +1 -0
- package/lib/core/harness-catalog.d.ts +35 -0
- package/lib/core/harness-catalog.d.ts.map +1 -0
- package/lib/core/harness-catalog.js +45 -0
- package/lib/core/harness-catalog.js.map +1 -0
- package/lib/core/harness-catalogs/antigravity.d.ts +11 -0
- package/lib/core/harness-catalogs/antigravity.d.ts.map +1 -0
- package/lib/core/harness-catalogs/antigravity.js +23 -0
- package/lib/core/harness-catalogs/antigravity.js.map +1 -0
- package/lib/core/harness-catalogs/claude-code.d.ts +12 -0
- package/lib/core/harness-catalogs/claude-code.d.ts.map +1 -0
- package/lib/core/harness-catalogs/claude-code.js +24 -0
- package/lib/core/harness-catalogs/claude-code.js.map +1 -0
- package/lib/core/harness-catalogs/codex.d.ts +11 -0
- package/lib/core/harness-catalogs/codex.d.ts.map +1 -0
- package/lib/core/harness-catalogs/codex.js +26 -0
- package/lib/core/harness-catalogs/codex.js.map +1 -0
- package/lib/core/harness-catalogs/copilot.d.ts +29 -0
- package/lib/core/harness-catalogs/copilot.d.ts.map +1 -0
- package/lib/core/harness-catalogs/copilot.js +45 -0
- package/lib/core/harness-catalogs/copilot.js.map +1 -0
- package/lib/core/harness-catalogs/cursor.d.ts +2 -0
- package/lib/core/harness-catalogs/cursor.d.ts.map +1 -0
- package/lib/core/harness-catalogs/cursor.js +13 -0
- package/lib/core/harness-catalogs/cursor.js.map +1 -0
- package/lib/core/harness-catalogs/grok-build.d.ts +7 -0
- package/lib/core/harness-catalogs/grok-build.d.ts.map +1 -0
- package/lib/core/harness-catalogs/grok-build.js +18 -0
- package/lib/core/harness-catalogs/grok-build.js.map +1 -0
- package/lib/core/harness-catalogs/index.d.ts +73 -0
- package/lib/core/harness-catalogs/index.d.ts.map +1 -0
- package/lib/core/harness-catalogs/index.js +17 -0
- package/lib/core/harness-catalogs/index.js.map +1 -0
- package/lib/core/harness-matrix.d.ts +154 -0
- package/lib/core/harness-matrix.d.ts.map +1 -0
- package/lib/core/harness-matrix.js +204 -0
- package/lib/core/harness-matrix.js.map +1 -0
- package/lib/core/index.d.ts +12 -0
- package/lib/core/index.d.ts.map +1 -1
- package/lib/core/index.js +12 -0
- package/lib/core/index.js.map +1 -1
- package/lib/core/lifecycle-fixtures.d.ts.map +1 -1
- package/lib/core/lifecycle-fixtures.js +4 -2
- package/lib/core/lifecycle-fixtures.js.map +1 -1
- package/lib/core/model-cli.d.ts +9 -11
- package/lib/core/model-cli.d.ts.map +1 -1
- package/lib/core/model-cli.js +39 -22
- package/lib/core/model-cli.js.map +1 -1
- package/lib/core/platform-process.d.ts +7 -2
- package/lib/core/platform-process.d.ts.map +1 -1
- package/lib/core/platform-process.js +54 -16
- package/lib/core/platform-process.js.map +1 -1
- package/lib/core/reference-defaults.d.ts +5 -0
- package/lib/core/reference-defaults.d.ts.map +1 -0
- package/lib/core/reference-defaults.js +42 -0
- package/lib/core/reference-defaults.js.map +1 -0
- package/lib/core/rubric.d.ts +1 -0
- package/lib/core/rubric.d.ts.map +1 -1
- package/lib/core/run.d.ts +292 -1
- package/lib/core/run.d.ts.map +1 -1
- package/lib/core/run.js +27 -0
- package/lib/core/run.js.map +1 -1
- package/lib/core/secure-copy.d.ts.map +1 -0
- package/lib/core/secure-copy.js.map +1 -0
- package/lib/core/usage.d.ts +5 -0
- package/lib/core/usage.d.ts.map +1 -1
- package/lib/core/usage.js +3 -0
- package/lib/core/usage.js.map +1 -1
- package/lib/core/wire.d.ts +11 -0
- package/lib/core/wire.d.ts.map +1 -1
- package/lib/drivers/antigravity/artifacts.d.ts +2 -0
- package/lib/drivers/antigravity/artifacts.d.ts.map +1 -0
- package/lib/drivers/antigravity/artifacts.js +33 -0
- package/lib/drivers/antigravity/artifacts.js.map +1 -0
- package/lib/drivers/antigravity/config.d.ts +88 -0
- package/lib/drivers/antigravity/config.d.ts.map +1 -0
- package/lib/drivers/antigravity/config.js +87 -0
- package/lib/drivers/antigravity/config.js.map +1 -0
- package/lib/drivers/antigravity/events.d.ts +4 -0
- package/lib/drivers/antigravity/events.d.ts.map +1 -0
- package/lib/drivers/antigravity/events.js +241 -0
- package/lib/drivers/antigravity/events.js.map +1 -0
- package/lib/drivers/antigravity/index.d.ts +114 -0
- package/lib/drivers/antigravity/index.d.ts.map +1 -0
- package/lib/drivers/antigravity/index.js +113 -0
- package/lib/drivers/antigravity/index.js.map +1 -0
- package/lib/drivers/antigravity/runtime.d.ts +7 -0
- package/lib/drivers/antigravity/runtime.d.ts.map +1 -0
- package/lib/drivers/antigravity/runtime.js +83 -0
- package/lib/drivers/antigravity/runtime.js.map +1 -0
- package/lib/drivers/claude-code/deferred-process.d.ts +17 -0
- package/lib/drivers/claude-code/deferred-process.d.ts.map +1 -0
- package/lib/drivers/claude-code/deferred-process.js +108 -0
- package/lib/drivers/claude-code/deferred-process.js.map +1 -0
- package/lib/drivers/claude-code/index.d.ts +187 -10
- package/lib/drivers/claude-code/index.d.ts.map +1 -1
- package/lib/drivers/claude-code/index.js +197 -252
- package/lib/drivers/claude-code/index.js.map +1 -1
- package/lib/drivers/claude-code/isolation.d.ts +8 -0
- package/lib/drivers/claude-code/isolation.d.ts.map +1 -0
- package/lib/drivers/claude-code/isolation.js +199 -0
- package/lib/drivers/claude-code/isolation.js.map +1 -0
- package/lib/drivers/claude-code/parse-events.d.ts.map +1 -1
- package/lib/drivers/claude-code/parse-events.js +73 -8
- package/lib/drivers/claude-code/parse-events.js.map +1 -1
- package/lib/drivers/codex/app-events.d.ts +9 -2
- package/lib/drivers/codex/app-events.d.ts.map +1 -1
- package/lib/drivers/codex/app-events.js +104 -12
- package/lib/drivers/codex/app-events.js.map +1 -1
- package/lib/drivers/codex/config.d.ts +69 -5
- package/lib/drivers/codex/config.d.ts.map +1 -1
- package/lib/drivers/codex/config.js +48 -8
- package/lib/drivers/codex/config.js.map +1 -1
- package/lib/drivers/codex/exec-events.d.ts +4 -0
- package/lib/drivers/codex/exec-events.d.ts.map +1 -0
- package/lib/drivers/codex/exec-events.js +230 -0
- package/lib/drivers/codex/exec-events.js.map +1 -0
- package/lib/drivers/codex/index.d.ts +142 -4
- package/lib/drivers/codex/index.d.ts.map +1 -1
- package/lib/drivers/codex/index.js +114 -276
- package/lib/drivers/codex/index.js.map +1 -1
- package/lib/drivers/copilot/config.d.ts +84 -0
- package/lib/drivers/copilot/config.d.ts.map +1 -0
- package/lib/drivers/copilot/config.js +57 -0
- package/lib/drivers/copilot/config.js.map +1 -0
- package/lib/drivers/copilot/index.d.ts +107 -0
- package/lib/drivers/copilot/index.d.ts.map +1 -0
- package/lib/drivers/copilot/index.js +268 -0
- package/lib/drivers/copilot/index.js.map +1 -0
- package/lib/drivers/cursor/config.d.ts +28 -0
- package/lib/drivers/cursor/config.d.ts.map +1 -0
- package/lib/drivers/cursor/config.js +28 -0
- package/lib/drivers/cursor/config.js.map +1 -0
- package/lib/drivers/cursor/index.d.ts +44 -0
- package/lib/drivers/cursor/index.d.ts.map +1 -0
- package/lib/drivers/cursor/index.js +286 -0
- package/lib/drivers/cursor/index.js.map +1 -0
- package/lib/drivers/cursor/interaction.d.ts +6 -0
- package/lib/drivers/cursor/interaction.d.ts.map +1 -0
- package/lib/drivers/cursor/interaction.js +55 -0
- package/lib/drivers/cursor/interaction.js.map +1 -0
- package/lib/drivers/grok-build/config.d.ts +68 -0
- package/lib/drivers/grok-build/config.d.ts.map +1 -0
- package/lib/drivers/grok-build/config.js +40 -0
- package/lib/drivers/grok-build/config.js.map +1 -0
- package/lib/drivers/grok-build/index.d.ts +87 -0
- package/lib/drivers/grok-build/index.d.ts.map +1 -0
- package/lib/drivers/grok-build/index.js +293 -0
- package/lib/drivers/grok-build/index.js.map +1 -0
- package/lib/drivers/mcp/index.d.ts +37 -4
- package/lib/drivers/mcp/index.d.ts.map +1 -1
- package/lib/drivers/mcp/index.js +25 -11
- package/lib/drivers/mcp/index.js.map +1 -1
- package/lib/drivers/shared/acp-events.d.ts +15 -0
- package/lib/drivers/shared/acp-events.d.ts.map +1 -0
- package/lib/drivers/shared/acp-events.js +83 -0
- package/lib/drivers/shared/acp-events.js.map +1 -0
- package/lib/drivers/shared/acp-interaction.d.ts +7 -0
- package/lib/drivers/shared/acp-interaction.d.ts.map +1 -0
- package/lib/drivers/shared/acp-interaction.js +20 -0
- package/lib/drivers/shared/acp-interaction.js.map +1 -0
- package/lib/drivers/shared/acp-session.d.ts +4 -0
- package/lib/drivers/shared/acp-session.d.ts.map +1 -0
- package/lib/drivers/shared/acp-session.js +23 -0
- package/lib/drivers/shared/acp-session.js.map +1 -0
- package/lib/drivers/shared/acp-transport.d.ts +40 -0
- package/lib/drivers/shared/acp-transport.d.ts.map +1 -0
- package/lib/drivers/shared/acp-transport.js +152 -0
- package/lib/drivers/shared/acp-transport.js.map +1 -0
- package/lib/drivers/shared/artifacts.d.ts.map +1 -1
- package/lib/drivers/shared/artifacts.js +3 -2
- package/lib/drivers/shared/artifacts.js.map +1 -1
- package/lib/drivers/shared/experiment-behavior.d.ts +13 -0
- package/lib/drivers/shared/experiment-behavior.d.ts.map +1 -0
- package/lib/drivers/shared/experiment-behavior.js +200 -0
- package/lib/drivers/shared/experiment-behavior.js.map +1 -0
- package/lib/drivers/shared/experiment-policy.d.ts +4 -0
- package/lib/drivers/shared/experiment-policy.d.ts.map +1 -0
- package/lib/drivers/shared/experiment-policy.js +144 -0
- package/lib/drivers/shared/experiment-policy.js.map +1 -0
- package/lib/drivers/shared/skills.d.ts +3 -0
- package/lib/drivers/shared/skills.d.ts.map +1 -0
- package/lib/drivers/shared/skills.js +73 -0
- package/lib/drivers/shared/skills.js.map +1 -0
- package/lib/drivers/subprocess/index.d.ts +39 -4
- package/lib/drivers/subprocess/index.d.ts.map +1 -1
- package/lib/drivers/subprocess/index.js +56 -12
- package/lib/drivers/subprocess/index.js.map +1 -1
- package/lib/index.d.ts +24 -8
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +20 -5
- package/lib/index.js.map +1 -1
- package/lib/optimization/reflective.d.ts +3 -1
- package/lib/optimization/reflective.d.ts.map +1 -1
- package/lib/optimization/reflective.js +3 -2
- package/lib/optimization/reflective.js.map +1 -1
- package/lib/orchestrator/annotation-loader.d.ts +6 -2
- package/lib/orchestrator/annotation-loader.d.ts.map +1 -1
- package/lib/orchestrator/annotation-loader.js +8 -0
- package/lib/orchestrator/annotation-loader.js.map +1 -1
- package/lib/orchestrator/annotation-store.d.ts.map +1 -1
- package/lib/orchestrator/annotation-store.js +16 -2
- package/lib/orchestrator/annotation-store.js.map +1 -1
- package/lib/orchestrator/artifact-origin.d.ts +3 -0
- package/lib/orchestrator/artifact-origin.d.ts.map +1 -0
- package/lib/orchestrator/artifact-origin.js +40 -0
- package/lib/orchestrator/artifact-origin.js.map +1 -0
- package/lib/orchestrator/campaign-lifecycle.d.ts +1 -0
- package/lib/orchestrator/campaign-lifecycle.d.ts.map +1 -1
- package/lib/orchestrator/campaign-lifecycle.js +4 -1
- package/lib/orchestrator/campaign-lifecycle.js.map +1 -1
- package/lib/orchestrator/candidates.d.ts.map +1 -1
- package/lib/orchestrator/candidates.js +3 -2
- package/lib/orchestrator/candidates.js.map +1 -1
- package/lib/orchestrator/compare.d.ts +23 -6
- package/lib/orchestrator/compare.d.ts.map +1 -1
- package/lib/orchestrator/compare.js +45 -12
- package/lib/orchestrator/compare.js.map +1 -1
- package/lib/orchestrator/comparison-identity.d.ts.map +1 -1
- package/lib/orchestrator/comparison-identity.js +2 -0
- package/lib/orchestrator/comparison-identity.js.map +1 -1
- package/lib/orchestrator/config.d.ts +180 -1
- package/lib/orchestrator/config.d.ts.map +1 -1
- package/lib/orchestrator/config.js +21 -2
- package/lib/orchestrator/config.js.map +1 -1
- package/lib/orchestrator/discover.d.ts +1 -0
- package/lib/orchestrator/discover.d.ts.map +1 -1
- package/lib/orchestrator/discover.js +7 -1
- package/lib/orchestrator/discover.js.map +1 -1
- package/lib/orchestrator/doctor.d.ts +1 -0
- package/lib/orchestrator/doctor.d.ts.map +1 -1
- package/lib/orchestrator/doctor.js +134 -69
- package/lib/orchestrator/doctor.js.map +1 -1
- package/lib/orchestrator/driver-capabilities.d.ts +4 -0
- package/lib/orchestrator/driver-capabilities.d.ts.map +1 -0
- package/lib/orchestrator/driver-capabilities.js +13 -0
- package/lib/orchestrator/driver-capabilities.js.map +1 -0
- package/lib/orchestrator/driver-identity.d.ts +5 -0
- package/lib/orchestrator/driver-identity.d.ts.map +1 -0
- package/lib/orchestrator/driver-identity.js +51 -0
- package/lib/orchestrator/driver-identity.js.map +1 -0
- package/lib/orchestrator/experiment-context.d.ts +51 -3
- package/lib/orchestrator/experiment-context.d.ts.map +1 -1
- package/lib/orchestrator/experiment-context.js +2 -1
- package/lib/orchestrator/experiment-context.js.map +1 -1
- package/lib/orchestrator/experiment-contracts.d.ts +31 -1
- package/lib/orchestrator/experiment-contracts.d.ts.map +1 -1
- package/lib/orchestrator/experiment-corpus.d.ts +2 -1
- package/lib/orchestrator/experiment-corpus.d.ts.map +1 -1
- package/lib/orchestrator/experiment-corpus.js +28 -22
- package/lib/orchestrator/experiment-corpus.js.map +1 -1
- package/lib/orchestrator/experiment-effects.d.ts +33 -0
- package/lib/orchestrator/experiment-effects.d.ts.map +1 -0
- package/lib/orchestrator/experiment-effects.js +46 -0
- package/lib/orchestrator/experiment-effects.js.map +1 -0
- package/lib/orchestrator/experiment-evaluation.d.ts.map +1 -1
- package/lib/orchestrator/experiment-evaluation.js +70 -5
- package/lib/orchestrator/experiment-evaluation.js.map +1 -1
- package/lib/orchestrator/experiment-identity.d.ts +3 -0
- package/lib/orchestrator/experiment-identity.d.ts.map +1 -1
- package/lib/orchestrator/experiment-identity.js +22 -33
- package/lib/orchestrator/experiment-identity.js.map +1 -1
- package/lib/orchestrator/experiment-loader.d.ts +8 -3
- package/lib/orchestrator/experiment-loader.d.ts.map +1 -1
- package/lib/orchestrator/experiment-preflight.d.ts +3 -0
- package/lib/orchestrator/experiment-preflight.d.ts.map +1 -0
- package/lib/orchestrator/experiment-preflight.js +21 -0
- package/lib/orchestrator/experiment-preflight.js.map +1 -0
- package/lib/orchestrator/experiment-report.d.ts +91 -1
- package/lib/orchestrator/experiment-report.d.ts.map +1 -1
- package/lib/orchestrator/experiment-report.js +9 -1
- package/lib/orchestrator/experiment-report.js.map +1 -1
- package/lib/orchestrator/experiment-runs.d.ts +63 -2
- package/lib/orchestrator/experiment-runs.d.ts.map +1 -1
- package/lib/orchestrator/experiment-runs.js +13 -5
- package/lib/orchestrator/experiment-runs.js.map +1 -1
- package/lib/orchestrator/experiment-runtime-identity.d.ts +0 -16
- package/lib/orchestrator/experiment-runtime-identity.d.ts.map +1 -1
- package/lib/orchestrator/experiment-runtime-identity.js +5 -220
- package/lib/orchestrator/experiment-runtime-identity.js.map +1 -1
- package/lib/orchestrator/experiment-selection.d.ts.map +1 -1
- package/lib/orchestrator/experiment-selection.js +151 -43
- package/lib/orchestrator/experiment-selection.js.map +1 -1
- package/lib/orchestrator/experiment.d.ts +3 -3
- package/lib/orchestrator/experiment.d.ts.map +1 -1
- package/lib/orchestrator/experiment.js +26 -21
- package/lib/orchestrator/experiment.js.map +1 -1
- package/lib/orchestrator/fixture-identity.d.ts +1 -0
- package/lib/orchestrator/fixture-identity.d.ts.map +1 -1
- package/lib/orchestrator/fixture-identity.js +6 -1
- package/lib/orchestrator/fixture-identity.js.map +1 -1
- package/lib/orchestrator/fs-read.d.ts +1 -6
- package/lib/orchestrator/fs-read.d.ts.map +1 -1
- package/lib/orchestrator/fs-read.js +2 -32
- package/lib/orchestrator/fs-read.js.map +1 -1
- package/lib/orchestrator/grader-experiment.d.ts +23 -5
- package/lib/orchestrator/grader-experiment.d.ts.map +1 -1
- package/lib/orchestrator/grader-experiment.js +54 -9
- package/lib/orchestrator/grader-experiment.js.map +1 -1
- package/lib/orchestrator/grading.d.ts +1 -0
- package/lib/orchestrator/grading.d.ts.map +1 -1
- package/lib/orchestrator/grading.js +41 -33
- package/lib/orchestrator/grading.js.map +1 -1
- package/lib/orchestrator/harness-catalog-drift.d.ts +22 -0
- package/lib/orchestrator/harness-catalog-drift.d.ts.map +1 -0
- package/lib/orchestrator/harness-catalog-drift.js +90 -0
- package/lib/orchestrator/harness-catalog-drift.js.map +1 -0
- package/lib/orchestrator/harness-registry.d.ts +4 -0
- package/lib/orchestrator/harness-registry.d.ts.map +1 -0
- package/lib/orchestrator/harness-registry.js +5 -0
- package/lib/orchestrator/harness-registry.js.map +1 -0
- package/lib/orchestrator/index.d.ts +1 -1
- package/lib/orchestrator/index.d.ts.map +1 -1
- package/lib/orchestrator/index.js +1 -1
- package/lib/orchestrator/index.js.map +1 -1
- package/lib/orchestrator/list-runs.d.ts +8 -1
- package/lib/orchestrator/list-runs.d.ts.map +1 -1
- package/lib/orchestrator/list-runs.js +83 -18
- package/lib/orchestrator/list-runs.js.map +1 -1
- package/lib/orchestrator/optimization-lifecycle.d.ts.map +1 -1
- package/lib/orchestrator/optimization-lifecycle.js +3 -1
- package/lib/orchestrator/optimization-lifecycle.js.map +1 -1
- package/lib/orchestrator/orchestrator.d.ts +17 -2
- package/lib/orchestrator/orchestrator.d.ts.map +1 -1
- package/lib/orchestrator/orchestrator.js +55 -177
- package/lib/orchestrator/orchestrator.js.map +1 -1
- package/lib/orchestrator/paired-effects.d.ts +34 -0
- package/lib/orchestrator/paired-effects.d.ts.map +1 -0
- package/lib/orchestrator/paired-effects.js +164 -0
- package/lib/orchestrator/paired-effects.js.map +1 -0
- package/lib/orchestrator/profile.d.ts +4 -0
- package/lib/orchestrator/profile.d.ts.map +1 -0
- package/lib/orchestrator/profile.js +10 -0
- package/lib/orchestrator/profile.js.map +1 -0
- package/lib/orchestrator/provenance.d.ts +5 -1
- package/lib/orchestrator/provenance.d.ts.map +1 -1
- package/lib/orchestrator/provenance.js +16 -3
- package/lib/orchestrator/provenance.js.map +1 -1
- package/lib/orchestrator/regrade.d.ts +4 -2
- package/lib/orchestrator/regrade.d.ts.map +1 -1
- package/lib/orchestrator/rubric-loader.js +1 -1
- package/lib/orchestrator/rubric-loader.js.map +1 -1
- package/lib/orchestrator/run-dir.d.ts +2 -0
- package/lib/orchestrator/run-dir.d.ts.map +1 -1
- package/lib/orchestrator/run-dir.js +1 -1
- package/lib/orchestrator/run-dir.js.map +1 -1
- package/lib/orchestrator/run-execution.d.ts +3 -0
- package/lib/orchestrator/run-execution.d.ts.map +1 -1
- package/lib/orchestrator/run-execution.js +64 -10
- package/lib/orchestrator/run-execution.js.map +1 -1
- package/lib/orchestrator/run-lifecycle.d.ts +1 -0
- package/lib/orchestrator/run-lifecycle.d.ts.map +1 -1
- package/lib/orchestrator/run-lifecycle.js +3 -2
- package/lib/orchestrator/run-lifecycle.js.map +1 -1
- package/lib/orchestrator/run-workspace.d.ts +10 -0
- package/lib/orchestrator/run-workspace.d.ts.map +1 -0
- package/lib/orchestrator/run-workspace.js +58 -0
- package/lib/orchestrator/run-workspace.js.map +1 -0
- package/lib/orchestrator/suite-compare.d.ts +37 -0
- package/lib/orchestrator/suite-compare.d.ts.map +1 -1
- package/lib/orchestrator/suite-compare.js +44 -22
- package/lib/orchestrator/suite-compare.js.map +1 -1
- package/lib/orchestrator/suite-runs.d.ts +6 -1
- package/lib/orchestrator/suite-runs.d.ts.map +1 -1
- package/lib/orchestrator/suite-runs.js +14 -2
- package/lib/orchestrator/suite-runs.js.map +1 -1
- package/lib/orchestrator/suite-summary.d.ts +79 -0
- package/lib/orchestrator/suite-summary.d.ts.map +1 -1
- package/lib/orchestrator/suite-summary.js +67 -14
- package/lib/orchestrator/suite-summary.js.map +1 -1
- package/lib/previewer/artifacts.d.ts +25 -0
- package/lib/previewer/artifacts.d.ts.map +1 -0
- package/lib/previewer/artifacts.js +180 -0
- package/lib/previewer/artifacts.js.map +1 -0
- package/lib/previewer/server.d.ts +111 -0
- package/lib/previewer/server.d.ts.map +1 -0
- package/lib/previewer/server.js +559 -0
- package/lib/previewer/server.js.map +1 -0
- package/package.json +4 -2
- package/skills/lux-answerer/SKILL.md +2 -2
- package/src/answerers/claude-code/index.ts +6 -5
- package/src/answerers/persona/index.ts +75 -34
- package/src/answerers/terminal/index.ts +7 -11
- package/src/assertions/authoring.ts +2 -0
- package/src/assertions/core/command-exits-zero.ts +82 -17
- package/src/assertions/core/event-checks.ts +64 -3
- package/src/assertions/core/index.ts +2 -0
- package/src/assertions/core/resource-checks.ts +38 -5
- package/src/assertions/core/safe-regex.ts +16 -6
- package/src/assertions/rubric/evidence.ts +85 -0
- package/src/assertions/rubric/index.ts +63 -46
- package/src/assertions/rubric/internal.ts +256 -41
- package/src/authoring.ts +346 -2
- package/src/cli/bin.ts +2 -12
- package/src/cli/command-runtime.ts +20 -9
- package/src/cli/doctor.ts +17 -0
- package/src/cli/harness-plan.ts +123 -0
- package/src/cli/init-templates.ts +238 -0
- package/src/cli/init.ts +53 -184
- package/src/cli/output-schemas.ts +109 -0
- package/src/cli/program.ts +697 -730
- package/src/cli/render/format.ts +1 -1
- package/src/cli/render/reporter.ts +36 -8
- package/src/cli/run-options.ts +131 -25
- package/src/cli/view.ts +129 -0
- package/src/container/driver.ts +213 -0
- package/src/container/paths.ts +29 -0
- package/src/container/protocol.ts +48 -0
- package/src/container/resources.ts +165 -0
- package/src/container/worker.ts +107 -0
- package/src/core/action-events.ts +59 -0
- package/src/core/agent-activity.ts +236 -0
- package/src/core/agent-environment.ts +64 -0
- package/src/core/agent-events.ts +17 -0
- package/src/core/annotation.ts +2 -1
- package/src/core/answerer.ts +9 -0
- package/src/core/artifact-evidence.ts +40 -5
- package/src/core/assertion.ts +8 -2
- package/src/core/case-defaults.ts +65 -0
- package/src/core/case.ts +79 -8
- package/src/core/command-verification.ts +74 -0
- package/src/core/driver-execution.ts +84 -0
- package/src/core/driver-metrics.ts +51 -0
- package/src/core/driver.ts +76 -0
- package/src/core/duration.ts +52 -0
- package/src/core/environment.ts +22 -0
- package/src/core/evaluation-clusters.ts +35 -0
- package/src/core/events.ts +13 -2
- package/src/core/experiment.ts +71 -25
- package/src/core/file-tree.ts +35 -0
- package/src/core/global-id.ts +26 -0
- package/src/core/harness-catalog.ts +100 -0
- package/src/core/harness-catalogs/antigravity.ts +24 -0
- package/src/core/harness-catalogs/claude-code.ts +25 -0
- package/src/core/harness-catalogs/codex.ts +27 -0
- package/src/core/harness-catalogs/copilot.ts +46 -0
- package/src/core/harness-catalogs/cursor.ts +13 -0
- package/src/core/harness-catalogs/grok-build.ts +19 -0
- package/src/core/harness-catalogs/index.ts +31 -0
- package/src/core/harness-matrix.ts +316 -0
- package/src/core/index.ts +12 -0
- package/src/core/lifecycle-fixtures.ts +4 -2
- package/src/core/model-cli.ts +60 -26
- package/src/core/platform-process.ts +65 -16
- package/src/core/reference-defaults.ts +47 -0
- package/src/core/run.ts +31 -0
- package/src/core/usage.ts +8 -0
- package/src/drivers/antigravity/README.md +171 -0
- package/src/drivers/antigravity/artifacts.ts +37 -0
- package/src/drivers/antigravity/config.ts +92 -0
- package/src/drivers/antigravity/events.ts +290 -0
- package/src/drivers/antigravity/index.ts +132 -0
- package/src/drivers/antigravity/runtime.ts +94 -0
- package/src/drivers/claude-code/deferred-process.ts +118 -0
- package/src/drivers/claude-code/index.ts +227 -281
- package/src/drivers/claude-code/isolation.ts +228 -0
- package/src/drivers/claude-code/parse-events.ts +98 -12
- package/src/drivers/codex/app-events.ts +126 -16
- package/src/drivers/codex/config.ts +59 -8
- package/src/drivers/codex/exec-events.ts +258 -0
- package/src/drivers/codex/index.ts +146 -312
- package/src/drivers/copilot/README.md +60 -0
- package/src/drivers/copilot/config.ts +61 -0
- package/src/drivers/copilot/index.ts +310 -0
- package/src/drivers/cursor/README.md +66 -0
- package/src/drivers/cursor/config.ts +31 -0
- package/src/drivers/cursor/index.ts +326 -0
- package/src/drivers/cursor/interaction.ts +74 -0
- package/src/drivers/grok-build/README.md +112 -0
- package/src/drivers/grok-build/config.ts +43 -0
- package/src/drivers/grok-build/index.ts +331 -0
- package/src/drivers/mcp/index.ts +25 -11
- package/src/drivers/shared/acp-events.ts +94 -0
- package/src/drivers/shared/acp-interaction.ts +27 -0
- package/src/drivers/shared/acp-session.ts +40 -0
- package/src/drivers/shared/acp-transport.ts +187 -0
- package/src/drivers/shared/artifacts.ts +6 -2
- package/src/drivers/shared/experiment-behavior.ts +217 -0
- package/src/drivers/shared/experiment-policy.ts +196 -0
- package/src/drivers/shared/skills.ts +74 -0
- package/src/drivers/subprocess/index.ts +55 -12
- package/src/index.ts +117 -6
- package/src/optimization/reflective.ts +3 -2
- package/src/orchestrator/annotation-loader.ts +10 -0
- package/src/orchestrator/annotation-store.ts +24 -3
- package/src/orchestrator/artifact-origin.ts +50 -0
- package/src/orchestrator/campaign-lifecycle.ts +5 -1
- package/src/orchestrator/candidates.ts +3 -2
- package/src/orchestrator/compare.ts +62 -13
- package/src/orchestrator/comparison-identity.ts +2 -0
- package/src/orchestrator/config.ts +23 -1
- package/src/orchestrator/discover.ts +8 -1
- package/src/orchestrator/doctor.ts +211 -65
- package/src/orchestrator/driver-capabilities.ts +21 -0
- package/src/orchestrator/driver-identity.ts +69 -0
- package/src/orchestrator/experiment-context.ts +5 -3
- package/src/orchestrator/experiment-corpus.ts +58 -37
- package/src/orchestrator/experiment-effects.ts +63 -0
- package/src/orchestrator/experiment-evaluation.ts +88 -10
- package/src/orchestrator/experiment-identity.ts +57 -48
- package/src/orchestrator/experiment-preflight.ts +27 -0
- package/src/orchestrator/experiment-report.ts +21 -1
- package/src/orchestrator/experiment-runs.ts +11 -4
- package/src/orchestrator/experiment-runtime-identity.ts +5 -243
- package/src/orchestrator/experiment-selection.ts +178 -18
- package/src/orchestrator/experiment.ts +29 -29
- package/src/orchestrator/fixture-identity.ts +9 -1
- package/src/orchestrator/fs-read.ts +2 -32
- package/src/orchestrator/grader-experiment.ts +74 -13
- package/src/orchestrator/grading.ts +60 -31
- package/src/orchestrator/harness-catalog-drift.ts +132 -0
- package/src/orchestrator/harness-registry.ts +12 -0
- package/src/orchestrator/index.ts +1 -0
- package/src/orchestrator/list-runs.ts +102 -18
- package/src/orchestrator/optimization-lifecycle.ts +4 -1
- package/src/orchestrator/orchestrator.ts +105 -224
- package/src/orchestrator/paired-effects.ts +214 -0
- package/src/orchestrator/profile.ts +24 -0
- package/src/orchestrator/provenance.ts +23 -3
- package/src/orchestrator/rubric-loader.ts +1 -1
- package/src/orchestrator/run-dir.ts +8 -1
- package/src/orchestrator/run-execution.ts +75 -10
- package/src/orchestrator/run-lifecycle.ts +4 -2
- package/src/orchestrator/run-workspace.ts +62 -0
- package/src/orchestrator/suite-compare.ts +47 -23
- package/src/orchestrator/suite-runs.ts +13 -2
- package/src/orchestrator/suite-summary.ts +129 -15
- package/src/previewer/artifacts.ts +221 -0
- package/src/previewer/server.ts +789 -0
- package/viewer/app.css +2111 -0
- package/viewer/app.js +627 -0
- package/viewer/artifact-view.js +48 -0
- package/viewer/charts.js +98 -0
- package/viewer/detail.js +1266 -0
- package/viewer/format.js +129 -0
- package/viewer/index.html +151 -0
- package/lib/cli/skills.d.ts +0 -38
- package/lib/cli/skills.d.ts.map +0 -1
- package/lib/cli/skills.js +0 -84
- package/lib/cli/skills.js.map +0 -1
- package/lib/drivers/shared/secure-copy.d.ts.map +0 -1
- package/lib/drivers/shared/secure-copy.js.map +0 -1
- package/skills/lux/SKILL.md +0 -89
- package/skills/lux/references/cli.md +0 -125
- package/skills/lux/references/eval-authoring.md +0 -257
- package/src/cli/skills.ts +0 -129
- /package/lib/{drivers/shared → core}/secure-copy.d.ts +0 -0
- /package/lib/{drivers/shared → core}/secure-copy.js +0 -0
- /package/src/{drivers/shared → core}/secure-copy.ts +0 -0
package/lib/core/assertion.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ export declare const AssertionResultSchema: z.ZodObject<{
|
|
|
6
6
|
score: z.ZodNumber;
|
|
7
7
|
comment: z.ZodExactOptional<z.ZodString>;
|
|
8
8
|
detail: z.ZodOptional<z.ZodUnknown>;
|
|
9
|
+
abstention: z.ZodExactOptional<z.ZodLiteral<"insufficient-evidence">>;
|
|
9
10
|
errorMessage: z.ZodExactOptional<z.ZodString>;
|
|
10
11
|
usage: z.ZodExactOptional<z.ZodObject<{
|
|
11
12
|
input: z.ZodNumber;
|
|
@@ -69,6 +70,7 @@ export declare const baseAssertionFields: {
|
|
|
69
70
|
readonly weight: z.ZodOptional<z.ZodNumber>;
|
|
70
71
|
readonly threshold: z.ZodOptional<z.ZodNumber>;
|
|
71
72
|
readonly name: z.ZodOptional<z.ZodString>;
|
|
73
|
+
readonly informational: z.ZodOptional<z.ZodBoolean>;
|
|
72
74
|
};
|
|
73
75
|
export declare const defineAssertion: <S extends z.ZodObject>(definition: AssertionPluginDefinition<S>) => AssertionPlugin<SchemaAssertion<S>, SchemaAssertionInput<S>>;
|
|
74
76
|
export declare const defineAssertionRegistry: <const Plugins extends readonly AssertionPlugin[]>(...plugins: Plugins) => Plugins;
|
|
@@ -76,13 +78,15 @@ export declare const GradingAssertionEntrySchema: z.ZodObject<{
|
|
|
76
78
|
id: z.ZodExactOptional<z.ZodString>;
|
|
77
79
|
type: z.ZodString;
|
|
78
80
|
name: z.ZodString;
|
|
79
|
-
passed: z.ZodBoolean
|
|
81
|
+
passed: z.ZodNullable<z.ZodBoolean>;
|
|
82
|
+
informational: z.ZodExactOptional<z.ZodBoolean>;
|
|
80
83
|
score: z.ZodNumber;
|
|
81
84
|
weight: z.ZodNumber;
|
|
82
|
-
threshold: z.ZodNumber
|
|
85
|
+
threshold: z.ZodExactOptional<z.ZodNumber>;
|
|
83
86
|
durationMs: z.ZodNumber;
|
|
84
87
|
comment: z.ZodExactOptional<z.ZodString>;
|
|
85
88
|
detail: z.ZodOptional<z.ZodUnknown>;
|
|
89
|
+
abstention: z.ZodExactOptional<z.ZodLiteral<"insufficient-evidence">>;
|
|
86
90
|
errorMessage: z.ZodExactOptional<z.ZodString>;
|
|
87
91
|
usage: z.ZodExactOptional<z.ZodObject<{
|
|
88
92
|
input: z.ZodNumber;
|
|
@@ -100,13 +104,15 @@ export declare const GradingReportSchema: z.ZodObject<{
|
|
|
100
104
|
id: z.ZodExactOptional<z.ZodString>;
|
|
101
105
|
type: z.ZodString;
|
|
102
106
|
name: z.ZodString;
|
|
103
|
-
passed: z.ZodBoolean
|
|
107
|
+
passed: z.ZodNullable<z.ZodBoolean>;
|
|
108
|
+
informational: z.ZodExactOptional<z.ZodBoolean>;
|
|
104
109
|
score: z.ZodNumber;
|
|
105
110
|
weight: z.ZodNumber;
|
|
106
|
-
threshold: z.ZodNumber
|
|
111
|
+
threshold: z.ZodExactOptional<z.ZodNumber>;
|
|
107
112
|
durationMs: z.ZodNumber;
|
|
108
113
|
comment: z.ZodExactOptional<z.ZodString>;
|
|
109
114
|
detail: z.ZodOptional<z.ZodUnknown>;
|
|
115
|
+
abstention: z.ZodExactOptional<z.ZodLiteral<"insufficient-evidence">>;
|
|
110
116
|
errorMessage: z.ZodExactOptional<z.ZodString>;
|
|
111
117
|
usage: z.ZodExactOptional<z.ZodObject<{
|
|
112
118
|
input: z.ZodNumber;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"assertion.d.ts","sourceRoot":"","sources":["../../src/core/assertion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAGpC,eAAO,MAAM,qBAAqB
|
|
1
|
+
{"version":3,"file":"assertion.d.ts","sourceRoot":"","sources":["../../src/core/assertion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAGpC,eAAO,MAAM,qBAAqB;;;;;;;;;;;;;iBAWhC,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,MAAM,MAAM,eAAe,GAAG;IAC5B;;;;OAIG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;OAGG;IACH,uBAAuB,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5D,CAAC;AAGF,MAAM,WAAW,aAAa,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS;IAC5D,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IAChB,YAAY,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,MAAM,eAAe,CACzB,CAAC,SAAS,SAAS,GAAG,SAAS,EAC/B,KAAK,SAAS,SAAS,GAAG,CAAC,IACzB,aAAa,CAAC,CAAC,CAAC,GAAG;IACrB,YAAY,EAAE,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACzC,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC;IACzB,gEAAgE;IAChE,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,KAAK,eAAe,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;AAClG,KAAK,oBAAoB,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAC7C,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;AAEpD,KAAK,oBAAoB,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAC7C,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,UAAU,CAAC,MAAM,OAAO,SAAS,MAAM,CAAC,GAAG,OAAO,GAAG,KAAK,CAAC;AAE1F,MAAM,MAAM,yBAAyB,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAAI;IAC7D,IAAI,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC;IAC9B,YAAY,EAAE,CAAC,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACjG,CAAC;AAEF,eAAO,MAAM,oBAAoB,eAAgB;IAC/C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,KAAG,MAAM,GAAG,SAsBZ,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,mBAAmB;aAC9B,EAAE;aACF,MAAM;aACN,SAAS;aACT,IAAI;aACJ,aAAa;CACL,CAAC;AAEX,eAAO,MAAM,eAAe,GAAI,CAAC,SAAS,CAAC,CAAC,SAAS,cACvC,yBAAyB,CAAC,CAAC,CAAC,KACvC,eAAe,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAI7D,CAAC;AAEF,eAAO,MAAM,uBAAuB,GAAI,KAAK,CAAC,OAAO,SAAS,SAAS,eAAe,EAAE,cAC1E,OAAO,KAClB,OAWF,CAAC;AAEF,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;iBAkBtC,CAAC;AACH,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAShF,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAgB9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}
|
package/lib/core/assertion.js
CHANGED
|
@@ -6,6 +6,8 @@ export const AssertionResultSchema = z.object({
|
|
|
6
6
|
score: z.number().min(0).max(1),
|
|
7
7
|
comment: z.exactOptional(z.string()),
|
|
8
8
|
detail: z.unknown().optional(),
|
|
9
|
+
/** The evaluator ran successfully but lacked evidence for a verdict. */
|
|
10
|
+
abstention: z.exactOptional(z.literal("insufficient-evidence")),
|
|
9
11
|
/** Present when the assertion could not produce a judgment. */
|
|
10
12
|
errorMessage: z.exactOptional(z.string()),
|
|
11
13
|
/** Tokens the assertion itself spent. Only LLM-judge assertions report this. */
|
|
@@ -41,6 +43,7 @@ export const baseAssertionFields = {
|
|
|
41
43
|
weight: z.number().positive().optional(),
|
|
42
44
|
threshold: z.number().min(0).max(1).optional(),
|
|
43
45
|
name: z.string().optional(),
|
|
46
|
+
informational: z.boolean().optional(),
|
|
44
47
|
};
|
|
45
48
|
export const defineAssertion = (definition) => {
|
|
46
49
|
const issue = assertionPluginIssue(definition);
|
|
@@ -66,14 +69,17 @@ export const GradingAssertionEntrySchema = z.object({
|
|
|
66
69
|
id: z.exactOptional(z.string()),
|
|
67
70
|
type: z.string(),
|
|
68
71
|
name: z.string(),
|
|
69
|
-
passed: z.boolean(),
|
|
72
|
+
passed: z.boolean().nullable(),
|
|
73
|
+
informational: z.exactOptional(z.boolean()),
|
|
70
74
|
score: z.number(),
|
|
71
75
|
weight: z.number(),
|
|
72
|
-
|
|
76
|
+
/** Acceptance floor; absent for informational measurements. */
|
|
77
|
+
threshold: z.exactOptional(z.number()),
|
|
73
78
|
/** Wall-clock the assertion took to evaluate. A `rubric` costs seconds; a file check, none. */
|
|
74
79
|
durationMs: z.number(),
|
|
75
80
|
comment: z.exactOptional(z.string()),
|
|
76
81
|
detail: z.unknown().optional(),
|
|
82
|
+
abstention: z.exactOptional(z.literal("insufficient-evidence")),
|
|
77
83
|
errorMessage: z.exactOptional(z.string()),
|
|
78
84
|
usage: z.exactOptional(TokenUsageSchema),
|
|
79
85
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"assertion.js","sourceRoot":"","sources":["../../src/core/assertion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAErD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,+DAA+D;IAC/D,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,gFAAgF;IAChF,KAAK,EAAE,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC;CACzC,CAAC,CAAC;AAsDH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,UAKpC,EAAsB,EAAE;IACvB,IAAI,OAAO,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,wCAAwC,CAAC;IAClD,CAAC;IACD,IAAI,CAAC,CAAC,UAAU,CAAC,YAAY,YAAY,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;QACtD,OAAO,2DAA2D,CAAC;IACrE,CAAC;IACD,MAAM,aAAa,GAAG,UAAU,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;IACzD,IACE,CAAC,CAAC,aAAa,YAAY,CAAC,CAAC,UAAU,CAAC;QACxC,aAAa,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC;QAC/B,aAAa,CAAC,KAAK,KAAK,UAAU,CAAC,IAAI,EACvC,CAAC;QACD,OAAO,wEAAwE,CAAC;IAClF,CAAC;IACD,IAAI,OAAO,UAAU,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC;QACrF,OAAO,kDAAkD,CAAC;IAC5D,CAAC;IACD,IAAI,OAAO,UAAU,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC9C,OAAO,8CAA8C,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAChC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACxC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;
|
|
1
|
+
{"version":3,"file":"assertion.js","sourceRoot":"","sources":["../../src/core/assertion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAErD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,wEAAwE;IACxE,UAAU,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC/D,+DAA+D;IAC/D,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,gFAAgF;IAChF,KAAK,EAAE,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC;CACzC,CAAC,CAAC;AAsDH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,UAKpC,EAAsB,EAAE;IACvB,IAAI,OAAO,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,wCAAwC,CAAC;IAClD,CAAC;IACD,IAAI,CAAC,CAAC,UAAU,CAAC,YAAY,YAAY,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;QACtD,OAAO,2DAA2D,CAAC;IACrE,CAAC;IACD,MAAM,aAAa,GAAG,UAAU,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;IACzD,IACE,CAAC,CAAC,aAAa,YAAY,CAAC,CAAC,UAAU,CAAC;QACxC,aAAa,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC;QAC/B,aAAa,CAAC,KAAK,KAAK,UAAU,CAAC,IAAI,EACvC,CAAC;QACD,OAAO,wEAAwE,CAAC;IAClF,CAAC;IACD,IAAI,OAAO,UAAU,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC;QACrF,OAAO,kDAAkD,CAAC;IAC5D,CAAC;IACD,IAAI,OAAO,UAAU,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC9C,OAAO,8CAA8C,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAChC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACxC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CAC7B,CAAC;AAEX,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,UAAwC,EACsB,EAAE;IAChE,MAAM,KAAK,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;IAC/C,IAAI,KAAK;QAAE,MAAM,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;IACtC,OAAO,UAAqF,CAAC;AAC/F,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,GAAG,OAAgB,EACV,EAAE;IACX,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,KAAK;YAAE,MAAM,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;QACtC,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,SAAS,CAAC,0CAA0C,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;QAChF,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD,yEAAyE;IACzE,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,aAAa,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IAC3C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,+DAA+D;IAC/D,SAAS,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACtC,+FAA+F;IAC/F,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,OAAO,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,UAAU,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAC/D,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,KAAK,EAAE,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC;CACzC,CAAC,CAAC;AAGH,MAAM,6BAA6B,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;IACd,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;CACpC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,2BAA2B,CAAC;IAChD,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC,QAAQ,EAAE;IACpE;;;OAGG;IACH,UAAU,EAAE,CAAC,CAAC,aAAa,CACzB,CAAC,CAAC,MAAM,CAAC;QACP,UAAU,EAAE,sBAAsB;QAClC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE;KAC3D,CAAC,CACH;CACF,CAAC,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Assertion, EvalCase } from "./case.js";
|
|
2
|
+
type DeepPartial<T> = T extends readonly unknown[] ? T : T extends (...args: never[]) => unknown ? T : T extends object ? {
|
|
3
|
+
[K in keyof T]?: DeepPartial<T[K]>;
|
|
4
|
+
} : T;
|
|
5
|
+
export type AssertionDefaults<A extends Assertion = Assertion> = {
|
|
6
|
+
[Type in A["type"]]?: DeepPartial<Omit<Extract<A, {
|
|
7
|
+
type: Type;
|
|
8
|
+
}>, "type" | "id">>;
|
|
9
|
+
};
|
|
10
|
+
export type EvalCaseDefaults<A extends Assertion = Assertion> = Omit<Partial<EvalCase>, "id" | "prompt" | "assertions"> & {
|
|
11
|
+
assertions?: A[] | undefined;
|
|
12
|
+
};
|
|
13
|
+
export declare const mergeCaseDefaults: (base: EvalCaseDefaults, local: EvalCaseDefaults) => EvalCaseDefaults;
|
|
14
|
+
export declare const applyAssertionDefaults: (assertions: Assertion[], defaults: AssertionDefaults) => Assertion[];
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=case-defaults.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"case-defaults.d.ts","sourceRoot":"","sources":["../../src/core/case-defaults.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAIrD,KAAK,WAAW,CAAC,CAAC,IAAI,CAAC,SAAS,SAAS,OAAO,EAAE,GAC9C,CAAC,GACD,CAAC,SAAS,CAAC,GAAG,IAAI,EAAE,KAAK,EAAE,KAAK,OAAO,GACrC,CAAC,GACD,CAAC,SAAS,MAAM,GACd;KAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;CAAE,GACtC,CAAC,CAAC;AAEV,MAAM,MAAM,iBAAiB,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS,IAAI;KAC9D,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE;QAAE,IAAI,EAAE,IAAI,CAAA;KAAE,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;CACnF,CAAC;AAEF,MAAM,MAAM,gBAAgB,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS,IAAI,IAAI,CAClE,OAAO,CAAC,QAAQ,CAAC,EACjB,IAAI,GAAG,QAAQ,GAAG,YAAY,CAC/B,GAAG;IAAE,UAAU,CAAC,EAAE,CAAC,EAAE,GAAG,SAAS,CAAA;CAAE,CAAC;AAmBrC,eAAO,MAAM,iBAAiB,SACtB,gBAAgB,SACf,gBAAgB,KACtB,gBAeD,CAAC;AAEH,eAAO,MAAM,sBAAsB,eACrB,SAAS,EAAE,YACb,iBAAiB,KAC1B,SAAS,EAGP,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { mergePluginConfig } from "./harness-matrix.js";
|
|
2
|
+
import { mergeAnswererReference, mergeDriverReference } from "./reference-defaults.js";
|
|
3
|
+
/** Named assertions replace inherited checks; unnamed checks append in declaration order. */
|
|
4
|
+
const mergeAssertions = (base, local) => {
|
|
5
|
+
const seen = new Set();
|
|
6
|
+
for (const assertion of local) {
|
|
7
|
+
if (!assertion.id)
|
|
8
|
+
continue;
|
|
9
|
+
if (seen.has(assertion.id))
|
|
10
|
+
throw new Error(`duplicate assertion id '${assertion.id}'`);
|
|
11
|
+
seen.add(assertion.id);
|
|
12
|
+
}
|
|
13
|
+
const result = [...base];
|
|
14
|
+
for (const assertion of local) {
|
|
15
|
+
const index = assertion.id ? result.findIndex((entry) => entry.id === assertion.id) : -1;
|
|
16
|
+
if (index < 0)
|
|
17
|
+
result.push(assertion);
|
|
18
|
+
else
|
|
19
|
+
result[index] = assertion;
|
|
20
|
+
}
|
|
21
|
+
return result;
|
|
22
|
+
};
|
|
23
|
+
export const mergeCaseDefaults = (base, local) => ({
|
|
24
|
+
...base,
|
|
25
|
+
...local,
|
|
26
|
+
...(base.driver || local.driver
|
|
27
|
+
? { driver: mergeDriverReference(base.driver, local.driver) }
|
|
28
|
+
: {}),
|
|
29
|
+
...(base.answerer || local.answerer
|
|
30
|
+
? { answerer: mergeAnswererReference(base.answerer, local.answerer) }
|
|
31
|
+
: {}),
|
|
32
|
+
...(base.meta || local.meta
|
|
33
|
+
? { meta: mergePluginConfig(base.meta, local.meta) }
|
|
34
|
+
: {}),
|
|
35
|
+
...(base.assertions || local.assertions
|
|
36
|
+
? { assertions: mergeAssertions(base.assertions ?? [], local.assertions ?? []) }
|
|
37
|
+
: {}),
|
|
38
|
+
});
|
|
39
|
+
export const applyAssertionDefaults = (assertions, defaults) => assertions.map((assertion) => ({
|
|
40
|
+
...mergePluginConfig(defaults[assertion.type], assertion),
|
|
41
|
+
}));
|
|
42
|
+
//# sourceMappingURL=case-defaults.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"case-defaults.js","sourceRoot":"","sources":["../../src/core/case-defaults.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAmBvF,6FAA6F;AAC7F,MAAM,eAAe,GAAG,CAAC,IAAiB,EAAE,KAAkB,EAAe,EAAE;IAC7E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,SAAS,CAAC,EAAE;YAAE,SAAS;QAC5B,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,SAAS,CAAC,EAAE,GAAG,CAAC,CAAC;QACxF,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACzB,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzF,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;;YACjC,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;IACjC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAC/B,IAAsB,EACtB,KAAuB,EACL,EAAE,CAAC,CAAC;IACtB,GAAG,IAAI;IACP,GAAG,KAAK;IACR,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM;QAC7B,CAAC,CAAC,EAAE,MAAM,EAAE,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE;QAC7D,CAAC,CAAC,EAAE,CAAC;IACP,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ;QACjC,CAAC,CAAC,EAAE,QAAQ,EAAE,sBAAsB,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE;QACrE,CAAC,CAAC,EAAE,CAAC;IACP,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI;QACzB,CAAC,CAAC,EAAE,IAAI,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAqB,EAAE;QACxE,CAAC,CAAC,EAAE,CAAC;IACP,GAAG,CAAC,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU;QACrC,CAAC,CAAC,EAAE,UAAU,EAAE,eAAe,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,EAAE,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE;QAChF,CAAC,CAAC,EAAE,CAAC;CACR,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACpC,UAAuB,EACvB,QAA2B,EACd,EAAE,CACf,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IAC7B,GAAI,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,SAAS,CAAe;CACzE,CAAC,CAAC,CAAC"}
|
package/lib/core/case.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import type { AssertionPlugin } from "./assertion.js";
|
|
3
|
+
import { type AssertionDefaults, type EvalCaseDefaults } from "./case-defaults.js";
|
|
3
4
|
import type { LifecycleHooksPlugin } from "./lifecycle-hooks.js";
|
|
4
5
|
export declare const FixturesRefSchema: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
5
6
|
kind: z.ZodLiteral<"dir">;
|
|
@@ -9,6 +10,46 @@ export type FixturesRef = z.infer<typeof FixturesRefSchema>;
|
|
|
9
10
|
export declare const DriverRefSchema: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
10
11
|
name: z.ZodString;
|
|
11
12
|
config: z.ZodOptional<z.ZodUnknown>;
|
|
13
|
+
execution: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
14
|
+
kind: z.ZodLiteral<"host">;
|
|
15
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
16
|
+
kind: z.ZodLiteral<"container">;
|
|
17
|
+
image: z.ZodString;
|
|
18
|
+
command: z.ZodOptional<z.ZodString>;
|
|
19
|
+
worker: z.ZodOptional<z.ZodString>;
|
|
20
|
+
driverModule: z.ZodOptional<z.ZodString>;
|
|
21
|
+
mounts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
22
|
+
source: z.ZodString;
|
|
23
|
+
target: z.ZodOptional<z.ZodString>;
|
|
24
|
+
readOnly: z.ZodOptional<z.ZodBoolean>;
|
|
25
|
+
}, z.core.$strict>>>;
|
|
26
|
+
passEnv: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
27
|
+
env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
28
|
+
homeFiles: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
29
|
+
destination: z.ZodString;
|
|
30
|
+
source: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
31
|
+
kind: z.ZodLiteral<"file">;
|
|
32
|
+
path: z.ZodString;
|
|
33
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
34
|
+
kind: z.ZodLiteral<"keychain">;
|
|
35
|
+
service: z.ZodString;
|
|
36
|
+
account: z.ZodString;
|
|
37
|
+
encoding: z.ZodOptional<z.ZodEnum<{
|
|
38
|
+
"go-keyring-base64": "go-keyring-base64";
|
|
39
|
+
utf8: "utf8";
|
|
40
|
+
}>>;
|
|
41
|
+
}, z.core.$strict>], "kind">;
|
|
42
|
+
}, z.core.$strict>>>;
|
|
43
|
+
network: z.ZodOptional<z.ZodEnum<{
|
|
44
|
+
bridge: "bridge";
|
|
45
|
+
none: "none";
|
|
46
|
+
}>>;
|
|
47
|
+
user: z.ZodOptional<z.ZodString>;
|
|
48
|
+
memory: z.ZodOptional<z.ZodString>;
|
|
49
|
+
cpus: z.ZodOptional<z.ZodNumber>;
|
|
50
|
+
pidsLimit: z.ZodOptional<z.ZodNumber>;
|
|
51
|
+
startupTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
52
|
+
}, z.core.$strict>], "kind">>;
|
|
12
53
|
}, z.core.$strict>]>;
|
|
13
54
|
export type DriverRef = z.infer<typeof DriverRefSchema>;
|
|
14
55
|
export declare const AnswererRefSchema: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -22,6 +63,7 @@ export declare const AssertionSchema: z.ZodObject<{
|
|
|
22
63
|
name: z.ZodOptional<z.ZodString>;
|
|
23
64
|
weight: z.ZodOptional<z.ZodNumber>;
|
|
24
65
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
66
|
+
informational: z.ZodOptional<z.ZodBoolean>;
|
|
25
67
|
}, z.core.$loose>;
|
|
26
68
|
export type Assertion = z.infer<typeof AssertionSchema>;
|
|
27
69
|
export declare const EvalCaseSchema: z.ZodObject<{
|
|
@@ -31,7 +73,48 @@ export declare const EvalCaseSchema: z.ZodObject<{
|
|
|
31
73
|
driver: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
32
74
|
name: z.ZodString;
|
|
33
75
|
config: z.ZodOptional<z.ZodUnknown>;
|
|
76
|
+
execution: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
77
|
+
kind: z.ZodLiteral<"host">;
|
|
78
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
79
|
+
kind: z.ZodLiteral<"container">;
|
|
80
|
+
image: z.ZodString;
|
|
81
|
+
command: z.ZodOptional<z.ZodString>;
|
|
82
|
+
worker: z.ZodOptional<z.ZodString>;
|
|
83
|
+
driverModule: z.ZodOptional<z.ZodString>;
|
|
84
|
+
mounts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
85
|
+
source: z.ZodString;
|
|
86
|
+
target: z.ZodOptional<z.ZodString>;
|
|
87
|
+
readOnly: z.ZodOptional<z.ZodBoolean>;
|
|
88
|
+
}, z.core.$strict>>>;
|
|
89
|
+
passEnv: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
90
|
+
env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
|
|
91
|
+
homeFiles: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
92
|
+
destination: z.ZodString;
|
|
93
|
+
source: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
94
|
+
kind: z.ZodLiteral<"file">;
|
|
95
|
+
path: z.ZodString;
|
|
96
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
97
|
+
kind: z.ZodLiteral<"keychain">;
|
|
98
|
+
service: z.ZodString;
|
|
99
|
+
account: z.ZodString;
|
|
100
|
+
encoding: z.ZodOptional<z.ZodEnum<{
|
|
101
|
+
"go-keyring-base64": "go-keyring-base64";
|
|
102
|
+
utf8: "utf8";
|
|
103
|
+
}>>;
|
|
104
|
+
}, z.core.$strict>], "kind">;
|
|
105
|
+
}, z.core.$strict>>>;
|
|
106
|
+
network: z.ZodOptional<z.ZodEnum<{
|
|
107
|
+
bridge: "bridge";
|
|
108
|
+
none: "none";
|
|
109
|
+
}>>;
|
|
110
|
+
user: z.ZodOptional<z.ZodString>;
|
|
111
|
+
memory: z.ZodOptional<z.ZodString>;
|
|
112
|
+
cpus: z.ZodOptional<z.ZodNumber>;
|
|
113
|
+
pidsLimit: z.ZodOptional<z.ZodNumber>;
|
|
114
|
+
startupTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
115
|
+
}, z.core.$strict>], "kind">>;
|
|
34
116
|
}, z.core.$strict>]>>;
|
|
117
|
+
matrix: z.ZodOptional<z.ZodArray<z.ZodCustom<import("./harness-matrix.js").HarnessMatrixEntry, import("./harness-matrix.js").HarnessMatrixEntry>>>;
|
|
35
118
|
answerer: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
36
119
|
name: z.ZodString;
|
|
37
120
|
config: z.ZodOptional<z.ZodUnknown>;
|
|
@@ -40,24 +123,42 @@ export declare const EvalCaseSchema: z.ZodObject<{
|
|
|
40
123
|
kind: z.ZodLiteral<"dir">;
|
|
41
124
|
path: z.ZodString;
|
|
42
125
|
}, z.core.$strict>]>>;
|
|
126
|
+
requires: z.ZodOptional<z.ZodArray<z.ZodEnum<{
|
|
127
|
+
artifacts: "artifacts";
|
|
128
|
+
cost: "cost";
|
|
129
|
+
interactive: "interactive";
|
|
130
|
+
isolation: "isolation";
|
|
131
|
+
subagents: "subagents";
|
|
132
|
+
"tool-calls": "tool-calls";
|
|
133
|
+
"tool-results": "tool-results";
|
|
134
|
+
usage: "usage";
|
|
135
|
+
}>>>;
|
|
43
136
|
assertions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
44
137
|
id: z.ZodOptional<z.ZodString>;
|
|
45
138
|
type: z.ZodString;
|
|
46
139
|
name: z.ZodOptional<z.ZodString>;
|
|
47
140
|
weight: z.ZodOptional<z.ZodNumber>;
|
|
48
141
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
142
|
+
informational: z.ZodOptional<z.ZodBoolean>;
|
|
49
143
|
}, z.core.$loose>>>;
|
|
50
|
-
meta: z.ZodOptional<z.
|
|
144
|
+
meta: z.ZodOptional<z.ZodObject<{
|
|
145
|
+
cluster: z.ZodOptional<z.ZodString>;
|
|
146
|
+
}, z.core.$catchall<z.ZodUnknown>>>;
|
|
51
147
|
}, z.core.$strict>;
|
|
52
148
|
export type EvalCase = z.infer<typeof EvalCaseSchema>;
|
|
53
|
-
type AssertionPlanFields = Pick<Assertion, "id" | "name" | "threshold" | "weight">;
|
|
149
|
+
type AssertionPlanFields = Pick<Assertion, "id" | "name" | "threshold" | "weight" | "informational">;
|
|
54
150
|
type AssertionInputOf<Plugin> = Plugin extends AssertionPlugin<Assertion, infer Input> ? Input & AssertionPlanFields : never;
|
|
55
151
|
export type EvalCaseInput<AllowedAssertion extends Assertion = Assertion> = Omit<EvalCase, "assertions"> & {
|
|
56
152
|
assertions?: AllowedAssertion[];
|
|
153
|
+
assertionDefaults?: AssertionDefaults<AllowedAssertion>;
|
|
154
|
+
/** Remove named inherited checks without replacing the rest of the suite plan. */
|
|
155
|
+
omitAssertions?: string[];
|
|
57
156
|
};
|
|
58
|
-
export type EvalCaseAuthoringOptions<Plugins extends readonly AssertionPlugin[] = readonly AssertionPlugin[]> = {
|
|
157
|
+
export type EvalCaseAuthoringOptions<Plugins extends readonly AssertionPlugin[] = readonly AssertionPlugin[], AllowedAssertion extends Assertion = Assertion> = {
|
|
59
158
|
assertions?: Plugins;
|
|
60
159
|
fixtures?: readonly LifecycleHooksPlugin[];
|
|
160
|
+
defaults?: EvalCaseDefaults<AllowedAssertion | AssertionInputOf<Plugins[number]>>;
|
|
161
|
+
assertionDefaults?: AssertionDefaults<AllowedAssertion | AssertionInputOf<Plugins[number]>>;
|
|
61
162
|
};
|
|
62
163
|
export type EvalCaseDefinition<AllowedAssertion extends Assertion = Assertion> = {
|
|
63
164
|
(input: EvalCaseInput<AllowedAssertion>): EvalCase;
|
|
@@ -65,9 +166,9 @@ export type EvalCaseDefinition<AllowedAssertion extends Assertion = Assertion> =
|
|
|
65
166
|
* Create a case definition with selectively activated lifecycle fixtures.
|
|
66
167
|
* Further calls to `extend` append fixtures in setup order.
|
|
67
168
|
*/
|
|
68
|
-
extend<const Plugins extends readonly AssertionPlugin[] = readonly []>(options: EvalCaseAuthoringOptions<Plugins>): EvalCaseDefinition<AllowedAssertion | AssertionInputOf<Plugins[number]>>;
|
|
169
|
+
extend<const Plugins extends readonly AssertionPlugin[] = readonly []>(options: EvalCaseAuthoringOptions<Plugins, AllowedAssertion>): EvalCaseDefinition<AllowedAssertion | AssertionInputOf<Plugins[number]>>;
|
|
69
170
|
};
|
|
70
|
-
export declare const createEvalCaseDefinition: <AllowedAssertion extends Assertion = Assertion>(inheritedAssertions?: readonly AssertionPlugin[], inheritedFixtures?: readonly LifecycleHooksPlugin[]) => EvalCaseDefinition<AllowedAssertion>;
|
|
171
|
+
export declare const createEvalCaseDefinition: <AllowedAssertion extends Assertion = Assertion>(inheritedAssertions?: readonly AssertionPlugin[], inheritedFixtures?: readonly LifecycleHooksPlugin[], defaults?: EvalCaseDefaults, assertionDefaults?: AssertionDefaults) => EvalCaseDefinition<AllowedAssertion>;
|
|
71
172
|
export declare const defineEvalCase: EvalCaseDefinition<{
|
|
72
173
|
[x: string]: unknown;
|
|
73
174
|
id?: string | undefined;
|
|
@@ -75,19 +176,31 @@ export declare const defineEvalCase: EvalCaseDefinition<{
|
|
|
75
176
|
name?: string | undefined;
|
|
76
177
|
weight?: number | undefined;
|
|
77
178
|
threshold?: number | undefined;
|
|
179
|
+
informational?: boolean | undefined;
|
|
78
180
|
}>;
|
|
79
181
|
export declare const PersistedCaseSchema: z.ZodObject<{
|
|
80
182
|
id: z.ZodOptional<z.ZodOptional<z.ZodString>>;
|
|
81
183
|
prompt: z.ZodOptional<z.ZodString>;
|
|
82
184
|
persona: z.ZodOptional<z.ZodOptional<z.ZodString>>;
|
|
185
|
+
matrix: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodCustom<import("./harness-matrix.js").HarnessMatrixEntry, import("./harness-matrix.js").HarnessMatrixEntry>>>>;
|
|
186
|
+
requires: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodEnum<{
|
|
187
|
+
artifacts: "artifacts";
|
|
188
|
+
cost: "cost";
|
|
189
|
+
interactive: "interactive";
|
|
190
|
+
isolation: "isolation";
|
|
191
|
+
subagents: "subagents";
|
|
192
|
+
"tool-calls": "tool-calls";
|
|
193
|
+
"tool-results": "tool-results";
|
|
194
|
+
usage: "usage";
|
|
195
|
+
}>>>>;
|
|
83
196
|
assertions: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
84
197
|
id: z.ZodOptional<z.ZodString>;
|
|
85
198
|
type: z.ZodString;
|
|
86
199
|
name: z.ZodOptional<z.ZodString>;
|
|
87
200
|
weight: z.ZodOptional<z.ZodNumber>;
|
|
88
201
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
202
|
+
informational: z.ZodOptional<z.ZodBoolean>;
|
|
89
203
|
}, z.core.$loose>>>>;
|
|
90
|
-
meta: z.ZodOptional<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
91
204
|
casePath: z.ZodExactOptional<z.ZodString>;
|
|
92
205
|
driver: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
93
206
|
name: z.ZodString;
|
|
@@ -101,6 +214,7 @@ export declare const PersistedCaseSchema: z.ZodObject<{
|
|
|
101
214
|
kind: z.ZodLiteral<"dir">;
|
|
102
215
|
path: z.ZodString;
|
|
103
216
|
}, z.core.$loose>]>>;
|
|
217
|
+
meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
104
218
|
}, z.core.$loose>;
|
|
105
219
|
export type PersistedCase = z.infer<typeof PersistedCaseSchema>;
|
|
106
220
|
export {};
|
package/lib/core/case.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"case.d.ts","sourceRoot":"","sources":["../../src/core/case.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEtD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAGjE,eAAO,MAAM,iBAAiB;;;oBAG5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAO5D,eAAO,MAAM,eAAe
|
|
1
|
+
{"version":3,"file":"case.d.ts","sourceRoot":"","sources":["../../src/core/case.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEtD,OAAO,EACL,KAAK,iBAAiB,EAEtB,KAAK,gBAAgB,EAEtB,MAAM,oBAAoB,CAAC;AAK5B,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAGjE,eAAO,MAAM,iBAAiB;;;oBAG5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAO5D,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;oBAS1B,CAAC;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,eAAO,MAAM,iBAAiB;;;oBAG5B,CAAC;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAE5D,eAAO,MAAM,eAAe;;;;;;;iBAUlB,CAAC;AACX,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAgBxD,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAiBvB,CAAC;AACL,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD,KAAK,mBAAmB,GAAG,IAAI,CAC7B,SAAS,EACT,IAAI,GAAG,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,eAAe,CACzD,CAAC;AACF,KAAK,gBAAgB,CAAC,MAAM,IAC1B,MAAM,SAAS,eAAe,CAAC,SAAS,EAAE,MAAM,KAAK,CAAC,GAAG,KAAK,GAAG,mBAAmB,GAAG,KAAK,CAAC;AAE/F,MAAM,MAAM,aAAa,CAAC,gBAAgB,SAAS,SAAS,GAAG,SAAS,IAAI,IAAI,CAC9E,QAAQ,EACR,YAAY,CACb,GAAG;IACF,UAAU,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAChC,iBAAiB,CAAC,EAAE,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;IACxD,kFAAkF;IAClF,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,wBAAwB,CAClC,OAAO,SAAS,SAAS,eAAe,EAAE,GAAG,SAAS,eAAe,EAAE,EACvE,gBAAgB,SAAS,SAAS,GAAG,SAAS,IAC5C;IACF,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,QAAQ,CAAC,EAAE,SAAS,oBAAoB,EAAE,CAAC;IAC3C,QAAQ,CAAC,EAAE,gBAAgB,CAAC,gBAAgB,GAAG,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAClF,iBAAiB,CAAC,EAAE,iBAAiB,CAAC,gBAAgB,GAAG,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;CAC7F,CAAC;AAEF,MAAM,MAAM,kBAAkB,CAAC,gBAAgB,SAAS,SAAS,GAAG,SAAS,IAAI;IAC/E,CAAC,KAAK,EAAE,aAAa,CAAC,gBAAgB,CAAC,GAAG,QAAQ,CAAC;IACnD;;;OAGG;IACH,MAAM,CAAC,KAAK,CAAC,OAAO,SAAS,SAAS,eAAe,EAAE,GAAG,SAAS,EAAE,EACnE,OAAO,EAAE,wBAAwB,CAAC,OAAO,EAAE,gBAAgB,CAAC,GAC3D,kBAAkB,CAAC,gBAAgB,GAAG,gBAAgB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;CAC7E,CAAC;AAEF,eAAO,MAAM,wBAAwB,GAAI,gBAAgB,SAAS,SAAS,GAAG,SAAS,wBAChE,SAAS,eAAe,EAAE,sBAC5B,SAAS,oBAAoB,EAAE,aACxC,gBAAgB,sBACP,iBAAiB,KACnC,kBAAkB,CAAC,gBAAgB,CA2DrC,CAAC;AAEF,eAAO,MAAM,cAAc;;;;;;;;EAA6B,CAAC;AAmBzD,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAQtB,CAAC;AACX,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}
|
package/lib/core/case.js
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { collectEvalCaseExtensions, setEvalCaseExtensions } from "./case-authoring-state.js";
|
|
3
|
+
import { applyAssertionDefaults, mergeCaseDefaults, } from "./case-defaults.js";
|
|
4
|
+
import { DriverCapabilitySchema } from "./driver.js";
|
|
5
|
+
import { DriverExecutionSchema } from "./driver-execution.js";
|
|
6
|
+
import { EvaluationMetadataSchema } from "./evaluation-clusters.js";
|
|
7
|
+
import { HarnessMatrixSchema, mergePluginConfig } from "./harness-matrix.js";
|
|
3
8
|
import { formatZodIssues } from "./zod.js";
|
|
4
9
|
export const FixturesRefSchema = z.union([
|
|
5
10
|
z.string(),
|
|
@@ -12,7 +17,13 @@ export const FixturesRefSchema = z.union([
|
|
|
12
17
|
// `defineEvalCase` discriminate the config type on `name`.
|
|
13
18
|
export const DriverRefSchema = z.union([
|
|
14
19
|
z.string(),
|
|
15
|
-
z
|
|
20
|
+
z
|
|
21
|
+
.object({
|
|
22
|
+
name: z.string(),
|
|
23
|
+
config: z.unknown().optional(),
|
|
24
|
+
execution: DriverExecutionSchema.optional(),
|
|
25
|
+
})
|
|
26
|
+
.strict(),
|
|
16
27
|
]);
|
|
17
28
|
export const AnswererRefSchema = z.union([
|
|
18
29
|
z.string(),
|
|
@@ -26,6 +37,7 @@ export const AssertionSchema = z
|
|
|
26
37
|
name: z.string().optional(),
|
|
27
38
|
weight: z.number().positive().optional(),
|
|
28
39
|
threshold: z.number().min(0).max(1).optional(),
|
|
40
|
+
informational: z.boolean().optional(),
|
|
29
41
|
})
|
|
30
42
|
.loose();
|
|
31
43
|
const EvalCaseObjectSchema = z.object({
|
|
@@ -33,10 +45,13 @@ const EvalCaseObjectSchema = z.object({
|
|
|
33
45
|
prompt: z.string().min(1, "prompt is required"),
|
|
34
46
|
persona: z.string().min(1).optional(),
|
|
35
47
|
driver: DriverRefSchema.optional(),
|
|
48
|
+
/** Harness permutations this case runs through instead of the project matrix. */
|
|
49
|
+
matrix: HarnessMatrixSchema.optional(),
|
|
36
50
|
answerer: AnswererRefSchema.optional(),
|
|
37
51
|
fixtures: FixturesRefSchema.optional(),
|
|
52
|
+
requires: z.array(DriverCapabilitySchema).optional(),
|
|
38
53
|
assertions: z.array(AssertionSchema).optional(),
|
|
39
|
-
meta:
|
|
54
|
+
meta: EvaluationMetadataSchema.optional(),
|
|
40
55
|
});
|
|
41
56
|
export const EvalCaseSchema = EvalCaseObjectSchema
|
|
42
57
|
// strict so a stray `driverConfig`/`answererConfig` (the old shape) or a
|
|
@@ -57,9 +72,32 @@ export const EvalCaseSchema = EvalCaseObjectSchema
|
|
|
57
72
|
seen.add(assertion.id);
|
|
58
73
|
}
|
|
59
74
|
});
|
|
60
|
-
export const createEvalCaseDefinition = (inheritedAssertions = [], inheritedFixtures = []) => {
|
|
75
|
+
export const createEvalCaseDefinition = (inheritedAssertions = [], inheritedFixtures = [], defaults = {}, assertionDefaults = {}) => {
|
|
61
76
|
const define = ((input) => {
|
|
62
|
-
const
|
|
77
|
+
const { omitAssertions = [], assertionDefaults: localAssertionDefaults = {}, ...local } = input;
|
|
78
|
+
for (const id of omitAssertions) {
|
|
79
|
+
if (!defaults.assertions?.some((assertion) => assertion.id === id)) {
|
|
80
|
+
throw new Error(`cannot omit unknown inherited assertion '${id}'`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const resolved = mergeCaseDefaults({
|
|
84
|
+
...defaults,
|
|
85
|
+
...(defaults.assertions
|
|
86
|
+
? {
|
|
87
|
+
assertions: defaults.assertions.filter((assertion) => !omitAssertions.includes(assertion.id ?? "")),
|
|
88
|
+
}
|
|
89
|
+
: {}),
|
|
90
|
+
}, local);
|
|
91
|
+
const parsed = EvalCaseSchema.safeParse({
|
|
92
|
+
...resolved,
|
|
93
|
+
...("id" in local ? { id: local.id } : {}),
|
|
94
|
+
prompt: local.prompt,
|
|
95
|
+
...(resolved.assertions
|
|
96
|
+
? {
|
|
97
|
+
assertions: applyAssertionDefaults(resolved.assertions, mergePluginConfig(assertionDefaults, localAssertionDefaults)),
|
|
98
|
+
}
|
|
99
|
+
: {}),
|
|
100
|
+
});
|
|
63
101
|
if (!parsed.success) {
|
|
64
102
|
throw new Error(`Invalid eval case: ${formatZodIssues(parsed.error)}`);
|
|
65
103
|
}
|
|
@@ -67,7 +105,14 @@ export const createEvalCaseDefinition = (inheritedAssertions = [], inheritedFixt
|
|
|
67
105
|
setEvalCaseExtensions(parsed.data, extensions);
|
|
68
106
|
return parsed.data;
|
|
69
107
|
});
|
|
70
|
-
define.extend = ((
|
|
108
|
+
define.extend = ((options) => {
|
|
109
|
+
const { assertions = [], fixtures = [] } = options;
|
|
110
|
+
const inherited = EvalCaseObjectSchema.omit({ id: true, prompt: true })
|
|
111
|
+
.partial()
|
|
112
|
+
.strict()
|
|
113
|
+
.parse(options.defaults ?? {});
|
|
114
|
+
return createEvalCaseDefinition([...inheritedAssertions, ...assertions], [...inheritedFixtures, ...fixtures], mergeCaseDefaults(defaults, inherited), mergePluginConfig(assertionDefaults, options.assertionDefaults ?? {}));
|
|
115
|
+
});
|
|
71
116
|
return define;
|
|
72
117
|
};
|
|
73
118
|
export const defineEvalCase = createEvalCaseDefinition();
|
|
@@ -93,6 +138,7 @@ export const PersistedCaseSchema = EvalCaseObjectSchema.partial()
|
|
|
93
138
|
driver: PersistedDriverRefSchema.optional(),
|
|
94
139
|
answerer: PersistedAnswererRefSchema.optional(),
|
|
95
140
|
fixtures: PersistedFixturesRefSchema.optional(),
|
|
141
|
+
meta: z.record(z.string(), z.unknown()).optional(),
|
|
96
142
|
})
|
|
97
143
|
.loose();
|
|
98
144
|
//# sourceMappingURL=case.js.map
|
package/lib/core/case.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"case.js","sourceRoot":"","sources":["../../src/core/case.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"case.js","sourceRoot":"","sources":["../../src/core/case.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,yBAAyB,EAAE,qBAAqB,EAAE,MAAM,2BAA2B,CAAC;AAC7F,OAAO,EAEL,sBAAsB,EAEtB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAE7E,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE3C,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC;IACvC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;CAChE,CAAC,CAAC;AAGH,6EAA6E;AAC7E,6EAA6E;AAC7E,8EAA8E;AAC9E,8EAA8E;AAC9E,2DAA2D;AAC3D,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC;IACrC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC;SACE,MAAM,CAAC;QACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;QAC9B,SAAS,EAAE,qBAAqB,CAAC,QAAQ,EAAE;KAC5C,CAAC;SACD,MAAM,EAAE;CACZ,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC;IACvC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE;CACxE,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,sFAAsF;IACtF,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAChC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACxC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACtC,CAAC;KACD,KAAK,EAAE,CAAC;AAGX,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,oBAAoB,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IACrC,MAAM,EAAE,eAAe,CAAC,QAAQ,EAAE;IAClC,iFAAiF;IACjF,MAAM,EAAE,mBAAmB,CAAC,QAAQ,EAAE;IACtC,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,EAAE;IACtC,QAAQ,EAAE,iBAAiB,CAAC,QAAQ,EAAE;IACtC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE;IACpD,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,QAAQ,EAAE;IAC/C,IAAI,EAAE,wBAAwB,CAAC,QAAQ,EAAE;CAC1C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,oBAAoB;IAChD,yEAAyE;IACzE,+DAA+D;KAC9D,MAAM,EAAE;KACR,WAAW,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACvE,IAAI,CAAC,SAAS,CAAC,EAAE;YAAE,SAAS;QAC5B,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;YAC3B,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,CAAC,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC;gBACjC,OAAO,EAAE,2BAA2B,SAAS,CAAC,EAAE,GAAG;aACpD,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACzB,CAAC;AACH,CAAC,CAAC,CAAC;AAyCL,MAAM,CAAC,MAAM,wBAAwB,GAAG,CACtC,mBAAmB,GAA+B,EAAE,EACpD,iBAAiB,GAAoC,EAAE,EACvD,QAAQ,GAAqB,EAAE,EAC/B,iBAAiB,GAAsB,EAAE,EACH,EAAE;IACxC,MAAM,MAAM,GAAG,CAAC,CAAC,KAAsC,EAAY,EAAE;QACnE,MAAM,EAAE,cAAc,GAAG,EAAE,EAAE,iBAAiB,EAAE,sBAAsB,GAAG,EAAE,EAAE,GAAG,KAAK,EAAE,GAAG,KAAK,CAAC;QAChG,KAAK,MAAM,EAAE,IAAI,cAAc,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;gBACnE,MAAM,IAAI,KAAK,CAAC,4CAA4C,EAAE,GAAG,CAAC,CAAC;YACrE,CAAC;QACH,CAAC;QACD,MAAM,QAAQ,GAAG,iBAAiB,CAChC;YACE,GAAG,QAAQ;YACX,GAAG,CAAC,QAAQ,CAAC,UAAU;gBACrB,CAAC,CAAC;oBACE,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,MAAM,CACpC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,CAAC,CAC5D;iBACF;gBACH,CAAC,CAAC,EAAE,CAAC;SACR,EACD,KAAK,CACN,CAAC;QACF,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC;YACtC,GAAG,QAAQ;YACX,GAAG,CAAC,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1C,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,GAAG,CAAC,QAAQ,CAAC,UAAU;gBACrB,CAAC,CAAC;oBACE,UAAU,EAAE,sBAAsB,CAChC,QAAQ,CAAC,UAAU,EACnB,iBAAiB,CAAC,iBAAiB,EAAE,sBAAsB,CAAsB,CAClF;iBACF;gBACH,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,sBAAsB,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,UAAU,GAAG,yBAAyB,CAC1C,MAAM,CAAC,IAAI,EACX,mBAAmB,EACnB,iBAAiB,CAClB,CAAC;QACF,qBAAqB,CAAC,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC,CAAyC,CAAC;IAC3C,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,OAAiC,EAAE,EAAE;QACrD,MAAM,EAAE,UAAU,GAAG,EAAE,EAAE,QAAQ,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC;QACnD,MAAM,SAAS,GAAG,oBAAoB,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;aACpE,OAAO,EAAE;aACT,MAAM,EAAE;aACR,KAAK,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QACjC,OAAO,wBAAwB,CAC7B,CAAC,GAAG,mBAAmB,EAAE,GAAG,UAAU,CAAC,EACvC,CAAC,GAAG,iBAAiB,EAAE,GAAG,QAAQ,CAAC,EACnC,iBAAiB,CAAC,QAAQ,EAAE,SAAS,CAAC,EACtC,iBAAiB,CAAC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,EAAE,CAAsB,CAC3F,CAAC;IACJ,CAAC,CAAmD,CAAC;IACrD,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;AAEzD,MAAM,0BAA0B,GAAG,CAAC,CAAC,KAAK,CAAC;IACzC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;CAC5D,CAAC,CAAC;AACH,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC;IACvC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;CACpE,CAAC,CAAC;AACH,MAAM,0BAA0B,GAAG,CAAC,CAAC,KAAK,CAAC;IACzC,CAAC,CAAC,MAAM,EAAE;IACV,CAAC,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC;CACpE,CAAC,CAAC;AAEH,8EAA8E;AAC9E,uEAAuE;AACvE,2EAA2E;AAC3E,0DAA0D;AAC1D,MAAM,CAAC,MAAM,mBAAmB,GAAG,oBAAoB,CAAC,OAAO,EAAE;KAC9D,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACrC,MAAM,EAAE,wBAAwB,CAAC,QAAQ,EAAE;IAC3C,QAAQ,EAAE,0BAA0B,CAAC,QAAQ,EAAE;IAC/C,QAAQ,EAAE,0BAA0B,CAAC,QAAQ,EAAE;IAC/C,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACnD,CAAC;KACD,KAAK,EAAE,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Run } from "./run.js";
|
|
2
|
+
/** Internal: live execution state is deliberately absent from persisted run evidence. */
|
|
3
|
+
export declare const bindRunExecutionWorkspace: (run: Run, root: string) => void;
|
|
4
|
+
export declare const runExecutionWorkspace: (run: Run) => string | undefined;
|
|
5
|
+
/** A disposable command tree preserves dependencies without exposing host symlink targets. */
|
|
6
|
+
export declare const copyCommandWorkspace: (source: string, signal?: AbortSignal) => Promise<{
|
|
7
|
+
root: string;
|
|
8
|
+
close: () => Promise<void>;
|
|
9
|
+
}>;
|
|
10
|
+
//# sourceMappingURL=command-verification.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"command-verification.d.ts","sourceRoot":"","sources":["../../src/core/command-verification.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAKpC,yFAAyF;AACzF,eAAO,MAAM,yBAAyB,QAAS,GAAG,QAAQ,MAAM,KAAG,IAElE,CAAC;AAEF,eAAO,MAAM,qBAAqB,QAAS,GAAG,KAAG,MAAM,GAAG,SAAyC,CAAC;AAOpG,8FAA8F;AAC9F,eAAO,MAAM,oBAAoB,WACvB,MAAM,WACL,WAAW,KACnB,OAAO,CAAC;IACT,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5B,CA8CA,CAAC"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { chmod, lstat, mkdir, mkdtemp, readdir, realpath, rm, symlink } from "node:fs/promises";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
4
|
+
import { copyRegularFileNoFollow } from "./secure-copy.js";
|
|
5
|
+
const executionWorkspaces = new WeakMap();
|
|
6
|
+
/** Internal: live execution state is deliberately absent from persisted run evidence. */
|
|
7
|
+
export const bindRunExecutionWorkspace = (run, root) => {
|
|
8
|
+
executionWorkspaces.set(run, resolve(root));
|
|
9
|
+
};
|
|
10
|
+
export const runExecutionWorkspace = (run) => executionWorkspaces.get(run);
|
|
11
|
+
const within = (root, path) => {
|
|
12
|
+
const rel = relative(root, path);
|
|
13
|
+
return rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel);
|
|
14
|
+
};
|
|
15
|
+
/** A disposable command tree preserves dependencies without exposing host symlink targets. */
|
|
16
|
+
export const copyCommandWorkspace = async (source, signal) => {
|
|
17
|
+
const sourceRoot = await realpath(source);
|
|
18
|
+
if (!(await lstat(source)).isDirectory() || (await lstat(source)).isSymbolicLink()) {
|
|
19
|
+
throw new Error(`verification workspace must be a real directory: ${source}`);
|
|
20
|
+
}
|
|
21
|
+
const root = await mkdtemp(join(tmpdir(), "lux-command-verification-"));
|
|
22
|
+
const close = () => rm(root, { recursive: true, force: true });
|
|
23
|
+
let entries = 0;
|
|
24
|
+
const copy = async (directory) => {
|
|
25
|
+
if (!within(sourceRoot, await realpath(directory))) {
|
|
26
|
+
throw new Error(`verification directory escapes its workspace: ${directory}`);
|
|
27
|
+
}
|
|
28
|
+
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
|
29
|
+
signal?.throwIfAborted();
|
|
30
|
+
if (++entries > 100_000)
|
|
31
|
+
throw new Error("verification workspace exceeds 100000 entries");
|
|
32
|
+
if (directory === sourceRoot && entry.name === ".lux-claude")
|
|
33
|
+
continue;
|
|
34
|
+
const path = join(directory, entry.name);
|
|
35
|
+
const target = join(root, relative(sourceRoot, path));
|
|
36
|
+
const metadata = await lstat(path);
|
|
37
|
+
if (metadata.isSymbolicLink()) {
|
|
38
|
+
const resolved = await realpath(path);
|
|
39
|
+
if (!within(sourceRoot, resolved)) {
|
|
40
|
+
throw new Error(`verification symlink escapes its workspace: ${path}`);
|
|
41
|
+
}
|
|
42
|
+
await symlink(relative(dirname(target), join(root, relative(sourceRoot, resolved))), target);
|
|
43
|
+
}
|
|
44
|
+
else if (metadata.isDirectory()) {
|
|
45
|
+
await mkdir(target, { mode: (metadata.mode & 0o777) | 0o700 });
|
|
46
|
+
await copy(path);
|
|
47
|
+
await chmod(target, metadata.mode & 0o777);
|
|
48
|
+
}
|
|
49
|
+
else if (metadata.isFile()) {
|
|
50
|
+
await copyRegularFileNoFollow(sourceRoot, path, target);
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
throw new Error(`verification workspace contains an unsupported entry: ${path}`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
try {
|
|
58
|
+
await copy(sourceRoot);
|
|
59
|
+
return { root, close };
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
await close();
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
//# sourceMappingURL=command-verification.js.map
|