@prismatic-io/lux 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/lib/answerers/claude-code/index.d.ts +17 -0
- package/lib/answerers/claude-code/index.d.ts.map +1 -0
- package/lib/answerers/claude-code/index.js +280 -0
- package/lib/answerers/claude-code/index.js.map +1 -0
- package/lib/answerers/persona/index.d.ts +40 -0
- package/lib/answerers/persona/index.d.ts.map +1 -0
- package/lib/answerers/persona/index.js +265 -0
- package/lib/answerers/persona/index.js.map +1 -0
- package/lib/answerers/scripted/index.d.ts +8 -0
- package/lib/answerers/scripted/index.d.ts.map +1 -0
- package/lib/answerers/scripted/index.js +163 -0
- package/lib/answerers/scripted/index.js.map +1 -0
- package/lib/answerers/shared.d.ts +13 -0
- package/lib/answerers/shared.d.ts.map +1 -0
- package/lib/answerers/shared.js +17 -0
- package/lib/answerers/shared.js.map +1 -0
- package/lib/answerers/terminal/index.d.ts +9 -0
- package/lib/answerers/terminal/index.d.ts.map +1 -0
- package/lib/answerers/terminal/index.js +97 -0
- package/lib/answerers/terminal/index.js.map +1 -0
- package/lib/assertions/core/command-exits-zero.d.ts +21 -0
- package/lib/assertions/core/command-exits-zero.d.ts.map +1 -0
- package/lib/assertions/core/command-exits-zero.js +87 -0
- package/lib/assertions/core/command-exits-zero.js.map +1 -0
- package/lib/assertions/core/event-checks.d.ts +57 -0
- package/lib/assertions/core/event-checks.d.ts.map +1 -0
- package/lib/assertions/core/event-checks.js +123 -0
- package/lib/assertions/core/event-checks.js.map +1 -0
- package/lib/assertions/core/file-checks.d.ts +103 -0
- package/lib/assertions/core/file-checks.d.ts.map +1 -0
- package/lib/assertions/core/file-checks.js +288 -0
- package/lib/assertions/core/file-checks.js.map +1 -0
- package/lib/assertions/core/helpers.d.ts +66 -0
- package/lib/assertions/core/helpers.d.ts.map +1 -0
- package/lib/assertions/core/helpers.js +170 -0
- package/lib/assertions/core/helpers.js.map +1 -0
- package/lib/assertions/core/index.d.ts +15 -0
- package/lib/assertions/core/index.d.ts.map +1 -0
- package/lib/assertions/core/index.js +56 -0
- package/lib/assertions/core/index.js.map +1 -0
- package/lib/assertions/core/json-pointer-equals.d.ts +19 -0
- package/lib/assertions/core/json-pointer-equals.d.ts.map +1 -0
- package/lib/assertions/core/json-pointer-equals.js +71 -0
- package/lib/assertions/core/json-pointer-equals.js.map +1 -0
- package/lib/assertions/core/output-checks.d.ts +46 -0
- package/lib/assertions/core/output-checks.d.ts.map +1 -0
- package/lib/assertions/core/output-checks.js +121 -0
- package/lib/assertions/core/output-checks.js.map +1 -0
- package/lib/assertions/core/predicate.d.ts +14 -0
- package/lib/assertions/core/predicate.d.ts.map +1 -0
- package/lib/assertions/core/predicate.js +55 -0
- package/lib/assertions/core/predicate.js.map +1 -0
- package/lib/assertions/core/resource-checks.d.ts +71 -0
- package/lib/assertions/core/resource-checks.d.ts.map +1 -0
- package/lib/assertions/core/resource-checks.js +159 -0
- package/lib/assertions/core/resource-checks.js.map +1 -0
- package/lib/assertions/core/safe-regex.d.ts +23 -0
- package/lib/assertions/core/safe-regex.d.ts.map +1 -0
- package/lib/assertions/core/safe-regex.js +102 -0
- package/lib/assertions/core/safe-regex.js.map +1 -0
- package/lib/assertions/core/tool-called-with.d.ts +21 -0
- package/lib/assertions/core/tool-called-with.d.ts.map +1 -0
- package/lib/assertions/core/tool-called-with.js +69 -0
- package/lib/assertions/core/tool-called-with.js.map +1 -0
- package/lib/assertions/core/tool-calls.d.ts +22 -0
- package/lib/assertions/core/tool-calls.d.ts.map +1 -0
- package/lib/assertions/core/tool-calls.js +17 -0
- package/lib/assertions/core/tool-calls.js.map +1 -0
- package/lib/assertions/core/tool-invocation.d.ts +36 -0
- package/lib/assertions/core/tool-invocation.d.ts.map +1 -0
- package/lib/assertions/core/tool-invocation.js +170 -0
- package/lib/assertions/core/tool-invocation.js.map +1 -0
- package/lib/assertions/core/tool-result-checks.d.ts +59 -0
- package/lib/assertions/core/tool-result-checks.d.ts.map +1 -0
- package/lib/assertions/core/tool-result-checks.js +193 -0
- package/lib/assertions/core/tool-result-checks.js.map +1 -0
- package/lib/assertions/mcp/discovery-checks.d.ts +36 -0
- package/lib/assertions/mcp/discovery-checks.d.ts.map +1 -0
- package/lib/assertions/mcp/discovery-checks.js +122 -0
- package/lib/assertions/mcp/discovery-checks.js.map +1 -0
- package/lib/assertions/mcp/index.d.ts +10 -0
- package/lib/assertions/mcp/index.d.ts.map +1 -0
- package/lib/assertions/mcp/index.js +25 -0
- package/lib/assertions/mcp/index.js.map +1 -0
- package/lib/assertions/rubric/index.d.ts +104 -0
- package/lib/assertions/rubric/index.d.ts.map +1 -0
- package/lib/assertions/rubric/index.js +191 -0
- package/lib/assertions/rubric/index.js.map +1 -0
- package/lib/assertions/rubric/internal.d.ts +14 -0
- package/lib/assertions/rubric/internal.d.ts.map +1 -0
- package/lib/assertions/rubric/internal.js +335 -0
- package/lib/assertions/rubric/internal.js.map +1 -0
- package/lib/authoring.d.ts +3 -0
- package/lib/authoring.d.ts.map +1 -0
- package/lib/authoring.js +3 -0
- package/lib/authoring.js.map +1 -0
- package/lib/cli/bin.d.ts +3 -0
- package/lib/cli/bin.d.ts.map +1 -0
- package/lib/cli/bin.js +15 -0
- package/lib/cli/bin.js.map +1 -0
- package/lib/cli/command-runtime.d.ts +5 -0
- package/lib/cli/command-runtime.d.ts.map +1 -0
- package/lib/cli/command-runtime.js +30 -0
- package/lib/cli/command-runtime.js.map +1 -0
- package/lib/cli/init.d.ts +21 -0
- package/lib/cli/init.d.ts.map +1 -0
- package/lib/cli/init.js +340 -0
- package/lib/cli/init.js.map +1 -0
- package/lib/cli/program.d.ts +22 -0
- package/lib/cli/program.d.ts.map +1 -0
- package/lib/cli/program.js +757 -0
- package/lib/cli/program.js.map +1 -0
- package/lib/cli/render/format.d.ts +14 -0
- package/lib/cli/render/format.d.ts.map +1 -0
- package/lib/cli/render/format.js +71 -0
- package/lib/cli/render/format.js.map +1 -0
- package/lib/cli/render/reporter.d.ts +73 -0
- package/lib/cli/render/reporter.d.ts.map +1 -0
- package/lib/cli/render/reporter.js +519 -0
- package/lib/cli/render/reporter.js.map +1 -0
- package/lib/cli/render/tty.d.ts +77 -0
- package/lib/cli/render/tty.d.ts.map +1 -0
- package/lib/cli/render/tty.js +283 -0
- package/lib/cli/render/tty.js.map +1 -0
- package/lib/cli/run-options.d.ts +27 -0
- package/lib/cli/run-options.d.ts.map +1 -0
- package/lib/cli/run-options.js +58 -0
- package/lib/cli/run-options.js.map +1 -0
- package/lib/cli/skills.d.ts +38 -0
- package/lib/cli/skills.d.ts.map +1 -0
- package/lib/cli/skills.js +84 -0
- package/lib/cli/skills.js.map +1 -0
- package/lib/core/annotation.d.ts +56 -0
- package/lib/core/annotation.d.ts.map +1 -0
- package/lib/core/annotation.js +92 -0
- package/lib/core/annotation.js.map +1 -0
- package/lib/core/answerer.d.ts +68 -0
- package/lib/core/answerer.d.ts.map +1 -0
- package/lib/core/answerer.js +17 -0
- package/lib/core/answerer.js.map +1 -0
- package/lib/core/artifact-evidence.d.ts +50 -0
- package/lib/core/artifact-evidence.d.ts.map +1 -0
- package/lib/core/artifact-evidence.js +185 -0
- package/lib/core/artifact-evidence.js.map +1 -0
- package/lib/core/artifact-snapshot.d.ts +37 -0
- package/lib/core/artifact-snapshot.d.ts.map +1 -0
- package/lib/core/artifact-snapshot.js +339 -0
- package/lib/core/artifact-snapshot.js.map +1 -0
- package/lib/core/assertion.d.ts +129 -0
- package/lib/core/assertion.d.ts.map +1 -0
- package/lib/core/assertion.js +75 -0
- package/lib/core/assertion.js.map +1 -0
- package/lib/core/case.d.ts +79 -0
- package/lib/core/case.d.ts.map +1 -0
- package/lib/core/case.js +90 -0
- package/lib/core/case.js.map +1 -0
- package/lib/core/conversation-turn.d.ts +23 -0
- package/lib/core/conversation-turn.d.ts.map +1 -0
- package/lib/core/conversation-turn.js +31 -0
- package/lib/core/conversation-turn.js.map +1 -0
- package/lib/core/driver.d.ts +64 -0
- package/lib/core/driver.d.ts.map +1 -0
- package/lib/core/driver.js +18 -0
- package/lib/core/driver.js.map +1 -0
- package/lib/core/emitter.d.ts +20 -0
- package/lib/core/emitter.d.ts.map +1 -0
- package/lib/core/emitter.js +34 -0
- package/lib/core/emitter.js.map +1 -0
- package/lib/core/events.d.ts +202 -0
- package/lib/core/events.d.ts.map +1 -0
- package/lib/core/events.js +107 -0
- package/lib/core/events.js.map +1 -0
- package/lib/core/exec.d.ts +82 -0
- package/lib/core/exec.d.ts.map +1 -0
- package/lib/core/exec.js +168 -0
- package/lib/core/exec.js.map +1 -0
- package/lib/core/experiment.d.ts +285 -0
- package/lib/core/experiment.d.ts.map +1 -0
- package/lib/core/experiment.js +389 -0
- package/lib/core/experiment.js.map +1 -0
- package/lib/core/glob.d.ts +3 -0
- package/lib/core/glob.d.ts.map +1 -0
- package/lib/core/glob.js +32 -0
- package/lib/core/glob.js.map +1 -0
- package/lib/core/hash.d.ts +8 -0
- package/lib/core/hash.d.ts.map +1 -0
- package/lib/core/hash.js +214 -0
- package/lib/core/hash.js.map +1 -0
- package/lib/core/index.d.ts +29 -0
- package/lib/core/index.d.ts.map +1 -0
- package/lib/core/index.js +29 -0
- package/lib/core/index.js.map +1 -0
- package/lib/core/lifecycle-fixtures.d.ts +20 -0
- package/lib/core/lifecycle-fixtures.d.ts.map +1 -0
- package/lib/core/lifecycle-fixtures.js +109 -0
- package/lib/core/lifecycle-fixtures.js.map +1 -0
- package/lib/core/lifecycle-hooks.d.ts +52 -0
- package/lib/core/lifecycle-hooks.d.ts.map +1 -0
- package/lib/core/lifecycle-hooks.js +42 -0
- package/lib/core/lifecycle-hooks.js.map +1 -0
- package/lib/core/mcp-events.d.ts +44 -0
- package/lib/core/mcp-events.d.ts.map +1 -0
- package/lib/core/mcp-events.js +23 -0
- package/lib/core/mcp-events.js.map +1 -0
- package/lib/core/model-cli.d.ts +57 -0
- package/lib/core/model-cli.d.ts.map +1 -0
- package/lib/core/model-cli.js +237 -0
- package/lib/core/model-cli.js.map +1 -0
- package/lib/core/observer.d.ts +31 -0
- package/lib/core/observer.d.ts.map +1 -0
- package/lib/core/observer.js +25 -0
- package/lib/core/observer.js.map +1 -0
- package/lib/core/platform-process.d.ts +33 -0
- package/lib/core/platform-process.d.ts.map +1 -0
- package/lib/core/platform-process.js +214 -0
- package/lib/core/platform-process.js.map +1 -0
- package/lib/core/registry.d.ts +22 -0
- package/lib/core/registry.d.ts.map +1 -0
- package/lib/core/registry.js +45 -0
- package/lib/core/registry.js.map +1 -0
- package/lib/core/rubric.d.ts +18 -0
- package/lib/core/rubric.d.ts.map +1 -0
- package/lib/core/rubric.js +49 -0
- package/lib/core/rubric.js.map +1 -0
- package/lib/core/run.d.ts +419 -0
- package/lib/core/run.d.ts.map +1 -0
- package/lib/core/run.js +126 -0
- package/lib/core/run.js.map +1 -0
- package/lib/core/statistics.d.ts +39 -0
- package/lib/core/statistics.d.ts.map +1 -0
- package/lib/core/statistics.js +163 -0
- package/lib/core/statistics.js.map +1 -0
- package/lib/core/subject.d.ts +4 -0
- package/lib/core/subject.d.ts.map +1 -0
- package/lib/core/subject.js +46 -0
- package/lib/core/subject.js.map +1 -0
- package/lib/core/tool-events.d.ts +69 -0
- package/lib/core/tool-events.d.ts.map +1 -0
- package/lib/core/tool-events.js +242 -0
- package/lib/core/tool-events.js.map +1 -0
- package/lib/core/usage.d.ts +78 -0
- package/lib/core/usage.d.ts.map +1 -0
- package/lib/core/usage.js +104 -0
- package/lib/core/usage.js.map +1 -0
- package/lib/core/wire.d.ts +83 -0
- package/lib/core/wire.d.ts.map +1 -0
- package/lib/core/wire.js +101 -0
- package/lib/core/wire.js.map +1 -0
- package/lib/core/zod.d.ts +10 -0
- package/lib/core/zod.d.ts.map +1 -0
- package/lib/core/zod.js +18 -0
- package/lib/core/zod.js.map +1 -0
- package/lib/drivers/claude-code/index.d.ts +55 -0
- package/lib/drivers/claude-code/index.d.ts.map +1 -0
- package/lib/drivers/claude-code/index.js +779 -0
- package/lib/drivers/claude-code/index.js.map +1 -0
- package/lib/drivers/claude-code/parse-events.d.ts +46 -0
- package/lib/drivers/claude-code/parse-events.d.ts.map +1 -0
- package/lib/drivers/claude-code/parse-events.js +387 -0
- package/lib/drivers/claude-code/parse-events.js.map +1 -0
- package/lib/drivers/codex/app-events.d.ts +16 -0
- package/lib/drivers/codex/app-events.d.ts.map +1 -0
- package/lib/drivers/codex/app-events.js +184 -0
- package/lib/drivers/codex/app-events.js.map +1 -0
- package/lib/drivers/codex/config.d.ts +55 -0
- package/lib/drivers/codex/config.d.ts.map +1 -0
- package/lib/drivers/codex/config.js +125 -0
- package/lib/drivers/codex/config.js.map +1 -0
- package/lib/drivers/codex/index.d.ts +8 -0
- package/lib/drivers/codex/index.d.ts.map +1 -0
- package/lib/drivers/codex/index.js +904 -0
- package/lib/drivers/codex/index.js.map +1 -0
- package/lib/drivers/mcp/index.d.ts +52 -0
- package/lib/drivers/mcp/index.d.ts.map +1 -0
- package/lib/drivers/mcp/index.js +282 -0
- package/lib/drivers/mcp/index.js.map +1 -0
- package/lib/drivers/mcp/transport.d.ts +50 -0
- package/lib/drivers/mcp/transport.d.ts.map +1 -0
- package/lib/drivers/mcp/transport.js +190 -0
- package/lib/drivers/mcp/transport.js.map +1 -0
- package/lib/drivers/shared/artifacts.d.ts +4 -0
- package/lib/drivers/shared/artifacts.d.ts.map +1 -0
- package/lib/drivers/shared/artifacts.js +81 -0
- package/lib/drivers/shared/artifacts.js.map +1 -0
- package/lib/drivers/shared/secure-copy.d.ts +2 -0
- package/lib/drivers/shared/secure-copy.d.ts.map +1 -0
- package/lib/drivers/shared/secure-copy.js +35 -0
- package/lib/drivers/shared/secure-copy.js.map +1 -0
- package/lib/drivers/subprocess/index.d.ts +24 -0
- package/lib/drivers/subprocess/index.d.ts.map +1 -0
- package/lib/drivers/subprocess/index.js +451 -0
- package/lib/drivers/subprocess/index.js.map +1 -0
- package/lib/index.d.ts +40 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +48 -0
- package/lib/index.js.map +1 -0
- package/lib/optimization/reflective.d.ts +27 -0
- package/lib/optimization/reflective.d.ts.map +1 -0
- package/lib/optimization/reflective.js +257 -0
- package/lib/optimization/reflective.js.map +1 -0
- package/lib/orchestrator/annotation-loader.d.ts +65 -0
- package/lib/orchestrator/annotation-loader.d.ts.map +1 -0
- package/lib/orchestrator/annotation-loader.js +73 -0
- package/lib/orchestrator/annotation-loader.js.map +1 -0
- package/lib/orchestrator/annotation-store.d.ts +3 -0
- package/lib/orchestrator/annotation-store.d.ts.map +1 -0
- package/lib/orchestrator/annotation-store.js +100 -0
- package/lib/orchestrator/annotation-store.js.map +1 -0
- package/lib/orchestrator/authored-dependencies.d.ts +3 -0
- package/lib/orchestrator/authored-dependencies.d.ts.map +1 -0
- package/lib/orchestrator/authored-dependencies.js +146 -0
- package/lib/orchestrator/authored-dependencies.js.map +1 -0
- package/lib/orchestrator/campaign-lifecycle.d.ts +105 -0
- package/lib/orchestrator/campaign-lifecycle.d.ts.map +1 -0
- package/lib/orchestrator/campaign-lifecycle.js +141 -0
- package/lib/orchestrator/campaign-lifecycle.js.map +1 -0
- package/lib/orchestrator/candidate-integrity.d.ts +4 -0
- package/lib/orchestrator/candidate-integrity.d.ts.map +1 -0
- package/lib/orchestrator/candidate-integrity.js +14 -0
- package/lib/orchestrator/candidate-integrity.js.map +1 -0
- package/lib/orchestrator/candidates.d.ts +13 -0
- package/lib/orchestrator/candidates.d.ts.map +1 -0
- package/lib/orchestrator/candidates.js +577 -0
- package/lib/orchestrator/candidates.js.map +1 -0
- package/lib/orchestrator/compare.d.ts +80 -0
- package/lib/orchestrator/compare.d.ts.map +1 -0
- package/lib/orchestrator/compare.js +270 -0
- package/lib/orchestrator/compare.js.map +1 -0
- package/lib/orchestrator/comparison-identity.d.ts +10 -0
- package/lib/orchestrator/comparison-identity.d.ts.map +1 -0
- package/lib/orchestrator/comparison-identity.js +39 -0
- package/lib/orchestrator/comparison-identity.js.map +1 -0
- package/lib/orchestrator/config.d.ts +121 -0
- package/lib/orchestrator/config.d.ts.map +1 -0
- package/lib/orchestrator/config.js +176 -0
- package/lib/orchestrator/config.js.map +1 -0
- package/lib/orchestrator/content-bound-json.d.ts +5 -0
- package/lib/orchestrator/content-bound-json.d.ts.map +1 -0
- package/lib/orchestrator/content-bound-json.js +59 -0
- package/lib/orchestrator/content-bound-json.js.map +1 -0
- package/lib/orchestrator/discover.d.ts +45 -0
- package/lib/orchestrator/discover.d.ts.map +1 -0
- package/lib/orchestrator/discover.js +98 -0
- package/lib/orchestrator/discover.js.map +1 -0
- package/lib/orchestrator/doctor.d.ts +141 -0
- package/lib/orchestrator/doctor.d.ts.map +1 -0
- package/lib/orchestrator/doctor.js +525 -0
- package/lib/orchestrator/doctor.js.map +1 -0
- package/lib/orchestrator/driver-session.d.ts +177 -0
- package/lib/orchestrator/driver-session.d.ts.map +1 -0
- package/lib/orchestrator/driver-session.js +396 -0
- package/lib/orchestrator/driver-session.js.map +1 -0
- package/lib/orchestrator/evaluator-identity.d.ts +7 -0
- package/lib/orchestrator/evaluator-identity.d.ts.map +1 -0
- package/lib/orchestrator/evaluator-identity.js +42 -0
- package/lib/orchestrator/evaluator-identity.js.map +1 -0
- package/lib/orchestrator/experiment-budget.d.ts +11 -0
- package/lib/orchestrator/experiment-budget.d.ts.map +1 -0
- package/lib/orchestrator/experiment-budget.js +51 -0
- package/lib/orchestrator/experiment-budget.js.map +1 -0
- package/lib/orchestrator/experiment-context.d.ts +43 -0
- package/lib/orchestrator/experiment-context.d.ts.map +1 -0
- package/lib/orchestrator/experiment-context.js +9 -0
- package/lib/orchestrator/experiment-context.js.map +1 -0
- package/lib/orchestrator/experiment-contracts.d.ts +290 -0
- package/lib/orchestrator/experiment-contracts.d.ts.map +1 -0
- package/lib/orchestrator/experiment-contracts.js +38 -0
- package/lib/orchestrator/experiment-contracts.js.map +1 -0
- package/lib/orchestrator/experiment-corpus.d.ts +21 -0
- package/lib/orchestrator/experiment-corpus.d.ts.map +1 -0
- package/lib/orchestrator/experiment-corpus.js +276 -0
- package/lib/orchestrator/experiment-corpus.js.map +1 -0
- package/lib/orchestrator/experiment-evaluation.d.ts +44 -0
- package/lib/orchestrator/experiment-evaluation.d.ts.map +1 -0
- package/lib/orchestrator/experiment-evaluation.js +542 -0
- package/lib/orchestrator/experiment-evaluation.js.map +1 -0
- package/lib/orchestrator/experiment-finish.d.ts +6 -0
- package/lib/orchestrator/experiment-finish.d.ts.map +1 -0
- package/lib/orchestrator/experiment-finish.js +19 -0
- package/lib/orchestrator/experiment-finish.js.map +1 -0
- package/lib/orchestrator/experiment-identity.d.ts +6 -0
- package/lib/orchestrator/experiment-identity.d.ts.map +1 -0
- package/lib/orchestrator/experiment-identity.js +257 -0
- package/lib/orchestrator/experiment-identity.js.map +1 -0
- package/lib/orchestrator/experiment-loader.d.ts +175 -0
- package/lib/orchestrator/experiment-loader.d.ts.map +1 -0
- package/lib/orchestrator/experiment-loader.js +77 -0
- package/lib/orchestrator/experiment-loader.js.map +1 -0
- package/lib/orchestrator/experiment-report.d.ts +450 -0
- package/lib/orchestrator/experiment-report.d.ts.map +1 -0
- package/lib/orchestrator/experiment-report.js +570 -0
- package/lib/orchestrator/experiment-report.js.map +1 -0
- package/lib/orchestrator/experiment-runs.d.ts +379 -0
- package/lib/orchestrator/experiment-runs.d.ts.map +1 -0
- package/lib/orchestrator/experiment-runs.js +796 -0
- package/lib/orchestrator/experiment-runs.js.map +1 -0
- package/lib/orchestrator/experiment-runtime-identity.d.ts +41 -0
- package/lib/orchestrator/experiment-runtime-identity.d.ts.map +1 -0
- package/lib/orchestrator/experiment-runtime-identity.js +411 -0
- package/lib/orchestrator/experiment-runtime-identity.js.map +1 -0
- package/lib/orchestrator/experiment-selection.d.ts +25 -0
- package/lib/orchestrator/experiment-selection.d.ts.map +1 -0
- package/lib/orchestrator/experiment-selection.js +286 -0
- package/lib/orchestrator/experiment-selection.js.map +1 -0
- package/lib/orchestrator/experiment.d.ts +11 -0
- package/lib/orchestrator/experiment.d.ts.map +1 -0
- package/lib/orchestrator/experiment.js +639 -0
- package/lib/orchestrator/experiment.js.map +1 -0
- package/lib/orchestrator/fixture-identity.d.ts +7 -0
- package/lib/orchestrator/fixture-identity.d.ts.map +1 -0
- package/lib/orchestrator/fixture-identity.js +55 -0
- package/lib/orchestrator/fixture-identity.js.map +1 -0
- package/lib/orchestrator/fs-read.d.ts +19 -0
- package/lib/orchestrator/fs-read.d.ts.map +1 -0
- package/lib/orchestrator/fs-read.js +113 -0
- package/lib/orchestrator/fs-read.js.map +1 -0
- package/lib/orchestrator/grader-experiment.d.ts +190 -0
- package/lib/orchestrator/grader-experiment.d.ts.map +1 -0
- package/lib/orchestrator/grader-experiment.js +579 -0
- package/lib/orchestrator/grader-experiment.js.map +1 -0
- package/lib/orchestrator/grading.d.ts +31 -0
- package/lib/orchestrator/grading.d.ts.map +1 -0
- package/lib/orchestrator/grading.js +186 -0
- package/lib/orchestrator/grading.js.map +1 -0
- package/lib/orchestrator/index.d.ts +22 -0
- package/lib/orchestrator/index.d.ts.map +1 -0
- package/lib/orchestrator/index.js +30 -0
- package/lib/orchestrator/index.js.map +1 -0
- package/lib/orchestrator/lifecycle-hook-execution.d.ts +4 -0
- package/lib/orchestrator/lifecycle-hook-execution.d.ts.map +1 -0
- package/lib/orchestrator/lifecycle-hook-execution.js +67 -0
- package/lib/orchestrator/lifecycle-hook-execution.js.map +1 -0
- package/lib/orchestrator/list-runs.d.ts +33 -0
- package/lib/orchestrator/list-runs.d.ts.map +1 -0
- package/lib/orchestrator/list-runs.js +308 -0
- package/lib/orchestrator/list-runs.js.map +1 -0
- package/lib/orchestrator/loader.d.ts +3 -0
- package/lib/orchestrator/loader.d.ts.map +1 -0
- package/lib/orchestrator/loader.js +17 -0
- package/lib/orchestrator/loader.js.map +1 -0
- package/lib/orchestrator/optimization-lifecycle.d.ts +12 -0
- package/lib/orchestrator/optimization-lifecycle.d.ts.map +1 -0
- package/lib/orchestrator/optimization-lifecycle.js +92 -0
- package/lib/orchestrator/optimization-lifecycle.js.map +1 -0
- package/lib/orchestrator/orchestrator.d.ts +144 -0
- package/lib/orchestrator/orchestrator.d.ts.map +1 -0
- package/lib/orchestrator/orchestrator.js +632 -0
- package/lib/orchestrator/orchestrator.js.map +1 -0
- package/lib/orchestrator/owned-lock.d.ts +17 -0
- package/lib/orchestrator/owned-lock.d.ts.map +1 -0
- package/lib/orchestrator/owned-lock.js +149 -0
- package/lib/orchestrator/owned-lock.js.map +1 -0
- package/lib/orchestrator/package-version.d.ts +2 -0
- package/lib/orchestrator/package-version.d.ts.map +1 -0
- package/lib/orchestrator/package-version.js +4 -0
- package/lib/orchestrator/package-version.js.map +1 -0
- package/lib/orchestrator/provenance.d.ts +5 -0
- package/lib/orchestrator/provenance.d.ts.map +1 -0
- package/lib/orchestrator/provenance.js +137 -0
- package/lib/orchestrator/provenance.js.map +1 -0
- package/lib/orchestrator/regrade.d.ts +80 -0
- package/lib/orchestrator/regrade.d.ts.map +1 -0
- package/lib/orchestrator/regrade.js +433 -0
- package/lib/orchestrator/regrade.js.map +1 -0
- package/lib/orchestrator/rubric-loader.d.ts +7 -0
- package/lib/orchestrator/rubric-loader.d.ts.map +1 -0
- package/lib/orchestrator/rubric-loader.js +65 -0
- package/lib/orchestrator/rubric-loader.js.map +1 -0
- package/lib/orchestrator/run-dir.d.ts +40 -0
- package/lib/orchestrator/run-dir.d.ts.map +1 -0
- package/lib/orchestrator/run-dir.js +161 -0
- package/lib/orchestrator/run-dir.js.map +1 -0
- package/lib/orchestrator/run-execution.d.ts +30 -0
- package/lib/orchestrator/run-execution.d.ts.map +1 -0
- package/lib/orchestrator/run-execution.js +531 -0
- package/lib/orchestrator/run-execution.js.map +1 -0
- package/lib/orchestrator/run-lifecycle.d.ts +115 -0
- package/lib/orchestrator/run-lifecycle.d.ts.map +1 -0
- package/lib/orchestrator/run-lifecycle.js +136 -0
- package/lib/orchestrator/run-lifecycle.js.map +1 -0
- package/lib/orchestrator/suite-compare.d.ts +76 -0
- package/lib/orchestrator/suite-compare.d.ts.map +1 -0
- package/lib/orchestrator/suite-compare.js +159 -0
- package/lib/orchestrator/suite-compare.js.map +1 -0
- package/lib/orchestrator/suite-lifecycle.d.ts +157 -0
- package/lib/orchestrator/suite-lifecycle.d.ts.map +1 -0
- package/lib/orchestrator/suite-lifecycle.js +253 -0
- package/lib/orchestrator/suite-lifecycle.js.map +1 -0
- package/lib/orchestrator/suite-runs.d.ts +91 -0
- package/lib/orchestrator/suite-runs.d.ts.map +1 -0
- package/lib/orchestrator/suite-runs.js +157 -0
- package/lib/orchestrator/suite-runs.js.map +1 -0
- package/lib/orchestrator/suite-summary.d.ts +109 -0
- package/lib/orchestrator/suite-summary.d.ts.map +1 -0
- package/lib/orchestrator/suite-summary.js +213 -0
- package/lib/orchestrator/suite-summary.js.map +1 -0
- package/lib/orchestrator/trace-limits.d.ts +20 -0
- package/lib/orchestrator/trace-limits.d.ts.map +1 -0
- package/lib/orchestrator/trace-limits.js +28 -0
- package/lib/orchestrator/trace-limits.js.map +1 -0
- package/lib/orchestrator/validate-plugin-config.d.ts +10 -0
- package/lib/orchestrator/validate-plugin-config.d.ts.map +1 -0
- package/lib/orchestrator/validate-plugin-config.js +26 -0
- package/lib/orchestrator/validate-plugin-config.js.map +1 -0
- package/package.json +84 -0
- package/skills/lux/SKILL.md +85 -0
- package/skills/lux/references/cli.md +102 -0
- package/skills/lux/references/eval-authoring.md +105 -0
- package/skills/lux-answerer/SKILL.md +169 -0
- package/src/answerers/claude-code/index.ts +342 -0
- package/src/answerers/persona/index.ts +327 -0
- package/src/answerers/scripted/index.ts +186 -0
- package/src/answerers/shared.ts +23 -0
- package/src/answerers/terminal/index.ts +117 -0
- package/src/assertions/core/command-exits-zero.ts +94 -0
- package/src/assertions/core/event-checks.ts +138 -0
- package/src/assertions/core/file-checks.ts +336 -0
- package/src/assertions/core/helpers.ts +203 -0
- package/src/assertions/core/index.ts +80 -0
- package/src/assertions/core/json-pointer-equals.ts +84 -0
- package/src/assertions/core/output-checks.ts +134 -0
- package/src/assertions/core/predicate.ts +67 -0
- package/src/assertions/core/resource-checks.ts +180 -0
- package/src/assertions/core/safe-regex.ts +196 -0
- package/src/assertions/core/tool-called-with.ts +68 -0
- package/src/assertions/core/tool-calls.ts +28 -0
- package/src/assertions/core/tool-invocation.ts +198 -0
- package/src/assertions/core/tool-result-checks.ts +211 -0
- package/src/assertions/mcp/discovery-checks.ts +139 -0
- package/src/assertions/mcp/index.ts +37 -0
- package/src/assertions/rubric/index.ts +258 -0
- package/src/assertions/rubric/internal.ts +367 -0
- package/src/authoring.ts +2 -0
- package/src/cli/bin.ts +14 -0
- package/src/cli/command-runtime.ts +34 -0
- package/src/cli/init.ts +410 -0
- package/src/cli/program.ts +992 -0
- package/src/cli/render/format.ts +69 -0
- package/src/cli/render/reporter.ts +643 -0
- package/src/cli/render/tty.ts +314 -0
- package/src/cli/run-options.ts +82 -0
- package/src/cli/skills.ts +129 -0
- package/src/core/annotation.ts +104 -0
- package/src/core/answerer.ts +75 -0
- package/src/core/artifact-evidence.ts +249 -0
- package/src/core/artifact-snapshot.ts +429 -0
- package/src/core/assertion.ts +156 -0
- package/src/core/case.ts +103 -0
- package/src/core/conversation-turn.ts +47 -0
- package/src/core/driver.ts +75 -0
- package/src/core/emitter.ts +38 -0
- package/src/core/events.ts +135 -0
- package/src/core/exec.ts +263 -0
- package/src/core/experiment.ts +467 -0
- package/src/core/glob.ts +31 -0
- package/src/core/hash.ts +237 -0
- package/src/core/index.ts +28 -0
- package/src/core/lifecycle-fixtures.ts +145 -0
- package/src/core/lifecycle-hooks.ts +101 -0
- package/src/core/mcp-events.ts +57 -0
- package/src/core/model-cli.ts +271 -0
- package/src/core/observer.ts +50 -0
- package/src/core/platform-process.ts +284 -0
- package/src/core/registry.ts +57 -0
- package/src/core/rubric.ts +52 -0
- package/src/core/run.ts +144 -0
- package/src/core/statistics.ts +194 -0
- package/src/core/subject.ts +50 -0
- package/src/core/tool-events.ts +292 -0
- package/src/core/usage.ts +123 -0
- package/src/core/wire.ts +135 -0
- package/src/core/zod.ts +20 -0
- package/src/drivers/claude-code/index.ts +866 -0
- package/src/drivers/claude-code/parse-events.ts +464 -0
- package/src/drivers/codex/app-events.ts +218 -0
- package/src/drivers/codex/config.ts +128 -0
- package/src/drivers/codex/index.ts +995 -0
- package/src/drivers/mcp/index.ts +351 -0
- package/src/drivers/mcp/transport.ts +222 -0
- package/src/drivers/shared/artifacts.ts +86 -0
- package/src/drivers/shared/secure-copy.ts +40 -0
- package/src/drivers/subprocess/index.ts +490 -0
- package/src/index.ts +594 -0
- package/src/optimization/reflective.ts +319 -0
- package/src/orchestrator/annotation-loader.ts +97 -0
- package/src/orchestrator/annotation-store.ts +112 -0
- package/src/orchestrator/authored-dependencies.ts +193 -0
- package/src/orchestrator/campaign-lifecycle.ts +181 -0
- package/src/orchestrator/candidate-integrity.ts +18 -0
- package/src/orchestrator/candidates.ts +683 -0
- package/src/orchestrator/compare.ts +358 -0
- package/src/orchestrator/comparison-identity.ts +45 -0
- package/src/orchestrator/config.ts +205 -0
- package/src/orchestrator/content-bound-json.ts +71 -0
- package/src/orchestrator/discover.ts +132 -0
- package/src/orchestrator/doctor.ts +781 -0
- package/src/orchestrator/driver-session.ts +507 -0
- package/src/orchestrator/evaluator-identity.ts +45 -0
- package/src/orchestrator/experiment-budget.ts +61 -0
- package/src/orchestrator/experiment-context.ts +41 -0
- package/src/orchestrator/experiment-contracts.ts +43 -0
- package/src/orchestrator/experiment-corpus.ts +416 -0
- package/src/orchestrator/experiment-evaluation.ts +749 -0
- package/src/orchestrator/experiment-finish.ts +33 -0
- package/src/orchestrator/experiment-identity.ts +356 -0
- package/src/orchestrator/experiment-loader.ts +105 -0
- package/src/orchestrator/experiment-report.ts +788 -0
- package/src/orchestrator/experiment-runs.ts +960 -0
- package/src/orchestrator/experiment-runtime-identity.ts +465 -0
- package/src/orchestrator/experiment-selection.ts +467 -0
- package/src/orchestrator/experiment.ts +991 -0
- package/src/orchestrator/fixture-identity.ts +67 -0
- package/src/orchestrator/fs-read.ts +126 -0
- package/src/orchestrator/grader-experiment.ts +856 -0
- package/src/orchestrator/grading.ts +260 -0
- package/src/orchestrator/index.ts +186 -0
- package/src/orchestrator/lifecycle-hook-execution.ts +90 -0
- package/src/orchestrator/list-runs.ts +384 -0
- package/src/orchestrator/loader.ts +19 -0
- package/src/orchestrator/optimization-lifecycle.ts +126 -0
- package/src/orchestrator/orchestrator.ts +919 -0
- package/src/orchestrator/owned-lock.ts +147 -0
- package/src/orchestrator/package-version.ts +5 -0
- package/src/orchestrator/provenance.ts +140 -0
- package/src/orchestrator/regrade.ts +580 -0
- package/src/orchestrator/rubric-loader.ts +74 -0
- package/src/orchestrator/run-dir.ts +209 -0
- package/src/orchestrator/run-execution.ts +694 -0
- package/src/orchestrator/run-lifecycle.ts +174 -0
- package/src/orchestrator/suite-compare.ts +195 -0
- package/src/orchestrator/suite-lifecycle.ts +335 -0
- package/src/orchestrator/suite-runs.ts +202 -0
- package/src/orchestrator/suite-summary.ts +276 -0
- package/src/orchestrator/trace-limits.ts +65 -0
- package/src/orchestrator/validate-plugin-config.ts +34 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { type GradingReport, type PersistedCase, type RunMetadata, type RunUsage } from "../core/index.js";
|
|
2
|
+
export type ComparisonInput = {
|
|
3
|
+
pathA: string;
|
|
4
|
+
pathB: string;
|
|
5
|
+
};
|
|
6
|
+
export type RunBundle = {
|
|
7
|
+
path: string;
|
|
8
|
+
caseJson: PersistedCase | null;
|
|
9
|
+
runJson: RunMetadata | null;
|
|
10
|
+
grading: GradingReport | null;
|
|
11
|
+
/** Relative artifact path → sha256 of its bytes. */
|
|
12
|
+
fileTree: Map<string, string>;
|
|
13
|
+
};
|
|
14
|
+
export type GradingDelta = {
|
|
15
|
+
id: string;
|
|
16
|
+
type: string;
|
|
17
|
+
name: string;
|
|
18
|
+
before: {
|
|
19
|
+
passed: boolean;
|
|
20
|
+
score: number;
|
|
21
|
+
} | null;
|
|
22
|
+
after: {
|
|
23
|
+
passed: boolean;
|
|
24
|
+
score: number;
|
|
25
|
+
} | null;
|
|
26
|
+
status: "added" | "removed" | "improved" | "regressed" | "unchanged";
|
|
27
|
+
};
|
|
28
|
+
export type RunComparison = {
|
|
29
|
+
pathA: string;
|
|
30
|
+
pathB: string;
|
|
31
|
+
caseMismatch: {
|
|
32
|
+
field: string;
|
|
33
|
+
a: unknown;
|
|
34
|
+
b: unknown;
|
|
35
|
+
}[];
|
|
36
|
+
runDelta: {
|
|
37
|
+
exitReason: {
|
|
38
|
+
a: string;
|
|
39
|
+
b: string;
|
|
40
|
+
};
|
|
41
|
+
durationMsA: number | null;
|
|
42
|
+
durationMsB: number | null;
|
|
43
|
+
interruptCounts: {
|
|
44
|
+
a: {
|
|
45
|
+
ask: number;
|
|
46
|
+
approve: number;
|
|
47
|
+
custom: number;
|
|
48
|
+
};
|
|
49
|
+
b: {
|
|
50
|
+
ask: number;
|
|
51
|
+
approve: number;
|
|
52
|
+
custom: number;
|
|
53
|
+
};
|
|
54
|
+
};
|
|
55
|
+
usage: {
|
|
56
|
+
a: RunUsage;
|
|
57
|
+
b: RunUsage;
|
|
58
|
+
};
|
|
59
|
+
} | null;
|
|
60
|
+
grading: {
|
|
61
|
+
casePassedA: boolean | null;
|
|
62
|
+
casePassedB: boolean | null;
|
|
63
|
+
caseScoreA: number | null;
|
|
64
|
+
caseScoreB: number | null;
|
|
65
|
+
deltas: GradingDelta[];
|
|
66
|
+
improvements: number;
|
|
67
|
+
regressions: number;
|
|
68
|
+
};
|
|
69
|
+
artifactsDelta: {
|
|
70
|
+
added: string[];
|
|
71
|
+
removed: string[];
|
|
72
|
+
changed: string[];
|
|
73
|
+
unchanged: string[];
|
|
74
|
+
};
|
|
75
|
+
};
|
|
76
|
+
export declare const loadRunBundle: (path: string) => Promise<RunBundle>;
|
|
77
|
+
export declare const compareRuns: (input: ComparisonInput) => Promise<RunComparison>;
|
|
78
|
+
export declare const hasRegression: (cmp: RunComparison) => boolean;
|
|
79
|
+
export declare const formatComparisonText: (cmp: RunComparison) => string;
|
|
80
|
+
//# sourceMappingURL=compare.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/orchestrator/compare.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,aAAa,EAGlB,KAAK,aAAa,EAElB,KAAK,WAAW,EAEhB,KAAK,QAAQ,EAGd,MAAM,kBAAkB,CAAC;AAI1B,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,SAAS,GAAG;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IAC/B,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,oDAAoD;IACpD,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,YAAY,GAAG;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAClD,KAAK,EAAE;QAAE,MAAM,EAAE,OAAO,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACjD,MAAM,EAAE,OAAO,GAAG,SAAS,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,CAAC;CACtE,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,OAAO,CAAC;QAAC,CAAC,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;IAC1D,QAAQ,EAAE;QACR,UAAU,EAAE;YAAE,CAAC,EAAE,MAAM,CAAC;YAAC,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACrC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;QAC3B,eAAe,EAAE;YACf,CAAC,EAAE;gBAAE,GAAG,EAAE,MAAM,CAAC;gBAAC,OAAO,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;YACpD,CAAC,EAAE;gBAAE,GAAG,EAAE,MAAM,CAAC;gBAAC,OAAO,EAAE,MAAM,CAAC;gBAAC,MAAM,EAAE,MAAM,CAAA;aAAE,CAAC;SACrD,CAAC;QACF,KAAK,EAAE;YAAE,CAAC,EAAE,QAAQ,CAAC;YAAC,CAAC,EAAE,QAAQ,CAAA;SAAE,CAAC;KACrC,GAAG,IAAI,CAAC;IACT,OAAO,EAAE;QACP,WAAW,EAAE,OAAO,GAAG,IAAI,CAAC;QAC5B,WAAW,EAAE,OAAO,GAAG,IAAI,CAAC;QAC5B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,MAAM,EAAE,YAAY,EAAE,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,cAAc,EAAE;QACd,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,SAAS,EAAE,MAAM,EAAE,CAAC;KACrB,CAAC;CACH,CAAC;AAgBF,eAAO,MAAM,aAAa,SAAgB,MAAM,KAAG,OAAO,CAAC,SAAS,CAMlE,CAAC;AAgGH,eAAO,MAAM,WAAW,UAAiB,eAAe,KAAG,OAAO,CAAC,aAAa,CAoG/E,CAAC;AAMF,eAAO,MAAM,aAAa,QAAS,aAAa,KAAG,OAIqB,CAAC;AAOzE,eAAO,MAAM,oBAAoB,QAAS,aAAa,KAAG,MAmDzD,CAAC"}
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFile } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { emptyRunUsage, GradingReportSchema, harnessUsage, PersistedCaseSchema, RunMetadataSchema, stableJson, totalTokens, } from "../core/index.js";
|
|
5
|
+
import { stableDriverIdentity } from "./comparison-identity.js";
|
|
6
|
+
import { safeReadJson, walkFiles } from "./fs-read.js";
|
|
7
|
+
// Content hashes, not sizes: a same-size edit must still read as changed.
|
|
8
|
+
const hashFileTree = async (root) => {
|
|
9
|
+
const out = new Map();
|
|
10
|
+
for (const rel of await walkFiles(root)) {
|
|
11
|
+
out.set(rel, createHash("sha256")
|
|
12
|
+
.update(await readFile(join(root, rel)))
|
|
13
|
+
.digest("hex"));
|
|
14
|
+
}
|
|
15
|
+
return out;
|
|
16
|
+
};
|
|
17
|
+
export const loadRunBundle = async (path) => ({
|
|
18
|
+
path,
|
|
19
|
+
caseJson: await safeReadJson(join(path, "case.json"), PersistedCaseSchema),
|
|
20
|
+
runJson: await safeReadJson(join(path, "run.json"), RunMetadataSchema),
|
|
21
|
+
grading: await safeReadJson(join(path, "grading.json"), GradingReportSchema),
|
|
22
|
+
fileTree: await hashFileTree(join(path, "artifacts")),
|
|
23
|
+
});
|
|
24
|
+
const durationFromMeta = (m) => {
|
|
25
|
+
if (!m)
|
|
26
|
+
return null;
|
|
27
|
+
const a = Date.parse(m.startedAt);
|
|
28
|
+
const b = Date.parse(m.endedAt);
|
|
29
|
+
if (Number.isNaN(a) || Number.isNaN(b))
|
|
30
|
+
return null;
|
|
31
|
+
return b - a;
|
|
32
|
+
};
|
|
33
|
+
const computeFileDelta = (a, b) => {
|
|
34
|
+
const added = [];
|
|
35
|
+
const removed = [];
|
|
36
|
+
const changed = [];
|
|
37
|
+
const unchanged = [];
|
|
38
|
+
for (const [path, hashB] of b) {
|
|
39
|
+
if (!a.has(path)) {
|
|
40
|
+
added.push(path);
|
|
41
|
+
}
|
|
42
|
+
else if (a.get(path) !== hashB) {
|
|
43
|
+
changed.push(path);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
unchanged.push(path);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
for (const path of a.keys()) {
|
|
50
|
+
if (!b.has(path))
|
|
51
|
+
removed.push(path);
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
added: added.sort(),
|
|
55
|
+
removed: removed.sort(),
|
|
56
|
+
changed: changed.sort(),
|
|
57
|
+
unchanged: unchanged.sort(),
|
|
58
|
+
};
|
|
59
|
+
};
|
|
60
|
+
// Duplicate assertion names would collapse into one map entry and silently
|
|
61
|
+
// drop deltas; later occurrences get an index suffix so each keeps its row.
|
|
62
|
+
const assertionsByIdentity = (report) => {
|
|
63
|
+
const occurrences = new Map();
|
|
64
|
+
const out = new Map();
|
|
65
|
+
for (const x of report?.assertions ?? []) {
|
|
66
|
+
if (x.id) {
|
|
67
|
+
out.set(x.id, x);
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
const n = (occurrences.get(x.name) ?? 0) + 1;
|
|
71
|
+
occurrences.set(x.name, n);
|
|
72
|
+
out.set(n === 1 ? x.name : `${x.name} (#${n})`, x);
|
|
73
|
+
}
|
|
74
|
+
return out;
|
|
75
|
+
};
|
|
76
|
+
const computeGradingDeltas = (a, b) => {
|
|
77
|
+
const aByName = assertionsByIdentity(a);
|
|
78
|
+
const bByName = assertionsByIdentity(b);
|
|
79
|
+
const all = new Set([...aByName.keys(), ...bByName.keys()]);
|
|
80
|
+
const deltas = [];
|
|
81
|
+
let improvements = 0;
|
|
82
|
+
let regressions = 0;
|
|
83
|
+
for (const name of [...all].sort()) {
|
|
84
|
+
const ax = aByName.get(name);
|
|
85
|
+
const bx = bByName.get(name);
|
|
86
|
+
let status;
|
|
87
|
+
if (!ax && bx)
|
|
88
|
+
status = "added";
|
|
89
|
+
else if (ax && !bx)
|
|
90
|
+
status = "removed";
|
|
91
|
+
else if (ax && bx) {
|
|
92
|
+
if (ax.passed === bx.passed && ax.score === bx.score)
|
|
93
|
+
status = "unchanged";
|
|
94
|
+
// A pass/fail flip dominates any score movement.
|
|
95
|
+
else if (ax.passed !== bx.passed)
|
|
96
|
+
status = bx.passed ? "improved" : "regressed";
|
|
97
|
+
else
|
|
98
|
+
status = ax.score < bx.score ? "improved" : "regressed";
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
status = "unchanged";
|
|
102
|
+
}
|
|
103
|
+
if (status === "improved")
|
|
104
|
+
improvements++;
|
|
105
|
+
if (status === "regressed")
|
|
106
|
+
regressions++;
|
|
107
|
+
deltas.push({
|
|
108
|
+
id: name,
|
|
109
|
+
type: ax?.type ?? bx?.type ?? "unknown",
|
|
110
|
+
name: ax?.id || bx?.id ? (ax?.name ?? bx?.name ?? name) : name,
|
|
111
|
+
before: ax ? { passed: ax.passed, score: ax.score } : null,
|
|
112
|
+
after: bx ? { passed: bx.passed, score: bx.score } : null,
|
|
113
|
+
status,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
return { deltas, improvements, regressions };
|
|
117
|
+
};
|
|
118
|
+
export const compareRuns = async (input) => {
|
|
119
|
+
const [a, b] = await Promise.all([loadRunBundle(input.pathA), loadRunBundle(input.pathB)]);
|
|
120
|
+
const caseMismatch = [];
|
|
121
|
+
for (const field of ["id", "prompt", "persona"]) {
|
|
122
|
+
const av = a.caseJson?.[field];
|
|
123
|
+
const bv = b.caseJson?.[field];
|
|
124
|
+
if (av !== bv)
|
|
125
|
+
caseMismatch.push({ field, a: av, b: bv });
|
|
126
|
+
}
|
|
127
|
+
for (const field of ["fixtures", "assertions"]) {
|
|
128
|
+
const av = a.caseJson?.[field];
|
|
129
|
+
const bv = b.caseJson?.[field];
|
|
130
|
+
if (stableJson(av) !== stableJson(bv))
|
|
131
|
+
caseMismatch.push({ field, a: av, b: bv });
|
|
132
|
+
}
|
|
133
|
+
const fixturesHashA = a.runJson?.provenance?.fixturesHash;
|
|
134
|
+
const fixturesHashB = b.runJson?.provenance?.fixturesHash;
|
|
135
|
+
if (fixturesHashA !== fixturesHashB) {
|
|
136
|
+
caseMismatch.push({ field: "fixturesHash", a: fixturesHashA, b: fixturesHashB });
|
|
137
|
+
}
|
|
138
|
+
const assertionsHashA = a.runJson?.provenance?.assertionsHash;
|
|
139
|
+
const assertionsHashB = b.runJson?.provenance?.assertionsHash;
|
|
140
|
+
if (assertionsHashA !== assertionsHashB) {
|
|
141
|
+
caseMismatch.push({ field: "assertionsHash", a: assertionsHashA, b: assertionsHashB });
|
|
142
|
+
}
|
|
143
|
+
for (const field of [
|
|
144
|
+
"caseHash",
|
|
145
|
+
"driverConfigHash",
|
|
146
|
+
"answererConfigHash",
|
|
147
|
+
"evaluatorConfigHash",
|
|
148
|
+
]) {
|
|
149
|
+
const av = a.runJson?.provenance?.[field];
|
|
150
|
+
const bv = b.runJson?.provenance?.[field];
|
|
151
|
+
if (av !== bv)
|
|
152
|
+
caseMismatch.push({ field, a: av, b: bv });
|
|
153
|
+
}
|
|
154
|
+
const runtimeA = a.runJson?.provenance?.runtime;
|
|
155
|
+
const runtimeB = b.runJson?.provenance?.runtime;
|
|
156
|
+
if (stableJson(runtimeA) !== stableJson(runtimeB)) {
|
|
157
|
+
caseMismatch.push({ field: "runtime", a: runtimeA, b: runtimeB });
|
|
158
|
+
}
|
|
159
|
+
const sourceA = a.runJson?.provenance?.source;
|
|
160
|
+
const sourceB = b.runJson?.provenance?.source;
|
|
161
|
+
const sourceHashUnavailable = (sourceA?.git?.dirty === true && !sourceA.git.workingTreeHash) ||
|
|
162
|
+
(sourceB?.git?.dirty === true && !sourceB.git.workingTreeHash);
|
|
163
|
+
if (sourceHashUnavailable) {
|
|
164
|
+
caseMismatch.push({
|
|
165
|
+
field: "sourceWorkingTreeHash",
|
|
166
|
+
a: sourceA?.git?.workingTreeHash,
|
|
167
|
+
b: sourceB?.git?.workingTreeHash,
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
else if (stableJson(sourceA) !== stableJson(sourceB)) {
|
|
171
|
+
caseMismatch.push({ field: "source", a: sourceA, b: sourceB });
|
|
172
|
+
}
|
|
173
|
+
const driverA = a.runJson ? stableDriverIdentity(a.runJson.driver) : undefined;
|
|
174
|
+
const driverB = b.runJson ? stableDriverIdentity(b.runJson.driver) : undefined;
|
|
175
|
+
if (stableJson(driverA) !== stableJson(driverB)) {
|
|
176
|
+
caseMismatch.push({ field: "driver", a: driverA, b: driverB });
|
|
177
|
+
}
|
|
178
|
+
const answererA = a.runJson?.answerer;
|
|
179
|
+
const answererB = b.runJson?.answerer;
|
|
180
|
+
if (stableJson(answererA) !== stableJson(answererB)) {
|
|
181
|
+
caseMismatch.push({ field: "answerer", a: answererA, b: answererB });
|
|
182
|
+
}
|
|
183
|
+
const runDelta = a.runJson || b.runJson
|
|
184
|
+
? {
|
|
185
|
+
exitReason: {
|
|
186
|
+
a: a.runJson?.exitReason ?? "unknown",
|
|
187
|
+
b: b.runJson?.exitReason ?? "unknown",
|
|
188
|
+
},
|
|
189
|
+
durationMsA: durationFromMeta(a.runJson),
|
|
190
|
+
durationMsB: durationFromMeta(b.runJson),
|
|
191
|
+
interruptCounts: {
|
|
192
|
+
a: a.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
|
|
193
|
+
b: b.runJson?.interruptCounts ?? { ask: 0, approve: 0, custom: 0 },
|
|
194
|
+
},
|
|
195
|
+
usage: {
|
|
196
|
+
a: a.runJson?.usage ?? emptyRunUsage(),
|
|
197
|
+
b: b.runJson?.usage ?? emptyRunUsage(),
|
|
198
|
+
},
|
|
199
|
+
}
|
|
200
|
+
: null;
|
|
201
|
+
const grading = computeGradingDeltas(a.grading, b.grading);
|
|
202
|
+
return {
|
|
203
|
+
pathA: input.pathA,
|
|
204
|
+
pathB: input.pathB,
|
|
205
|
+
caseMismatch,
|
|
206
|
+
runDelta,
|
|
207
|
+
grading: {
|
|
208
|
+
casePassedA: a.grading?.casePassed ?? null,
|
|
209
|
+
casePassedB: b.grading?.casePassed ?? null,
|
|
210
|
+
caseScoreA: a.grading?.caseScore ?? null,
|
|
211
|
+
caseScoreB: b.grading?.caseScore ?? null,
|
|
212
|
+
...grading,
|
|
213
|
+
},
|
|
214
|
+
artifactsDelta: computeFileDelta(a.fileTree, b.fileTree),
|
|
215
|
+
};
|
|
216
|
+
};
|
|
217
|
+
// The CI gate. Per-assertion counting alone misses whole-run regressions: a
|
|
218
|
+
// candidate that crashed before grading has every assertion "removed" (0
|
|
219
|
+
// regressed), and one downgraded by `incomplete` keeps all assertions passing
|
|
220
|
+
// — so a case that went from passed to anything-but-passed must also gate.
|
|
221
|
+
export const hasRegression = (cmp) => cmp.caseMismatch.length > 0 ||
|
|
222
|
+
cmp.grading.regressions > 0 ||
|
|
223
|
+
cmp.grading.deltas.some((delta) => delta.status === "removed") ||
|
|
224
|
+
(cmp.grading.casePassedA === true && cmp.grading.casePassedB !== true);
|
|
225
|
+
const formatMismatchValue = (value) => {
|
|
226
|
+
const rendered = JSON.stringify(value) ?? String(value);
|
|
227
|
+
return rendered.length > 500 ? `${rendered.slice(0, 499)}…` : rendered;
|
|
228
|
+
};
|
|
229
|
+
export const formatComparisonText = (cmp) => {
|
|
230
|
+
const lines = [];
|
|
231
|
+
lines.push(`A: ${cmp.pathA}`);
|
|
232
|
+
lines.push(`B: ${cmp.pathB}`);
|
|
233
|
+
lines.push("");
|
|
234
|
+
if (cmp.caseMismatch.length > 0) {
|
|
235
|
+
lines.push("Case mismatch:");
|
|
236
|
+
for (const mismatch of cmp.caseMismatch) {
|
|
237
|
+
lines.push(` ${mismatch.field}: A=${formatMismatchValue(mismatch.a)} | B=${formatMismatchValue(mismatch.b)}`);
|
|
238
|
+
}
|
|
239
|
+
lines.push("");
|
|
240
|
+
}
|
|
241
|
+
if (cmp.runDelta) {
|
|
242
|
+
lines.push("Run delta:");
|
|
243
|
+
lines.push(` exit: ${cmp.runDelta.exitReason.a} → ${cmp.runDelta.exitReason.b}`);
|
|
244
|
+
lines.push(` duration ms: ${cmp.runDelta.durationMsA ?? "?"} → ${cmp.runDelta.durationMsB ?? "?"}`);
|
|
245
|
+
const ai = cmp.runDelta.interruptCounts.a;
|
|
246
|
+
const bi = cmp.runDelta.interruptCounts.b;
|
|
247
|
+
lines.push(` interrupts: ask=${ai.ask}/approve=${ai.approve}/custom=${ai.custom} → ` +
|
|
248
|
+
`ask=${bi.ask}/approve=${bi.approve}/custom=${bi.custom}`);
|
|
249
|
+
const au = cmp.runDelta.usage.a;
|
|
250
|
+
const bu = cmp.runDelta.usage.b;
|
|
251
|
+
lines.push(` agent tokens: ${totalTokens(au.agent)} → ${totalTokens(bu.agent)}`);
|
|
252
|
+
lines.push(` harness tokens: ${totalTokens(harnessUsage(au))} → ${totalTokens(harnessUsage(bu))}`);
|
|
253
|
+
lines.push("");
|
|
254
|
+
}
|
|
255
|
+
lines.push("Grading:");
|
|
256
|
+
lines.push(` passed: ${String(cmp.grading.casePassedA)} → ${String(cmp.grading.casePassedB)}; ` +
|
|
257
|
+
`score: ${cmp.grading.caseScoreA ?? "?"} → ${cmp.grading.caseScoreB ?? "?"}`);
|
|
258
|
+
lines.push(` ${cmp.grading.improvements} improved, ${cmp.grading.regressions} regressed`);
|
|
259
|
+
for (const d of cmp.grading.deltas) {
|
|
260
|
+
if (d.status === "unchanged")
|
|
261
|
+
continue;
|
|
262
|
+
const arrow = { added: "+", removed: "-", improved: "↑", regressed: "↓" }[d.status];
|
|
263
|
+
lines.push(` ${arrow} ${d.name}`);
|
|
264
|
+
}
|
|
265
|
+
lines.push("");
|
|
266
|
+
lines.push("Artifacts:");
|
|
267
|
+
lines.push(` +${cmp.artifactsDelta.added.length} -${cmp.artifactsDelta.removed.length} ~${cmp.artifactsDelta.changed.length}`);
|
|
268
|
+
return lines.join("\n");
|
|
269
|
+
};
|
|
270
|
+
//# sourceMappingURL=compare.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compare.js","sourceRoot":"","sources":["../../src/orchestrator/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EACL,aAAa,EAEb,mBAAmB,EACnB,YAAY,EAEZ,mBAAmB,EAEnB,iBAAiB,EAEjB,UAAU,EACV,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAwDvD,0EAA0E;AAC1E,MAAM,YAAY,GAAG,KAAK,EAAE,IAAY,EAAgC,EAAE;IACxE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,KAAK,MAAM,GAAG,IAAI,MAAM,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,GAAG,CAAC,GAAG,CACL,GAAG,EACH,UAAU,CAAC,QAAQ,CAAC;aACjB,MAAM,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;aACvC,MAAM,CAAC,KAAK,CAAC,CACjB,CAAC;IACJ,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAAE,IAAY,EAAsB,EAAE,CAAC,CAAC;IACxE,IAAI;IACJ,QAAQ,EAAE,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,mBAAmB,CAAC;IAC1E,OAAO,EAAE,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,iBAAiB,CAAC;IACtE,OAAO,EAAE,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,mBAAmB,CAAC;IAC5E,QAAQ,EAAE,MAAM,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;CACtD,CAAC,CAAC;AAEH,MAAM,gBAAgB,GAAG,CAAC,CAAqB,EAAiB,EAAE;IAChE,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAClC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAChC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,CACvB,CAAsB,EACtB,CAAsB,EACW,EAAE;IACnC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;aAAM,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC;YACjC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IACD,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;QACnB,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;QACvB,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE;QACvB,SAAS,EAAE,SAAS,CAAC,IAAI,EAAE;KAC5B,CAAC;AACJ,CAAC,CAAC;AAEF,2EAA2E;AAC3E,4EAA4E;AAC5E,MAAM,oBAAoB,GAAG,CAC3B,MAA4B,EACsB,EAAE;IACpD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,MAAM,GAAG,GAAG,IAAI,GAAG,EAA+C,CAAC;IACnE,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,UAAU,IAAI,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;YACT,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YACjB,SAAS;QACX,CAAC;QACD,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7C,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAC3B,CAAuB,EACvB,CAAuB,EACgD,EAAE;IACzE,MAAM,OAAO,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC;IAExC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,EAAE,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,MAA8B,CAAC;QACnC,IAAI,CAAC,EAAE,IAAI,EAAE;YAAE,MAAM,GAAG,OAAO,CAAC;aAC3B,IAAI,EAAE,IAAI,CAAC,EAAE;YAAE,MAAM,GAAG,SAAS,CAAC;aAClC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;YAClB,IAAI,EAAE,CAAC,MAAM,KAAK,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC,KAAK;gBAAE,MAAM,GAAG,WAAW,CAAC;YAC3E,iDAAiD;iBAC5C,IAAI,EAAE,CAAC,MAAM,KAAK,EAAE,CAAC,MAAM;gBAAE,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC;;gBAC3E,MAAM,GAAG,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,WAAW,CAAC;QACvB,CAAC;QACD,IAAI,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC1C,IAAI,MAAM,KAAK,WAAW;YAAE,WAAW,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,IAAI;YACR,IAAI,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,EAAE,IAAI,IAAI,SAAS;YACvC,IAAI,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,IAAI,EAAE,EAAE,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;YAC9D,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YAC1D,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YACzD,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC;AAC/C,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,WAAW,GAAG,KAAK,EAAE,KAAsB,EAA0B,EAAE;IAClF,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAE3F,MAAM,YAAY,GAAgD,EAAE,CAAC;IACrE,KAAK,MAAM,KAAK,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAU,EAAE,CAAC;QACzD,MAAM,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,EAAE,KAAK,EAAE;YAAE,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,YAAY,CAAU,EAAE,CAAC;QACxD,MAAM,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC;QAC/B,MAAM,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,UAAU,CAAC,EAAE,CAAC,KAAK,UAAU,CAAC,EAAE,CAAC;YAAE,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,aAAa,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,YAAY,CAAC;IAC1D,MAAM,aAAa,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,YAAY,CAAC;IAC1D,IAAI,aAAa,KAAK,aAAa,EAAE,CAAC;QACpC,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC;IACnF,CAAC;IACD,MAAM,eAAe,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,cAAc,CAAC;IAC9D,MAAM,eAAe,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,cAAc,CAAC;IAC9D,IAAI,eAAe,KAAK,eAAe,EAAE,CAAC;QACxC,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,KAAK,MAAM,KAAK,IAAI;QAClB,UAAU;QACV,kBAAkB;QAClB,oBAAoB;QACpB,qBAAqB;KACb,EAAE,CAAC;QACX,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC;QAC1C,MAAM,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,EAAE,KAAK,EAAE;YAAE,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC;IAChD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC;IAChD,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClD,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC;IAC9C,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC;IAC9C,MAAM,qBAAqB,GACzB,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;QAC9D,CAAC,OAAO,EAAE,GAAG,EAAE,KAAK,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACjE,IAAI,qBAAqB,EAAE,CAAC;QAC1B,YAAY,CAAC,IAAI,CAAC;YAChB,KAAK,EAAE,uBAAuB;YAC9B,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,eAAe;YAChC,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,eAAe;SACjC,CAAC,CAAC;IACL,CAAC;SAAM,IAAI,UAAU,CAAC,OAAO,CAAC,KAAK,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACvD,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/E,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC/E,IAAI,UAAU,CAAC,OAAO,CAAC,KAAK,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QAChD,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC;IACtC,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC;IACtC,IAAI,UAAU,CAAC,SAAS,CAAC,KAAK,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QACpD,YAAY,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,QAAQ,GACZ,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO;QACpB,CAAC,CAAC;YACE,UAAU,EAAE;gBACV,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,UAAU,IAAI,SAAS;gBACrC,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,UAAU,IAAI,SAAS;aACtC;YACD,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC;YACxC,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC;YACxC,eAAe,EAAE;gBACf,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,eAAe,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;gBAClE,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,eAAe,IAAI,EAAE,GAAG,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE;aACnE;YACD,KAAK,EAAE;gBACL,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,IAAI,aAAa,EAAE;gBACtC,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,KAAK,IAAI,aAAa,EAAE;aACvC;SACF;QACH,CAAC,CAAC,IAAI,CAAC;IAEX,MAAM,OAAO,GAAG,oBAAoB,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;IAE3D,OAAO;QACL,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,YAAY;QACZ,QAAQ;QACR,OAAO,EAAE;YACP,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,UAAU,IAAI,IAAI;YAC1C,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE,UAAU,IAAI,IAAI;YAC1C,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,IAAI,IAAI;YACxC,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,SAAS,IAAI,IAAI;YACxC,GAAG,OAAO;SACX;QACD,cAAc,EAAE,gBAAgB,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC;KACzD,CAAC;AACJ,CAAC,CAAC;AAEF,4EAA4E;AAC5E,yEAAyE;AACzE,8EAA8E;AAC9E,2EAA2E;AAC3E,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,GAAkB,EAAW,EAAE,CAC3D,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;IAC3B,GAAG,CAAC,OAAO,CAAC,WAAW,GAAG,CAAC;IAC3B,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC;IAC9D,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,IAAI,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC;AAEzE,MAAM,mBAAmB,GAAG,CAAC,KAAc,EAAU,EAAE;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IACxD,OAAO,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;AACzE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAkB,EAAU,EAAE;IACjE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IAC9B,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IAC9B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,IAAI,GAAG,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC7B,KAAK,MAAM,QAAQ,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CACR,KAAK,QAAQ,CAAC,KAAK,OAAO,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,mBAAmB,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CACnG,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,kBAAkB,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC;QACzF,KAAK,CAAC,IAAI,CACR,kBAAkB,GAAG,CAAC,QAAQ,CAAC,WAAW,IAAI,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,WAAW,IAAI,GAAG,EAAE,CACzF,CAAC;QACF,MAAM,EAAE,GAAG,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;QAC1C,MAAM,EAAE,GAAG,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;QAC1C,KAAK,CAAC,IAAI,CACR,sBAAsB,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,OAAO,WAAW,EAAE,CAAC,MAAM,KAAK;YACzE,OAAO,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,OAAO,WAAW,EAAE,CAAC,MAAM,EAAE,CAC5D,CAAC;QACF,MAAM,EAAE,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QAChC,MAAM,EAAE,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,mBAAmB,WAAW,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,WAAW,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAClF,KAAK,CAAC,IAAI,CACR,qBAAqB,WAAW,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,MAAM,WAAW,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,EAAE,CACxF,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACvB,KAAK,CAAC,IAAI,CACR,aAAa,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI;QACnF,UAAU,GAAG,CAAC,OAAO,CAAC,UAAU,IAAI,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,UAAU,IAAI,GAAG,EAAE,CAC/E,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,YAAY,cAAc,GAAG,CAAC,OAAO,CAAC,WAAW,YAAY,CAAC,CAAC;IAC3F,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACnC,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;YAAE,SAAS;QACvC,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACpF,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACvC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzB,KAAK,CAAC,IAAI,CACR,MAAM,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,KAAK,GAAG,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,KAAK,GAAG,CAAC,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,CACpH,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { RunMetadata } from "../core/index.js";
|
|
2
|
+
declare const stableDriverIdentity: (driver: RunMetadata["driver"]) => unknown;
|
|
3
|
+
/**
|
|
4
|
+
* Identity that must match before suite samples can be treated as repeated
|
|
5
|
+
* measurements. Subject/candidate hashes are intentionally excluded: those
|
|
6
|
+
* are the independent variable a comparison is meant to evaluate.
|
|
7
|
+
*/
|
|
8
|
+
export declare const runComparisonIdentity: (metadata: RunMetadata) => string;
|
|
9
|
+
export { stableDriverIdentity };
|
|
10
|
+
//# sourceMappingURL=comparison-identity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"comparison-identity.d.ts","sourceRoot":"","sources":["../../src/orchestrator/comparison-identity.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,QAAA,MAAM,oBAAoB,WAAY,WAAW,CAAC,QAAQ,CAAC,KAAG,OAW7D,CAAC;AAUF;;;;GAIG;AACH,eAAO,MAAM,qBAAqB,aAAc,WAAW,KAAG,MAa1D,CAAC;AAEL,OAAO,EAAE,oBAAoB,EAAE,CAAC"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { valueHash } from "../core/index.js";
|
|
2
|
+
const stableDriverIdentity = (driver) => {
|
|
3
|
+
const { detail: _detail, ...identity } = driver;
|
|
4
|
+
const detail = driver.detail
|
|
5
|
+
? Object.fromEntries(Object.entries(driver.detail).filter(([key]) => key !== "sessionId" && key !== "threadId"))
|
|
6
|
+
: undefined;
|
|
7
|
+
return {
|
|
8
|
+
...identity,
|
|
9
|
+
...(detail && Object.keys(detail).length > 0 ? { detail } : {}),
|
|
10
|
+
};
|
|
11
|
+
};
|
|
12
|
+
const sourceIdentity = (metadata) => {
|
|
13
|
+
const source = metadata.provenance?.source;
|
|
14
|
+
if (!source?.git?.dirty || source.git.workingTreeHash)
|
|
15
|
+
return source;
|
|
16
|
+
// Old or oversized dirty runs have no content identity. Keep them from
|
|
17
|
+
// becoming false repetitions merely because commit + `dirty: true` match.
|
|
18
|
+
return { ...source, dirtySnapshotUnavailableForRun: metadata.startedAt };
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Identity that must match before suite samples can be treated as repeated
|
|
22
|
+
* measurements. Subject/candidate hashes are intentionally excluded: those
|
|
23
|
+
* are the independent variable a comparison is meant to evaluate.
|
|
24
|
+
*/
|
|
25
|
+
export const runComparisonIdentity = (metadata) => valueHash({
|
|
26
|
+
source: sourceIdentity(metadata),
|
|
27
|
+
runtime: metadata.provenance?.runtime,
|
|
28
|
+
caseHash: metadata.provenance?.caseHash,
|
|
29
|
+
assertionsHash: metadata.provenance?.assertionsHash,
|
|
30
|
+
fixturesHash: metadata.provenance?.fixturesHash,
|
|
31
|
+
profileId: metadata.provenance?.profileId,
|
|
32
|
+
driverConfigHash: metadata.provenance?.driverConfigHash,
|
|
33
|
+
answererConfigHash: metadata.provenance?.answererConfigHash,
|
|
34
|
+
evaluatorConfigHash: metadata.provenance?.evaluatorConfigHash,
|
|
35
|
+
driver: stableDriverIdentity(metadata.driver),
|
|
36
|
+
answerer: metadata.answerer,
|
|
37
|
+
});
|
|
38
|
+
export { stableDriverIdentity };
|
|
39
|
+
//# sourceMappingURL=comparison-identity.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"comparison-identity.js","sourceRoot":"","sources":["../../src/orchestrator/comparison-identity.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAE7C,MAAM,oBAAoB,GAAG,CAAC,MAA6B,EAAW,EAAE;IACtE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;IAChD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM;QAC1B,CAAC,CAAC,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,WAAW,IAAI,GAAG,KAAK,UAAU,CAAC,CAC3F;QACH,CAAC,CAAC,SAAS,CAAC;IACd,OAAO;QACL,GAAG,QAAQ;QACX,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAChE,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,QAAqB,EAAW,EAAE;IACxD,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC3C,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,eAAe;QAAE,OAAO,MAAM,CAAC;IACrE,uEAAuE;IACvE,0EAA0E;IAC1E,OAAO,EAAE,GAAG,MAAM,EAAE,8BAA8B,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC;AAC3E,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,QAAqB,EAAU,EAAE,CACrE,SAAS,CAAC;IACR,MAAM,EAAE,cAAc,CAAC,QAAQ,CAAC;IAChC,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,OAAO;IACrC,QAAQ,EAAE,QAAQ,CAAC,UAAU,EAAE,QAAQ;IACvC,cAAc,EAAE,QAAQ,CAAC,UAAU,EAAE,cAAc;IACnD,YAAY,EAAE,QAAQ,CAAC,UAAU,EAAE,YAAY;IAC/C,SAAS,EAAE,QAAQ,CAAC,UAAU,EAAE,SAAS;IACzC,gBAAgB,EAAE,QAAQ,CAAC,UAAU,EAAE,gBAAgB;IACvD,kBAAkB,EAAE,QAAQ,CAAC,UAAU,EAAE,kBAAkB;IAC3D,mBAAmB,EAAE,QAAQ,CAAC,UAAU,EAAE,mBAAmB;IAC7D,MAAM,EAAE,oBAAoB,CAAC,QAAQ,CAAC,MAAM,CAAC;IAC7C,QAAQ,EAAE,QAAQ,CAAC,QAAQ;CAC5B,CAAC,CAAC;AAEL,OAAO,EAAE,oBAAoB,EAAE,CAAC"}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { type AnswererPlugin, type AssertionPlugin, type DriverPlugin, type LifecycleHooksPlugin, Registry } from "../core/index.js";
|
|
3
|
+
export declare const ExperimentProfileSchema: z.ZodObject<{
|
|
4
|
+
driver: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
5
|
+
name: z.ZodString;
|
|
6
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
7
|
+
}, z.core.$strict>]>>;
|
|
8
|
+
answerer: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
9
|
+
name: z.ZodString;
|
|
10
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
11
|
+
}, z.core.$strict>]>>;
|
|
12
|
+
harness: z.ZodOptional<z.ZodObject<{
|
|
13
|
+
provider: z.ZodEnum<{
|
|
14
|
+
"claude-code": "claude-code";
|
|
15
|
+
codex: "codex";
|
|
16
|
+
}>;
|
|
17
|
+
model: z.ZodString;
|
|
18
|
+
command: z.ZodOptional<z.ZodString>;
|
|
19
|
+
reasoningEffort: z.ZodOptional<z.ZodEnum<{
|
|
20
|
+
high: "high";
|
|
21
|
+
low: "low";
|
|
22
|
+
medium: "medium";
|
|
23
|
+
minimal: "minimal";
|
|
24
|
+
none: "none";
|
|
25
|
+
xhigh: "xhigh";
|
|
26
|
+
}>>;
|
|
27
|
+
}, z.core.$strict>>;
|
|
28
|
+
}, z.core.$strict>;
|
|
29
|
+
export type ExperimentProfile = z.infer<typeof ExperimentProfileSchema>;
|
|
30
|
+
declare const LuxConfigSchema: z.ZodObject<{
|
|
31
|
+
casesRoot: z.ZodOptional<z.ZodString>;
|
|
32
|
+
fixturesRoot: z.ZodOptional<z.ZodString>;
|
|
33
|
+
runsRoot: z.ZodDefault<z.ZodString>;
|
|
34
|
+
subjectRoot: z.ZodOptional<z.ZodString>;
|
|
35
|
+
defaultAnswerer: z.ZodDefault<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
36
|
+
name: z.ZodString;
|
|
37
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
38
|
+
}, z.core.$strict>]>>;
|
|
39
|
+
defaultDriver: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
40
|
+
name: z.ZodString;
|
|
41
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
42
|
+
}, z.core.$strict>]>>;
|
|
43
|
+
harness: z.ZodOptional<z.ZodObject<{
|
|
44
|
+
provider: z.ZodEnum<{
|
|
45
|
+
"claude-code": "claude-code";
|
|
46
|
+
codex: "codex";
|
|
47
|
+
}>;
|
|
48
|
+
model: z.ZodString;
|
|
49
|
+
command: z.ZodOptional<z.ZodString>;
|
|
50
|
+
reasoningEffort: z.ZodOptional<z.ZodEnum<{
|
|
51
|
+
high: "high";
|
|
52
|
+
low: "low";
|
|
53
|
+
medium: "medium";
|
|
54
|
+
minimal: "minimal";
|
|
55
|
+
none: "none";
|
|
56
|
+
xhigh: "xhigh";
|
|
57
|
+
}>>;
|
|
58
|
+
}, z.core.$strict>>;
|
|
59
|
+
concurrency: z.ZodDefault<z.ZodNumber>;
|
|
60
|
+
profiles: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
61
|
+
driver: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
62
|
+
name: z.ZodString;
|
|
63
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
64
|
+
}, z.core.$strict>]>>;
|
|
65
|
+
answerer: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
66
|
+
name: z.ZodString;
|
|
67
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
68
|
+
}, z.core.$strict>]>>;
|
|
69
|
+
harness: z.ZodOptional<z.ZodObject<{
|
|
70
|
+
provider: z.ZodEnum<{
|
|
71
|
+
"claude-code": "claude-code";
|
|
72
|
+
codex: "codex";
|
|
73
|
+
}>;
|
|
74
|
+
model: z.ZodString;
|
|
75
|
+
command: z.ZodOptional<z.ZodString>;
|
|
76
|
+
reasoningEffort: z.ZodOptional<z.ZodEnum<{
|
|
77
|
+
high: "high";
|
|
78
|
+
low: "low";
|
|
79
|
+
medium: "medium";
|
|
80
|
+
minimal: "minimal";
|
|
81
|
+
none: "none";
|
|
82
|
+
xhigh: "xhigh";
|
|
83
|
+
}>>;
|
|
84
|
+
}, z.core.$strict>>;
|
|
85
|
+
}, z.core.$strict>>>;
|
|
86
|
+
drivers: z.ZodOptional<z.ZodArray<z.ZodCustom<DriverPlugin<never>, DriverPlugin<never>>>>;
|
|
87
|
+
answerers: z.ZodOptional<z.ZodArray<z.ZodCustom<AnswererPlugin<never>, AnswererPlugin<never>>>>;
|
|
88
|
+
assertions: z.ZodOptional<z.ZodArray<z.ZodCustom<AssertionPlugin<{
|
|
89
|
+
[x: string]: unknown;
|
|
90
|
+
id?: string | undefined;
|
|
91
|
+
type: string;
|
|
92
|
+
name?: string | undefined;
|
|
93
|
+
weight?: number | undefined;
|
|
94
|
+
threshold?: number | undefined;
|
|
95
|
+
}>, AssertionPlugin<{
|
|
96
|
+
[x: string]: unknown;
|
|
97
|
+
id?: string | undefined;
|
|
98
|
+
type: string;
|
|
99
|
+
name?: string | undefined;
|
|
100
|
+
weight?: number | undefined;
|
|
101
|
+
threshold?: number | undefined;
|
|
102
|
+
}>>>>;
|
|
103
|
+
lifecycleFixtures: z.ZodOptional<z.ZodArray<z.ZodCustom<LifecycleHooksPlugin, LifecycleHooksPlugin>>>;
|
|
104
|
+
}, z.core.$strict>;
|
|
105
|
+
/**
|
|
106
|
+
* Author-facing config before schema defaults are applied.
|
|
107
|
+
*
|
|
108
|
+
* Keep this as the schema input: using `z.infer`/`z.output` here makes
|
|
109
|
+
* defaulted fields look required in otherwise-valid `lux.config.ts` files.
|
|
110
|
+
*/
|
|
111
|
+
export type LuxConfig = z.input<typeof LuxConfigSchema>;
|
|
112
|
+
type ParsedLuxConfig = z.output<typeof LuxConfigSchema>;
|
|
113
|
+
export type ResolvedLuxConfig = ParsedLuxConfig & {
|
|
114
|
+
configPath: string | null;
|
|
115
|
+
rootDir: string;
|
|
116
|
+
};
|
|
117
|
+
export declare const findConfigFile: (cwd: string) => string | null;
|
|
118
|
+
export declare const loadLuxConfig: (cwd?: string) => Promise<ResolvedLuxConfig>;
|
|
119
|
+
export declare const buildRegistryFromConfig: (config: ResolvedLuxConfig) => Registry;
|
|
120
|
+
export {};
|
|
121
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/orchestrator/config.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,OAAO,EACL,KAAK,cAAc,EAGnB,KAAK,eAAe,EAEpB,KAAK,YAAY,EAIjB,KAAK,oBAAoB,EAEzB,QAAQ,EACT,MAAM,kBAAkB,CAAC;AAW1B,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;kBAMzB,CAAC;AACZ,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AA4BxE,QAAA,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAqCV,CAAC;AAEZ;;;;;GAKG;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,KAAK,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD,MAAM,MAAM,iBAAiB,GAAG,eAAe,GAAG;IAChD,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAIF,eAAO,MAAM,cAAc,QAAS,MAAM,KAAG,MAAM,GAAG,IAMrD,CAAC;AAEF,eAAO,MAAM,aAAa,SAAe,MAAM,KAAmB,OAAO,CAAC,iBAAiB,CAwC1F,CAAC;AAcF,eAAO,MAAM,uBAAuB,WAAY,iBAAiB,KAAG,QAiBnE,CAAC"}
|