@prismatic-io/lux 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/lib/answerers/claude-code/index.d.ts +17 -0
- package/lib/answerers/claude-code/index.d.ts.map +1 -0
- package/lib/answerers/claude-code/index.js +280 -0
- package/lib/answerers/claude-code/index.js.map +1 -0
- package/lib/answerers/persona/index.d.ts +40 -0
- package/lib/answerers/persona/index.d.ts.map +1 -0
- package/lib/answerers/persona/index.js +265 -0
- package/lib/answerers/persona/index.js.map +1 -0
- package/lib/answerers/scripted/index.d.ts +8 -0
- package/lib/answerers/scripted/index.d.ts.map +1 -0
- package/lib/answerers/scripted/index.js +163 -0
- package/lib/answerers/scripted/index.js.map +1 -0
- package/lib/answerers/shared.d.ts +13 -0
- package/lib/answerers/shared.d.ts.map +1 -0
- package/lib/answerers/shared.js +17 -0
- package/lib/answerers/shared.js.map +1 -0
- package/lib/answerers/terminal/index.d.ts +9 -0
- package/lib/answerers/terminal/index.d.ts.map +1 -0
- package/lib/answerers/terminal/index.js +97 -0
- package/lib/answerers/terminal/index.js.map +1 -0
- package/lib/assertions/core/command-exits-zero.d.ts +21 -0
- package/lib/assertions/core/command-exits-zero.d.ts.map +1 -0
- package/lib/assertions/core/command-exits-zero.js +87 -0
- package/lib/assertions/core/command-exits-zero.js.map +1 -0
- package/lib/assertions/core/event-checks.d.ts +57 -0
- package/lib/assertions/core/event-checks.d.ts.map +1 -0
- package/lib/assertions/core/event-checks.js +123 -0
- package/lib/assertions/core/event-checks.js.map +1 -0
- package/lib/assertions/core/file-checks.d.ts +103 -0
- package/lib/assertions/core/file-checks.d.ts.map +1 -0
- package/lib/assertions/core/file-checks.js +288 -0
- package/lib/assertions/core/file-checks.js.map +1 -0
- package/lib/assertions/core/helpers.d.ts +66 -0
- package/lib/assertions/core/helpers.d.ts.map +1 -0
- package/lib/assertions/core/helpers.js +170 -0
- package/lib/assertions/core/helpers.js.map +1 -0
- package/lib/assertions/core/index.d.ts +15 -0
- package/lib/assertions/core/index.d.ts.map +1 -0
- package/lib/assertions/core/index.js +56 -0
- package/lib/assertions/core/index.js.map +1 -0
- package/lib/assertions/core/json-pointer-equals.d.ts +19 -0
- package/lib/assertions/core/json-pointer-equals.d.ts.map +1 -0
- package/lib/assertions/core/json-pointer-equals.js +71 -0
- package/lib/assertions/core/json-pointer-equals.js.map +1 -0
- package/lib/assertions/core/output-checks.d.ts +46 -0
- package/lib/assertions/core/output-checks.d.ts.map +1 -0
- package/lib/assertions/core/output-checks.js +121 -0
- package/lib/assertions/core/output-checks.js.map +1 -0
- package/lib/assertions/core/predicate.d.ts +14 -0
- package/lib/assertions/core/predicate.d.ts.map +1 -0
- package/lib/assertions/core/predicate.js +55 -0
- package/lib/assertions/core/predicate.js.map +1 -0
- package/lib/assertions/core/resource-checks.d.ts +71 -0
- package/lib/assertions/core/resource-checks.d.ts.map +1 -0
- package/lib/assertions/core/resource-checks.js +159 -0
- package/lib/assertions/core/resource-checks.js.map +1 -0
- package/lib/assertions/core/safe-regex.d.ts +23 -0
- package/lib/assertions/core/safe-regex.d.ts.map +1 -0
- package/lib/assertions/core/safe-regex.js +102 -0
- package/lib/assertions/core/safe-regex.js.map +1 -0
- package/lib/assertions/core/tool-called-with.d.ts +21 -0
- package/lib/assertions/core/tool-called-with.d.ts.map +1 -0
- package/lib/assertions/core/tool-called-with.js +69 -0
- package/lib/assertions/core/tool-called-with.js.map +1 -0
- package/lib/assertions/core/tool-calls.d.ts +22 -0
- package/lib/assertions/core/tool-calls.d.ts.map +1 -0
- package/lib/assertions/core/tool-calls.js +17 -0
- package/lib/assertions/core/tool-calls.js.map +1 -0
- package/lib/assertions/core/tool-invocation.d.ts +36 -0
- package/lib/assertions/core/tool-invocation.d.ts.map +1 -0
- package/lib/assertions/core/tool-invocation.js +170 -0
- package/lib/assertions/core/tool-invocation.js.map +1 -0
- package/lib/assertions/core/tool-result-checks.d.ts +59 -0
- package/lib/assertions/core/tool-result-checks.d.ts.map +1 -0
- package/lib/assertions/core/tool-result-checks.js +193 -0
- package/lib/assertions/core/tool-result-checks.js.map +1 -0
- package/lib/assertions/mcp/discovery-checks.d.ts +36 -0
- package/lib/assertions/mcp/discovery-checks.d.ts.map +1 -0
- package/lib/assertions/mcp/discovery-checks.js +122 -0
- package/lib/assertions/mcp/discovery-checks.js.map +1 -0
- package/lib/assertions/mcp/index.d.ts +10 -0
- package/lib/assertions/mcp/index.d.ts.map +1 -0
- package/lib/assertions/mcp/index.js +25 -0
- package/lib/assertions/mcp/index.js.map +1 -0
- package/lib/assertions/rubric/index.d.ts +104 -0
- package/lib/assertions/rubric/index.d.ts.map +1 -0
- package/lib/assertions/rubric/index.js +191 -0
- package/lib/assertions/rubric/index.js.map +1 -0
- package/lib/assertions/rubric/internal.d.ts +14 -0
- package/lib/assertions/rubric/internal.d.ts.map +1 -0
- package/lib/assertions/rubric/internal.js +335 -0
- package/lib/assertions/rubric/internal.js.map +1 -0
- package/lib/authoring.d.ts +3 -0
- package/lib/authoring.d.ts.map +1 -0
- package/lib/authoring.js +3 -0
- package/lib/authoring.js.map +1 -0
- package/lib/cli/bin.d.ts +3 -0
- package/lib/cli/bin.d.ts.map +1 -0
- package/lib/cli/bin.js +15 -0
- package/lib/cli/bin.js.map +1 -0
- package/lib/cli/command-runtime.d.ts +5 -0
- package/lib/cli/command-runtime.d.ts.map +1 -0
- package/lib/cli/command-runtime.js +30 -0
- package/lib/cli/command-runtime.js.map +1 -0
- package/lib/cli/init.d.ts +21 -0
- package/lib/cli/init.d.ts.map +1 -0
- package/lib/cli/init.js +340 -0
- package/lib/cli/init.js.map +1 -0
- package/lib/cli/program.d.ts +22 -0
- package/lib/cli/program.d.ts.map +1 -0
- package/lib/cli/program.js +757 -0
- package/lib/cli/program.js.map +1 -0
- package/lib/cli/render/format.d.ts +14 -0
- package/lib/cli/render/format.d.ts.map +1 -0
- package/lib/cli/render/format.js +71 -0
- package/lib/cli/render/format.js.map +1 -0
- package/lib/cli/render/reporter.d.ts +73 -0
- package/lib/cli/render/reporter.d.ts.map +1 -0
- package/lib/cli/render/reporter.js +519 -0
- package/lib/cli/render/reporter.js.map +1 -0
- package/lib/cli/render/tty.d.ts +77 -0
- package/lib/cli/render/tty.d.ts.map +1 -0
- package/lib/cli/render/tty.js +283 -0
- package/lib/cli/render/tty.js.map +1 -0
- package/lib/cli/run-options.d.ts +27 -0
- package/lib/cli/run-options.d.ts.map +1 -0
- package/lib/cli/run-options.js +58 -0
- package/lib/cli/run-options.js.map +1 -0
- package/lib/cli/skills.d.ts +38 -0
- package/lib/cli/skills.d.ts.map +1 -0
- package/lib/cli/skills.js +84 -0
- package/lib/cli/skills.js.map +1 -0
- package/lib/core/annotation.d.ts +56 -0
- package/lib/core/annotation.d.ts.map +1 -0
- package/lib/core/annotation.js +92 -0
- package/lib/core/annotation.js.map +1 -0
- package/lib/core/answerer.d.ts +68 -0
- package/lib/core/answerer.d.ts.map +1 -0
- package/lib/core/answerer.js +17 -0
- package/lib/core/answerer.js.map +1 -0
- package/lib/core/artifact-evidence.d.ts +50 -0
- package/lib/core/artifact-evidence.d.ts.map +1 -0
- package/lib/core/artifact-evidence.js +185 -0
- package/lib/core/artifact-evidence.js.map +1 -0
- package/lib/core/artifact-snapshot.d.ts +37 -0
- package/lib/core/artifact-snapshot.d.ts.map +1 -0
- package/lib/core/artifact-snapshot.js +339 -0
- package/lib/core/artifact-snapshot.js.map +1 -0
- package/lib/core/assertion.d.ts +129 -0
- package/lib/core/assertion.d.ts.map +1 -0
- package/lib/core/assertion.js +75 -0
- package/lib/core/assertion.js.map +1 -0
- package/lib/core/case.d.ts +79 -0
- package/lib/core/case.d.ts.map +1 -0
- package/lib/core/case.js +90 -0
- package/lib/core/case.js.map +1 -0
- package/lib/core/conversation-turn.d.ts +23 -0
- package/lib/core/conversation-turn.d.ts.map +1 -0
- package/lib/core/conversation-turn.js +31 -0
- package/lib/core/conversation-turn.js.map +1 -0
- package/lib/core/driver.d.ts +64 -0
- package/lib/core/driver.d.ts.map +1 -0
- package/lib/core/driver.js +18 -0
- package/lib/core/driver.js.map +1 -0
- package/lib/core/emitter.d.ts +20 -0
- package/lib/core/emitter.d.ts.map +1 -0
- package/lib/core/emitter.js +34 -0
- package/lib/core/emitter.js.map +1 -0
- package/lib/core/events.d.ts +202 -0
- package/lib/core/events.d.ts.map +1 -0
- package/lib/core/events.js +107 -0
- package/lib/core/events.js.map +1 -0
- package/lib/core/exec.d.ts +82 -0
- package/lib/core/exec.d.ts.map +1 -0
- package/lib/core/exec.js +168 -0
- package/lib/core/exec.js.map +1 -0
- package/lib/core/experiment.d.ts +285 -0
- package/lib/core/experiment.d.ts.map +1 -0
- package/lib/core/experiment.js +389 -0
- package/lib/core/experiment.js.map +1 -0
- package/lib/core/glob.d.ts +3 -0
- package/lib/core/glob.d.ts.map +1 -0
- package/lib/core/glob.js +32 -0
- package/lib/core/glob.js.map +1 -0
- package/lib/core/hash.d.ts +8 -0
- package/lib/core/hash.d.ts.map +1 -0
- package/lib/core/hash.js +214 -0
- package/lib/core/hash.js.map +1 -0
- package/lib/core/index.d.ts +29 -0
- package/lib/core/index.d.ts.map +1 -0
- package/lib/core/index.js +29 -0
- package/lib/core/index.js.map +1 -0
- package/lib/core/lifecycle-fixtures.d.ts +20 -0
- package/lib/core/lifecycle-fixtures.d.ts.map +1 -0
- package/lib/core/lifecycle-fixtures.js +109 -0
- package/lib/core/lifecycle-fixtures.js.map +1 -0
- package/lib/core/lifecycle-hooks.d.ts +52 -0
- package/lib/core/lifecycle-hooks.d.ts.map +1 -0
- package/lib/core/lifecycle-hooks.js +42 -0
- package/lib/core/lifecycle-hooks.js.map +1 -0
- package/lib/core/mcp-events.d.ts +44 -0
- package/lib/core/mcp-events.d.ts.map +1 -0
- package/lib/core/mcp-events.js +23 -0
- package/lib/core/mcp-events.js.map +1 -0
- package/lib/core/model-cli.d.ts +57 -0
- package/lib/core/model-cli.d.ts.map +1 -0
- package/lib/core/model-cli.js +237 -0
- package/lib/core/model-cli.js.map +1 -0
- package/lib/core/observer.d.ts +31 -0
- package/lib/core/observer.d.ts.map +1 -0
- package/lib/core/observer.js +25 -0
- package/lib/core/observer.js.map +1 -0
- package/lib/core/platform-process.d.ts +33 -0
- package/lib/core/platform-process.d.ts.map +1 -0
- package/lib/core/platform-process.js +214 -0
- package/lib/core/platform-process.js.map +1 -0
- package/lib/core/registry.d.ts +22 -0
- package/lib/core/registry.d.ts.map +1 -0
- package/lib/core/registry.js +45 -0
- package/lib/core/registry.js.map +1 -0
- package/lib/core/rubric.d.ts +18 -0
- package/lib/core/rubric.d.ts.map +1 -0
- package/lib/core/rubric.js +49 -0
- package/lib/core/rubric.js.map +1 -0
- package/lib/core/run.d.ts +419 -0
- package/lib/core/run.d.ts.map +1 -0
- package/lib/core/run.js +126 -0
- package/lib/core/run.js.map +1 -0
- package/lib/core/statistics.d.ts +39 -0
- package/lib/core/statistics.d.ts.map +1 -0
- package/lib/core/statistics.js +163 -0
- package/lib/core/statistics.js.map +1 -0
- package/lib/core/subject.d.ts +4 -0
- package/lib/core/subject.d.ts.map +1 -0
- package/lib/core/subject.js +46 -0
- package/lib/core/subject.js.map +1 -0
- package/lib/core/tool-events.d.ts +69 -0
- package/lib/core/tool-events.d.ts.map +1 -0
- package/lib/core/tool-events.js +242 -0
- package/lib/core/tool-events.js.map +1 -0
- package/lib/core/usage.d.ts +78 -0
- package/lib/core/usage.d.ts.map +1 -0
- package/lib/core/usage.js +104 -0
- package/lib/core/usage.js.map +1 -0
- package/lib/core/wire.d.ts +83 -0
- package/lib/core/wire.d.ts.map +1 -0
- package/lib/core/wire.js +101 -0
- package/lib/core/wire.js.map +1 -0
- package/lib/core/zod.d.ts +10 -0
- package/lib/core/zod.d.ts.map +1 -0
- package/lib/core/zod.js +18 -0
- package/lib/core/zod.js.map +1 -0
- package/lib/drivers/claude-code/index.d.ts +55 -0
- package/lib/drivers/claude-code/index.d.ts.map +1 -0
- package/lib/drivers/claude-code/index.js +779 -0
- package/lib/drivers/claude-code/index.js.map +1 -0
- package/lib/drivers/claude-code/parse-events.d.ts +46 -0
- package/lib/drivers/claude-code/parse-events.d.ts.map +1 -0
- package/lib/drivers/claude-code/parse-events.js +387 -0
- package/lib/drivers/claude-code/parse-events.js.map +1 -0
- package/lib/drivers/codex/app-events.d.ts +16 -0
- package/lib/drivers/codex/app-events.d.ts.map +1 -0
- package/lib/drivers/codex/app-events.js +184 -0
- package/lib/drivers/codex/app-events.js.map +1 -0
- package/lib/drivers/codex/config.d.ts +55 -0
- package/lib/drivers/codex/config.d.ts.map +1 -0
- package/lib/drivers/codex/config.js +125 -0
- package/lib/drivers/codex/config.js.map +1 -0
- package/lib/drivers/codex/index.d.ts +8 -0
- package/lib/drivers/codex/index.d.ts.map +1 -0
- package/lib/drivers/codex/index.js +904 -0
- package/lib/drivers/codex/index.js.map +1 -0
- package/lib/drivers/mcp/index.d.ts +52 -0
- package/lib/drivers/mcp/index.d.ts.map +1 -0
- package/lib/drivers/mcp/index.js +282 -0
- package/lib/drivers/mcp/index.js.map +1 -0
- package/lib/drivers/mcp/transport.d.ts +50 -0
- package/lib/drivers/mcp/transport.d.ts.map +1 -0
- package/lib/drivers/mcp/transport.js +190 -0
- package/lib/drivers/mcp/transport.js.map +1 -0
- package/lib/drivers/shared/artifacts.d.ts +4 -0
- package/lib/drivers/shared/artifacts.d.ts.map +1 -0
- package/lib/drivers/shared/artifacts.js +81 -0
- package/lib/drivers/shared/artifacts.js.map +1 -0
- package/lib/drivers/shared/secure-copy.d.ts +2 -0
- package/lib/drivers/shared/secure-copy.d.ts.map +1 -0
- package/lib/drivers/shared/secure-copy.js +35 -0
- package/lib/drivers/shared/secure-copy.js.map +1 -0
- package/lib/drivers/subprocess/index.d.ts +24 -0
- package/lib/drivers/subprocess/index.d.ts.map +1 -0
- package/lib/drivers/subprocess/index.js +451 -0
- package/lib/drivers/subprocess/index.js.map +1 -0
- package/lib/index.d.ts +40 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +48 -0
- package/lib/index.js.map +1 -0
- package/lib/optimization/reflective.d.ts +27 -0
- package/lib/optimization/reflective.d.ts.map +1 -0
- package/lib/optimization/reflective.js +257 -0
- package/lib/optimization/reflective.js.map +1 -0
- package/lib/orchestrator/annotation-loader.d.ts +65 -0
- package/lib/orchestrator/annotation-loader.d.ts.map +1 -0
- package/lib/orchestrator/annotation-loader.js +73 -0
- package/lib/orchestrator/annotation-loader.js.map +1 -0
- package/lib/orchestrator/annotation-store.d.ts +3 -0
- package/lib/orchestrator/annotation-store.d.ts.map +1 -0
- package/lib/orchestrator/annotation-store.js +100 -0
- package/lib/orchestrator/annotation-store.js.map +1 -0
- package/lib/orchestrator/authored-dependencies.d.ts +3 -0
- package/lib/orchestrator/authored-dependencies.d.ts.map +1 -0
- package/lib/orchestrator/authored-dependencies.js +146 -0
- package/lib/orchestrator/authored-dependencies.js.map +1 -0
- package/lib/orchestrator/campaign-lifecycle.d.ts +105 -0
- package/lib/orchestrator/campaign-lifecycle.d.ts.map +1 -0
- package/lib/orchestrator/campaign-lifecycle.js +141 -0
- package/lib/orchestrator/campaign-lifecycle.js.map +1 -0
- package/lib/orchestrator/candidate-integrity.d.ts +4 -0
- package/lib/orchestrator/candidate-integrity.d.ts.map +1 -0
- package/lib/orchestrator/candidate-integrity.js +14 -0
- package/lib/orchestrator/candidate-integrity.js.map +1 -0
- package/lib/orchestrator/candidates.d.ts +13 -0
- package/lib/orchestrator/candidates.d.ts.map +1 -0
- package/lib/orchestrator/candidates.js +577 -0
- package/lib/orchestrator/candidates.js.map +1 -0
- package/lib/orchestrator/compare.d.ts +80 -0
- package/lib/orchestrator/compare.d.ts.map +1 -0
- package/lib/orchestrator/compare.js +270 -0
- package/lib/orchestrator/compare.js.map +1 -0
- package/lib/orchestrator/comparison-identity.d.ts +10 -0
- package/lib/orchestrator/comparison-identity.d.ts.map +1 -0
- package/lib/orchestrator/comparison-identity.js +39 -0
- package/lib/orchestrator/comparison-identity.js.map +1 -0
- package/lib/orchestrator/config.d.ts +121 -0
- package/lib/orchestrator/config.d.ts.map +1 -0
- package/lib/orchestrator/config.js +176 -0
- package/lib/orchestrator/config.js.map +1 -0
- package/lib/orchestrator/content-bound-json.d.ts +5 -0
- package/lib/orchestrator/content-bound-json.d.ts.map +1 -0
- package/lib/orchestrator/content-bound-json.js +59 -0
- package/lib/orchestrator/content-bound-json.js.map +1 -0
- package/lib/orchestrator/discover.d.ts +45 -0
- package/lib/orchestrator/discover.d.ts.map +1 -0
- package/lib/orchestrator/discover.js +98 -0
- package/lib/orchestrator/discover.js.map +1 -0
- package/lib/orchestrator/doctor.d.ts +141 -0
- package/lib/orchestrator/doctor.d.ts.map +1 -0
- package/lib/orchestrator/doctor.js +525 -0
- package/lib/orchestrator/doctor.js.map +1 -0
- package/lib/orchestrator/driver-session.d.ts +177 -0
- package/lib/orchestrator/driver-session.d.ts.map +1 -0
- package/lib/orchestrator/driver-session.js +396 -0
- package/lib/orchestrator/driver-session.js.map +1 -0
- package/lib/orchestrator/evaluator-identity.d.ts +7 -0
- package/lib/orchestrator/evaluator-identity.d.ts.map +1 -0
- package/lib/orchestrator/evaluator-identity.js +42 -0
- package/lib/orchestrator/evaluator-identity.js.map +1 -0
- package/lib/orchestrator/experiment-budget.d.ts +11 -0
- package/lib/orchestrator/experiment-budget.d.ts.map +1 -0
- package/lib/orchestrator/experiment-budget.js +51 -0
- package/lib/orchestrator/experiment-budget.js.map +1 -0
- package/lib/orchestrator/experiment-context.d.ts +43 -0
- package/lib/orchestrator/experiment-context.d.ts.map +1 -0
- package/lib/orchestrator/experiment-context.js +9 -0
- package/lib/orchestrator/experiment-context.js.map +1 -0
- package/lib/orchestrator/experiment-contracts.d.ts +290 -0
- package/lib/orchestrator/experiment-contracts.d.ts.map +1 -0
- package/lib/orchestrator/experiment-contracts.js +38 -0
- package/lib/orchestrator/experiment-contracts.js.map +1 -0
- package/lib/orchestrator/experiment-corpus.d.ts +21 -0
- package/lib/orchestrator/experiment-corpus.d.ts.map +1 -0
- package/lib/orchestrator/experiment-corpus.js +276 -0
- package/lib/orchestrator/experiment-corpus.js.map +1 -0
- package/lib/orchestrator/experiment-evaluation.d.ts +44 -0
- package/lib/orchestrator/experiment-evaluation.d.ts.map +1 -0
- package/lib/orchestrator/experiment-evaluation.js +542 -0
- package/lib/orchestrator/experiment-evaluation.js.map +1 -0
- package/lib/orchestrator/experiment-finish.d.ts +6 -0
- package/lib/orchestrator/experiment-finish.d.ts.map +1 -0
- package/lib/orchestrator/experiment-finish.js +19 -0
- package/lib/orchestrator/experiment-finish.js.map +1 -0
- package/lib/orchestrator/experiment-identity.d.ts +6 -0
- package/lib/orchestrator/experiment-identity.d.ts.map +1 -0
- package/lib/orchestrator/experiment-identity.js +257 -0
- package/lib/orchestrator/experiment-identity.js.map +1 -0
- package/lib/orchestrator/experiment-loader.d.ts +175 -0
- package/lib/orchestrator/experiment-loader.d.ts.map +1 -0
- package/lib/orchestrator/experiment-loader.js +77 -0
- package/lib/orchestrator/experiment-loader.js.map +1 -0
- package/lib/orchestrator/experiment-report.d.ts +450 -0
- package/lib/orchestrator/experiment-report.d.ts.map +1 -0
- package/lib/orchestrator/experiment-report.js +570 -0
- package/lib/orchestrator/experiment-report.js.map +1 -0
- package/lib/orchestrator/experiment-runs.d.ts +379 -0
- package/lib/orchestrator/experiment-runs.d.ts.map +1 -0
- package/lib/orchestrator/experiment-runs.js +796 -0
- package/lib/orchestrator/experiment-runs.js.map +1 -0
- package/lib/orchestrator/experiment-runtime-identity.d.ts +41 -0
- package/lib/orchestrator/experiment-runtime-identity.d.ts.map +1 -0
- package/lib/orchestrator/experiment-runtime-identity.js +411 -0
- package/lib/orchestrator/experiment-runtime-identity.js.map +1 -0
- package/lib/orchestrator/experiment-selection.d.ts +25 -0
- package/lib/orchestrator/experiment-selection.d.ts.map +1 -0
- package/lib/orchestrator/experiment-selection.js +286 -0
- package/lib/orchestrator/experiment-selection.js.map +1 -0
- package/lib/orchestrator/experiment.d.ts +11 -0
- package/lib/orchestrator/experiment.d.ts.map +1 -0
- package/lib/orchestrator/experiment.js +639 -0
- package/lib/orchestrator/experiment.js.map +1 -0
- package/lib/orchestrator/fixture-identity.d.ts +7 -0
- package/lib/orchestrator/fixture-identity.d.ts.map +1 -0
- package/lib/orchestrator/fixture-identity.js +55 -0
- package/lib/orchestrator/fixture-identity.js.map +1 -0
- package/lib/orchestrator/fs-read.d.ts +19 -0
- package/lib/orchestrator/fs-read.d.ts.map +1 -0
- package/lib/orchestrator/fs-read.js +113 -0
- package/lib/orchestrator/fs-read.js.map +1 -0
- package/lib/orchestrator/grader-experiment.d.ts +190 -0
- package/lib/orchestrator/grader-experiment.d.ts.map +1 -0
- package/lib/orchestrator/grader-experiment.js +579 -0
- package/lib/orchestrator/grader-experiment.js.map +1 -0
- package/lib/orchestrator/grading.d.ts +31 -0
- package/lib/orchestrator/grading.d.ts.map +1 -0
- package/lib/orchestrator/grading.js +186 -0
- package/lib/orchestrator/grading.js.map +1 -0
- package/lib/orchestrator/index.d.ts +22 -0
- package/lib/orchestrator/index.d.ts.map +1 -0
- package/lib/orchestrator/index.js +30 -0
- package/lib/orchestrator/index.js.map +1 -0
- package/lib/orchestrator/lifecycle-hook-execution.d.ts +4 -0
- package/lib/orchestrator/lifecycle-hook-execution.d.ts.map +1 -0
- package/lib/orchestrator/lifecycle-hook-execution.js +67 -0
- package/lib/orchestrator/lifecycle-hook-execution.js.map +1 -0
- package/lib/orchestrator/list-runs.d.ts +33 -0
- package/lib/orchestrator/list-runs.d.ts.map +1 -0
- package/lib/orchestrator/list-runs.js +308 -0
- package/lib/orchestrator/list-runs.js.map +1 -0
- package/lib/orchestrator/loader.d.ts +3 -0
- package/lib/orchestrator/loader.d.ts.map +1 -0
- package/lib/orchestrator/loader.js +17 -0
- package/lib/orchestrator/loader.js.map +1 -0
- package/lib/orchestrator/optimization-lifecycle.d.ts +12 -0
- package/lib/orchestrator/optimization-lifecycle.d.ts.map +1 -0
- package/lib/orchestrator/optimization-lifecycle.js +92 -0
- package/lib/orchestrator/optimization-lifecycle.js.map +1 -0
- package/lib/orchestrator/orchestrator.d.ts +144 -0
- package/lib/orchestrator/orchestrator.d.ts.map +1 -0
- package/lib/orchestrator/orchestrator.js +632 -0
- package/lib/orchestrator/orchestrator.js.map +1 -0
- package/lib/orchestrator/owned-lock.d.ts +17 -0
- package/lib/orchestrator/owned-lock.d.ts.map +1 -0
- package/lib/orchestrator/owned-lock.js +149 -0
- package/lib/orchestrator/owned-lock.js.map +1 -0
- package/lib/orchestrator/package-version.d.ts +2 -0
- package/lib/orchestrator/package-version.d.ts.map +1 -0
- package/lib/orchestrator/package-version.js +4 -0
- package/lib/orchestrator/package-version.js.map +1 -0
- package/lib/orchestrator/provenance.d.ts +5 -0
- package/lib/orchestrator/provenance.d.ts.map +1 -0
- package/lib/orchestrator/provenance.js +137 -0
- package/lib/orchestrator/provenance.js.map +1 -0
- package/lib/orchestrator/regrade.d.ts +80 -0
- package/lib/orchestrator/regrade.d.ts.map +1 -0
- package/lib/orchestrator/regrade.js +433 -0
- package/lib/orchestrator/regrade.js.map +1 -0
- package/lib/orchestrator/rubric-loader.d.ts +7 -0
- package/lib/orchestrator/rubric-loader.d.ts.map +1 -0
- package/lib/orchestrator/rubric-loader.js +65 -0
- package/lib/orchestrator/rubric-loader.js.map +1 -0
- package/lib/orchestrator/run-dir.d.ts +40 -0
- package/lib/orchestrator/run-dir.d.ts.map +1 -0
- package/lib/orchestrator/run-dir.js +161 -0
- package/lib/orchestrator/run-dir.js.map +1 -0
- package/lib/orchestrator/run-execution.d.ts +30 -0
- package/lib/orchestrator/run-execution.d.ts.map +1 -0
- package/lib/orchestrator/run-execution.js +531 -0
- package/lib/orchestrator/run-execution.js.map +1 -0
- package/lib/orchestrator/run-lifecycle.d.ts +115 -0
- package/lib/orchestrator/run-lifecycle.d.ts.map +1 -0
- package/lib/orchestrator/run-lifecycle.js +136 -0
- package/lib/orchestrator/run-lifecycle.js.map +1 -0
- package/lib/orchestrator/suite-compare.d.ts +76 -0
- package/lib/orchestrator/suite-compare.d.ts.map +1 -0
- package/lib/orchestrator/suite-compare.js +159 -0
- package/lib/orchestrator/suite-compare.js.map +1 -0
- package/lib/orchestrator/suite-lifecycle.d.ts +157 -0
- package/lib/orchestrator/suite-lifecycle.d.ts.map +1 -0
- package/lib/orchestrator/suite-lifecycle.js +253 -0
- package/lib/orchestrator/suite-lifecycle.js.map +1 -0
- package/lib/orchestrator/suite-runs.d.ts +91 -0
- package/lib/orchestrator/suite-runs.d.ts.map +1 -0
- package/lib/orchestrator/suite-runs.js +157 -0
- package/lib/orchestrator/suite-runs.js.map +1 -0
- package/lib/orchestrator/suite-summary.d.ts +109 -0
- package/lib/orchestrator/suite-summary.d.ts.map +1 -0
- package/lib/orchestrator/suite-summary.js +213 -0
- package/lib/orchestrator/suite-summary.js.map +1 -0
- package/lib/orchestrator/trace-limits.d.ts +20 -0
- package/lib/orchestrator/trace-limits.d.ts.map +1 -0
- package/lib/orchestrator/trace-limits.js +28 -0
- package/lib/orchestrator/trace-limits.js.map +1 -0
- package/lib/orchestrator/validate-plugin-config.d.ts +10 -0
- package/lib/orchestrator/validate-plugin-config.d.ts.map +1 -0
- package/lib/orchestrator/validate-plugin-config.js +26 -0
- package/lib/orchestrator/validate-plugin-config.js.map +1 -0
- package/package.json +84 -0
- package/skills/lux/SKILL.md +85 -0
- package/skills/lux/references/cli.md +102 -0
- package/skills/lux/references/eval-authoring.md +105 -0
- package/skills/lux-answerer/SKILL.md +169 -0
- package/src/answerers/claude-code/index.ts +342 -0
- package/src/answerers/persona/index.ts +327 -0
- package/src/answerers/scripted/index.ts +186 -0
- package/src/answerers/shared.ts +23 -0
- package/src/answerers/terminal/index.ts +117 -0
- package/src/assertions/core/command-exits-zero.ts +94 -0
- package/src/assertions/core/event-checks.ts +138 -0
- package/src/assertions/core/file-checks.ts +336 -0
- package/src/assertions/core/helpers.ts +203 -0
- package/src/assertions/core/index.ts +80 -0
- package/src/assertions/core/json-pointer-equals.ts +84 -0
- package/src/assertions/core/output-checks.ts +134 -0
- package/src/assertions/core/predicate.ts +67 -0
- package/src/assertions/core/resource-checks.ts +180 -0
- package/src/assertions/core/safe-regex.ts +196 -0
- package/src/assertions/core/tool-called-with.ts +68 -0
- package/src/assertions/core/tool-calls.ts +28 -0
- package/src/assertions/core/tool-invocation.ts +198 -0
- package/src/assertions/core/tool-result-checks.ts +211 -0
- package/src/assertions/mcp/discovery-checks.ts +139 -0
- package/src/assertions/mcp/index.ts +37 -0
- package/src/assertions/rubric/index.ts +258 -0
- package/src/assertions/rubric/internal.ts +367 -0
- package/src/authoring.ts +2 -0
- package/src/cli/bin.ts +14 -0
- package/src/cli/command-runtime.ts +34 -0
- package/src/cli/init.ts +410 -0
- package/src/cli/program.ts +992 -0
- package/src/cli/render/format.ts +69 -0
- package/src/cli/render/reporter.ts +643 -0
- package/src/cli/render/tty.ts +314 -0
- package/src/cli/run-options.ts +82 -0
- package/src/cli/skills.ts +129 -0
- package/src/core/annotation.ts +104 -0
- package/src/core/answerer.ts +75 -0
- package/src/core/artifact-evidence.ts +249 -0
- package/src/core/artifact-snapshot.ts +429 -0
- package/src/core/assertion.ts +156 -0
- package/src/core/case.ts +103 -0
- package/src/core/conversation-turn.ts +47 -0
- package/src/core/driver.ts +75 -0
- package/src/core/emitter.ts +38 -0
- package/src/core/events.ts +135 -0
- package/src/core/exec.ts +263 -0
- package/src/core/experiment.ts +467 -0
- package/src/core/glob.ts +31 -0
- package/src/core/hash.ts +237 -0
- package/src/core/index.ts +28 -0
- package/src/core/lifecycle-fixtures.ts +145 -0
- package/src/core/lifecycle-hooks.ts +101 -0
- package/src/core/mcp-events.ts +57 -0
- package/src/core/model-cli.ts +271 -0
- package/src/core/observer.ts +50 -0
- package/src/core/platform-process.ts +284 -0
- package/src/core/registry.ts +57 -0
- package/src/core/rubric.ts +52 -0
- package/src/core/run.ts +144 -0
- package/src/core/statistics.ts +194 -0
- package/src/core/subject.ts +50 -0
- package/src/core/tool-events.ts +292 -0
- package/src/core/usage.ts +123 -0
- package/src/core/wire.ts +135 -0
- package/src/core/zod.ts +20 -0
- package/src/drivers/claude-code/index.ts +866 -0
- package/src/drivers/claude-code/parse-events.ts +464 -0
- package/src/drivers/codex/app-events.ts +218 -0
- package/src/drivers/codex/config.ts +128 -0
- package/src/drivers/codex/index.ts +995 -0
- package/src/drivers/mcp/index.ts +351 -0
- package/src/drivers/mcp/transport.ts +222 -0
- package/src/drivers/shared/artifacts.ts +86 -0
- package/src/drivers/shared/secure-copy.ts +40 -0
- package/src/drivers/subprocess/index.ts +490 -0
- package/src/index.ts +594 -0
- package/src/optimization/reflective.ts +319 -0
- package/src/orchestrator/annotation-loader.ts +97 -0
- package/src/orchestrator/annotation-store.ts +112 -0
- package/src/orchestrator/authored-dependencies.ts +193 -0
- package/src/orchestrator/campaign-lifecycle.ts +181 -0
- package/src/orchestrator/candidate-integrity.ts +18 -0
- package/src/orchestrator/candidates.ts +683 -0
- package/src/orchestrator/compare.ts +358 -0
- package/src/orchestrator/comparison-identity.ts +45 -0
- package/src/orchestrator/config.ts +205 -0
- package/src/orchestrator/content-bound-json.ts +71 -0
- package/src/orchestrator/discover.ts +132 -0
- package/src/orchestrator/doctor.ts +781 -0
- package/src/orchestrator/driver-session.ts +507 -0
- package/src/orchestrator/evaluator-identity.ts +45 -0
- package/src/orchestrator/experiment-budget.ts +61 -0
- package/src/orchestrator/experiment-context.ts +41 -0
- package/src/orchestrator/experiment-contracts.ts +43 -0
- package/src/orchestrator/experiment-corpus.ts +416 -0
- package/src/orchestrator/experiment-evaluation.ts +749 -0
- package/src/orchestrator/experiment-finish.ts +33 -0
- package/src/orchestrator/experiment-identity.ts +356 -0
- package/src/orchestrator/experiment-loader.ts +105 -0
- package/src/orchestrator/experiment-report.ts +788 -0
- package/src/orchestrator/experiment-runs.ts +960 -0
- package/src/orchestrator/experiment-runtime-identity.ts +465 -0
- package/src/orchestrator/experiment-selection.ts +467 -0
- package/src/orchestrator/experiment.ts +991 -0
- package/src/orchestrator/fixture-identity.ts +67 -0
- package/src/orchestrator/fs-read.ts +126 -0
- package/src/orchestrator/grader-experiment.ts +856 -0
- package/src/orchestrator/grading.ts +260 -0
- package/src/orchestrator/index.ts +186 -0
- package/src/orchestrator/lifecycle-hook-execution.ts +90 -0
- package/src/orchestrator/list-runs.ts +384 -0
- package/src/orchestrator/loader.ts +19 -0
- package/src/orchestrator/optimization-lifecycle.ts +126 -0
- package/src/orchestrator/orchestrator.ts +919 -0
- package/src/orchestrator/owned-lock.ts +147 -0
- package/src/orchestrator/package-version.ts +5 -0
- package/src/orchestrator/provenance.ts +140 -0
- package/src/orchestrator/regrade.ts +580 -0
- package/src/orchestrator/rubric-loader.ts +74 -0
- package/src/orchestrator/run-dir.ts +209 -0
- package/src/orchestrator/run-execution.ts +694 -0
- package/src/orchestrator/run-lifecycle.ts +174 -0
- package/src/orchestrator/suite-compare.ts +195 -0
- package/src/orchestrator/suite-lifecycle.ts +335 -0
- package/src/orchestrator/suite-runs.ts +202 -0
- package/src/orchestrator/suite-summary.ts +276 -0
- package/src/orchestrator/trace-limits.ts +65 -0
- package/src/orchestrator/validate-plugin-config.ts +34 -0
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { chmod, constants, copyFile, lstat, mkdir, mkdtemp, open, readFile, realpath, rename, rm, unlink, } from "node:fs/promises";
|
|
3
|
+
import { dirname, join, posix, relative, resolve, sep } from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
const DEFAULT_MAX_FILES = 100_000;
|
|
6
|
+
const DEFAULT_MAX_BYTES = 256 * 1024 * 1024;
|
|
7
|
+
const DEFAULT_WRITE_CONCURRENCY = 16;
|
|
8
|
+
const MAX_MANIFEST_BYTES = 64 * 1024 * 1024;
|
|
9
|
+
const MAX_SNAPSHOT_PATH_BYTES = 4_096;
|
|
10
|
+
export const ARTIFACT_SNAPSHOT_MANIFEST = ".lux-snapshot.json";
|
|
11
|
+
const SNAPSHOT_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
|
|
12
|
+
export const ArtifactSnapshotFileSchema = z.object({
|
|
13
|
+
path: z
|
|
14
|
+
.string()
|
|
15
|
+
.min(1)
|
|
16
|
+
.refine((path) => {
|
|
17
|
+
try {
|
|
18
|
+
return snapshotPath(path) === path;
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
}, "invalid snapshot path"),
|
|
24
|
+
bytes: z.number().int().nonnegative(),
|
|
25
|
+
sha256: z.string().regex(/^[a-f0-9]{64}$/),
|
|
26
|
+
});
|
|
27
|
+
export const ArtifactSnapshotManifestSchema = z
|
|
28
|
+
.object({
|
|
29
|
+
schemaVersion: z.literal(1),
|
|
30
|
+
name: z.string().regex(SNAPSHOT_NAME),
|
|
31
|
+
files: z.array(ArtifactSnapshotFileSchema),
|
|
32
|
+
fileCount: z.number().int().nonnegative(),
|
|
33
|
+
bytes: z.number().int().nonnegative(),
|
|
34
|
+
baseSnapshot: z.string().regex(SNAPSHOT_NAME).optional(),
|
|
35
|
+
reusedFiles: z.number().int().nonnegative(),
|
|
36
|
+
reusedBytes: z.number().int().nonnegative(),
|
|
37
|
+
})
|
|
38
|
+
.superRefine((manifest, ctx) => {
|
|
39
|
+
const paths = new Set(manifest.files.map((file) => file.path));
|
|
40
|
+
const bytes = manifest.files.reduce((total, file) => total + file.bytes, 0);
|
|
41
|
+
if (paths.size !== manifest.files.length) {
|
|
42
|
+
ctx.addIssue({ code: "custom", path: ["files"], message: "paths must be unique" });
|
|
43
|
+
}
|
|
44
|
+
if (manifest.fileCount !== manifest.files.length) {
|
|
45
|
+
ctx.addIssue({
|
|
46
|
+
code: "custom",
|
|
47
|
+
path: ["fileCount"],
|
|
48
|
+
message: "fileCount must equal files.length",
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
if (manifest.bytes !== bytes) {
|
|
52
|
+
ctx.addIssue({ code: "custom", path: ["bytes"], message: "bytes must equal file sizes" });
|
|
53
|
+
}
|
|
54
|
+
if (manifest.reusedFiles > manifest.fileCount || manifest.reusedBytes > manifest.bytes) {
|
|
55
|
+
ctx.addIssue({
|
|
56
|
+
code: "custom",
|
|
57
|
+
path: ["reusedFiles"],
|
|
58
|
+
message: "reused totals cannot exceed snapshot totals",
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
const positiveInteger = (value, fallback, label) => {
|
|
63
|
+
const resolved = value ?? fallback;
|
|
64
|
+
if (!Number.isSafeInteger(resolved) || resolved <= 0) {
|
|
65
|
+
throw new RangeError(`${label} must be a positive safe integer`);
|
|
66
|
+
}
|
|
67
|
+
return resolved;
|
|
68
|
+
};
|
|
69
|
+
const snapshotPath = (source) => {
|
|
70
|
+
if (!source || source.includes("\0") || source.includes("\\")) {
|
|
71
|
+
throw new Error(`snapshot file path is invalid: ${JSON.stringify(source)}`);
|
|
72
|
+
}
|
|
73
|
+
const withoutRoot = source.replace(/^\/+/, "");
|
|
74
|
+
if (Buffer.byteLength(withoutRoot) > MAX_SNAPSHOT_PATH_BYTES) {
|
|
75
|
+
throw new Error(`snapshot file path is too long: ${JSON.stringify(source)}`);
|
|
76
|
+
}
|
|
77
|
+
const normalized = posix.normalize(withoutRoot);
|
|
78
|
+
if (!withoutRoot ||
|
|
79
|
+
normalized === "." ||
|
|
80
|
+
normalized === ".." ||
|
|
81
|
+
normalized.startsWith("../") ||
|
|
82
|
+
posix.isAbsolute(normalized) ||
|
|
83
|
+
normalized !== withoutRoot) {
|
|
84
|
+
throw new Error(`snapshot file path is unsafe: ${JSON.stringify(source)}`);
|
|
85
|
+
}
|
|
86
|
+
return normalized;
|
|
87
|
+
};
|
|
88
|
+
const contentBuffer = (content) => typeof content === "string" ? Buffer.from(content, "utf8") : Buffer.from(content);
|
|
89
|
+
const prepareFiles = (files, maxFiles, maxBytes) => {
|
|
90
|
+
const paths = new Set();
|
|
91
|
+
const entries = Object.entries(files);
|
|
92
|
+
if (entries.length > maxFiles) {
|
|
93
|
+
throw new Error(`snapshot has ${entries.length} files, exceeding the ${maxFiles}-file limit`);
|
|
94
|
+
}
|
|
95
|
+
const prepared = [];
|
|
96
|
+
let bytes = 0;
|
|
97
|
+
for (const [sourcePath, content] of entries) {
|
|
98
|
+
const path = snapshotPath(sourcePath);
|
|
99
|
+
if (path === ARTIFACT_SNAPSHOT_MANIFEST) {
|
|
100
|
+
throw new Error(`snapshot file path is reserved: ${path}`);
|
|
101
|
+
}
|
|
102
|
+
if (paths.has(path)) {
|
|
103
|
+
throw new Error(`snapshot contains duplicate path: ${path}`);
|
|
104
|
+
}
|
|
105
|
+
paths.add(path);
|
|
106
|
+
const contentBytes = typeof content === "string" ? Buffer.byteLength(content, "utf8") : content.byteLength;
|
|
107
|
+
bytes += contentBytes;
|
|
108
|
+
if (!Number.isSafeInteger(bytes) || bytes > maxBytes) {
|
|
109
|
+
throw new Error(`snapshot has ${bytes} bytes, exceeding the ${maxBytes}-byte limit`);
|
|
110
|
+
}
|
|
111
|
+
const buffer = contentBuffer(content);
|
|
112
|
+
prepared.push({
|
|
113
|
+
path,
|
|
114
|
+
content: buffer,
|
|
115
|
+
bytes: buffer.byteLength,
|
|
116
|
+
sha256: createHash("sha256").update(buffer).digest("hex"),
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
prepared.sort((left, right) => {
|
|
120
|
+
if (left.path < right.path)
|
|
121
|
+
return -1;
|
|
122
|
+
if (left.path > right.path)
|
|
123
|
+
return 1;
|
|
124
|
+
return 0;
|
|
125
|
+
});
|
|
126
|
+
return prepared;
|
|
127
|
+
};
|
|
128
|
+
const ensurePrivateParents = async (root, target) => {
|
|
129
|
+
const resolvedRoot = resolve(root);
|
|
130
|
+
const parent = dirname(target);
|
|
131
|
+
const path = relative(resolvedRoot, parent);
|
|
132
|
+
let cursor = resolvedRoot;
|
|
133
|
+
for (const part of path.split(sep).filter(Boolean)) {
|
|
134
|
+
cursor = resolve(cursor, part);
|
|
135
|
+
try {
|
|
136
|
+
await mkdir(cursor, { mode: 0o700 });
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
if (!(error instanceof Error && "code" in error && error.code === "EEXIST"))
|
|
140
|
+
throw error;
|
|
141
|
+
}
|
|
142
|
+
const stats = await lstat(cursor);
|
|
143
|
+
if (stats.isSymbolicLink() || !stats.isDirectory()) {
|
|
144
|
+
throw new Error(`snapshot parent must be a real directory: ${cursor}`);
|
|
145
|
+
}
|
|
146
|
+
const canonical = await realpath(cursor);
|
|
147
|
+
if (canonical !== resolvedRoot && !canonical.startsWith(`${resolvedRoot}${sep}`)) {
|
|
148
|
+
throw new Error(`snapshot parent escaped its root: ${cursor}`);
|
|
149
|
+
}
|
|
150
|
+
await chmod(cursor, 0o700);
|
|
151
|
+
}
|
|
152
|
+
};
|
|
153
|
+
const writePrivateFile = async (root, path, content) => {
|
|
154
|
+
const target = resolve(root, path);
|
|
155
|
+
if (!target.startsWith(`${resolve(root)}${sep}`)) {
|
|
156
|
+
throw new Error(`snapshot path escaped its root: ${path}`);
|
|
157
|
+
}
|
|
158
|
+
await ensurePrivateParents(root, target);
|
|
159
|
+
const handle = await open(target, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600);
|
|
160
|
+
try {
|
|
161
|
+
await handle.writeFile(content);
|
|
162
|
+
await handle.chmod(0o400);
|
|
163
|
+
}
|
|
164
|
+
finally {
|
|
165
|
+
await handle.close();
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
export const readArtifactSnapshotManifest = async (snapshotRoot) => {
|
|
169
|
+
const rootStats = await lstat(snapshotRoot);
|
|
170
|
+
if (rootStats.isSymbolicLink() || !rootStats.isDirectory()) {
|
|
171
|
+
throw new Error(`artifact snapshot must be a real directory: ${snapshotRoot}`);
|
|
172
|
+
}
|
|
173
|
+
const manifestPath = join(snapshotRoot, ARTIFACT_SNAPSHOT_MANIFEST);
|
|
174
|
+
const manifestStats = await lstat(manifestPath).catch((error) => {
|
|
175
|
+
if (error instanceof Error && "code" in error && error.code === "ENOENT")
|
|
176
|
+
return undefined;
|
|
177
|
+
throw error;
|
|
178
|
+
});
|
|
179
|
+
if (!manifestStats)
|
|
180
|
+
return undefined;
|
|
181
|
+
if (manifestStats.isSymbolicLink() ||
|
|
182
|
+
!manifestStats.isFile() ||
|
|
183
|
+
manifestStats.size > MAX_MANIFEST_BYTES) {
|
|
184
|
+
throw new Error(`artifact snapshot manifest is invalid: ${snapshotRoot}`);
|
|
185
|
+
}
|
|
186
|
+
return ArtifactSnapshotManifestSchema.parse(JSON.parse(await readFile(manifestPath, "utf8")));
|
|
187
|
+
};
|
|
188
|
+
const loadSnapshotBase = async (root, name) => {
|
|
189
|
+
if (name === undefined)
|
|
190
|
+
return undefined;
|
|
191
|
+
if (!SNAPSHOT_NAME.test(name)) {
|
|
192
|
+
throw new Error(`artifact base snapshot name is invalid: ${name}`);
|
|
193
|
+
}
|
|
194
|
+
const baseRoot = join(root, name);
|
|
195
|
+
const stats = await lstat(baseRoot);
|
|
196
|
+
if (stats.isSymbolicLink() || !stats.isDirectory()) {
|
|
197
|
+
throw new Error(`artifact base snapshot must be a real directory: ${name}`);
|
|
198
|
+
}
|
|
199
|
+
const manifest = await readArtifactSnapshotManifest(baseRoot);
|
|
200
|
+
if (!manifest)
|
|
201
|
+
throw new Error(`artifact base snapshot manifest is missing: ${name}`);
|
|
202
|
+
if (manifest.name !== name) {
|
|
203
|
+
throw new Error(`artifact base snapshot manifest names ${manifest.name}, expected ${name}`);
|
|
204
|
+
}
|
|
205
|
+
return {
|
|
206
|
+
root: baseRoot,
|
|
207
|
+
files: new Map(manifest.files.map((file) => [file.path, file])),
|
|
208
|
+
};
|
|
209
|
+
};
|
|
210
|
+
const writeSnapshotFile = async (root, file, base) => {
|
|
211
|
+
const previous = base?.files.get(file.path);
|
|
212
|
+
if (base && previous?.bytes === file.bytes && previous.sha256 === file.sha256) {
|
|
213
|
+
const source = resolve(base.root, file.path);
|
|
214
|
+
const target = resolve(root, file.path);
|
|
215
|
+
if (!source.startsWith(`${resolve(base.root)}${sep}`) ||
|
|
216
|
+
!target.startsWith(`${resolve(root)}${sep}`)) {
|
|
217
|
+
throw new Error(`snapshot path escaped its root: ${file.path}`);
|
|
218
|
+
}
|
|
219
|
+
let sourceMatches = false;
|
|
220
|
+
try {
|
|
221
|
+
const sourceHandle = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
222
|
+
try {
|
|
223
|
+
const sourceStats = await sourceHandle.stat();
|
|
224
|
+
if (sourceStats.isFile() && sourceStats.size === file.bytes) {
|
|
225
|
+
const sourceContent = await sourceHandle.readFile();
|
|
226
|
+
sourceMatches = createHash("sha256").update(sourceContent).digest("hex") === file.sha256;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
finally {
|
|
230
|
+
await sourceHandle.close();
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
catch (error) {
|
|
234
|
+
if (error instanceof Error &&
|
|
235
|
+
"code" in error &&
|
|
236
|
+
(error.code === "ENOENT" || error.code === "ELOOP")) {
|
|
237
|
+
await writePrivateFile(root, file.path, file.content);
|
|
238
|
+
return false;
|
|
239
|
+
}
|
|
240
|
+
throw error;
|
|
241
|
+
}
|
|
242
|
+
if (sourceMatches) {
|
|
243
|
+
await ensurePrivateParents(root, target);
|
|
244
|
+
await copyFile(source, target, constants.COPYFILE_FICLONE);
|
|
245
|
+
await chmod(target, 0o400);
|
|
246
|
+
const copied = await readFile(target);
|
|
247
|
+
if (copied.byteLength === file.bytes &&
|
|
248
|
+
createHash("sha256").update(copied).digest("hex") === file.sha256) {
|
|
249
|
+
return true;
|
|
250
|
+
}
|
|
251
|
+
try {
|
|
252
|
+
await unlink(target);
|
|
253
|
+
}
|
|
254
|
+
catch (error) {
|
|
255
|
+
if (!(error instanceof Error && "code" in error && error.code === "ENOENT"))
|
|
256
|
+
throw error;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
await writePrivateFile(root, file.path, file.content);
|
|
261
|
+
return false;
|
|
262
|
+
};
|
|
263
|
+
const writeFiles = async (root, files, concurrency, base) => {
|
|
264
|
+
let cursor = 0;
|
|
265
|
+
const reused = new Set();
|
|
266
|
+
const worker = async () => {
|
|
267
|
+
while (cursor < files.length) {
|
|
268
|
+
const file = files[cursor];
|
|
269
|
+
cursor += 1;
|
|
270
|
+
if (file && (await writeSnapshotFile(root, file, base))) {
|
|
271
|
+
reused.add(file.path);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
};
|
|
275
|
+
await Promise.all(Array.from({ length: Math.min(concurrency, Math.max(1, files.length)) }, worker));
|
|
276
|
+
return reused;
|
|
277
|
+
};
|
|
278
|
+
const ensureArtifactRoot = async (artifactsDir) => {
|
|
279
|
+
await mkdir(artifactsDir, { recursive: true, mode: 0o700 });
|
|
280
|
+
const stats = await lstat(artifactsDir);
|
|
281
|
+
if (stats.isSymbolicLink() || !stats.isDirectory()) {
|
|
282
|
+
throw new Error(`artifact snapshot root must be a real directory: ${artifactsDir}`);
|
|
283
|
+
}
|
|
284
|
+
await chmod(artifactsDir, 0o700);
|
|
285
|
+
return await realpath(artifactsDir);
|
|
286
|
+
};
|
|
287
|
+
export const writeArtifactSnapshot = async ({ artifactsDir, name, files, baseSnapshot, maxFiles: requestedMaxFiles, maxBytes: requestedMaxBytes, writeConcurrency: requestedWriteConcurrency, }) => {
|
|
288
|
+
if (!SNAPSHOT_NAME.test(name)) {
|
|
289
|
+
throw new Error("artifact snapshot name must start with a letter or number and contain only letters, numbers, dots, underscores, or hyphens");
|
|
290
|
+
}
|
|
291
|
+
const maxFiles = positiveInteger(requestedMaxFiles, DEFAULT_MAX_FILES, "snapshot maxFiles");
|
|
292
|
+
const maxBytes = positiveInteger(requestedMaxBytes, DEFAULT_MAX_BYTES, "snapshot maxBytes");
|
|
293
|
+
const writeConcurrency = positiveInteger(requestedWriteConcurrency, DEFAULT_WRITE_CONCURRENCY, "snapshot writeConcurrency");
|
|
294
|
+
const prepared = prepareFiles(files, maxFiles, maxBytes);
|
|
295
|
+
const root = await ensureArtifactRoot(artifactsDir);
|
|
296
|
+
if (baseSnapshot === name) {
|
|
297
|
+
throw new Error("artifact snapshot cannot use itself as its base");
|
|
298
|
+
}
|
|
299
|
+
const base = await loadSnapshotBase(root, baseSnapshot);
|
|
300
|
+
const target = join(root, name);
|
|
301
|
+
try {
|
|
302
|
+
await lstat(target);
|
|
303
|
+
throw new Error(`artifact snapshot already exists: ${name}`);
|
|
304
|
+
}
|
|
305
|
+
catch (error) {
|
|
306
|
+
if (!(error instanceof Error && "code" in error && error.code === "ENOENT")) {
|
|
307
|
+
throw error;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
const staging = await mkdtemp(join(root, `.lux-snapshot-${name}-`));
|
|
311
|
+
await chmod(staging, 0o700);
|
|
312
|
+
try {
|
|
313
|
+
const reused = await writeFiles(staging, prepared, writeConcurrency, base);
|
|
314
|
+
const manifest = {
|
|
315
|
+
schemaVersion: 1,
|
|
316
|
+
name,
|
|
317
|
+
files: prepared.map(({ path, bytes, sha256 }) => ({
|
|
318
|
+
path,
|
|
319
|
+
bytes,
|
|
320
|
+
sha256,
|
|
321
|
+
})),
|
|
322
|
+
fileCount: prepared.length,
|
|
323
|
+
bytes: prepared.reduce((total, file) => total + file.bytes, 0),
|
|
324
|
+
...(baseSnapshot ? { baseSnapshot } : {}),
|
|
325
|
+
reusedFiles: reused.size,
|
|
326
|
+
reusedBytes: prepared
|
|
327
|
+
.filter((file) => reused.has(file.path))
|
|
328
|
+
.reduce((total, file) => total + file.bytes, 0),
|
|
329
|
+
};
|
|
330
|
+
await writePrivateFile(staging, ARTIFACT_SNAPSHOT_MANIFEST, Buffer.from(`${JSON.stringify(manifest, null, 2)}\n`, "utf8"));
|
|
331
|
+
await rename(staging, target);
|
|
332
|
+
return manifest;
|
|
333
|
+
}
|
|
334
|
+
catch (error) {
|
|
335
|
+
await rm(staging, { recursive: true, force: true });
|
|
336
|
+
throw error;
|
|
337
|
+
}
|
|
338
|
+
};
|
|
339
|
+
//# sourceMappingURL=artifact-snapshot.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"artifact-snapshot.js","sourceRoot":"","sources":["../../src/core/artifact-snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,KAAK,EACL,SAAS,EACT,QAAQ,EACR,KAAK,EACL,KAAK,EACL,OAAO,EACP,IAAI,EACJ,QAAQ,EACR,QAAQ,EACR,MAAM,EACN,EAAE,EACF,MAAM,GACP,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACzE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,iBAAiB,GAAG,OAAO,CAAC;AAClC,MAAM,iBAAiB,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAC5C,MAAM,yBAAyB,GAAG,EAAE,CAAC;AACrC,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAC5C,MAAM,uBAAuB,GAAG,KAAK,CAAC;AACtC,MAAM,CAAC,MAAM,0BAA0B,GAAG,oBAAoB,CAAC;AAC/D,MAAM,aAAa,GAAG,oCAAoC,CAAC;AAE3D,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,IAAI,EAAE,CAAC;SACJ,MAAM,EAAE;SACR,GAAG,CAAC,CAAC,CAAC;SACN,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACf,IAAI,CAAC;YACH,OAAO,YAAY,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC,EAAE,uBAAuB,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACrC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC;CAC3C,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC;KAC5C,MAAM,CAAC;IACN,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,0BAA0B,CAAC;IAC1C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACzC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IACrC,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE;IACxD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;IAC3C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE;CAC5C,CAAC;KACD,WAAW,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;IAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC5E,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACzC,GAAG,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,sBAAsB,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,QAAQ,CAAC,SAAS,KAAK,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACjD,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,CAAC,WAAW,CAAC;YACnB,OAAO,EAAE,mCAAmC;SAC7C,CAAC,CAAC;IACL,CAAC;IACD,IAAI,QAAQ,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;QAC7B,GAAG,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,6BAA6B,EAAE,CAAC,CAAC;IAC5F,CAAC;IACD,IAAI,QAAQ,CAAC,WAAW,GAAG,QAAQ,CAAC,SAAS,IAAI,QAAQ,CAAC,WAAW,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;QACvF,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,QAAQ;YACd,IAAI,EAAE,CAAC,aAAa,CAAC;YACrB,OAAO,EAAE,6CAA6C;SACvD,CAAC,CAAC;IACL,CAAC;AACH,CAAC,CAAC,CAAC;AAoBL,MAAM,eAAe,GAAG,CAAC,KAAyB,EAAE,QAAgB,EAAE,KAAa,EAAU,EAAE;IAC7F,MAAM,QAAQ,GAAG,KAAK,IAAI,QAAQ,CAAC;IACnC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,UAAU,CAAC,GAAG,KAAK,kCAAkC,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,MAAc,EAAU,EAAE;IAC9C,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC/C,IAAI,MAAM,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,uBAAuB,EAAE,CAAC;QAC7D,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IAChD,IACE,CAAC,WAAW;QACZ,UAAU,KAAK,GAAG;QAClB,UAAU,KAAK,IAAI;QACnB,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC;QAC5B,KAAK,CAAC,UAAU,CAAC,UAAU,CAAC;QAC5B,UAAU,KAAK,WAAW,EAC1B,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,OAAgC,EAAU,EAAE,CACjE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAEpF,MAAM,YAAY,GAAG,CACnB,KAA4B,EAC5B,QAAgB,EAChB,QAAgB,EACQ,EAAE;IAC1B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACtC,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,gBAAgB,OAAO,CAAC,MAAM,yBAAyB,QAAQ,aAAa,CAAC,CAAC;IAChG,CAAC;IACD,MAAM,QAAQ,GAA2B,EAAE,CAAC;IAC5C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;QAC5C,MAAM,IAAI,GAAG,YAAY,CAAC,UAAU,CAAC,CAAC;QACtC,IAAI,IAAI,KAAK,0BAA0B,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChB,MAAM,YAAY,GAChB,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC;QACxF,KAAK,IAAI,YAAY,CAAC;QACtB,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,gBAAgB,KAAK,yBAAyB,QAAQ,aAAa,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;QACtC,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI;YACJ,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,MAAM,CAAC,UAAU;YACxB,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC1D,CAAC,CAAC;IACL,CAAC;IACD,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,IAAI,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC,CAAC;QACtC,IAAI,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QACrC,OAAO,CAAC,CAAC;IACX,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,oBAAoB,GAAG,KAAK,EAAE,IAAY,EAAE,MAAc,EAAiB,EAAE;IACjF,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC5C,IAAI,MAAM,GAAG,YAAY,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACnD,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC/B,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;gBAAE,MAAM,KAAK,CAAC;QAC3F,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CAAC,6CAA6C,MAAM,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,SAAS,KAAK,YAAY,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,YAAY,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC;YACjF,MAAM,IAAI,KAAK,CAAC,qCAAqC,MAAM,EAAE,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAAE,IAAY,EAAE,IAAY,EAAE,OAAmB,EAAiB,EAAE;IAChG,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,EAAE,CAAC,CAAC;IAC7D,CAAC;IACD,MAAM,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzC,MAAM,MAAM,GAAG,MAAM,IAAI,CACvB,MAAM,EACN,SAAS,CAAC,OAAO,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,UAAU,EAChF,KAAK,CACN,CAAC;IACF,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAChC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC,CAAC;AAOF,MAAM,CAAC,MAAM,4BAA4B,GAAG,KAAK,EAC/C,YAAoB,EAC2B,EAAE;IACjD,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,CAAC;IAC5C,IAAI,SAAS,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,+CAA+C,YAAY,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,EAAE,0BAA0B,CAAC,CAAC;IACpE,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;QACvE,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QAC3F,MAAM,KAAK,CAAC;IACd,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,aAAa;QAAE,OAAO,SAAS,CAAC;IACrC,IACE,aAAa,CAAC,cAAc,EAAE;QAC9B,CAAC,aAAa,CAAC,MAAM,EAAE;QACvB,aAAa,CAAC,IAAI,GAAG,kBAAkB,EACvC,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,0CAA0C,YAAY,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,8BAA8B,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AAChG,CAAC,CAAC;AAEF,MAAM,gBAAgB,GAAG,KAAK,EAC5B,IAAY,EACZ,IAAwB,EACW,EAAE;IACrC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACzC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,2CAA2C,IAAI,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,oDAAoD,IAAI,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,4BAA4B,CAAC,QAAQ,CAAC,CAAC;IAC9D,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,EAAE,CAAC,CAAC;IACtF,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,yCAAyC,QAAQ,CAAC,IAAI,cAAc,IAAI,EAAE,CAAC,CAAC;IAC9F,CAAC;IACD,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;KAChE,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,KAAK,EAC7B,IAAY,EACZ,IAA0B,EAC1B,IAA8B,EACZ,EAAE;IACpB,MAAM,QAAQ,GAAG,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,IAAI,IAAI,QAAQ,EAAE,KAAK,KAAK,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAC9E,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,IACE,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;YACjD,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC,EAC5C,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC;QACD,IAAI,aAAa,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,SAAS,CAAC,QAAQ,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC;YACnF,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,CAAC;gBAC9C,IAAI,WAAW,CAAC,MAAM,EAAE,IAAI,WAAW,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC5D,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,QAAQ,EAAE,CAAC;oBACpD,aAAa,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC;gBAC3F,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,YAAY,CAAC,KAAK,EAAE,CAAC;YAC7B,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IACE,KAAK,YAAY,KAAK;gBACtB,MAAM,IAAI,KAAK;gBACf,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,CAAC,EACnD,CAAC;gBACD,MAAM,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;gBACtD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;QACD,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,oBAAoB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACzC,MAAM,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC;YAC3D,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;YACtC,IACE,MAAM,CAAC,UAAU,KAAK,IAAI,CAAC,KAAK;gBAChC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,MAAM,EACjE,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;oBAAE,MAAM,KAAK,CAAC;YAC3F,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,gBAAgB,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;IACtD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,KAAK,EACtB,IAAY,EACZ,KAA6B,EAC7B,WAAmB,EACnB,IAA8B,EACR,EAAE;IACxB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;IACjC,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YAC3B,MAAM,IAAI,CAAC,CAAC;YACZ,IAAI,IAAI,IAAI,CAAC,MAAM,iBAAiB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;gBACxD,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,CACjF,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,kBAAkB,GAAG,KAAK,EAAE,YAAoB,EAAmB,EAAE;IACzE,MAAM,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,CAAC;IACxC,IAAI,KAAK,CAAC,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,oDAAoD,YAAY,EAAE,CAAC,CAAC;IACtF,CAAC;IACD,MAAM,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;IACjC,OAAO,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;AACtC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EAAE,EAC1C,YAAY,EACZ,IAAI,EACJ,KAAK,EACL,YAAY,EACZ,QAAQ,EAAE,iBAAiB,EAC3B,QAAQ,EAAE,iBAAiB,EAC3B,gBAAgB,EAAE,yBAAyB,GACd,EAAqC,EAAE;IACpE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,4HAA4H,CAC7H,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,eAAe,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,mBAAmB,CAAC,CAAC;IAC5F,MAAM,QAAQ,GAAG,eAAe,CAAC,iBAAiB,EAAE,iBAAiB,EAAE,mBAAmB,CAAC,CAAC;IAC5F,MAAM,gBAAgB,GAAG,eAAe,CACtC,yBAAyB,EACzB,yBAAyB,EACzB,2BAA2B,CAC5B,CAAC;IACF,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,CAAC;IACpD,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,EAAE,CAAC,CAAC;IAC/D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,CAAC,CAAC,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC;YAC5E,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,IAAI,GAAG,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;QAC3E,MAAM,QAAQ,GAA6B;YACzC,aAAa,EAAE,CAAC;YAChB,IAAI;YACJ,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;gBAChD,IAAI;gBACJ,KAAK;gBACL,MAAM;aACP,CAAC,CAAC;YACH,SAAS,EAAE,QAAQ,CAAC,MAAM;YAC1B,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;YAC9D,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzC,WAAW,EAAE,MAAM,CAAC,IAAI;YACxB,WAAW,EAAE,QAAQ;iBAClB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;iBACvC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;SAClD,CAAC;QACF,MAAM,gBAAgB,CACpB,OAAO,EACP,0BAA0B,EAC1B,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAC9D,CAAC;QACF,MAAM,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC9B,OAAO,QAAQ,CAAC;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC,CAAC"}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { Assertion } from "./case.js";
|
|
3
|
+
import type { Run } from "./run.js";
|
|
4
|
+
export declare const AssertionResultSchema: z.ZodObject<{
|
|
5
|
+
passed: z.ZodBoolean;
|
|
6
|
+
score: z.ZodNumber;
|
|
7
|
+
comment: z.ZodExactOptional<z.ZodString>;
|
|
8
|
+
detail: z.ZodOptional<z.ZodUnknown>;
|
|
9
|
+
errorMessage: z.ZodExactOptional<z.ZodString>;
|
|
10
|
+
usage: z.ZodExactOptional<z.ZodObject<{
|
|
11
|
+
input: z.ZodNumber;
|
|
12
|
+
output: z.ZodNumber;
|
|
13
|
+
cacheRead: z.ZodNumber;
|
|
14
|
+
cacheCreation: z.ZodNumber;
|
|
15
|
+
}, z.core.$strip>>;
|
|
16
|
+
}, z.core.$strip>;
|
|
17
|
+
export type AssertionResult = z.infer<typeof AssertionResultSchema>;
|
|
18
|
+
export type EvaluateContext = {
|
|
19
|
+
/**
|
|
20
|
+
* Fires when the run is being torn down. Long-running assertions (an LLM
|
|
21
|
+
* judge, a shell command) must cancel their in-flight work, or a Ctrl-C
|
|
22
|
+
* leaves billable subprocesses running past the run's end.
|
|
23
|
+
*/
|
|
24
|
+
abortSignal?: AbortSignal;
|
|
25
|
+
/**
|
|
26
|
+
* Persists a judge-attempt start before an assertion invokes its provider.
|
|
27
|
+
* Grading marks it complete only after assertion spend is journaled.
|
|
28
|
+
*/
|
|
29
|
+
recordModelAttemptStart?: (kind: "judge") => Promise<void>;
|
|
30
|
+
};
|
|
31
|
+
export interface AssertionImpl<C extends Assertion = Assertion> {
|
|
32
|
+
type: C["type"];
|
|
33
|
+
configSchema?: z.ZodTypeAny;
|
|
34
|
+
evaluate(config: C, run: Run, ctx?: EvaluateContext): Promise<AssertionResult>;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Public assertion extension contract. Unlike the lower-level
|
|
38
|
+
* `AssertionImpl`, plugins must describe their case configuration so Lux can
|
|
39
|
+
* reject invalid plans before starting a subject run.
|
|
40
|
+
*/
|
|
41
|
+
export type AssertionPlugin<C extends Assertion = Assertion> = AssertionImpl<C> & {
|
|
42
|
+
configSchema: z.ZodObject & z.ZodType<C>;
|
|
43
|
+
/** Author-maintained identity for evaluator/schema behavior. */
|
|
44
|
+
pluginVersion: string;
|
|
45
|
+
};
|
|
46
|
+
type SchemaAssertion<S extends z.ZodObject> = z.output<S> extends Assertion ? z.output<S> : never;
|
|
47
|
+
type IsUnion<T, Whole = T> = T extends Whole ? ([Whole] extends [T] ? false : true) : never;
|
|
48
|
+
type SingleStringLiteral<T> = T extends string ? string extends T ? never : true extends IsUnion<T> ? never : T : never;
|
|
49
|
+
type LiteralAssertionType<S extends z.ZodObject> = SchemaAssertion<S> extends {
|
|
50
|
+
type: infer OutputType;
|
|
51
|
+
} ? z.input<S> extends {
|
|
52
|
+
type: infer InputType;
|
|
53
|
+
} ? SingleStringLiteral<OutputType> extends infer Literal extends string ? [InputType, Literal] extends [Literal, InputType] ? Literal : never : never : never : never;
|
|
54
|
+
export type AssertionPluginDefinition<S extends z.ZodObject> = {
|
|
55
|
+
type: LiteralAssertionType<S>;
|
|
56
|
+
configSchema: S;
|
|
57
|
+
pluginVersion: string;
|
|
58
|
+
evaluate(config: SchemaAssertion<S>, run: Run, ctx?: EvaluateContext): Promise<AssertionResult>;
|
|
59
|
+
};
|
|
60
|
+
export declare const assertionPluginIssue: (definition: {
|
|
61
|
+
type?: unknown;
|
|
62
|
+
configSchema?: unknown;
|
|
63
|
+
pluginVersion?: unknown;
|
|
64
|
+
evaluate?: unknown;
|
|
65
|
+
}) => string | undefined;
|
|
66
|
+
/**
|
|
67
|
+
* Define a schema-validated assertion plugin while inferring the config passed
|
|
68
|
+
* to `evaluate` from the schema.
|
|
69
|
+
*/
|
|
70
|
+
export declare const defineAssertion: <S extends z.ZodObject>(definition: AssertionPluginDefinition<S>) => AssertionPlugin<SchemaAssertion<S>>;
|
|
71
|
+
export declare const GradingAssertionEntrySchema: z.ZodObject<{
|
|
72
|
+
id: z.ZodExactOptional<z.ZodString>;
|
|
73
|
+
type: z.ZodString;
|
|
74
|
+
name: z.ZodString;
|
|
75
|
+
passed: z.ZodBoolean;
|
|
76
|
+
score: z.ZodNumber;
|
|
77
|
+
weight: z.ZodNumber;
|
|
78
|
+
threshold: z.ZodNumber;
|
|
79
|
+
durationMs: z.ZodNumber;
|
|
80
|
+
comment: z.ZodExactOptional<z.ZodString>;
|
|
81
|
+
detail: z.ZodOptional<z.ZodUnknown>;
|
|
82
|
+
errorMessage: z.ZodExactOptional<z.ZodString>;
|
|
83
|
+
usage: z.ZodExactOptional<z.ZodObject<{
|
|
84
|
+
input: z.ZodNumber;
|
|
85
|
+
output: z.ZodNumber;
|
|
86
|
+
cacheRead: z.ZodNumber;
|
|
87
|
+
cacheCreation: z.ZodNumber;
|
|
88
|
+
}, z.core.$strip>>;
|
|
89
|
+
}, z.core.$strip>;
|
|
90
|
+
export type GradingAssertionEntry = z.infer<typeof GradingAssertionEntrySchema>;
|
|
91
|
+
export declare const GradingReportSchema: z.ZodObject<{
|
|
92
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
93
|
+
casePassed: z.ZodNullable<z.ZodBoolean>;
|
|
94
|
+
caseScore: z.ZodNullable<z.ZodNumber>;
|
|
95
|
+
assertions: z.ZodArray<z.ZodObject<{
|
|
96
|
+
id: z.ZodExactOptional<z.ZodString>;
|
|
97
|
+
type: z.ZodString;
|
|
98
|
+
name: z.ZodString;
|
|
99
|
+
passed: z.ZodBoolean;
|
|
100
|
+
score: z.ZodNumber;
|
|
101
|
+
weight: z.ZodNumber;
|
|
102
|
+
threshold: z.ZodNumber;
|
|
103
|
+
durationMs: z.ZodNumber;
|
|
104
|
+
comment: z.ZodExactOptional<z.ZodString>;
|
|
105
|
+
detail: z.ZodOptional<z.ZodUnknown>;
|
|
106
|
+
errorMessage: z.ZodExactOptional<z.ZodString>;
|
|
107
|
+
usage: z.ZodExactOptional<z.ZodObject<{
|
|
108
|
+
input: z.ZodNumber;
|
|
109
|
+
output: z.ZodNumber;
|
|
110
|
+
cacheRead: z.ZodNumber;
|
|
111
|
+
cacheCreation: z.ZodNumber;
|
|
112
|
+
}, z.core.$strip>>;
|
|
113
|
+
}, z.core.$strip>>;
|
|
114
|
+
incomplete: z.ZodExactOptional<z.ZodObject<{
|
|
115
|
+
exitReason: z.ZodEnum<{
|
|
116
|
+
error: "error";
|
|
117
|
+
"idle-timeout": "idle-timeout";
|
|
118
|
+
"max-interactions": "max-interactions";
|
|
119
|
+
}>;
|
|
120
|
+
phase: z.ZodOptional<z.ZodEnum<{
|
|
121
|
+
collection: "collection";
|
|
122
|
+
grading: "grading";
|
|
123
|
+
run: "run";
|
|
124
|
+
}>>;
|
|
125
|
+
}, z.core.$strip>>;
|
|
126
|
+
}, z.core.$strip>;
|
|
127
|
+
export type GradingReport = z.infer<typeof GradingReportSchema>;
|
|
128
|
+
export {};
|
|
129
|
+
//# sourceMappingURL=assertion.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assertion.d.ts","sourceRoot":"","sources":["../../src/core/assertion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAGpC,eAAO,MAAM,qBAAqB;;;;;;;;;;;;iBAShC,CAAC;AACH,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AAEpE,MAAM,MAAM,eAAe,GAAG;IAC5B;;;;OAIG;IACH,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;OAGG;IACH,uBAAuB,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5D,CAAC;AAGF,MAAM,WAAW,aAAa,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS;IAC5D,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IAChB,YAAY,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,MAAM,eAAe,CAAC,CAAC,SAAS,SAAS,GAAG,SAAS,IAAI,aAAa,CAAC,CAAC,CAAC,GAAG;IAChF,YAAY,EAAE,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACzC,gEAAgE;IAChE,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,KAAK,eAAe,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC;AAElG,KAAK,OAAO,CAAC,CAAC,EAAE,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC;AAE5F,KAAK,mBAAmB,CAAC,CAAC,IAAI,CAAC,SAAS,MAAM,GAC1C,MAAM,SAAS,CAAC,GACd,KAAK,GACL,IAAI,SAAS,OAAO,CAAC,CAAC,CAAC,GACrB,KAAK,GACL,CAAC,GACL,KAAK,CAAC;AAEV,KAAK,oBAAoB,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAC7C,eAAe,CAAC,CAAC,CAAC,SAAS;IAAE,IAAI,EAAE,MAAM,UAAU,CAAA;CAAE,GACjD,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;IAAE,IAAI,EAAE,MAAM,SAAS,CAAA;CAAE,GAC1C,mBAAmB,CAAC,UAAU,CAAC,SAAS,MAAM,OAAO,SAAS,MAAM,GAClE,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,GAC/C,OAAO,GACP,KAAK,GACP,KAAK,GACP,KAAK,GACP,KAAK,CAAC;AAEZ,MAAM,MAAM,yBAAyB,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS,IAAI;IAC7D,IAAI,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC;IAC9B,YAAY,EAAE,CAAC,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACjG,CAAC;AAEF,eAAO,MAAM,oBAAoB,eAAgB;IAC/C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB,KAAG,MAAM,GAAG,SAsBZ,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,eAAe,GAAI,CAAC,SAAS,CAAC,CAAC,SAAS,cACvC,yBAAyB,CAAC,CAAC,CAAC,KACvC,eAAe,CAAC,eAAe,CAAC,CAAC,CAAC,CAIpC,CAAC;AAEF,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;iBAetC,CAAC;AACH,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAEhF,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAe9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { ErrorEventReasonSchema } from "./events.js";
|
|
3
|
+
import { TokenUsageSchema } from "./usage.js";
|
|
4
|
+
export const AssertionResultSchema = z.object({
|
|
5
|
+
passed: z.boolean(),
|
|
6
|
+
score: z.number().min(0).max(1),
|
|
7
|
+
comment: z.exactOptional(z.string()),
|
|
8
|
+
detail: z.unknown().optional(),
|
|
9
|
+
/** Present when the assertion could not produce a judgment. */
|
|
10
|
+
errorMessage: z.exactOptional(z.string()),
|
|
11
|
+
/** Tokens the assertion itself spent. Only LLM-judge assertions report this. */
|
|
12
|
+
usage: z.exactOptional(TokenUsageSchema),
|
|
13
|
+
});
|
|
14
|
+
export const assertionPluginIssue = (definition) => {
|
|
15
|
+
if (typeof definition.type !== "string") {
|
|
16
|
+
return "assertion plugin type must be a string";
|
|
17
|
+
}
|
|
18
|
+
if (!(definition.configSchema instanceof z.ZodObject)) {
|
|
19
|
+
return "assertion plugin configSchema must be a direct Zod object";
|
|
20
|
+
}
|
|
21
|
+
const discriminator = definition.configSchema.shape.type;
|
|
22
|
+
if (!(discriminator instanceof z.ZodLiteral) ||
|
|
23
|
+
discriminator.values.size !== 1 ||
|
|
24
|
+
discriminator.value !== definition.type) {
|
|
25
|
+
return "assertion plugin configSchema.type must be the matching string literal";
|
|
26
|
+
}
|
|
27
|
+
if (typeof definition.pluginVersion !== "string" || !definition.pluginVersion.trim()) {
|
|
28
|
+
return "assertion plugin pluginVersion must not be empty";
|
|
29
|
+
}
|
|
30
|
+
if (typeof definition.evaluate !== "function") {
|
|
31
|
+
return "assertion plugin evaluate must be a function";
|
|
32
|
+
}
|
|
33
|
+
return undefined;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Define a schema-validated assertion plugin while inferring the config passed
|
|
37
|
+
* to `evaluate` from the schema.
|
|
38
|
+
*/
|
|
39
|
+
export const defineAssertion = (definition) => {
|
|
40
|
+
const issue = assertionPluginIssue(definition);
|
|
41
|
+
if (issue)
|
|
42
|
+
throw new TypeError(issue);
|
|
43
|
+
return definition;
|
|
44
|
+
};
|
|
45
|
+
export const GradingAssertionEntrySchema = z.object({
|
|
46
|
+
/** Stable author-supplied id, or a deterministic positional fallback. */
|
|
47
|
+
id: z.exactOptional(z.string()),
|
|
48
|
+
type: z.string(),
|
|
49
|
+
name: z.string(),
|
|
50
|
+
passed: z.boolean(),
|
|
51
|
+
score: z.number(),
|
|
52
|
+
weight: z.number(),
|
|
53
|
+
threshold: z.number(),
|
|
54
|
+
/** Wall-clock the assertion took to evaluate. A `rubric` costs seconds; a file check, none. */
|
|
55
|
+
durationMs: z.number(),
|
|
56
|
+
comment: z.exactOptional(z.string()),
|
|
57
|
+
detail: z.unknown().optional(),
|
|
58
|
+
errorMessage: z.exactOptional(z.string()),
|
|
59
|
+
usage: z.exactOptional(TokenUsageSchema),
|
|
60
|
+
});
|
|
61
|
+
export const GradingReportSchema = z.object({
|
|
62
|
+
schemaVersion: z.literal(1),
|
|
63
|
+
casePassed: z.boolean().nullable(),
|
|
64
|
+
caseScore: z.number().nullable(),
|
|
65
|
+
assertions: z.array(GradingAssertionEntrySchema),
|
|
66
|
+
/**
|
|
67
|
+
* Present when the subject run or grading did not complete. Assertions may
|
|
68
|
+
* still contain partial diagnostics, but the report is not a clean verdict.
|
|
69
|
+
*/
|
|
70
|
+
incomplete: z.exactOptional(z.object({
|
|
71
|
+
exitReason: ErrorEventReasonSchema,
|
|
72
|
+
phase: z.enum(["run", "collection", "grading"]).optional(),
|
|
73
|
+
})),
|
|
74
|
+
});
|
|
75
|
+
//# sourceMappingURL=assertion.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"assertion.js","sourceRoot":"","sources":["../../src/core/assertion.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAErD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,+DAA+D;IAC/D,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,gFAAgF;IAChF,KAAK,EAAE,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC;CACzC,CAAC,CAAC;AAiEH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,UAKpC,EAAsB,EAAE;IACvB,IAAI,OAAO,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,wCAAwC,CAAC;IAClD,CAAC;IACD,IAAI,CAAC,CAAC,UAAU,CAAC,YAAY,YAAY,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;QACtD,OAAO,2DAA2D,CAAC;IACrE,CAAC;IACD,MAAM,aAAa,GAAG,UAAU,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;IACzD,IACE,CAAC,CAAC,aAAa,YAAY,CAAC,CAAC,UAAU,CAAC;QACxC,aAAa,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC;QAC/B,aAAa,CAAC,KAAK,KAAK,UAAU,CAAC,IAAI,EACvC,CAAC;QACD,OAAO,wEAAwE,CAAC;IAClF,CAAC;IACD,IAAI,OAAO,UAAU,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,EAAE,EAAE,CAAC;QACrF,OAAO,kDAAkD,CAAC;IAC5D,CAAC;IACD,IAAI,OAAO,UAAU,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QAC9C,OAAO,8CAA8C,CAAC;IACxD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,CAC7B,UAAwC,EACH,EAAE;IACvC,MAAM,KAAK,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;IAC/C,IAAI,KAAK;QAAE,MAAM,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;IACtC,OAAO,UAA4D,CAAC;AACtE,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD,yEAAyE;IACzE,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE;IACnB,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,+FAA+F;IAC/F,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;IACtB,OAAO,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACpC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAC9B,YAAY,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IACzC,KAAK,EAAE,CAAC,CAAC,aAAa,CAAC,gBAAgB,CAAC;CACzC,CAAC,CAAC;AAGH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1C,aAAa,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC3B,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,2BAA2B,CAAC;IAChD;;;OAGG;IACH,UAAU,EAAE,CAAC,CAAC,aAAa,CACzB,CAAC,CAAC,MAAM,CAAC;QACP,UAAU,EAAE,sBAAsB;QAClC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE;KAC3D,CAAC,CACH;CACF,CAAC,CAAC"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const FixturesRefSchema: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
3
|
+
kind: z.ZodLiteral<"dir">;
|
|
4
|
+
path: z.ZodString;
|
|
5
|
+
}, z.core.$strict>]>;
|
|
6
|
+
export type FixturesRef = z.infer<typeof FixturesRefSchema>;
|
|
7
|
+
export declare const DriverRefSchema: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
8
|
+
name: z.ZodString;
|
|
9
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
10
|
+
}, z.core.$strict>]>;
|
|
11
|
+
export type DriverRef = z.infer<typeof DriverRefSchema>;
|
|
12
|
+
export declare const AnswererRefSchema: z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
13
|
+
name: z.ZodString;
|
|
14
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
15
|
+
}, z.core.$strict>]>;
|
|
16
|
+
export type AnswererRef = z.infer<typeof AnswererRefSchema>;
|
|
17
|
+
export declare const AssertionSchema: z.ZodObject<{
|
|
18
|
+
id: z.ZodOptional<z.ZodString>;
|
|
19
|
+
type: z.ZodString;
|
|
20
|
+
name: z.ZodOptional<z.ZodString>;
|
|
21
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
22
|
+
threshold: z.ZodOptional<z.ZodNumber>;
|
|
23
|
+
}, z.core.$loose>;
|
|
24
|
+
export type Assertion = z.infer<typeof AssertionSchema>;
|
|
25
|
+
export declare const EvalCaseSchema: z.ZodObject<{
|
|
26
|
+
id: z.ZodOptional<z.ZodString>;
|
|
27
|
+
prompt: z.ZodString;
|
|
28
|
+
persona: z.ZodOptional<z.ZodString>;
|
|
29
|
+
driver: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
30
|
+
name: z.ZodString;
|
|
31
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
32
|
+
}, z.core.$strict>]>>;
|
|
33
|
+
answerer: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
34
|
+
name: z.ZodString;
|
|
35
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
36
|
+
}, z.core.$strict>]>>;
|
|
37
|
+
fixtures: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
38
|
+
kind: z.ZodLiteral<"dir">;
|
|
39
|
+
path: z.ZodString;
|
|
40
|
+
}, z.core.$strict>]>>;
|
|
41
|
+
assertions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
42
|
+
id: z.ZodOptional<z.ZodString>;
|
|
43
|
+
type: z.ZodString;
|
|
44
|
+
name: z.ZodOptional<z.ZodString>;
|
|
45
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
46
|
+
threshold: z.ZodOptional<z.ZodNumber>;
|
|
47
|
+
}, z.core.$loose>>>;
|
|
48
|
+
meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
49
|
+
}, z.core.$strict>;
|
|
50
|
+
export type EvalCase = z.infer<typeof EvalCaseSchema>;
|
|
51
|
+
export declare const defineEvalCase: (input: EvalCase) => EvalCase;
|
|
52
|
+
export declare const PersistedCaseSchema: z.ZodObject<{
|
|
53
|
+
id: z.ZodOptional<z.ZodOptional<z.ZodString>>;
|
|
54
|
+
prompt: z.ZodOptional<z.ZodString>;
|
|
55
|
+
persona: z.ZodOptional<z.ZodOptional<z.ZodString>>;
|
|
56
|
+
assertions: z.ZodOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
57
|
+
id: z.ZodOptional<z.ZodString>;
|
|
58
|
+
type: z.ZodString;
|
|
59
|
+
name: z.ZodOptional<z.ZodString>;
|
|
60
|
+
weight: z.ZodOptional<z.ZodNumber>;
|
|
61
|
+
threshold: z.ZodOptional<z.ZodNumber>;
|
|
62
|
+
}, z.core.$loose>>>>;
|
|
63
|
+
meta: z.ZodOptional<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
64
|
+
casePath: z.ZodExactOptional<z.ZodString>;
|
|
65
|
+
driver: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
66
|
+
name: z.ZodString;
|
|
67
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
68
|
+
}, z.core.$loose>]>>;
|
|
69
|
+
answerer: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
70
|
+
name: z.ZodString;
|
|
71
|
+
config: z.ZodOptional<z.ZodUnknown>;
|
|
72
|
+
}, z.core.$loose>]>>;
|
|
73
|
+
fixtures: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
74
|
+
kind: z.ZodLiteral<"dir">;
|
|
75
|
+
path: z.ZodString;
|
|
76
|
+
}, z.core.$loose>]>>;
|
|
77
|
+
}, z.core.$loose>;
|
|
78
|
+
export type PersistedCase = z.infer<typeof PersistedCaseSchema>;
|
|
79
|
+
//# sourceMappingURL=case.d.ts.map
|