agent-hitch 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -3
- package/README.zh-CN.md +14 -2
- package/dist/bin/hitch.js +25 -3
- package/dist/bin/hitch.js.map +1 -1
- package/dist/scripts/canary-benchmark.js +54 -0
- package/dist/scripts/canary-benchmark.js.map +1 -0
- package/dist/scripts/canary-buildkit-secrets.js +166 -0
- package/dist/scripts/canary-buildkit-secrets.js.map +1 -0
- package/dist/scripts/canary-collision-domains.js +77 -0
- package/dist/scripts/canary-collision-domains.js.map +1 -0
- package/dist/scripts/canary-docker-image-observation.js +181 -0
- package/dist/scripts/canary-docker-image-observation.js.map +1 -0
- package/dist/scripts/canary-eval-scheduler-throughput.js +47 -0
- package/dist/scripts/canary-eval-scheduler-throughput.js.map +1 -0
- package/dist/scripts/canary-gpu-hardware.js +95 -0
- package/dist/scripts/canary-gpu-hardware.js.map +1 -0
- package/dist/scripts/canary-harbor-load.js +365 -0
- package/dist/scripts/canary-harbor-load.js.map +1 -0
- package/dist/scripts/canary-packaged-harness.js +75 -0
- package/dist/scripts/canary-packaged-harness.js.map +1 -0
- package/dist/scripts/canary-resource-load.js +188 -0
- package/dist/scripts/canary-resource-load.js.map +1 -0
- package/dist/scripts/check-architecture.js +10 -5
- package/dist/scripts/check-architecture.js.map +1 -1
- package/dist/scripts/docker-reaper-crash-child.js +21 -0
- package/dist/scripts/docker-reaper-crash-child.js.map +1 -0
- package/dist/src/adapters/catalog.js +3 -1
- package/dist/src/adapters/catalog.js.map +1 -1
- package/dist/src/adapters/discovery.js +1 -1
- package/dist/src/adapters/discovery.js.map +1 -1
- package/dist/src/adapters/providers/claude.js +6 -0
- package/dist/src/adapters/providers/claude.js.map +1 -1
- package/dist/src/adapters/providers/codex-auth.js +29 -0
- package/dist/src/adapters/providers/codex-auth.js.map +1 -0
- package/dist/src/adapters/providers/codex.js +29 -6
- package/dist/src/adapters/providers/codex.js.map +1 -1
- package/dist/src/adapters/providers/deepseek.js +6 -0
- package/dist/src/adapters/providers/deepseek.js.map +1 -1
- package/dist/src/adapters/providers/model-call.js +26 -0
- package/dist/src/adapters/providers/model-call.js.map +1 -0
- package/dist/src/adapters/providers/opencode.js +6 -0
- package/dist/src/adapters/providers/opencode.js.map +1 -1
- package/dist/src/adapters/providers/pi.js +6 -0
- package/dist/src/adapters/providers/pi.js.map +1 -1
- package/dist/src/artifacts/index.js +1 -0
- package/dist/src/artifacts/index.js.map +1 -1
- package/dist/src/artifacts/preparer.js +7 -2
- package/dist/src/artifacts/preparer.js.map +1 -1
- package/dist/src/backends/harbor/agent-budget.js +31 -0
- package/dist/src/backends/harbor/agent-budget.js.map +1 -0
- package/dist/src/backends/harbor/backend.js +59 -107
- package/dist/src/backends/harbor/backend.js.map +1 -1
- package/dist/src/backends/harbor/environment-config.js +92 -0
- package/dist/src/backends/harbor/environment-config.js.map +1 -0
- package/dist/src/backends/harbor/index.js +6 -1
- package/dist/src/backends/harbor/index.js.map +1 -1
- package/dist/src/backends/harbor/model-proxy-config.js +23 -0
- package/dist/src/backends/harbor/model-proxy-config.js.map +1 -0
- package/dist/src/backends/harbor/process.js +177 -0
- package/dist/src/backends/harbor/process.js.map +1 -0
- package/dist/src/backends/harbor/regrade.js +77 -0
- package/dist/src/backends/harbor/regrade.js.map +1 -0
- package/dist/src/backends/harbor/runtime-toolchain.js +30 -0
- package/dist/src/backends/harbor/runtime-toolchain.js.map +1 -0
- package/dist/src/backends/harbor/supervisor.js +74 -0
- package/dist/src/backends/harbor/supervisor.js.map +1 -0
- package/dist/src/backends/harbor/task-resources.js +206 -0
- package/dist/src/backends/harbor/task-resources.js.map +1 -0
- package/dist/src/backends/harbor/tools.js +6 -29
- package/dist/src/backends/harbor/tools.js.map +1 -1
- package/dist/src/backends/index.js +5 -1
- package/dist/src/backends/index.js.map +1 -1
- package/dist/src/benchmarks/index.js +3 -0
- package/dist/src/benchmarks/index.js.map +1 -0
- package/dist/src/benchmarks/loader.js +271 -0
- package/dist/src/benchmarks/loader.js.map +1 -0
- package/dist/src/benchmarks/metrics.js +16 -0
- package/dist/src/benchmarks/metrics.js.map +1 -0
- package/dist/src/benchmarks/toml.js +13 -0
- package/dist/src/benchmarks/toml.js.map +1 -0
- package/dist/src/benchmarks/validation.js +242 -0
- package/dist/src/benchmarks/validation.js.map +1 -0
- package/dist/src/cli/arguments.js +6 -4
- package/dist/src/cli/arguments.js.map +1 -1
- package/dist/src/cli/commands/benchmark.js +22 -0
- package/dist/src/cli/commands/benchmark.js.map +1 -0
- package/dist/src/cli/commands/capabilities.js +18 -0
- package/dist/src/cli/commands/capabilities.js.map +1 -0
- package/dist/src/cli/commands/daemon.js +135 -8
- package/dist/src/cli/commands/daemon.js.map +1 -1
- package/dist/src/cli/commands/eval.js +228 -6
- package/dist/src/cli/commands/eval.js.map +1 -1
- package/dist/src/cli/commands/images.js +40 -0
- package/dist/src/cli/commands/images.js.map +1 -0
- package/dist/src/cli/commands/run.js +42 -17
- package/dist/src/cli/commands/run.js.map +1 -1
- package/dist/src/cli/commands/runs.js +25 -3
- package/dist/src/cli/commands/runs.js.map +1 -1
- package/dist/src/cli/commands/trajectory.js +96 -5
- package/dist/src/cli/commands/trajectory.js.map +1 -1
- package/dist/src/cli/commands/verifier.js +36 -0
- package/dist/src/cli/commands/verifier.js.map +1 -0
- package/dist/src/cli/commands/worker.js +79 -0
- package/dist/src/cli/commands/worker.js.map +1 -0
- package/dist/src/cli/main.js +10 -0
- package/dist/src/cli/main.js.map +1 -1
- package/dist/src/cli/output.js +84 -4
- package/dist/src/cli/output.js.map +1 -1
- package/dist/src/control-plane/build-admission.js +69 -0
- package/dist/src/control-plane/build-admission.js.map +1 -0
- package/dist/src/control-plane/build-records.js +6 -0
- package/dist/src/control-plane/build-records.js.map +1 -0
- package/dist/src/control-plane/collisions.js +40 -0
- package/dist/src/control-plane/collisions.js.map +1 -0
- package/dist/src/control-plane/eval-control-state.js +22 -0
- package/dist/src/control-plane/eval-control-state.js.map +1 -0
- package/dist/src/control-plane/eval-control-work.js +44 -0
- package/dist/src/control-plane/eval-control-work.js.map +1 -0
- package/dist/src/control-plane/eval-image-resolution.js +52 -0
- package/dist/src/control-plane/eval-image-resolution.js.map +1 -0
- package/dist/src/control-plane/eval-image-services.js +30 -0
- package/dist/src/control-plane/eval-image-services.js.map +1 -0
- package/dist/src/control-plane/eval-records.js +252 -0
- package/dist/src/control-plane/eval-records.js.map +1 -0
- package/dist/src/control-plane/eval-recovery.js +181 -0
- package/dist/src/control-plane/eval-recovery.js.map +1 -0
- package/dist/src/control-plane/eval-scheduler.js +450 -0
- package/dist/src/control-plane/eval-scheduler.js.map +1 -0
- package/dist/src/control-plane/index.js +20 -0
- package/dist/src/control-plane/index.js.map +1 -0
- package/dist/src/control-plane/local-worker.js +49 -0
- package/dist/src/control-plane/local-worker.js.map +1 -0
- package/dist/src/control-plane/model-capture-planning.js +12 -0
- package/dist/src/control-plane/model-capture-planning.js.map +1 -0
- package/dist/src/control-plane/remote-result-transport.js +175 -0
- package/dist/src/control-plane/remote-result-transport.js.map +1 -0
- package/dist/src/control-plane/remote-work-coordinator.js +333 -0
- package/dist/src/control-plane/remote-work-coordinator.js.map +1 -0
- package/dist/src/control-plane/remote-work-inputs.js +266 -0
- package/dist/src/control-plane/remote-work-inputs.js.map +1 -0
- package/dist/src/control-plane/remote-work-item.js +62 -0
- package/dist/src/control-plane/remote-work-item.js.map +1 -0
- package/dist/src/control-plane/remote-work-recovery.js +220 -0
- package/dist/src/control-plane/remote-work-recovery.js.map +1 -0
- package/dist/src/control-plane/remote-worker-artifacts.js +135 -0
- package/dist/src/control-plane/remote-worker-artifacts.js.map +1 -0
- package/dist/src/control-plane/remote-worker-client.js +202 -0
- package/dist/src/control-plane/remote-worker-client.js.map +1 -0
- package/dist/src/control-plane/remote-worker-configuration.js +34 -0
- package/dist/src/control-plane/remote-worker-configuration.js.map +1 -0
- package/dist/src/control-plane/remote-worker-credentials.js +58 -0
- package/dist/src/control-plane/remote-worker-credentials.js.map +1 -0
- package/dist/src/control-plane/remote-worker-protocol.js +489 -0
- package/dist/src/control-plane/remote-worker-protocol.js.map +1 -0
- package/dist/src/control-plane/remote-worker-runner.js +214 -0
- package/dist/src/control-plane/remote-worker-runner.js.map +1 -0
- package/dist/src/control-plane/remote-workers.js +377 -0
- package/dist/src/control-plane/remote-workers.js.map +1 -0
- package/dist/src/control-plane/rerun-scheduler.js +448 -0
- package/dist/src/control-plane/rerun-scheduler.js.map +1 -0
- package/dist/src/control-plane/rerun-submission.js +62 -0
- package/dist/src/control-plane/rerun-submission.js.map +1 -0
- package/dist/src/control-plane/resources.js +182 -0
- package/dist/src/control-plane/resources.js.map +1 -0
- package/dist/src/control-plane/scheduler-eval-entry.js +32 -0
- package/dist/src/control-plane/scheduler-eval-entry.js.map +1 -0
- package/dist/src/control-plane/synthetic-result.js +21 -0
- package/dist/src/control-plane/synthetic-result.js.map +1 -0
- package/dist/src/control-plane/work-admission.js +23 -0
- package/dist/src/control-plane/work-admission.js.map +1 -0
- package/dist/src/control-plane/work-dispatcher.js +237 -0
- package/dist/src/control-plane/work-dispatcher.js.map +1 -0
- package/dist/src/controller-runtime/hash.js +19 -3
- package/dist/src/controller-runtime/hash.js.map +1 -1
- package/dist/src/controller-runtime/index.js +1 -1
- package/dist/src/controller-runtime/index.js.map +1 -1
- package/dist/src/controller-runtime/store.js +15 -6
- package/dist/src/controller-runtime/store.js.map +1 -1
- package/dist/src/daemon/auth.js +5 -0
- package/dist/src/daemon/auth.js.map +1 -1
- package/dist/src/daemon/client.js.map +1 -1
- package/dist/src/daemon/event-stream.js +46 -0
- package/dist/src/daemon/event-stream.js.map +1 -0
- package/dist/src/daemon/health.js +66 -0
- package/dist/src/daemon/health.js.map +1 -0
- package/dist/src/daemon/launcher.js +28 -1
- package/dist/src/daemon/launcher.js.map +1 -1
- package/dist/src/daemon/logging.js +23 -0
- package/dist/src/daemon/logging.js.map +1 -0
- package/dist/src/daemon/scheduler.js +26 -5
- package/dist/src/daemon/scheduler.js.map +1 -1
- package/dist/src/daemon/server.js +255 -98
- package/dist/src/daemon/server.js.map +1 -1
- package/dist/src/daemon/telemetry.js +163 -0
- package/dist/src/daemon/telemetry.js.map +1 -0
- package/dist/src/daemon/worker-routes.js +219 -0
- package/dist/src/daemon/worker-routes.js.map +1 -0
- package/dist/src/domain/benchmarks.js +2 -0
- package/dist/src/domain/benchmarks.js.map +1 -0
- package/dist/src/domain/bundles.js +2 -0
- package/dist/src/domain/bundles.js.map +1 -0
- package/dist/src/domain/control-plane.js +2 -0
- package/dist/src/domain/control-plane.js.map +1 -0
- package/dist/src/domain/execution-evidence.js +2 -0
- package/dist/src/domain/execution-evidence.js.map +1 -0
- package/dist/src/domain/execution-plan.js +2 -0
- package/dist/src/domain/execution-plan.js.map +1 -0
- package/dist/src/domain/images.js +2 -0
- package/dist/src/domain/images.js.map +1 -0
- package/dist/src/domain/index.js +13 -0
- package/dist/src/domain/index.js.map +1 -1
- package/dist/src/domain/interactions.js +2 -0
- package/dist/src/domain/interactions.js.map +1 -0
- package/dist/src/domain/providers.js +2 -0
- package/dist/src/domain/providers.js.map +1 -0
- package/dist/src/domain/resources.js +2 -0
- package/dist/src/domain/resources.js.map +1 -0
- package/dist/src/domain/runs.js.map +1 -1
- package/dist/src/domain/validation.js +11 -4
- package/dist/src/domain/validation.js.map +1 -1
- package/dist/src/domain/verifier-evidence-validation.js +214 -0
- package/dist/src/domain/verifier-evidence-validation.js.map +1 -0
- package/dist/src/domain/verifier-evidence.js +3 -0
- package/dist/src/domain/verifier-evidence.js.map +1 -0
- package/dist/src/domain/verifier-score-contract.js +232 -0
- package/dist/src/domain/verifier-score-contract.js.map +1 -0
- package/dist/src/domain/workers.js +2 -0
- package/dist/src/domain/workers.js.map +1 -0
- package/dist/src/evals/benchmark-adapter-manifest.js +219 -0
- package/dist/src/evals/benchmark-adapter-manifest.js.map +1 -0
- package/dist/src/evals/benchmark-candidate.js +36 -0
- package/dist/src/evals/benchmark-candidate.js.map +1 -0
- package/dist/src/evals/benchmark-run.js +166 -0
- package/dist/src/evals/benchmark-run.js.map +1 -0
- package/dist/src/evals/collect-only-rerun.js +183 -0
- package/dist/src/evals/collect-only-rerun.js.map +1 -0
- package/dist/src/evals/directory.js +30 -0
- package/dist/src/evals/directory.js.map +1 -0
- package/dist/src/evals/docker-engine-stats.js +89 -0
- package/dist/src/evals/docker-engine-stats.js.map +1 -0
- package/dist/src/evals/docker-ownership.js +51 -0
- package/dist/src/evals/docker-ownership.js.map +1 -0
- package/dist/src/evals/docker-reaper.js +144 -0
- package/dist/src/evals/docker-reaper.js.map +1 -0
- package/dist/src/evals/docker-resource-observer.js +207 -0
- package/dist/src/evals/docker-resource-observer.js.map +1 -0
- package/dist/src/evals/duration-estimator.js +126 -0
- package/dist/src/evals/duration-estimator.js.map +1 -0
- package/dist/src/evals/environment-image-planning.js +164 -0
- package/dist/src/evals/environment-image-planning.js.map +1 -0
- package/dist/src/evals/eval-artifact-planning.js +56 -0
- package/dist/src/evals/eval-artifact-planning.js.map +1 -0
- package/dist/src/evals/eval-finalization.js +10 -0
- package/dist/src/evals/eval-finalization.js.map +1 -0
- package/dist/src/evals/eval-lifecycle-events.js +29 -0
- package/dist/src/evals/eval-lifecycle-events.js.map +1 -0
- package/dist/src/evals/eval-logical-plan.js +51 -0
- package/dist/src/evals/eval-logical-plan.js.map +1 -0
- package/dist/src/evals/eval-result-builder.js +81 -0
- package/dist/src/evals/eval-result-builder.js.map +1 -0
- package/dist/src/evals/events.js +53 -5
- package/dist/src/evals/events.js.map +1 -1
- package/dist/src/evals/evolution-baseline.js +191 -0
- package/dist/src/evals/evolution-baseline.js.map +1 -0
- package/dist/src/evals/execution-evidence.js +116 -0
- package/dist/src/evals/execution-evidence.js.map +1 -0
- package/dist/src/evals/execution-leases.js +342 -0
- package/dist/src/evals/execution-leases.js.map +1 -0
- package/dist/src/evals/execution-plan-artifacts.js +72 -0
- package/dist/src/evals/execution-plan-artifacts.js.map +1 -0
- package/dist/src/evals/execution-plan-images.js +69 -0
- package/dist/src/evals/execution-plan-images.js.map +1 -0
- package/dist/src/evals/execution-plan-resources.js +174 -0
- package/dist/src/evals/execution-plan-resources.js.map +1 -0
- package/dist/src/evals/execution-plan.js +451 -0
- package/dist/src/evals/execution-plan.js.map +1 -0
- package/dist/src/evals/failure-classifier.js +61 -0
- package/dist/src/evals/failure-classifier.js.map +1 -0
- package/dist/src/evals/harbor-artifact-builder-image.js +99 -0
- package/dist/src/evals/harbor-artifact-builder-image.js.map +1 -0
- package/dist/src/evals/harbor-artifact-builder.js +358 -0
- package/dist/src/evals/harbor-artifact-builder.js.map +1 -0
- package/dist/src/evals/harbor-bridge-error.js +58 -3
- package/dist/src/evals/harbor-bridge-error.js.map +1 -1
- package/dist/src/evals/harbor-node-runtime.js +146 -0
- package/dist/src/evals/harbor-node-runtime.js.map +1 -0
- package/dist/src/evals/harbor-phase-timings.js +60 -0
- package/dist/src/evals/harbor-phase-timings.js.map +1 -0
- package/dist/src/evals/index.js +40 -2
- package/dist/src/evals/index.js.map +1 -1
- package/dist/src/evals/infrastructure-retry.js +208 -84
- package/dist/src/evals/infrastructure-retry.js.map +1 -1
- package/dist/src/evals/interaction-capture-import.js +49 -0
- package/dist/src/evals/interaction-capture-import.js.map +1 -0
- package/dist/src/evals/local-docker-provider.js +372 -0
- package/dist/src/evals/local-docker-provider.js.map +1 -0
- package/dist/src/evals/local-eval-planning.js +80 -0
- package/dist/src/evals/local-eval-planning.js.map +1 -0
- package/dist/src/evals/model-capture-plan.js +22 -0
- package/dist/src/evals/model-capture-plan.js.map +1 -0
- package/dist/src/evals/model-capture-runtime.js +114 -0
- package/dist/src/evals/model-capture-runtime.js.map +1 -0
- package/dist/src/evals/model-proxy-runtime-state.js +53 -0
- package/dist/src/evals/model-proxy-runtime-state.js.map +1 -0
- package/dist/src/evals/native-phase-evidence.js +440 -0
- package/dist/src/evals/native-phase-evidence.js.map +1 -0
- package/dist/src/evals/physical-retry-work.js +31 -0
- package/dist/src/evals/physical-retry-work.js.map +1 -0
- package/dist/src/evals/planned-execution-support.js +66 -0
- package/dist/src/evals/planned-execution-support.js.map +1 -0
- package/dist/src/evals/planned-execution.js +469 -0
- package/dist/src/evals/planned-execution.js.map +1 -0
- package/dist/src/evals/planned-progress-publisher.js +62 -0
- package/dist/src/evals/planned-progress-publisher.js.map +1 -0
- package/dist/src/evals/planned-remote-execution.js +40 -0
- package/dist/src/evals/planned-remote-execution.js.map +1 -0
- package/dist/src/evals/planned-retry-execution.js +174 -0
- package/dist/src/evals/planned-retry-execution.js.map +1 -0
- package/dist/src/evals/planned-retry-lifecycle.js +188 -0
- package/dist/src/evals/planned-retry-lifecycle.js.map +1 -0
- package/dist/src/evals/preparation-rerun.js +225 -0
- package/dist/src/evals/preparation-rerun.js.map +1 -0
- package/dist/src/evals/prepared-harness.js +76 -0
- package/dist/src/evals/prepared-harness.js.map +1 -0
- package/dist/src/evals/progress.js +52 -12
- package/dist/src/evals/progress.js.map +1 -1
- package/dist/src/evals/records.js +20 -3
- package/dist/src/evals/records.js.map +1 -1
- package/dist/src/evals/recovery.js +242 -0
- package/dist/src/evals/recovery.js.map +1 -0
- package/dist/src/evals/regrade-evidence.js +77 -0
- package/dist/src/evals/regrade-evidence.js.map +1 -0
- package/dist/src/evals/remote-infrastructure-retry.js +165 -0
- package/dist/src/evals/remote-infrastructure-retry.js.map +1 -0
- package/dist/src/evals/request.js +21 -3
- package/dist/src/evals/request.js.map +1 -1
- package/dist/src/evals/rerun-inputs.js +108 -0
- package/dist/src/evals/rerun-inputs.js.map +1 -0
- package/dist/src/evals/rerun-slots.js +4 -3
- package/dist/src/evals/rerun-slots.js.map +1 -1
- package/dist/src/evals/rerun-types.js +45 -0
- package/dist/src/evals/rerun-types.js.map +1 -0
- package/dist/src/evals/rerun.js +133 -88
- package/dist/src/evals/rerun.js.map +1 -1
- package/dist/src/evals/result-helpers.js +78 -0
- package/dist/src/evals/result-helpers.js.map +1 -0
- package/dist/src/evals/resume-state.js +36 -0
- package/dist/src/evals/resume-state.js.map +1 -0
- package/dist/src/evals/retry-backoff.js +20 -0
- package/dist/src/evals/retry-backoff.js.map +1 -0
- package/dist/src/evals/retry-state.js +195 -0
- package/dist/src/evals/retry-state.js.map +1 -0
- package/dist/src/evals/scheduler-metrics.js +95 -0
- package/dist/src/evals/scheduler-metrics.js.map +1 -0
- package/dist/src/evals/scheduler-trace-replay.js +107 -0
- package/dist/src/evals/scheduler-trace-replay.js.map +1 -0
- package/dist/src/evals/service-types.js +2 -0
- package/dist/src/evals/service-types.js.map +1 -0
- package/dist/src/evals/service.js +331 -347
- package/dist/src/evals/service.js.map +1 -1
- package/dist/src/evals/task-resources.js +209 -0
- package/dist/src/evals/task-resources.js.map +1 -0
- package/dist/src/evals/trial-environment-evidence.js +87 -0
- package/dist/src/evals/trial-environment-evidence.js.map +1 -0
- package/dist/src/evals/trial-execution-evidence.js +13 -0
- package/dist/src/evals/trial-execution-evidence.js.map +1 -0
- package/dist/src/evals/trial-import-identity.js +31 -0
- package/dist/src/evals/trial-import-identity.js.map +1 -0
- package/dist/src/evals/trial-import.js +183 -188
- package/dist/src/evals/trial-import.js.map +1 -1
- package/dist/src/evals/trial-publication-recovery.js +144 -0
- package/dist/src/evals/trial-publication-recovery.js.map +1 -0
- package/dist/src/evals/trial-publication.js +25 -0
- package/dist/src/evals/trial-publication.js.map +1 -0
- package/dist/src/evals/trial-reference-validation.js +49 -0
- package/dist/src/evals/trial-reference-validation.js.map +1 -0
- package/dist/src/evals/verifier-artifacts.js +317 -0
- package/dist/src/evals/verifier-artifacts.js.map +1 -0
- package/dist/src/evals/verifier-eligibility.js +25 -0
- package/dist/src/evals/verifier-eligibility.js.map +1 -0
- package/dist/src/evals/verifier-only-rerun.js +210 -0
- package/dist/src/evals/verifier-only-rerun.js.map +1 -0
- package/dist/src/evals/verifier-runtime.js +61 -0
- package/dist/src/evals/verifier-runtime.js.map +1 -0
- package/dist/src/evals/verifier-score-artifacts.js +119 -0
- package/dist/src/evals/verifier-score-artifacts.js.map +1 -0
- package/dist/src/evals/work-item-artifacts.js +23 -0
- package/dist/src/evals/work-item-artifacts.js.map +1 -0
- package/dist/src/foundation/config.js +17 -0
- package/dist/src/foundation/config.js.map +1 -1
- package/dist/src/foundation/contained-file.js +85 -0
- package/dist/src/foundation/contained-file.js.map +1 -0
- package/dist/src/foundation/credential-redaction.js +162 -0
- package/dist/src/foundation/credential-redaction.js.map +1 -0
- package/dist/src/foundation/environment-image-references.js +59 -0
- package/dist/src/foundation/environment-image-references.js.map +1 -0
- package/dist/src/foundation/executable.js +45 -5
- package/dist/src/foundation/executable.js.map +1 -1
- package/dist/src/foundation/fs.js +2 -1
- package/dist/src/foundation/fs.js.map +1 -1
- package/dist/src/foundation/index.js +6 -2
- package/dist/src/foundation/index.js.map +1 -1
- package/dist/src/foundation/process-identity.js +69 -0
- package/dist/src/foundation/process-identity.js.map +1 -0
- package/dist/src/images/context.js +47 -0
- package/dist/src/images/context.js.map +1 -0
- package/dist/src/images/docker-buildkit.js +137 -0
- package/dist/src/images/docker-buildkit.js.map +1 -0
- package/dist/src/images/dockerfile.js +67 -0
- package/dist/src/images/dockerfile.js.map +1 -0
- package/dist/src/images/gc.js +261 -0
- package/dist/src/images/gc.js.map +1 -0
- package/dist/src/images/index.js +10 -0
- package/dist/src/images/index.js.map +1 -0
- package/dist/src/images/manifest.js +109 -0
- package/dist/src/images/manifest.js.map +1 -0
- package/dist/src/images/ownership.js +5 -0
- package/dist/src/images/ownership.js.map +1 -0
- package/dist/src/images/records.js +32 -0
- package/dist/src/images/records.js.map +1 -0
- package/dist/src/images/registry.js +154 -0
- package/dist/src/images/registry.js.map +1 -0
- package/dist/src/images/service.js +224 -0
- package/dist/src/images/service.js.map +1 -0
- package/dist/src/model-access/capture.js +255 -0
- package/dist/src/model-access/capture.js.map +1 -0
- package/dist/src/model-access/index.js +5 -0
- package/dist/src/model-access/index.js.map +1 -0
- package/dist/src/model-access/policy.js +49 -0
- package/dist/src/model-access/policy.js.map +1 -0
- package/dist/src/model-access/proxy.js +336 -0
- package/dist/src/model-access/proxy.js.map +1 -0
- package/dist/src/model-access/records.js +167 -0
- package/dist/src/model-access/records.js.map +1 -0
- package/dist/src/runs/adapter-process.js +12 -0
- package/dist/src/runs/adapter-process.js.map +1 -0
- package/dist/src/runs/bundle.js +450 -0
- package/dist/src/runs/bundle.js.map +1 -0
- package/dist/src/runs/events.js +12 -1
- package/dist/src/runs/events.js.map +1 -1
- package/dist/src/runs/executor.js +46 -45
- package/dist/src/runs/executor.js.map +1 -1
- package/dist/src/runs/finalizer.js +17 -0
- package/dist/src/runs/finalizer.js.map +1 -1
- package/dist/src/runs/index.js +8 -1
- package/dist/src/runs/index.js.map +1 -1
- package/dist/src/runs/manifest.js +5 -5
- package/dist/src/runs/manifest.js.map +1 -1
- package/dist/src/runs/phase-bundle.js +86 -0
- package/dist/src/runs/phase-bundle.js.map +1 -0
- package/dist/src/runs/phase-cancellation.js +107 -0
- package/dist/src/runs/phase-cancellation.js.map +1 -0
- package/dist/src/runs/phase-group.js +113 -0
- package/dist/src/runs/phase-group.js.map +1 -0
- package/dist/src/runs/query.js +1 -1
- package/dist/src/runs/query.js.map +1 -1
- package/dist/src/runs/records.js +3 -0
- package/dist/src/runs/records.js.map +1 -1
- package/dist/src/runs/request.js +10 -3
- package/dist/src/runs/request.js.map +1 -1
- package/dist/src/runs/training-candidate.js +161 -0
- package/dist/src/runs/training-candidate.js.map +1 -0
- package/dist/src/runs/verifier-evidence-redaction.js +61 -0
- package/dist/src/runs/verifier-evidence-redaction.js.map +1 -0
- package/dist/src/runs/verifier-evidence.js +459 -0
- package/dist/src/runs/verifier-evidence.js.map +1 -0
- package/dist/src/runs/verifier-structured-evidence.js +83 -0
- package/dist/src/runs/verifier-structured-evidence.js.map +1 -0
- package/dist/src/trajectories/analysis.js +288 -0
- package/dist/src/trajectories/analysis.js.map +1 -0
- package/dist/src/trajectories/chunk-projection.js +148 -0
- package/dist/src/trajectories/chunk-projection.js.map +1 -0
- package/dist/src/trajectories/content-projection.js +389 -0
- package/dist/src/trajectories/content-projection.js.map +1 -0
- package/dist/src/trajectories/dsh-chunk-contract.js +176 -0
- package/dist/src/trajectories/dsh-chunk-contract.js.map +1 -0
- package/dist/src/trajectories/dsh-contract.js +284 -0
- package/dist/src/trajectories/dsh-contract.js.map +1 -0
- package/dist/src/trajectories/events-chunk-drill.js +95 -0
- package/dist/src/trajectories/events-chunk-drill.js.map +1 -0
- package/dist/src/trajectories/events-page.js +367 -0
- package/dist/src/trajectories/events-page.js.map +1 -0
- package/dist/src/trajectories/index.js +3 -0
- package/dist/src/trajectories/index.js.map +1 -1
- package/dist/src/trajectories/provider-capture.js +15 -24
- package/dist/src/trajectories/provider-capture.js.map +1 -1
- package/dist/src/trajectories/providers/deepseek.js +3 -3
- package/dist/src/trajectories/providers/deepseek.js.map +1 -1
- package/dist/src/trajectories/request-attempt.js +47 -0
- package/dist/src/trajectories/request-attempt.js.map +1 -0
- package/dist/src/trajectories/stream-reader.js +311 -0
- package/dist/src/trajectories/stream-reader.js.map +1 -0
- package/dist/src/trajectories/surface-fold.js +126 -0
- package/dist/src/trajectories/surface-fold.js.map +1 -0
- package/dist/src/workers/index.js +3 -0
- package/dist/src/workers/index.js.map +1 -0
- package/dist/src/workers/remote-harbor-work-spec.js +97 -0
- package/dist/src/workers/remote-harbor-work-spec.js.map +1 -0
- package/dist/src/workers/remote-harbor-worker.js +223 -0
- package/dist/src/workers/remote-harbor-worker.js.map +1 -0
- package/docs/schemas/benchmark-hook-request-v1.schema.json +65 -0
- package/docs/schemas/benchmark-hook-response-v1.schema.json +75 -0
- package/docs/schemas/benchmark-hook-v1.schema.json +33 -0
- package/docs/schemas/benchmark-lock-v1.schema.json +351 -0
- package/docs/schemas/benchmark-metric-v1.schema.json +37 -0
- package/docs/schemas/benchmark-package-v1.schema.json +167 -0
- package/docs/schemas/benchmark-phase-group.schema.json +41 -0
- package/docs/schemas/benchmark-profile-v1.schema.json +141 -0
- package/docs/schemas/benchmark-task-v1.schema.json +384 -0
- package/docs/schemas/benchmark-tool-result-v1.schema.json +30 -0
- package/docs/schemas/environment-build-index.schema.json +13 -0
- package/docs/schemas/environment-build-record.schema.json +25 -0
- package/docs/schemas/environment-image-manifest.schema.json +62 -0
- package/docs/schemas/error.schema.json +22 -0
- package/docs/schemas/eval-control.schema.json +36 -0
- package/docs/schemas/eval-execution-plan.schema.json +216 -0
- package/docs/schemas/eval-progress.schema.json +1 -15
- package/docs/schemas/eval-rerun-result.schema.json +58 -0
- package/docs/schemas/eval-rerun-state.schema.json +61 -0
- package/docs/schemas/eval-rerun-submission.schema.json +60 -0
- package/docs/schemas/eval-result.schema.json +32 -13
- package/docs/schemas/eval-submission-input.schema.json +14 -0
- package/docs/schemas/eval-submission.schema.json +68 -0
- package/docs/schemas/eval-trial-publication.schema.json +15 -0
- package/docs/schemas/eval-trial-reference.schema.json +129 -0
- package/docs/schemas/execution-evidence.schema.json +96 -0
- package/docs/schemas/execution-lease.schema.json +50 -0
- package/docs/schemas/execution-provider-status.schema.json +65 -0
- package/docs/schemas/execution-worker.schema.json +53 -0
- package/docs/schemas/harbor-process-exit.schema.json +14 -0
- package/docs/schemas/interaction-capture-ref.schema.json +45 -0
- package/docs/schemas/local-provider-execution.schema.json +33 -0
- package/docs/schemas/model-interaction.schema.json +41 -0
- package/docs/schemas/regrade-assessment-reference.schema.json +11 -0
- package/docs/schemas/remote-result-envelope.schema.json +32 -0
- package/docs/schemas/remote-tree-envelope.schema.json +25 -0
- package/docs/schemas/remote-work-offer.schema.json +58 -0
- package/docs/schemas/remote-work-receipt.schema.json +50 -0
- package/docs/schemas/remote-worker-artifact.schema.json +17 -0
- package/docs/schemas/remote-worker-event.schema.json +18 -0
- package/docs/schemas/remote-worker-heartbeat.schema.json +37 -0
- package/docs/schemas/remote-worker-registration.schema.json +52 -0
- package/docs/schemas/result-bundle-index.schema.json +133 -0
- package/docs/schemas/run-context.schema.json +15 -0
- package/docs/schemas/run-environment-images.schema.json +39 -0
- package/docs/schemas/run-manifest.schema.json +4 -0
- package/docs/schemas/run-request.schema.json +4 -0
- package/docs/schemas/training-data-candidate.schema.json +46 -0
- package/docs/schemas/trajectory-analysis.schema.json +258 -0
- package/docs/schemas/trajectory-events-page.schema.json +40 -0
- package/docs/schemas/verifier-assessment.schema.json +77 -0
- package/docs/schemas/verifier-evidence.schema.json +276 -0
- package/integrations/harbor/hitch_benchmark.py +252 -0
- package/integrations/harbor/hitch_candidate_recycle.py +262 -0
- package/integrations/harbor/hitch_harbor_agent.py +698 -568
- package/integrations/harbor/hitch_harbor_environment.py +469 -0
- package/integrations/harbor/hitch_harbor_task_resources.py +275 -0
- package/integrations/harbor/hitch_harbor_verifier.py +83 -2
- package/integrations/harbor/hitch_phase_supervisor.py +452 -0
- package/integrations/harbor/hitch_tool_client.mjs +91 -0
- package/integrations/model-call/cli.js +57 -0
- package/package.json +30 -6
|
@@ -4,47 +4,85 @@ import { runHarborBackend } from "../backends/index.js";
|
|
|
4
4
|
import { HitchError } from "../foundation/index.js";
|
|
5
5
|
import { importEvalTrialRun, importEvalTrialRuns, TrialBundlePendingError, validateEvalTrialReferences } from "./trial-import.js";
|
|
6
6
|
import { replaceInvalidEvalProgressTrial, writeEvalProgress } from "./progress.js";
|
|
7
|
+
import { ensurePhysicalRetryDecision, transitionRetryDecision } from "./retry-state.js";
|
|
8
|
+
import { physicalRetryWorkItem } from "./physical-retry-work.js";
|
|
9
|
+
import { classifyTrialFailure, physicalRetryAllowed } from "./failure-classifier.js";
|
|
10
|
+
import { retryBackoffMs } from "./retry-backoff.js";
|
|
11
|
+
import { harborPhaseTimingEvents } from "./harbor-phase-timings.js";
|
|
7
12
|
const INFRASTRUCTURE_REASONS = new Set([
|
|
8
13
|
"infrastructure_failure",
|
|
9
14
|
"verifier_infrastructure_failure",
|
|
10
15
|
"verifier_result_missing",
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
"
|
|
16
|
+
"provider_quota_exhausted",
|
|
17
|
+
"provider_auth_failed",
|
|
18
|
+
"provider_configuration_invalid",
|
|
19
|
+
"provider_rate_limited",
|
|
20
|
+
"provider_transport_transient",
|
|
21
|
+
"worker_lost_before_candidate",
|
|
22
|
+
"sandbox_setup_failed",
|
|
23
|
+
"execution_state_ambiguous",
|
|
18
24
|
]);
|
|
19
25
|
export async function runInfrastructureRetries(options) {
|
|
20
26
|
let progress = options.progress;
|
|
21
27
|
let retryCandidates = retryableInfrastructureTrials(options.initialRefs);
|
|
22
28
|
const runs = [];
|
|
23
|
-
|
|
29
|
+
const firstRetryIndex = options.firstRetryIndex ?? 1;
|
|
30
|
+
if (!Number.isSafeInteger(firstRetryIndex) || firstRetryIndex < 1)
|
|
31
|
+
throw new TypeError("first infrastructure retry index is invalid");
|
|
32
|
+
for (let retry = firstRetryIndex; retry <= options.request.infrastructure_retries && retryCandidates.length > 0; retry += 1) {
|
|
24
33
|
if (options.signal?.aborted)
|
|
25
34
|
break;
|
|
26
35
|
const retryTriggers = [...retryCandidates];
|
|
27
36
|
const taskNames = [...new Set(retryCandidates.map((ref) => ref.task_id))].sort();
|
|
28
|
-
const
|
|
37
|
+
const retryWork = options.originWorkItem ? physicalRetryWorkItem(options.originWorkItem, retry, retryTriggers) : undefined;
|
|
38
|
+
const backoffMs = retryBackoffMs(options.request.infrastructure_retry_backoff_ms, retry, retryWork?.work_id ?? retryTriggers.map((trial) => trial.trial_id).join("\0"));
|
|
39
|
+
const notBefore = new Date(Date.now() + backoffMs).toISOString();
|
|
40
|
+
const retryDecisions = options.originWorkItem
|
|
41
|
+
? await Promise.all(retryTriggers.map((trigger) => ensurePhysicalRetryDecision({
|
|
42
|
+
evalDirectory: options.evalDirectory, evalId: options.evalId, item: options.originWorkItem,
|
|
43
|
+
retryIndex: retry, trigger, notBefore,
|
|
44
|
+
})))
|
|
45
|
+
: [];
|
|
46
|
+
if (retryWork && retryDecisions.some((decision) => decision.retry_work_id !== retryWork.work_id)) {
|
|
47
|
+
throw new Error(`persisted retry work identity conflicts with execution: ${retryWork.work_id}`);
|
|
48
|
+
}
|
|
49
|
+
if (retryWork)
|
|
50
|
+
await options.onRetryWorkPlanned?.(retryWork.work_id);
|
|
29
51
|
options.sink.emit({
|
|
30
|
-
type: "eval.
|
|
52
|
+
type: "eval.retry.decision",
|
|
53
|
+
execution_kind: "physical-infrastructure-retry",
|
|
54
|
+
candidate_executes: true,
|
|
55
|
+
...(retryWork ? { work_id: retryWork.work_id } : {}),
|
|
56
|
+
decision_ids: retryDecisions.map((decision) => decision.decision_id),
|
|
31
57
|
attempt: options.logicalAttempt,
|
|
32
58
|
retry,
|
|
33
59
|
tasks: taskNames,
|
|
34
60
|
backoff_ms: backoffMs,
|
|
35
61
|
});
|
|
36
|
-
|
|
62
|
+
const remainingBackoffMs = Math.max(0, Date.parse(retryDecisions[0]?.not_before ?? notBefore) - Date.now());
|
|
63
|
+
options.onRetryBackoff?.(remainingBackoffMs);
|
|
64
|
+
if (remainingBackoffMs > 0) {
|
|
37
65
|
if (options.signal)
|
|
38
|
-
await delay(
|
|
66
|
+
await delay(remainingBackoffMs, undefined, { signal: options.signal });
|
|
39
67
|
else
|
|
40
|
-
await delay(
|
|
68
|
+
await delay(remainingBackoffMs);
|
|
41
69
|
}
|
|
42
70
|
if (options.signal?.aborted)
|
|
43
71
|
break;
|
|
44
|
-
|
|
72
|
+
if (retryWork)
|
|
73
|
+
options.sink.emit({
|
|
74
|
+
type: "eval.retry.ready", work_id: retryWork.work_id, decision_ids: retryDecisions.map((decision) => decision.decision_id),
|
|
75
|
+
retry, tasks: taskNames, priority: options.originWorkItem?.scheduling?.remaining_path_ms ?? 0,
|
|
76
|
+
});
|
|
77
|
+
const backendDirectory = path.join(options.backendBaseDirectory || path.join(options.evalDirectory, "infrastructure-retries"), `retry-${String(retry).padStart(4, "0")}`, options.request.attempts === 1 ? "harbor" : attemptDirectoryName(options.logicalAttempt));
|
|
45
78
|
const harborJobDirectory = path.join(backendDirectory, "job");
|
|
46
79
|
const retryRefs = [];
|
|
47
80
|
const selectedTasks = new Set(taskNames);
|
|
81
|
+
let lifecycle;
|
|
82
|
+
const retryEnvironmentImages = () => lifecycle?.environmentImages ?? options.environmentImages;
|
|
83
|
+
const executionEvidence = async () => lifecycle?.captureExecutionEvidence
|
|
84
|
+
? { executionEvidence: await lifecycle.captureExecutionEvidence() }
|
|
85
|
+
: {};
|
|
48
86
|
const publish = async (ref) => {
|
|
49
87
|
if (ref.attempt !== options.logicalAttempt || !selectedTasks.has(ref.task_id)) {
|
|
50
88
|
throw new HitchError(`Harbor infrastructure retry returned an unselected trial: ${ref.task_id}#${ref.attempt}`, {
|
|
@@ -59,13 +97,26 @@ export async function runInfrastructureRetries(options) {
|
|
|
59
97
|
return;
|
|
60
98
|
}
|
|
61
99
|
retryRefs.push(ref);
|
|
100
|
+
if (ref.invalid_reason === "candidate_evidence_unavailable")
|
|
101
|
+
options.sink.emit({
|
|
102
|
+
type: "eval.verifier.skipped", work_id: retryWork?.work_id, trial_id: ref.trial_id, task_id: ref.task_id,
|
|
103
|
+
reason: "candidate_evidence_unavailable", candidate_executes: true, verifier_executes: false,
|
|
104
|
+
});
|
|
62
105
|
await validateEvalTrialReferences(options.root, options.evalId, [ref], {
|
|
63
106
|
benchmarkId: options.request.benchmark_id,
|
|
64
107
|
benchmarkRevision: options.request.benchmark_revision,
|
|
65
108
|
});
|
|
66
109
|
if (ref.observation_status === "valid") {
|
|
67
|
-
|
|
68
|
-
|
|
110
|
+
if (options.replaceProgressTrial) {
|
|
111
|
+
if (!lifecycle)
|
|
112
|
+
throw new Error("planned infrastructure retry is missing its lifecycle identity");
|
|
113
|
+
await options.replaceProgressTrial(ref, lifecycle.workId);
|
|
114
|
+
progress = options.currentProgress?.() ?? progress;
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
progress = replaceInvalidEvalProgressTrial(progress, ref);
|
|
118
|
+
await writeEvalProgress(options.evalDirectory, progress);
|
|
119
|
+
}
|
|
69
120
|
}
|
|
70
121
|
options.sink.emit({
|
|
71
122
|
type: ref.observation_status === "valid"
|
|
@@ -78,73 +129,148 @@ export async function runInfrastructureRetries(options) {
|
|
|
78
129
|
run_id: ref.run_id,
|
|
79
130
|
observation_status: ref.observation_status,
|
|
80
131
|
...(ref.invalid_reason ? { invalid_reason: ref.invalid_reason } : {}),
|
|
81
|
-
generation: progress.generation,
|
|
132
|
+
generation: (options.currentProgress?.() ?? progress).generation,
|
|
82
133
|
});
|
|
83
134
|
};
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
taskNames,
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
135
|
+
let run;
|
|
136
|
+
let nextCandidates = [];
|
|
137
|
+
let runRecord;
|
|
138
|
+
let executionStarted = false;
|
|
139
|
+
try {
|
|
140
|
+
lifecycle = await options.beginRetry?.({ retry, logicalAttempt: options.logicalAttempt, taskNames, triggers: retryTriggers, backendDirectory });
|
|
141
|
+
if (retryWork && lifecycle?.workId !== retryWork.work_id)
|
|
142
|
+
throw new Error(`retry lifecycle work identity conflicts with persisted decision: ${retryWork.work_id}`);
|
|
143
|
+
for (const decision of retryDecisions)
|
|
144
|
+
await transitionRetryDecision({
|
|
145
|
+
evalDirectory: options.evalDirectory, evalId: options.evalId, decisionId: decision.decision_id, state: "running",
|
|
146
|
+
});
|
|
147
|
+
if (retryWork)
|
|
148
|
+
options.sink.emit({ type: "eval.retry.admitted", work_id: retryWork.work_id, decision_ids: retryDecisions.map((decision) => decision.decision_id) });
|
|
149
|
+
if (retryWork) {
|
|
150
|
+
options.onRetryExecutionStarted?.(retryWork.work_id);
|
|
151
|
+
executionStarted = true;
|
|
152
|
+
}
|
|
153
|
+
run = await runHarborBackend({
|
|
154
|
+
evalId: options.evalId,
|
|
155
|
+
evalDirectory: options.evalDirectory,
|
|
156
|
+
backendDirectory,
|
|
157
|
+
logicalAttempt: options.logicalAttempt,
|
|
158
|
+
taskNames,
|
|
159
|
+
request: { ...options.request, attempts: 1 },
|
|
160
|
+
root: options.root,
|
|
161
|
+
resolvedRevision: options.resolvedRevision,
|
|
162
|
+
runtimeDirectory: options.controllerRuntime.directory,
|
|
163
|
+
runtimeId: options.controllerRuntime.runtime_id,
|
|
164
|
+
preparedArtifact: options.preparedArtifact,
|
|
165
|
+
...(options.executionResources ? { executionResources: options.executionResources } : {}),
|
|
166
|
+
...(options.resolvedImages ? { resolvedImages: options.resolvedImages } : {}),
|
|
167
|
+
...(options.interactionCaptureExporter ? { modelProxy: options.interactionCaptureExporter.route } : {}),
|
|
168
|
+
...lifecycle?.backend,
|
|
169
|
+
env: options.env,
|
|
170
|
+
...(options.harborExecutable !== undefined ? { harborExecutable: options.harborExecutable } : {}),
|
|
171
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
172
|
+
...(options.trialBundleGraceMs === undefined ? {} : { trialBundleGraceMs: options.trialBundleGraceMs }),
|
|
173
|
+
emit: (event) => options.sink.emit({ ...event, execution_kind: "physical-infrastructure-retry", infrastructure_retry: retry, logical_attempt: options.logicalAttempt, ...(lifecycle ? { work_id: lifecycle.workId, lease_id: lifecycle.leaseId } : {}) }),
|
|
174
|
+
onTrialSettled: async (trial, context) => {
|
|
175
|
+
try {
|
|
176
|
+
const environmentImages = retryEnvironmentImages();
|
|
177
|
+
const ref = await importEvalTrialRun({
|
|
178
|
+
root: options.root,
|
|
179
|
+
evalId: options.evalId,
|
|
180
|
+
evalDirectory: options.evalDirectory,
|
|
181
|
+
harborJobDirectory,
|
|
182
|
+
expectedAttempt: options.logicalAttempt,
|
|
183
|
+
request: options.request,
|
|
184
|
+
resolvedRevision: options.resolvedRevision,
|
|
185
|
+
benchmarkId: options.request.benchmark_id,
|
|
186
|
+
benchmarkRevision: options.request.benchmark_revision,
|
|
187
|
+
publicationMode: "replace-invalid",
|
|
188
|
+
runtimeId: options.controllerRuntime.runtime_id,
|
|
189
|
+
env: options.env,
|
|
190
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
191
|
+
...(options.modelCapturePlan ? { modelCapturePlan: options.modelCapturePlan } : {}),
|
|
192
|
+
...(options.interactionCaptureExporter ? {
|
|
193
|
+
interactionCaptureExporter: options.interactionCaptureExporter,
|
|
194
|
+
} : {}),
|
|
195
|
+
...await executionEvidence(),
|
|
196
|
+
...(environmentImages ? { environmentImages } : {}),
|
|
197
|
+
requireCompleteMarker: true,
|
|
198
|
+
allowMissingBundleDiagnostic: context.bundleWaitExpired,
|
|
199
|
+
}, trial, retryRefs.length, retryRefs);
|
|
200
|
+
await publish(ref);
|
|
201
|
+
return true;
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
if (error instanceof TrialBundlePendingError)
|
|
205
|
+
return false;
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
},
|
|
209
|
+
});
|
|
210
|
+
for (const event of await harborPhaseTimingEvents(harborJobDirectory, run.rawResult)) {
|
|
211
|
+
options.sink.emit(event);
|
|
212
|
+
if (event.type === "eval.verifier.completed" && typeof event.duration_ms === "number")
|
|
213
|
+
options.onVerifierDuration?.(event.duration_ms);
|
|
214
|
+
}
|
|
215
|
+
const environmentImages = retryEnvironmentImages();
|
|
216
|
+
const terminalRefs = await importEvalTrialRuns({
|
|
217
|
+
root: options.root,
|
|
218
|
+
evalId: options.evalId,
|
|
219
|
+
evalDirectory: options.evalDirectory,
|
|
220
|
+
harborJobDirectory,
|
|
221
|
+
expectedAttempt: options.logicalAttempt,
|
|
222
|
+
request: options.request,
|
|
223
|
+
resolvedRevision: options.resolvedRevision,
|
|
224
|
+
benchmarkId: options.request.benchmark_id,
|
|
225
|
+
benchmarkRevision: options.request.benchmark_revision,
|
|
226
|
+
publicationMode: "replace-invalid",
|
|
227
|
+
runtimeId: options.controllerRuntime.runtime_id,
|
|
228
|
+
env: options.env,
|
|
229
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
230
|
+
...(options.modelCapturePlan ? { modelCapturePlan: options.modelCapturePlan } : {}),
|
|
231
|
+
...(options.interactionCaptureExporter ? {
|
|
232
|
+
interactionCaptureExporter: options.interactionCaptureExporter,
|
|
233
|
+
} : {}),
|
|
234
|
+
...await executionEvidence(),
|
|
235
|
+
...(environmentImages ? { environmentImages } : {}),
|
|
236
|
+
rawResult: run.rawResult,
|
|
237
|
+
}, retryRefs);
|
|
238
|
+
for (const ref of terminalRefs)
|
|
239
|
+
await publish(ref);
|
|
240
|
+
if (run.rawResult !== null)
|
|
241
|
+
assertBackendTrialSet(run.rawResult, retryRefs);
|
|
242
|
+
runRecord = { attempt: options.logicalAttempt, retry, tasks: taskNames, triggers: retryTriggers, refs: retryRefs, run, ...(lifecycle ? { leaseId: lifecycle.leaseId, workId: lifecycle.workId } : {}) };
|
|
243
|
+
runs.push(runRecord);
|
|
244
|
+
nextCandidates = retryableInfrastructureTrials(retryRefs);
|
|
245
|
+
if (options.originWorkItem && retry < options.request.infrastructure_retries && nextCandidates.length > 0) {
|
|
246
|
+
const nextWork = physicalRetryWorkItem(options.originWorkItem, retry + 1, nextCandidates);
|
|
247
|
+
const nextNotBefore = new Date(Date.now() + retryBackoffMs(options.request.infrastructure_retry_backoff_ms, retry + 1, nextWork.work_id)).toISOString();
|
|
248
|
+
await Promise.all(nextCandidates.map((trigger) => ensurePhysicalRetryDecision({
|
|
249
|
+
evalDirectory: options.evalDirectory, evalId: options.evalId, item: options.originWorkItem,
|
|
250
|
+
retryIndex: retry + 1, trigger, notBefore: nextNotBefore,
|
|
251
|
+
})));
|
|
252
|
+
await options.onRetryWorkPlanned?.(nextWork.work_id);
|
|
253
|
+
}
|
|
254
|
+
const retryState = retryRefs.some((ref) => ref.observation_status === "valid")
|
|
255
|
+
? "repaired"
|
|
256
|
+
: retry < options.request.infrastructure_retries && nextCandidates.length > 0 ? "invalid" : "exhausted";
|
|
257
|
+
for (const decision of retryDecisions)
|
|
258
|
+
await transitionRetryDecision({
|
|
259
|
+
evalDirectory: options.evalDirectory, evalId: options.evalId, decisionId: decision.decision_id, state: retryState,
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
finally {
|
|
263
|
+
await lifecycle?.close();
|
|
264
|
+
if (executionStarted && retryWork) {
|
|
265
|
+
const durationMs = options.onRetryExecutionFinished?.(retryWork.work_id);
|
|
266
|
+
if (runRecord && durationMs !== undefined)
|
|
267
|
+
runRecord.durationMs = durationMs;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
146
270
|
options.sink.emit({
|
|
147
271
|
type: "eval.infrastructure-retry.completed",
|
|
272
|
+
execution_kind: "physical-infrastructure-retry",
|
|
273
|
+
candidate_executes: true,
|
|
148
274
|
attempt: options.logicalAttempt,
|
|
149
275
|
retry,
|
|
150
276
|
tasks: taskNames,
|
|
@@ -153,14 +279,12 @@ export async function runInfrastructureRetries(options) {
|
|
|
153
279
|
});
|
|
154
280
|
if (run.backend.process_exit_code !== 0 || run.rawResult === null || options.stopAfterResult?.(run.rawResult))
|
|
155
281
|
break;
|
|
156
|
-
retryCandidates =
|
|
282
|
+
retryCandidates = nextCandidates;
|
|
157
283
|
}
|
|
158
|
-
return { progress, runs };
|
|
284
|
+
return { progress: options.currentProgress?.() ?? progress, runs };
|
|
159
285
|
}
|
|
160
286
|
export function retryableInfrastructureTrials(trials) {
|
|
161
|
-
return trials.filter((trial) => trial
|
|
162
|
-
&& trial.invalid_reason !== undefined
|
|
163
|
-
&& RETRYABLE_TRIAL_INFRASTRUCTURE_REASONS.has(trial.invalid_reason));
|
|
287
|
+
return trials.filter((trial) => physicalRetryAllowed(classifyTrialFailure(trial)));
|
|
164
288
|
}
|
|
165
289
|
export function infrastructureFailureTrials(trials) {
|
|
166
290
|
return trials.filter((trial) => trial.observation_status === "invalid"
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"infrastructure-retry.js","sourceRoot":"","sources":["../../../src/evals/infrastructure-retry.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAIxD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAClI,OAAO,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAEnF,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACrC,wBAAwB;IACxB,iCAAiC;IACjC,yBAAyB;CAC1B,CAAC,CAAC;AAEH,6EAA6E;AAC7E,2EAA2E;AAC3E,4EAA4E;AAC5E,0CAA0C;AAC1C,MAAM,sCAAsC,GAAG,IAAI,GAAG,CAAC;IACrD,wBAAwB;CACzB,CAAC,CAAC;AA+BH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,OAAwC;IAExC,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAChC,IAAI,eAAe,GAAG,6BAA6B,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACzE,MAAM,IAAI,GAA6B,EAAE,CAAC;IAC1C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC9G,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACnC,MAAM,aAAa,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,+BAA+B,GAAG,KAAK,CAAC;QAC1E,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qCAAqC;YAC3C,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QACH,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;;gBAC7E,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QAEnC,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAChC,OAAO,CAAC,aAAa,EACrB,wBAAwB,EACxB,SAAS,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EACzC,OAAO,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,cAAc,CAAC,CACzF,CAAC;QACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAqB,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,KAAK,EAAE,GAAmB,EAAiB,EAAE;YAC3D,IAAI,GAAG,CAAC,OAAO,KAAK,OAAO,CAAC,cAAc,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9E,MAAM,IAAI,UAAU,CAAC,6DAA6D,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE;oBAC9G,IAAI,EAAE,0CAA0C;oBAChD,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;YACL,CAAC;YACD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAC;YAChF,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtI,OAAO;YACT,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,MAAM,2BAA2B,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE;gBACrE,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;gBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;aACtD,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACvC,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAC1D,MAAM,iBAAiB,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,GAAG,CAAC,kBAAkB,KAAK,OAAO;oBACtC,CAAC,CAAC,oCAAoC;oBACtC,CAAC,CAAC,kCAAkC;gBACtC,OAAO,EAAE,OAAO,CAAC,cAAc;gBAC/B,KAAK;gBACL,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;gBAC1C,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrE,UAAU,EAAE,QAAQ,CAAC,UAAU;aAChC,CAAC,CAAC;QACL,CAAC,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC;YACjC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,gBAAgB;YAChB,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,SAAS;YACT,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE;YAC5C,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,gBAAgB,EAAE,OAAO,CAAC,iBAAiB,CAAC,SAAS;YACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;YAC/C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7E,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC;YACvG,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,oBAAoB,EAAE,KAAK,EAAE,eAAe,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC;YACtH,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAoB,EAAE;gBACzD,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC;wBACnC,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;wBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;wBACpC,kBAAkB;wBAClB,eAAe,EAAE,OAAO,CAAC,cAAc;wBACvC,OAAO,EAAE,OAAO,CAAC,OAAO;wBACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;wBAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;wBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;wBACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;wBAC/C,qBAAqB,EAAE,IAAI;wBAC3B,4BAA4B,EAAE,OAAO,CAAC,iBAAiB;qBACxD,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;oBACvC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;oBACnB,OAAO,IAAI,CAAC;gBACd,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,uBAAuB;wBAAE,OAAO,KAAK,CAAC;oBAC3D,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QACH,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC;YAC7C,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,kBAAkB;YAClB,eAAe,EAAE,OAAO,CAAC,cAAc;YACvC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;YACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;YACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;YAC/C,SAAS,EAAE,GAAG,CAAC,SAAS;SACzB,EAAE,SAAS,CAAC,CAAC;QACd,KAAK,MAAM,GAAG,IAAI,YAAY;YAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;QACnD,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI;YAAE,qBAAqB,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAC5E,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QACvH,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qCAAqC;YAC3C,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;YAC9G,eAAe,EAAE,6BAA6B,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;SAC3F,CAAC,CAAC;QACH,IAAI,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,MAAM;QACrH,eAAe,GAAG,6BAA6B,CAAC,SAAS,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAAiC;IAC7E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS;WACjE,KAAK,CAAC,cAAc,KAAK,SAAS;WAClC,sCAAsC,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,MAAiC;IAC3E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS;WACjE,KAAK,CAAC,cAAc,KAAK,SAAS;WAClC,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,qBAAqB,CAAC,SAAkC,EAAE,IAA+B;IAChG,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,aAA0C,CAAC,CAAC,CAAC,EAAE,CAAC;IAClH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC;IACrK,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxD,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC1H,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IAClG,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAe;IAC3C,OAAO,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvD,CAAC"}
|
|
1
|
+
{"version":3,"file":"infrastructure-retry.js","sourceRoot":"","sources":["../../../src/evals/infrastructure-retry.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAIxD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAClI,OAAO,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAGnF,OAAO,EAAE,2BAA2B,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AACxF,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACrC,wBAAwB;IACxB,iCAAiC;IACjC,yBAAyB;IACzB,0BAA0B;IAC1B,sBAAsB;IACtB,gCAAgC;IAChC,uBAAuB;IACvB,8BAA8B;IAC9B,8BAA8B;IAC9B,sBAAsB;IACtB,2BAA2B;CAC5B,CAAC,CAAC;AAuEH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,OAAwC;IAExC,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAChC,IAAI,eAAe,GAAG,6BAA6B,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACzE,MAAM,IAAI,GAA6B,EAAE,CAAC;IAC1C,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,CAAC,CAAC;IACrD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,IAAI,eAAe,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IACtI,KAAK,IAAI,KAAK,GAAG,eAAe,EAAE,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC5H,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACnC,MAAM,aAAa,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjF,MAAM,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,cAAc,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3H,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,+BAA+B,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACxK,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc;YAC3C,CAAC,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,2BAA2B,CAAC;gBAC7E,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,cAAmC;gBAC/G,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS;aACtC,CAAC,CAAC,CAAC;YACJ,CAAC,CAAC,EAAE,CAAC;QACP,IAAI,SAAS,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;YACjG,MAAM,IAAI,KAAK,CAAC,2DAA2D,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,IAAI,SAAS;YAAE,MAAM,OAAO,CAAC,kBAAkB,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qBAAqB;YAC3B,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,YAAY,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC;YACpE,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QACH,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5G,OAAO,CAAC,cAAc,EAAE,CAAC,kBAAkB,CAAC,CAAC;QAC7C,IAAI,kBAAkB,GAAG,CAAC,EAAE,CAAC;YAC3B,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,KAAK,CAAC,kBAAkB,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;;gBACtF,MAAM,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACnC,IAAI,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC/B,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,YAAY,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAC1H,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,cAAc,EAAE,UAAU,EAAE,iBAAiB,IAAI,CAAC;aAC9F,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAChC,OAAO,CAAC,oBAAoB,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,wBAAwB,CAAC,EAC1F,SAAS,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EACzC,OAAO,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,cAAc,CAAC,CACzF,CAAC;QACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAqB,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,SAAmD,CAAC;QACxD,MAAM,sBAAsB,GAAG,GAAyC,EAAE,CAAC,SAAS,EAAE,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,CAAC;QACrI,MAAM,iBAAiB,GAAG,KAAK,IAA0D,EAAE,CAAC,SAAS,EAAE,wBAAwB;YAC7H,CAAC,CAAC,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC,wBAAwB,EAAE,EAAE;YACnE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,OAAO,GAAG,KAAK,EAAE,GAAmB,EAAiB,EAAE;YAC3D,IAAI,GAAG,CAAC,OAAO,KAAK,OAAO,CAAC,cAAc,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9E,MAAM,IAAI,UAAU,CAAC,6DAA6D,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE;oBAC9G,IAAI,EAAE,0CAA0C;oBAChD,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;YACL,CAAC;YACD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAC;YAChF,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtI,OAAO;YACT,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,IAAI,GAAG,CAAC,cAAc,KAAK,gCAAgC;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;oBAC7E,IAAI,EAAE,uBAAuB,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO;oBACxG,MAAM,EAAE,gCAAgC,EAAE,kBAAkB,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK;iBAC7F,CAAC,CAAC;YACH,MAAM,2BAA2B,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE;gBACrE,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;gBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;aACtD,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACvC,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;oBACjC,IAAI,CAAC,SAAS;wBAAE,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;oBAClG,MAAM,OAAO,CAAC,oBAAoB,CAAC,GAAG,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;oBAC1D,QAAQ,GAAG,OAAO,CAAC,eAAe,EAAE,EAAE,IAAI,QAAQ,CAAC;gBACrD,CAAC;qBAAM,CAAC;oBACN,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;oBAC1D,MAAM,iBAAiB,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;gBAC3D,CAAC;YACH,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,GAAG,CAAC,kBAAkB,KAAK,OAAO;oBACtC,CAAC,CAAC,oCAAoC;oBACtC,CAAC,CAAC,kCAAkC;gBACtC,OAAO,EAAE,OAAO,CAAC,cAAc;gBAC/B,KAAK;gBACL,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;gBAC1C,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrE,UAAU,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,IAAI,QAAQ,CAAC,CAAC,UAAU;aACjE,CAAC,CAAC;QACL,CAAC,CAAC;QACF,IAAI,GAAwB,CAAC;QAC7B,IAAI,cAAc,GAAqB,EAAE,CAAC;QAC1C,IAAI,SAA6C,CAAC;QAClD,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC;YACH,SAAS,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAChJ,IAAI,SAAS,IAAI,SAAS,EAAE,MAAM,KAAK,SAAS,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,oEAAoE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YACnK,KAAK,MAAM,QAAQ,IAAI,cAAc;gBAAE,MAAM,uBAAuB,CAAC;oBACnE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,SAAS;iBACjH,CAAC,CAAC;YACH,IAAI,SAAS;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,YAAY,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YACpK,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,uBAAuB,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBACrD,gBAAgB,GAAG,IAAI,CAAC;YAC1B,CAAC;YACD,GAAG,GAAG,MAAM,gBAAgB,CAAC;gBAC3B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,gBAAgB;gBAChB,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,SAAS;gBACT,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE;gBAC5C,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,gBAAgB,EAAE,OAAO,CAAC,iBAAiB,CAAC,SAAS;gBACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;gBAC/C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzF,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7E,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,0BAA0B,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvG,GAAG,SAAS,EAAE,OAAO;gBACrB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC;gBACvG,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,cAAc,EAAE,+BAA+B,EAAE,oBAAoB,EAAE,KAAK,EAAE,eAAe,EAAE,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;gBACzP,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAoB,EAAE;oBACzD,IAAI,CAAC;wBACH,MAAM,iBAAiB,GAAG,sBAAsB,EAAE,CAAC;wBACnD,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC;4BACnC,IAAI,EAAE,OAAO,CAAC,IAAI;4BAClB,MAAM,EAAE,OAAO,CAAC,MAAM;4BACtB,aAAa,EAAE,OAAO,CAAC,aAAa;4BACpC,kBAAkB;4BAClB,eAAe,EAAE,OAAO,CAAC,cAAc;4BACvC,OAAO,EAAE,OAAO,CAAC,OAAO;4BACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;4BAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;4BACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;4BACrD,eAAe,EAAE,iBAAiB;4BAClC,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;4BAC/C,GAAG,EAAE,OAAO,CAAC,GAAG;4BAChB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;4BACrD,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;4BACnF,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC;gCACvC,0BAA0B,EAAE,OAAO,CAAC,0BAA0B;6BAC/D,CAAC,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,MAAM,iBAAiB,EAAE;4BAC5B,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;4BACnD,qBAAqB,EAAE,IAAI;4BAC3B,4BAA4B,EAAE,OAAO,CAAC,iBAAiB;yBACxD,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;wBACvC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;wBACnB,OAAO,IAAI,CAAC;oBACd,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,KAAK,YAAY,uBAAuB;4BAAE,OAAO,KAAK,CAAC;wBAC3D,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YACH,KAAK,MAAM,KAAK,IAAI,MAAM,uBAAuB,CAAC,kBAAkB,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACrF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACzB,IAAI,KAAK,CAAC,IAAI,KAAK,yBAAyB,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;oBAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACzI,CAAC;YACD,MAAM,iBAAiB,GAAG,sBAAsB,EAAE,CAAC;YACnD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC;gBAC7C,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,kBAAkB;gBAClB,eAAe,EAAE,OAAO,CAAC,cAAc;gBACvC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;gBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;gBACrD,eAAe,EAAE,iBAAiB;gBAClC,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;gBAC/C,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC;oBACvC,0BAA0B,EAAE,OAAO,CAAC,0BAA0B;iBAC/D,CAAC,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,MAAM,iBAAiB,EAAE;gBAC5B,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnD,SAAS,EAAE,GAAG,CAAC,SAAS;aACzB,EAAE,SAAS,CAAC,CAAC;YACd,KAAK,MAAM,GAAG,IAAI,YAAY;gBAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YACnD,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI;gBAAE,qBAAqB,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC5E,SAAS,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YACxM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrB,cAAc,GAAG,6BAA6B,CAAC,SAAS,CAAC,CAAC;YAC1D,IAAI,OAAO,CAAC,cAAc,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1G,MAAM,QAAQ,GAAG,qBAAqB,CAAC,OAAO,CAAC,cAAc,EAAE,KAAK,GAAG,CAAC,EAAE,cAAc,CAAC,CAAC;gBAC1F,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,+BAA+B,EAAE,KAAK,GAAG,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBACxJ,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,2BAA2B,CAAC;oBAC5E,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,cAAmC;oBAC/G,UAAU,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa;iBACzD,CAAC,CAAC,CAAC,CAAC;gBACL,MAAM,OAAO,CAAC,kBAAkB,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACvD,CAAC;YACD,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC;gBAC5E,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC;YAC1G,KAAK,MAAM,QAAQ,IAAI,cAAc;gBAAE,MAAM,uBAAuB,CAAC;oBACnE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,UAAU;iBAClH,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,MAAM,SAAS,EAAE,KAAK,EAAE,CAAC;YACzB,IAAI,gBAAgB,IAAI,SAAS,EAAE,CAAC;gBAClC,MAAM,UAAU,GAAG,OAAO,CAAC,wBAAwB,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBACzE,IAAI,SAAS,IAAI,UAAU,KAAK,SAAS;oBAAE,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC;YAC/E,CAAC;QACH,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qCAAqC;YAC3C,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;YAC9G,eAAe,EAAE,6BAA6B,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;SAC3F,CAAC,CAAC;QACH,IAAI,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,MAAM;QACrH,eAAe,GAAG,cAAc,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,eAAe,EAAE,EAAE,IAAI,QAAQ,EAAE,IAAI,EAAE,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAAiC;IAC7E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,oBAAoB,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,MAAiC;IAC3E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS;WACjE,KAAK,CAAC,cAAc,KAAK,SAAS;WAClC,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,qBAAqB,CAAC,SAAkC,EAAE,IAA+B;IAChG,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,aAA0C,CAAC,CAAC,CAAC,EAAE,CAAC;IAClH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC;IACrK,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxD,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC1H,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IAClG,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAe;IAC3C,OAAO,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvD,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { HitchError, atomicWriteJSON, ensureDir } from "../foundation/index.js";
|
|
2
|
+
import { loadInteractionCapture } from "../model-access/index.js";
|
|
3
|
+
export async function importTrialInteractionCapture(input, runId, staging) {
|
|
4
|
+
const plan = input.modelCapturePlan;
|
|
5
|
+
await writeTrialCapturePolicy(staging, plan);
|
|
6
|
+
const expectsProxy = plan?.effective_mode === "proxy" || plan?.effective_mode === "hybrid";
|
|
7
|
+
const existing = expectsProxy
|
|
8
|
+
? await loadInteractionCapture(staging).catch((error) => {
|
|
9
|
+
if (error.code === "ENOENT")
|
|
10
|
+
return null;
|
|
11
|
+
throw error;
|
|
12
|
+
})
|
|
13
|
+
: null;
|
|
14
|
+
if (existing) {
|
|
15
|
+
const ref = existing.ref;
|
|
16
|
+
if (ref.run_id !== runId || ref.mode !== plan?.effective_mode || ref.required !== plan.required
|
|
17
|
+
|| ref.topology !== plan.topology || plan.required && ref.completeness !== "complete") {
|
|
18
|
+
throw new HitchError("remote model interaction capture does not match its sealed plan", { code: "model_capture_incomplete", exitCode: 12 });
|
|
19
|
+
}
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
if (!input.interactionCaptureExporter) {
|
|
23
|
+
if (expectsProxy && plan.required) {
|
|
24
|
+
throw new HitchError("required model interaction capture is unavailable", { code: "model_capture_incomplete", exitCode: 12 });
|
|
25
|
+
}
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
const ref = await input.interactionCaptureExporter.finalizeRun(runId, staging);
|
|
29
|
+
if (ref.run_id !== runId || ref.mode !== input.interactionCaptureExporter.route.mode) {
|
|
30
|
+
throw new HitchError("model interaction capture identity does not match the run", { code: "model_capture_incomplete", exitCode: 12 });
|
|
31
|
+
}
|
|
32
|
+
if (input.interactionCaptureExporter.plan.required && ref.completeness !== "complete") {
|
|
33
|
+
throw new HitchError("required model interaction capture is incomplete", { code: "model_capture_incomplete", exitCode: 12 });
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export async function writeTrialCapturePolicy(staging, plan) {
|
|
37
|
+
if (!plan)
|
|
38
|
+
return;
|
|
39
|
+
await ensureDir(`${staging}/interactions`);
|
|
40
|
+
await atomicWriteJSON(`${staging}/interactions/capture.policy.json`, {
|
|
41
|
+
schema_version: "1",
|
|
42
|
+
requested_mode: plan.requested_mode,
|
|
43
|
+
effective_mode: plan.effective_mode,
|
|
44
|
+
required: plan.required,
|
|
45
|
+
...(plan.topology ? { topology: plan.topology } : {}),
|
|
46
|
+
...(plan.degraded_reason ? { degraded_reason: plan.degraded_reason } : {}),
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=interaction-capture-import.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"interaction-capture-import.js","sourceRoot":"","sources":["../../../src/evals/interaction-capture-import.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAChF,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAGlE,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,KAGnD,EAAE,KAAa,EAAE,OAAe;IAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,gBAAgB,CAAC;IACpC,MAAM,uBAAuB,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,IAAI,EAAE,cAAc,KAAK,OAAO,IAAI,IAAI,EAAE,cAAc,KAAK,QAAQ,CAAC;IAC3F,MAAM,QAAQ,GAAG,YAAY;QAC3B,CAAC,CAAC,MAAM,sBAAsB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACtD,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAC;YACpE,MAAM,KAAK,CAAC;QACd,CAAC,CAAC;QACF,CAAC,CAAC,IAAI,CAAC;IACT,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC;QACzB,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,EAAE,cAAc,IAAI,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ;eAC1F,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YACxF,MAAM,IAAI,UAAU,CAAC,iEAAiE,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9I,CAAC;QACD,OAAO;IACT,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,0BAA0B,EAAE,CAAC;QACtC,IAAI,YAAY,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,MAAM,IAAI,UAAU,CAAC,mDAAmD,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAChI,CAAC;QACD,OAAO;IACT,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,0BAA0B,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/E,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,KAAK,CAAC,0BAA0B,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACrF,MAAM,IAAI,UAAU,CAAC,2DAA2D,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IACxI,CAAC;IACD,IAAI,KAAK,CAAC,0BAA0B,CAAC,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;QACtF,MAAM,IAAI,UAAU,CAAC,kDAAkD,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/H,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,OAAe,EAAE,IAAyB;IACtF,IAAI,CAAC,IAAI;QAAE,OAAO;IAClB,MAAM,SAAS,CAAC,GAAG,OAAO,eAAe,CAAC,CAAC;IAC3C,MAAM,eAAe,CAAC,GAAG,OAAO,mCAAmC,EAAE;QACnE,cAAc,EAAE,GAAG;QACnB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3E,CAAC,CAAC;AACL,CAAC"}
|