agent-hitch 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -3
- package/README.zh-CN.md +14 -2
- package/dist/bin/hitch.js +25 -3
- package/dist/bin/hitch.js.map +1 -1
- package/dist/scripts/canary-benchmark.js +54 -0
- package/dist/scripts/canary-benchmark.js.map +1 -0
- package/dist/scripts/canary-buildkit-secrets.js +166 -0
- package/dist/scripts/canary-buildkit-secrets.js.map +1 -0
- package/dist/scripts/canary-collision-domains.js +77 -0
- package/dist/scripts/canary-collision-domains.js.map +1 -0
- package/dist/scripts/canary-docker-image-observation.js +181 -0
- package/dist/scripts/canary-docker-image-observation.js.map +1 -0
- package/dist/scripts/canary-eval-scheduler-throughput.js +47 -0
- package/dist/scripts/canary-eval-scheduler-throughput.js.map +1 -0
- package/dist/scripts/canary-gpu-hardware.js +95 -0
- package/dist/scripts/canary-gpu-hardware.js.map +1 -0
- package/dist/scripts/canary-harbor-load.js +365 -0
- package/dist/scripts/canary-harbor-load.js.map +1 -0
- package/dist/scripts/canary-packaged-harness.js +75 -0
- package/dist/scripts/canary-packaged-harness.js.map +1 -0
- package/dist/scripts/canary-resource-load.js +188 -0
- package/dist/scripts/canary-resource-load.js.map +1 -0
- package/dist/scripts/check-architecture.js +10 -5
- package/dist/scripts/check-architecture.js.map +1 -1
- package/dist/scripts/docker-reaper-crash-child.js +21 -0
- package/dist/scripts/docker-reaper-crash-child.js.map +1 -0
- package/dist/src/adapters/catalog.js +3 -1
- package/dist/src/adapters/catalog.js.map +1 -1
- package/dist/src/adapters/discovery.js +1 -1
- package/dist/src/adapters/discovery.js.map +1 -1
- package/dist/src/adapters/providers/claude.js +6 -0
- package/dist/src/adapters/providers/claude.js.map +1 -1
- package/dist/src/adapters/providers/codex-auth.js +29 -0
- package/dist/src/adapters/providers/codex-auth.js.map +1 -0
- package/dist/src/adapters/providers/codex.js +29 -6
- package/dist/src/adapters/providers/codex.js.map +1 -1
- package/dist/src/adapters/providers/deepseek.js +6 -0
- package/dist/src/adapters/providers/deepseek.js.map +1 -1
- package/dist/src/adapters/providers/model-call.js +26 -0
- package/dist/src/adapters/providers/model-call.js.map +1 -0
- package/dist/src/adapters/providers/opencode.js +6 -0
- package/dist/src/adapters/providers/opencode.js.map +1 -1
- package/dist/src/adapters/providers/pi.js +6 -0
- package/dist/src/adapters/providers/pi.js.map +1 -1
- package/dist/src/artifacts/index.js +1 -0
- package/dist/src/artifacts/index.js.map +1 -1
- package/dist/src/artifacts/preparer.js +7 -2
- package/dist/src/artifacts/preparer.js.map +1 -1
- package/dist/src/backends/harbor/agent-budget.js +31 -0
- package/dist/src/backends/harbor/agent-budget.js.map +1 -0
- package/dist/src/backends/harbor/backend.js +59 -107
- package/dist/src/backends/harbor/backend.js.map +1 -1
- package/dist/src/backends/harbor/environment-config.js +92 -0
- package/dist/src/backends/harbor/environment-config.js.map +1 -0
- package/dist/src/backends/harbor/index.js +6 -1
- package/dist/src/backends/harbor/index.js.map +1 -1
- package/dist/src/backends/harbor/model-proxy-config.js +23 -0
- package/dist/src/backends/harbor/model-proxy-config.js.map +1 -0
- package/dist/src/backends/harbor/process.js +177 -0
- package/dist/src/backends/harbor/process.js.map +1 -0
- package/dist/src/backends/harbor/regrade.js +77 -0
- package/dist/src/backends/harbor/regrade.js.map +1 -0
- package/dist/src/backends/harbor/runtime-toolchain.js +30 -0
- package/dist/src/backends/harbor/runtime-toolchain.js.map +1 -0
- package/dist/src/backends/harbor/supervisor.js +74 -0
- package/dist/src/backends/harbor/supervisor.js.map +1 -0
- package/dist/src/backends/harbor/task-resources.js +206 -0
- package/dist/src/backends/harbor/task-resources.js.map +1 -0
- package/dist/src/backends/harbor/tools.js +6 -29
- package/dist/src/backends/harbor/tools.js.map +1 -1
- package/dist/src/backends/index.js +5 -1
- package/dist/src/backends/index.js.map +1 -1
- package/dist/src/benchmarks/index.js +3 -0
- package/dist/src/benchmarks/index.js.map +1 -0
- package/dist/src/benchmarks/loader.js +271 -0
- package/dist/src/benchmarks/loader.js.map +1 -0
- package/dist/src/benchmarks/metrics.js +16 -0
- package/dist/src/benchmarks/metrics.js.map +1 -0
- package/dist/src/benchmarks/toml.js +13 -0
- package/dist/src/benchmarks/toml.js.map +1 -0
- package/dist/src/benchmarks/validation.js +242 -0
- package/dist/src/benchmarks/validation.js.map +1 -0
- package/dist/src/cli/arguments.js +6 -4
- package/dist/src/cli/arguments.js.map +1 -1
- package/dist/src/cli/commands/benchmark.js +22 -0
- package/dist/src/cli/commands/benchmark.js.map +1 -0
- package/dist/src/cli/commands/capabilities.js +18 -0
- package/dist/src/cli/commands/capabilities.js.map +1 -0
- package/dist/src/cli/commands/daemon.js +135 -8
- package/dist/src/cli/commands/daemon.js.map +1 -1
- package/dist/src/cli/commands/eval.js +228 -6
- package/dist/src/cli/commands/eval.js.map +1 -1
- package/dist/src/cli/commands/images.js +40 -0
- package/dist/src/cli/commands/images.js.map +1 -0
- package/dist/src/cli/commands/run.js +42 -17
- package/dist/src/cli/commands/run.js.map +1 -1
- package/dist/src/cli/commands/runs.js +25 -3
- package/dist/src/cli/commands/runs.js.map +1 -1
- package/dist/src/cli/commands/trajectory.js +96 -5
- package/dist/src/cli/commands/trajectory.js.map +1 -1
- package/dist/src/cli/commands/verifier.js +36 -0
- package/dist/src/cli/commands/verifier.js.map +1 -0
- package/dist/src/cli/commands/worker.js +79 -0
- package/dist/src/cli/commands/worker.js.map +1 -0
- package/dist/src/cli/main.js +10 -0
- package/dist/src/cli/main.js.map +1 -1
- package/dist/src/cli/output.js +84 -4
- package/dist/src/cli/output.js.map +1 -1
- package/dist/src/control-plane/build-admission.js +69 -0
- package/dist/src/control-plane/build-admission.js.map +1 -0
- package/dist/src/control-plane/build-records.js +6 -0
- package/dist/src/control-plane/build-records.js.map +1 -0
- package/dist/src/control-plane/collisions.js +40 -0
- package/dist/src/control-plane/collisions.js.map +1 -0
- package/dist/src/control-plane/eval-control-state.js +22 -0
- package/dist/src/control-plane/eval-control-state.js.map +1 -0
- package/dist/src/control-plane/eval-control-work.js +44 -0
- package/dist/src/control-plane/eval-control-work.js.map +1 -0
- package/dist/src/control-plane/eval-image-resolution.js +52 -0
- package/dist/src/control-plane/eval-image-resolution.js.map +1 -0
- package/dist/src/control-plane/eval-image-services.js +30 -0
- package/dist/src/control-plane/eval-image-services.js.map +1 -0
- package/dist/src/control-plane/eval-records.js +252 -0
- package/dist/src/control-plane/eval-records.js.map +1 -0
- package/dist/src/control-plane/eval-recovery.js +181 -0
- package/dist/src/control-plane/eval-recovery.js.map +1 -0
- package/dist/src/control-plane/eval-scheduler.js +450 -0
- package/dist/src/control-plane/eval-scheduler.js.map +1 -0
- package/dist/src/control-plane/index.js +20 -0
- package/dist/src/control-plane/index.js.map +1 -0
- package/dist/src/control-plane/local-worker.js +49 -0
- package/dist/src/control-plane/local-worker.js.map +1 -0
- package/dist/src/control-plane/model-capture-planning.js +12 -0
- package/dist/src/control-plane/model-capture-planning.js.map +1 -0
- package/dist/src/control-plane/remote-result-transport.js +175 -0
- package/dist/src/control-plane/remote-result-transport.js.map +1 -0
- package/dist/src/control-plane/remote-work-coordinator.js +333 -0
- package/dist/src/control-plane/remote-work-coordinator.js.map +1 -0
- package/dist/src/control-plane/remote-work-inputs.js +266 -0
- package/dist/src/control-plane/remote-work-inputs.js.map +1 -0
- package/dist/src/control-plane/remote-work-item.js +62 -0
- package/dist/src/control-plane/remote-work-item.js.map +1 -0
- package/dist/src/control-plane/remote-work-recovery.js +220 -0
- package/dist/src/control-plane/remote-work-recovery.js.map +1 -0
- package/dist/src/control-plane/remote-worker-artifacts.js +135 -0
- package/dist/src/control-plane/remote-worker-artifacts.js.map +1 -0
- package/dist/src/control-plane/remote-worker-client.js +202 -0
- package/dist/src/control-plane/remote-worker-client.js.map +1 -0
- package/dist/src/control-plane/remote-worker-configuration.js +34 -0
- package/dist/src/control-plane/remote-worker-configuration.js.map +1 -0
- package/dist/src/control-plane/remote-worker-credentials.js +58 -0
- package/dist/src/control-plane/remote-worker-credentials.js.map +1 -0
- package/dist/src/control-plane/remote-worker-protocol.js +489 -0
- package/dist/src/control-plane/remote-worker-protocol.js.map +1 -0
- package/dist/src/control-plane/remote-worker-runner.js +214 -0
- package/dist/src/control-plane/remote-worker-runner.js.map +1 -0
- package/dist/src/control-plane/remote-workers.js +377 -0
- package/dist/src/control-plane/remote-workers.js.map +1 -0
- package/dist/src/control-plane/rerun-scheduler.js +448 -0
- package/dist/src/control-plane/rerun-scheduler.js.map +1 -0
- package/dist/src/control-plane/rerun-submission.js +62 -0
- package/dist/src/control-plane/rerun-submission.js.map +1 -0
- package/dist/src/control-plane/resources.js +182 -0
- package/dist/src/control-plane/resources.js.map +1 -0
- package/dist/src/control-plane/scheduler-eval-entry.js +32 -0
- package/dist/src/control-plane/scheduler-eval-entry.js.map +1 -0
- package/dist/src/control-plane/synthetic-result.js +21 -0
- package/dist/src/control-plane/synthetic-result.js.map +1 -0
- package/dist/src/control-plane/work-admission.js +23 -0
- package/dist/src/control-plane/work-admission.js.map +1 -0
- package/dist/src/control-plane/work-dispatcher.js +237 -0
- package/dist/src/control-plane/work-dispatcher.js.map +1 -0
- package/dist/src/controller-runtime/hash.js +19 -3
- package/dist/src/controller-runtime/hash.js.map +1 -1
- package/dist/src/controller-runtime/index.js +1 -1
- package/dist/src/controller-runtime/index.js.map +1 -1
- package/dist/src/controller-runtime/store.js +15 -6
- package/dist/src/controller-runtime/store.js.map +1 -1
- package/dist/src/daemon/auth.js +5 -0
- package/dist/src/daemon/auth.js.map +1 -1
- package/dist/src/daemon/client.js.map +1 -1
- package/dist/src/daemon/event-stream.js +46 -0
- package/dist/src/daemon/event-stream.js.map +1 -0
- package/dist/src/daemon/health.js +66 -0
- package/dist/src/daemon/health.js.map +1 -0
- package/dist/src/daemon/launcher.js +28 -1
- package/dist/src/daemon/launcher.js.map +1 -1
- package/dist/src/daemon/logging.js +23 -0
- package/dist/src/daemon/logging.js.map +1 -0
- package/dist/src/daemon/scheduler.js +26 -5
- package/dist/src/daemon/scheduler.js.map +1 -1
- package/dist/src/daemon/server.js +255 -98
- package/dist/src/daemon/server.js.map +1 -1
- package/dist/src/daemon/telemetry.js +163 -0
- package/dist/src/daemon/telemetry.js.map +1 -0
- package/dist/src/daemon/worker-routes.js +219 -0
- package/dist/src/daemon/worker-routes.js.map +1 -0
- package/dist/src/domain/benchmarks.js +2 -0
- package/dist/src/domain/benchmarks.js.map +1 -0
- package/dist/src/domain/bundles.js +2 -0
- package/dist/src/domain/bundles.js.map +1 -0
- package/dist/src/domain/control-plane.js +2 -0
- package/dist/src/domain/control-plane.js.map +1 -0
- package/dist/src/domain/execution-evidence.js +2 -0
- package/dist/src/domain/execution-evidence.js.map +1 -0
- package/dist/src/domain/execution-plan.js +2 -0
- package/dist/src/domain/execution-plan.js.map +1 -0
- package/dist/src/domain/images.js +2 -0
- package/dist/src/domain/images.js.map +1 -0
- package/dist/src/domain/index.js +13 -0
- package/dist/src/domain/index.js.map +1 -1
- package/dist/src/domain/interactions.js +2 -0
- package/dist/src/domain/interactions.js.map +1 -0
- package/dist/src/domain/providers.js +2 -0
- package/dist/src/domain/providers.js.map +1 -0
- package/dist/src/domain/resources.js +2 -0
- package/dist/src/domain/resources.js.map +1 -0
- package/dist/src/domain/runs.js.map +1 -1
- package/dist/src/domain/validation.js +11 -4
- package/dist/src/domain/validation.js.map +1 -1
- package/dist/src/domain/verifier-evidence-validation.js +214 -0
- package/dist/src/domain/verifier-evidence-validation.js.map +1 -0
- package/dist/src/domain/verifier-evidence.js +3 -0
- package/dist/src/domain/verifier-evidence.js.map +1 -0
- package/dist/src/domain/verifier-score-contract.js +232 -0
- package/dist/src/domain/verifier-score-contract.js.map +1 -0
- package/dist/src/domain/workers.js +2 -0
- package/dist/src/domain/workers.js.map +1 -0
- package/dist/src/evals/benchmark-adapter-manifest.js +219 -0
- package/dist/src/evals/benchmark-adapter-manifest.js.map +1 -0
- package/dist/src/evals/benchmark-candidate.js +36 -0
- package/dist/src/evals/benchmark-candidate.js.map +1 -0
- package/dist/src/evals/benchmark-run.js +166 -0
- package/dist/src/evals/benchmark-run.js.map +1 -0
- package/dist/src/evals/collect-only-rerun.js +183 -0
- package/dist/src/evals/collect-only-rerun.js.map +1 -0
- package/dist/src/evals/directory.js +30 -0
- package/dist/src/evals/directory.js.map +1 -0
- package/dist/src/evals/docker-engine-stats.js +89 -0
- package/dist/src/evals/docker-engine-stats.js.map +1 -0
- package/dist/src/evals/docker-ownership.js +51 -0
- package/dist/src/evals/docker-ownership.js.map +1 -0
- package/dist/src/evals/docker-reaper.js +144 -0
- package/dist/src/evals/docker-reaper.js.map +1 -0
- package/dist/src/evals/docker-resource-observer.js +207 -0
- package/dist/src/evals/docker-resource-observer.js.map +1 -0
- package/dist/src/evals/duration-estimator.js +126 -0
- package/dist/src/evals/duration-estimator.js.map +1 -0
- package/dist/src/evals/environment-image-planning.js +164 -0
- package/dist/src/evals/environment-image-planning.js.map +1 -0
- package/dist/src/evals/eval-artifact-planning.js +56 -0
- package/dist/src/evals/eval-artifact-planning.js.map +1 -0
- package/dist/src/evals/eval-finalization.js +10 -0
- package/dist/src/evals/eval-finalization.js.map +1 -0
- package/dist/src/evals/eval-lifecycle-events.js +29 -0
- package/dist/src/evals/eval-lifecycle-events.js.map +1 -0
- package/dist/src/evals/eval-logical-plan.js +51 -0
- package/dist/src/evals/eval-logical-plan.js.map +1 -0
- package/dist/src/evals/eval-result-builder.js +81 -0
- package/dist/src/evals/eval-result-builder.js.map +1 -0
- package/dist/src/evals/events.js +53 -5
- package/dist/src/evals/events.js.map +1 -1
- package/dist/src/evals/evolution-baseline.js +191 -0
- package/dist/src/evals/evolution-baseline.js.map +1 -0
- package/dist/src/evals/execution-evidence.js +116 -0
- package/dist/src/evals/execution-evidence.js.map +1 -0
- package/dist/src/evals/execution-leases.js +342 -0
- package/dist/src/evals/execution-leases.js.map +1 -0
- package/dist/src/evals/execution-plan-artifacts.js +72 -0
- package/dist/src/evals/execution-plan-artifacts.js.map +1 -0
- package/dist/src/evals/execution-plan-images.js +69 -0
- package/dist/src/evals/execution-plan-images.js.map +1 -0
- package/dist/src/evals/execution-plan-resources.js +174 -0
- package/dist/src/evals/execution-plan-resources.js.map +1 -0
- package/dist/src/evals/execution-plan.js +451 -0
- package/dist/src/evals/execution-plan.js.map +1 -0
- package/dist/src/evals/failure-classifier.js +61 -0
- package/dist/src/evals/failure-classifier.js.map +1 -0
- package/dist/src/evals/harbor-artifact-builder-image.js +99 -0
- package/dist/src/evals/harbor-artifact-builder-image.js.map +1 -0
- package/dist/src/evals/harbor-artifact-builder.js +358 -0
- package/dist/src/evals/harbor-artifact-builder.js.map +1 -0
- package/dist/src/evals/harbor-bridge-error.js +58 -3
- package/dist/src/evals/harbor-bridge-error.js.map +1 -1
- package/dist/src/evals/harbor-node-runtime.js +146 -0
- package/dist/src/evals/harbor-node-runtime.js.map +1 -0
- package/dist/src/evals/harbor-phase-timings.js +60 -0
- package/dist/src/evals/harbor-phase-timings.js.map +1 -0
- package/dist/src/evals/index.js +40 -2
- package/dist/src/evals/index.js.map +1 -1
- package/dist/src/evals/infrastructure-retry.js +208 -84
- package/dist/src/evals/infrastructure-retry.js.map +1 -1
- package/dist/src/evals/interaction-capture-import.js +49 -0
- package/dist/src/evals/interaction-capture-import.js.map +1 -0
- package/dist/src/evals/local-docker-provider.js +372 -0
- package/dist/src/evals/local-docker-provider.js.map +1 -0
- package/dist/src/evals/local-eval-planning.js +80 -0
- package/dist/src/evals/local-eval-planning.js.map +1 -0
- package/dist/src/evals/model-capture-plan.js +22 -0
- package/dist/src/evals/model-capture-plan.js.map +1 -0
- package/dist/src/evals/model-capture-runtime.js +114 -0
- package/dist/src/evals/model-capture-runtime.js.map +1 -0
- package/dist/src/evals/model-proxy-runtime-state.js +53 -0
- package/dist/src/evals/model-proxy-runtime-state.js.map +1 -0
- package/dist/src/evals/native-phase-evidence.js +440 -0
- package/dist/src/evals/native-phase-evidence.js.map +1 -0
- package/dist/src/evals/physical-retry-work.js +31 -0
- package/dist/src/evals/physical-retry-work.js.map +1 -0
- package/dist/src/evals/planned-execution-support.js +66 -0
- package/dist/src/evals/planned-execution-support.js.map +1 -0
- package/dist/src/evals/planned-execution.js +469 -0
- package/dist/src/evals/planned-execution.js.map +1 -0
- package/dist/src/evals/planned-progress-publisher.js +62 -0
- package/dist/src/evals/planned-progress-publisher.js.map +1 -0
- package/dist/src/evals/planned-remote-execution.js +40 -0
- package/dist/src/evals/planned-remote-execution.js.map +1 -0
- package/dist/src/evals/planned-retry-execution.js +174 -0
- package/dist/src/evals/planned-retry-execution.js.map +1 -0
- package/dist/src/evals/planned-retry-lifecycle.js +188 -0
- package/dist/src/evals/planned-retry-lifecycle.js.map +1 -0
- package/dist/src/evals/preparation-rerun.js +225 -0
- package/dist/src/evals/preparation-rerun.js.map +1 -0
- package/dist/src/evals/prepared-harness.js +76 -0
- package/dist/src/evals/prepared-harness.js.map +1 -0
- package/dist/src/evals/progress.js +52 -12
- package/dist/src/evals/progress.js.map +1 -1
- package/dist/src/evals/records.js +20 -3
- package/dist/src/evals/records.js.map +1 -1
- package/dist/src/evals/recovery.js +242 -0
- package/dist/src/evals/recovery.js.map +1 -0
- package/dist/src/evals/regrade-evidence.js +77 -0
- package/dist/src/evals/regrade-evidence.js.map +1 -0
- package/dist/src/evals/remote-infrastructure-retry.js +165 -0
- package/dist/src/evals/remote-infrastructure-retry.js.map +1 -0
- package/dist/src/evals/request.js +21 -3
- package/dist/src/evals/request.js.map +1 -1
- package/dist/src/evals/rerun-inputs.js +108 -0
- package/dist/src/evals/rerun-inputs.js.map +1 -0
- package/dist/src/evals/rerun-slots.js +4 -3
- package/dist/src/evals/rerun-slots.js.map +1 -1
- package/dist/src/evals/rerun-types.js +45 -0
- package/dist/src/evals/rerun-types.js.map +1 -0
- package/dist/src/evals/rerun.js +133 -88
- package/dist/src/evals/rerun.js.map +1 -1
- package/dist/src/evals/result-helpers.js +78 -0
- package/dist/src/evals/result-helpers.js.map +1 -0
- package/dist/src/evals/resume-state.js +36 -0
- package/dist/src/evals/resume-state.js.map +1 -0
- package/dist/src/evals/retry-backoff.js +20 -0
- package/dist/src/evals/retry-backoff.js.map +1 -0
- package/dist/src/evals/retry-state.js +195 -0
- package/dist/src/evals/retry-state.js.map +1 -0
- package/dist/src/evals/scheduler-metrics.js +95 -0
- package/dist/src/evals/scheduler-metrics.js.map +1 -0
- package/dist/src/evals/scheduler-trace-replay.js +107 -0
- package/dist/src/evals/scheduler-trace-replay.js.map +1 -0
- package/dist/src/evals/service-types.js +2 -0
- package/dist/src/evals/service-types.js.map +1 -0
- package/dist/src/evals/service.js +331 -347
- package/dist/src/evals/service.js.map +1 -1
- package/dist/src/evals/task-resources.js +209 -0
- package/dist/src/evals/task-resources.js.map +1 -0
- package/dist/src/evals/trial-environment-evidence.js +87 -0
- package/dist/src/evals/trial-environment-evidence.js.map +1 -0
- package/dist/src/evals/trial-execution-evidence.js +13 -0
- package/dist/src/evals/trial-execution-evidence.js.map +1 -0
- package/dist/src/evals/trial-import-identity.js +31 -0
- package/dist/src/evals/trial-import-identity.js.map +1 -0
- package/dist/src/evals/trial-import.js +183 -188
- package/dist/src/evals/trial-import.js.map +1 -1
- package/dist/src/evals/trial-publication-recovery.js +144 -0
- package/dist/src/evals/trial-publication-recovery.js.map +1 -0
- package/dist/src/evals/trial-publication.js +25 -0
- package/dist/src/evals/trial-publication.js.map +1 -0
- package/dist/src/evals/trial-reference-validation.js +49 -0
- package/dist/src/evals/trial-reference-validation.js.map +1 -0
- package/dist/src/evals/verifier-artifacts.js +317 -0
- package/dist/src/evals/verifier-artifacts.js.map +1 -0
- package/dist/src/evals/verifier-eligibility.js +25 -0
- package/dist/src/evals/verifier-eligibility.js.map +1 -0
- package/dist/src/evals/verifier-only-rerun.js +210 -0
- package/dist/src/evals/verifier-only-rerun.js.map +1 -0
- package/dist/src/evals/verifier-runtime.js +61 -0
- package/dist/src/evals/verifier-runtime.js.map +1 -0
- package/dist/src/evals/verifier-score-artifacts.js +119 -0
- package/dist/src/evals/verifier-score-artifacts.js.map +1 -0
- package/dist/src/evals/work-item-artifacts.js +23 -0
- package/dist/src/evals/work-item-artifacts.js.map +1 -0
- package/dist/src/foundation/config.js +17 -0
- package/dist/src/foundation/config.js.map +1 -1
- package/dist/src/foundation/contained-file.js +85 -0
- package/dist/src/foundation/contained-file.js.map +1 -0
- package/dist/src/foundation/credential-redaction.js +162 -0
- package/dist/src/foundation/credential-redaction.js.map +1 -0
- package/dist/src/foundation/environment-image-references.js +59 -0
- package/dist/src/foundation/environment-image-references.js.map +1 -0
- package/dist/src/foundation/executable.js +45 -5
- package/dist/src/foundation/executable.js.map +1 -1
- package/dist/src/foundation/fs.js +2 -1
- package/dist/src/foundation/fs.js.map +1 -1
- package/dist/src/foundation/index.js +6 -2
- package/dist/src/foundation/index.js.map +1 -1
- package/dist/src/foundation/process-identity.js +69 -0
- package/dist/src/foundation/process-identity.js.map +1 -0
- package/dist/src/images/context.js +47 -0
- package/dist/src/images/context.js.map +1 -0
- package/dist/src/images/docker-buildkit.js +137 -0
- package/dist/src/images/docker-buildkit.js.map +1 -0
- package/dist/src/images/dockerfile.js +67 -0
- package/dist/src/images/dockerfile.js.map +1 -0
- package/dist/src/images/gc.js +261 -0
- package/dist/src/images/gc.js.map +1 -0
- package/dist/src/images/index.js +10 -0
- package/dist/src/images/index.js.map +1 -0
- package/dist/src/images/manifest.js +109 -0
- package/dist/src/images/manifest.js.map +1 -0
- package/dist/src/images/ownership.js +5 -0
- package/dist/src/images/ownership.js.map +1 -0
- package/dist/src/images/records.js +32 -0
- package/dist/src/images/records.js.map +1 -0
- package/dist/src/images/registry.js +154 -0
- package/dist/src/images/registry.js.map +1 -0
- package/dist/src/images/service.js +224 -0
- package/dist/src/images/service.js.map +1 -0
- package/dist/src/model-access/capture.js +255 -0
- package/dist/src/model-access/capture.js.map +1 -0
- package/dist/src/model-access/index.js +5 -0
- package/dist/src/model-access/index.js.map +1 -0
- package/dist/src/model-access/policy.js +49 -0
- package/dist/src/model-access/policy.js.map +1 -0
- package/dist/src/model-access/proxy.js +336 -0
- package/dist/src/model-access/proxy.js.map +1 -0
- package/dist/src/model-access/records.js +167 -0
- package/dist/src/model-access/records.js.map +1 -0
- package/dist/src/runs/adapter-process.js +12 -0
- package/dist/src/runs/adapter-process.js.map +1 -0
- package/dist/src/runs/bundle.js +450 -0
- package/dist/src/runs/bundle.js.map +1 -0
- package/dist/src/runs/events.js +12 -1
- package/dist/src/runs/events.js.map +1 -1
- package/dist/src/runs/executor.js +46 -45
- package/dist/src/runs/executor.js.map +1 -1
- package/dist/src/runs/finalizer.js +17 -0
- package/dist/src/runs/finalizer.js.map +1 -1
- package/dist/src/runs/index.js +8 -1
- package/dist/src/runs/index.js.map +1 -1
- package/dist/src/runs/manifest.js +5 -5
- package/dist/src/runs/manifest.js.map +1 -1
- package/dist/src/runs/phase-bundle.js +86 -0
- package/dist/src/runs/phase-bundle.js.map +1 -0
- package/dist/src/runs/phase-cancellation.js +107 -0
- package/dist/src/runs/phase-cancellation.js.map +1 -0
- package/dist/src/runs/phase-group.js +113 -0
- package/dist/src/runs/phase-group.js.map +1 -0
- package/dist/src/runs/query.js +1 -1
- package/dist/src/runs/query.js.map +1 -1
- package/dist/src/runs/records.js +3 -0
- package/dist/src/runs/records.js.map +1 -1
- package/dist/src/runs/request.js +10 -3
- package/dist/src/runs/request.js.map +1 -1
- package/dist/src/runs/training-candidate.js +161 -0
- package/dist/src/runs/training-candidate.js.map +1 -0
- package/dist/src/runs/verifier-evidence-redaction.js +61 -0
- package/dist/src/runs/verifier-evidence-redaction.js.map +1 -0
- package/dist/src/runs/verifier-evidence.js +459 -0
- package/dist/src/runs/verifier-evidence.js.map +1 -0
- package/dist/src/runs/verifier-structured-evidence.js +83 -0
- package/dist/src/runs/verifier-structured-evidence.js.map +1 -0
- package/dist/src/trajectories/analysis.js +288 -0
- package/dist/src/trajectories/analysis.js.map +1 -0
- package/dist/src/trajectories/chunk-projection.js +148 -0
- package/dist/src/trajectories/chunk-projection.js.map +1 -0
- package/dist/src/trajectories/content-projection.js +389 -0
- package/dist/src/trajectories/content-projection.js.map +1 -0
- package/dist/src/trajectories/dsh-chunk-contract.js +176 -0
- package/dist/src/trajectories/dsh-chunk-contract.js.map +1 -0
- package/dist/src/trajectories/dsh-contract.js +284 -0
- package/dist/src/trajectories/dsh-contract.js.map +1 -0
- package/dist/src/trajectories/events-chunk-drill.js +95 -0
- package/dist/src/trajectories/events-chunk-drill.js.map +1 -0
- package/dist/src/trajectories/events-page.js +367 -0
- package/dist/src/trajectories/events-page.js.map +1 -0
- package/dist/src/trajectories/index.js +3 -0
- package/dist/src/trajectories/index.js.map +1 -1
- package/dist/src/trajectories/provider-capture.js +15 -24
- package/dist/src/trajectories/provider-capture.js.map +1 -1
- package/dist/src/trajectories/providers/deepseek.js +3 -3
- package/dist/src/trajectories/providers/deepseek.js.map +1 -1
- package/dist/src/trajectories/request-attempt.js +47 -0
- package/dist/src/trajectories/request-attempt.js.map +1 -0
- package/dist/src/trajectories/stream-reader.js +311 -0
- package/dist/src/trajectories/stream-reader.js.map +1 -0
- package/dist/src/trajectories/surface-fold.js +126 -0
- package/dist/src/trajectories/surface-fold.js.map +1 -0
- package/dist/src/workers/index.js +3 -0
- package/dist/src/workers/index.js.map +1 -0
- package/dist/src/workers/remote-harbor-work-spec.js +97 -0
- package/dist/src/workers/remote-harbor-work-spec.js.map +1 -0
- package/dist/src/workers/remote-harbor-worker.js +223 -0
- package/dist/src/workers/remote-harbor-worker.js.map +1 -0
- package/docs/schemas/benchmark-hook-request-v1.schema.json +65 -0
- package/docs/schemas/benchmark-hook-response-v1.schema.json +75 -0
- package/docs/schemas/benchmark-hook-v1.schema.json +33 -0
- package/docs/schemas/benchmark-lock-v1.schema.json +351 -0
- package/docs/schemas/benchmark-metric-v1.schema.json +37 -0
- package/docs/schemas/benchmark-package-v1.schema.json +167 -0
- package/docs/schemas/benchmark-phase-group.schema.json +41 -0
- package/docs/schemas/benchmark-profile-v1.schema.json +141 -0
- package/docs/schemas/benchmark-task-v1.schema.json +384 -0
- package/docs/schemas/benchmark-tool-result-v1.schema.json +30 -0
- package/docs/schemas/environment-build-index.schema.json +13 -0
- package/docs/schemas/environment-build-record.schema.json +25 -0
- package/docs/schemas/environment-image-manifest.schema.json +62 -0
- package/docs/schemas/error.schema.json +22 -0
- package/docs/schemas/eval-control.schema.json +36 -0
- package/docs/schemas/eval-execution-plan.schema.json +216 -0
- package/docs/schemas/eval-progress.schema.json +1 -15
- package/docs/schemas/eval-rerun-result.schema.json +58 -0
- package/docs/schemas/eval-rerun-state.schema.json +61 -0
- package/docs/schemas/eval-rerun-submission.schema.json +60 -0
- package/docs/schemas/eval-result.schema.json +32 -13
- package/docs/schemas/eval-submission-input.schema.json +14 -0
- package/docs/schemas/eval-submission.schema.json +68 -0
- package/docs/schemas/eval-trial-publication.schema.json +15 -0
- package/docs/schemas/eval-trial-reference.schema.json +129 -0
- package/docs/schemas/execution-evidence.schema.json +96 -0
- package/docs/schemas/execution-lease.schema.json +50 -0
- package/docs/schemas/execution-provider-status.schema.json +65 -0
- package/docs/schemas/execution-worker.schema.json +53 -0
- package/docs/schemas/harbor-process-exit.schema.json +14 -0
- package/docs/schemas/interaction-capture-ref.schema.json +45 -0
- package/docs/schemas/local-provider-execution.schema.json +33 -0
- package/docs/schemas/model-interaction.schema.json +41 -0
- package/docs/schemas/regrade-assessment-reference.schema.json +11 -0
- package/docs/schemas/remote-result-envelope.schema.json +32 -0
- package/docs/schemas/remote-tree-envelope.schema.json +25 -0
- package/docs/schemas/remote-work-offer.schema.json +58 -0
- package/docs/schemas/remote-work-receipt.schema.json +50 -0
- package/docs/schemas/remote-worker-artifact.schema.json +17 -0
- package/docs/schemas/remote-worker-event.schema.json +18 -0
- package/docs/schemas/remote-worker-heartbeat.schema.json +37 -0
- package/docs/schemas/remote-worker-registration.schema.json +52 -0
- package/docs/schemas/result-bundle-index.schema.json +133 -0
- package/docs/schemas/run-context.schema.json +15 -0
- package/docs/schemas/run-environment-images.schema.json +39 -0
- package/docs/schemas/run-manifest.schema.json +4 -0
- package/docs/schemas/run-request.schema.json +4 -0
- package/docs/schemas/training-data-candidate.schema.json +46 -0
- package/docs/schemas/trajectory-analysis.schema.json +258 -0
- package/docs/schemas/trajectory-events-page.schema.json +40 -0
- package/docs/schemas/verifier-assessment.schema.json +77 -0
- package/docs/schemas/verifier-evidence.schema.json +276 -0
- package/integrations/harbor/hitch_benchmark.py +252 -0
- package/integrations/harbor/hitch_candidate_recycle.py +262 -0
- package/integrations/harbor/hitch_harbor_agent.py +698 -568
- package/integrations/harbor/hitch_harbor_environment.py +469 -0
- package/integrations/harbor/hitch_harbor_task_resources.py +275 -0
- package/integrations/harbor/hitch_harbor_verifier.py +83 -2
- package/integrations/harbor/hitch_phase_supervisor.py +452 -0
- package/integrations/harbor/hitch_tool_client.mjs +91 -0
- package/integrations/model-call/cli.js +57 -0
- package/package.json +30 -6
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
2
|
+
import { chmod, lstat, mkdir, readFile, readdir, readlink, rename, rm, symlink, writeFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { HitchError, ensureDir, statePaths, withFileLock } from "../foundation/index.js";
|
|
5
|
+
const MAX_INPUT_BYTES = 256 * 1024 * 1024;
|
|
6
|
+
const MAX_FILES = 100_000;
|
|
7
|
+
const SHA256 = /^sha256:([a-f0-9]{64})$/;
|
|
8
|
+
export class RemoteWorkInputStore {
|
|
9
|
+
directory;
|
|
10
|
+
locks;
|
|
11
|
+
constructor(root) {
|
|
12
|
+
if (!root)
|
|
13
|
+
throw new TypeError("remote work input root is required");
|
|
14
|
+
const paths = statePaths(root);
|
|
15
|
+
this.directory = paths.workerInputs;
|
|
16
|
+
this.locks = paths.workerProtocolLocks;
|
|
17
|
+
}
|
|
18
|
+
async initialize() { await Promise.all([ensureDir(this.directory), ensureDir(this.locks)]); }
|
|
19
|
+
async put(kind, format, content) {
|
|
20
|
+
if (content.length > MAX_INPUT_BYTES)
|
|
21
|
+
throw inputError("remote work input exceeds its size limit");
|
|
22
|
+
const digest = hash(content);
|
|
23
|
+
const file = this.pathFor(digest);
|
|
24
|
+
await withFileLock(this.locks, `input-${digest.slice("sha256:".length)}`, async () => {
|
|
25
|
+
const existing = await readFile(file).catch((error) => {
|
|
26
|
+
if (error.code === "ENOENT")
|
|
27
|
+
return null;
|
|
28
|
+
throw error;
|
|
29
|
+
});
|
|
30
|
+
if (existing) {
|
|
31
|
+
if (hash(existing) !== digest || !existing.equals(content))
|
|
32
|
+
throw inputError("remote work input store identity conflict");
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
await ensureDir(path.dirname(file));
|
|
36
|
+
const temporary = `${file}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`;
|
|
37
|
+
try {
|
|
38
|
+
await writeFile(temporary, content, { flag: "wx", mode: 0o600 });
|
|
39
|
+
await rename(temporary, file);
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
await rm(temporary, { force: true });
|
|
43
|
+
throw error;
|
|
44
|
+
}
|
|
45
|
+
}, { timeoutCode: "worker_input_locked", timeoutExitCode: 12 });
|
|
46
|
+
return { kind, format, digest, size: content.length };
|
|
47
|
+
}
|
|
48
|
+
async verify(ref) {
|
|
49
|
+
validateRef(ref);
|
|
50
|
+
const file = this.pathFor(ref.digest);
|
|
51
|
+
const info = await lstat(file).catch(() => null);
|
|
52
|
+
if (!info?.isFile() || info.isSymbolicLink() || info.nlink !== 1 || info.size !== ref.size)
|
|
53
|
+
throw inputError("remote work input is missing or unsafe");
|
|
54
|
+
if (hash(await readFile(file)) !== ref.digest)
|
|
55
|
+
throw inputError("remote work input failed digest verification");
|
|
56
|
+
return { path: file, size: ref.size };
|
|
57
|
+
}
|
|
58
|
+
pathFor(digest) {
|
|
59
|
+
const match = digest.match(SHA256);
|
|
60
|
+
if (!match)
|
|
61
|
+
throw inputError("remote work input digest is invalid");
|
|
62
|
+
return path.join(this.directory, `${match[1]}.blob`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
export async function prepareRemoteWorkInputs(input) {
|
|
66
|
+
const store = new RemoteWorkInputStore(input.root);
|
|
67
|
+
await store.initialize();
|
|
68
|
+
const taskId = input.work.task_ids[0];
|
|
69
|
+
if (!taskId || input.work.task_ids.length !== 1)
|
|
70
|
+
throw inputError("remote work input requires exactly one task");
|
|
71
|
+
const taskDirectory = await resolveTaskDirectory(input.request.dataset, taskId);
|
|
72
|
+
const { storage: _hostStorage, ...portableArtifact } = input.preparedArtifact;
|
|
73
|
+
const spec = Buffer.from(`${JSON.stringify({
|
|
74
|
+
schema_version: "1", request: { ...input.request, dataset: "task-input" }, plan: input.plan,
|
|
75
|
+
work: input.work, resolution: input.resolvedRevision,
|
|
76
|
+
harness_artifact: { ...portableArtifact, directory: "harness-artifact" },
|
|
77
|
+
controller_runtime: { runtime_id: input.runtimeId, directory: "controller-runtime" },
|
|
78
|
+
task: { task_id: taskId, directory: "task-input" },
|
|
79
|
+
credential_names: [...(input.credentialNames ?? [])].sort(),
|
|
80
|
+
})}\n`);
|
|
81
|
+
const [workSpec, harness, runtime, task] = await Promise.all([
|
|
82
|
+
store.put("work-spec", "json", spec),
|
|
83
|
+
encodeTree(input.preparedArtifact.directory).then((body) => store.put("harness-artifact", "hitch-tree-v1", body)),
|
|
84
|
+
encodeTree(input.runtimeDirectory).then((body) => store.put("controller-runtime", "hitch-tree-v1", body)),
|
|
85
|
+
encodeTree(taskDirectory).then((body) => store.put("task-input", "hitch-tree-v1", body)),
|
|
86
|
+
]);
|
|
87
|
+
return [workSpec, harness, runtime, task];
|
|
88
|
+
}
|
|
89
|
+
export function parseRemoteTreeEnvelope(value) {
|
|
90
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
91
|
+
throw inputError("remote tree envelope is invalid");
|
|
92
|
+
const record = value;
|
|
93
|
+
if (Object.keys(record).some((key) => !["schema_version", "directories", "files", "symlinks"].includes(key)) || record.schema_version !== "1"
|
|
94
|
+
|| !Array.isArray(record.files) || !Array.isArray(record.directories ?? []) || !Array.isArray(record.symlinks ?? []))
|
|
95
|
+
throw inputError("remote tree envelope is invalid");
|
|
96
|
+
const files = record.files.map(parseFile);
|
|
97
|
+
const directories = (record.directories ?? []).map(parseDirectory);
|
|
98
|
+
const symlinks = (record.symlinks ?? []).map(parseSymlink);
|
|
99
|
+
const entries = [...directories, ...files, ...symlinks];
|
|
100
|
+
if (entries.length < 1 || entries.length > MAX_FILES || !canonical(directories) || !canonical(files) || !canonical(symlinks)
|
|
101
|
+
|| new Set(entries.map((entry) => entry.path)).size !== entries.length)
|
|
102
|
+
throw inputError("remote tree entries are not canonical and unique");
|
|
103
|
+
return { schema_version: "1", directories, files, symlinks };
|
|
104
|
+
}
|
|
105
|
+
export async function materializeRemoteTreeEnvelope(value, destination) {
|
|
106
|
+
const envelope = parseRemoteTreeEnvelope(value);
|
|
107
|
+
await mkdir(destination, { recursive: false, mode: 0o700 });
|
|
108
|
+
try {
|
|
109
|
+
for (const directory of envelope.directories)
|
|
110
|
+
await mkdir(path.join(destination, ...directory.path.split("/")), { recursive: false, mode: 0o700 });
|
|
111
|
+
for (const file of envelope.files) {
|
|
112
|
+
const target = path.join(destination, ...file.path.split("/"));
|
|
113
|
+
await ensureDir(path.dirname(target));
|
|
114
|
+
await writeFile(target, Buffer.from(file.content_base64, "base64"), { flag: "wx", mode: 0o600 });
|
|
115
|
+
await chmod(target, file.mode);
|
|
116
|
+
}
|
|
117
|
+
for (const link of envelope.symlinks) {
|
|
118
|
+
const target = path.join(destination, ...link.path.split("/"));
|
|
119
|
+
await ensureDir(path.dirname(target));
|
|
120
|
+
assertSafeSymlink(destination, target, link.target);
|
|
121
|
+
await symlink(link.target, target);
|
|
122
|
+
}
|
|
123
|
+
for (const directory of [...envelope.directories].reverse())
|
|
124
|
+
await chmod(path.join(destination, ...directory.path.split("/")), directory.mode);
|
|
125
|
+
}
|
|
126
|
+
catch (error) {
|
|
127
|
+
await removeMaterializedTree(destination);
|
|
128
|
+
throw error;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
function validateRef(ref) {
|
|
132
|
+
if (!new Set(["work-spec", "harness-artifact", "controller-runtime", "task-input"]).has(ref.kind)
|
|
133
|
+
|| !new Set(["json", "hitch-tree-v1"]).has(ref.format) || !SHA256.test(ref.digest)
|
|
134
|
+
|| !Number.isSafeInteger(ref.size) || ref.size < 1 || ref.size > MAX_INPUT_BYTES)
|
|
135
|
+
throw inputError("remote work input ref is invalid");
|
|
136
|
+
}
|
|
137
|
+
async function encodeTree(root) {
|
|
138
|
+
const entries = await walk(root);
|
|
139
|
+
const body = Buffer.from(`${JSON.stringify({ schema_version: "1", ...entries })}\n`);
|
|
140
|
+
if (body.length > MAX_INPUT_BYTES)
|
|
141
|
+
throw inputError("remote work input tree exceeds its size limit");
|
|
142
|
+
return body;
|
|
143
|
+
}
|
|
144
|
+
async function walk(root, relative = "") {
|
|
145
|
+
const directory = relative ? path.join(root, ...relative.split("/")) : root;
|
|
146
|
+
const entries = await readdir(directory, { withFileTypes: true });
|
|
147
|
+
const directories = [];
|
|
148
|
+
const files = [];
|
|
149
|
+
const symlinks = [];
|
|
150
|
+
for (const entry of entries.sort((left, right) => Buffer.from(left.name).compare(Buffer.from(right.name)))) {
|
|
151
|
+
const child = relative ? `${relative}/${entry.name}` : entry.name;
|
|
152
|
+
safePath(child);
|
|
153
|
+
const target = path.join(root, ...child.split("/"));
|
|
154
|
+
const info = await lstat(target);
|
|
155
|
+
if (info.isDirectory()) {
|
|
156
|
+
directories.push({ path: child, mode: safeMode(info.mode) });
|
|
157
|
+
const nested = await walk(root, child);
|
|
158
|
+
directories.push(...nested.directories);
|
|
159
|
+
files.push(...nested.files);
|
|
160
|
+
symlinks.push(...nested.symlinks);
|
|
161
|
+
}
|
|
162
|
+
else if (info.isFile() && !info.isSymbolicLink() && info.nlink === 1) {
|
|
163
|
+
const content = await readFile(target);
|
|
164
|
+
files.push({ path: child, mode: safeMode(info.mode), size: content.length, sha256: hash(content), content_base64: content.toString("base64") });
|
|
165
|
+
}
|
|
166
|
+
else if (info.isSymbolicLink()) {
|
|
167
|
+
const linkTarget = await readlink(target);
|
|
168
|
+
assertSafeSymlink(root, target, linkTarget);
|
|
169
|
+
symlinks.push({ path: child, target: linkTarget });
|
|
170
|
+
}
|
|
171
|
+
else
|
|
172
|
+
throw inputError(`remote work input contains an unsafe entry: ${child}`);
|
|
173
|
+
if (directories.length + files.length + symlinks.length > MAX_FILES)
|
|
174
|
+
throw inputError("remote work input has too many entries");
|
|
175
|
+
}
|
|
176
|
+
return { directories: sort(directories), files: sort(files), symlinks: sort(symlinks) };
|
|
177
|
+
}
|
|
178
|
+
function parseFile(value) {
|
|
179
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
180
|
+
throw inputError("remote tree file is invalid");
|
|
181
|
+
const file = value;
|
|
182
|
+
if (Object.keys(file).some((key) => !["path", "mode", "size", "sha256", "content_base64"].includes(key)))
|
|
183
|
+
throw inputError("remote tree file is invalid");
|
|
184
|
+
const relative = safePath(file.path);
|
|
185
|
+
if (!validMode(file.mode) || !Number.isSafeInteger(file.size) || file.size < 0 || file.size > MAX_INPUT_BYTES
|
|
186
|
+
|| typeof file.sha256 !== "string" || !SHA256.test(file.sha256)
|
|
187
|
+
|| typeof file.content_base64 !== "string" || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(file.content_base64))
|
|
188
|
+
throw inputError("remote tree file is invalid");
|
|
189
|
+
const content = Buffer.from(file.content_base64, "base64");
|
|
190
|
+
if (content.length !== file.size || hash(content) !== file.sha256)
|
|
191
|
+
throw inputError(`remote tree file integrity failed: ${relative}`);
|
|
192
|
+
return { path: relative, mode: file.mode, size: file.size, sha256: file.sha256, content_base64: file.content_base64 };
|
|
193
|
+
}
|
|
194
|
+
function parseDirectory(value) {
|
|
195
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
196
|
+
throw inputError("remote tree directory is invalid");
|
|
197
|
+
const entry = value;
|
|
198
|
+
if (Object.keys(entry).some((key) => !["path", "mode"].includes(key)) || !validMode(entry.mode))
|
|
199
|
+
throw inputError("remote tree directory is invalid");
|
|
200
|
+
return { path: safePath(entry.path), mode: entry.mode };
|
|
201
|
+
}
|
|
202
|
+
function parseSymlink(value) {
|
|
203
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
204
|
+
throw inputError("remote tree symlink is invalid");
|
|
205
|
+
const entry = value;
|
|
206
|
+
if (Object.keys(entry).some((key) => !["path", "target"].includes(key)) || typeof entry.target !== "string"
|
|
207
|
+
|| !entry.target || entry.target.length > 4_096 || entry.target.includes("\\") || path.posix.isAbsolute(entry.target) || /[\0\r\n]/.test(entry.target)) {
|
|
208
|
+
throw inputError("remote tree symlink is invalid");
|
|
209
|
+
}
|
|
210
|
+
return { path: safePath(entry.path), target: entry.target };
|
|
211
|
+
}
|
|
212
|
+
function assertSafeSymlink(root, link, target) {
|
|
213
|
+
if (!target || target.includes("\\") || path.isAbsolute(target) || /[\0\r\n]/.test(target))
|
|
214
|
+
throw inputError("remote tree symlink target is unsafe");
|
|
215
|
+
const resolvedRoot = path.resolve(root);
|
|
216
|
+
const resolved = path.resolve(path.dirname(link), target);
|
|
217
|
+
if (resolved !== resolvedRoot && !resolved.startsWith(`${resolvedRoot}${path.sep}`))
|
|
218
|
+
throw inputError("remote tree symlink escapes its root");
|
|
219
|
+
}
|
|
220
|
+
async function removeMaterializedTree(directory) {
|
|
221
|
+
const info = await lstat(directory).catch(() => null);
|
|
222
|
+
if (!info)
|
|
223
|
+
return;
|
|
224
|
+
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
|
225
|
+
const target = path.join(directory, entry.name);
|
|
226
|
+
if (entry.isDirectory()) {
|
|
227
|
+
await removeMaterializedTree(target);
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
if (!entry.isSymbolicLink())
|
|
231
|
+
await chmod(target, 0o600);
|
|
232
|
+
await rm(target, { force: true });
|
|
233
|
+
}
|
|
234
|
+
await chmod(directory, 0o700);
|
|
235
|
+
await rm(directory, { recursive: true, force: true });
|
|
236
|
+
}
|
|
237
|
+
function safeMode(value) {
|
|
238
|
+
const mode = value & 0o7777;
|
|
239
|
+
if (!validMode(mode))
|
|
240
|
+
throw inputError("remote tree entry mode is unsafe");
|
|
241
|
+
return mode;
|
|
242
|
+
}
|
|
243
|
+
function validMode(value) {
|
|
244
|
+
return Number.isSafeInteger(value) && value >= 0o400 && value <= 0o777 && (value & 0o400) !== 0;
|
|
245
|
+
}
|
|
246
|
+
function canonical(entries) { return JSON.stringify(entries) === JSON.stringify(sort(entries)); }
|
|
247
|
+
function sort(entries) { return [...entries].sort((left, right) => Buffer.from(left.path).compare(Buffer.from(right.path))); }
|
|
248
|
+
async function resolveTaskDirectory(dataset, taskId) {
|
|
249
|
+
if (!taskId || taskId.includes("/") || taskId.includes("\\") || taskId === "." || taskId === "..")
|
|
250
|
+
throw inputError("remote task id is unsafe");
|
|
251
|
+
const root = path.resolve(dataset);
|
|
252
|
+
const rootTask = path.join(root, "task.toml");
|
|
253
|
+
const directory = (await lstat(rootTask).catch(() => null))?.isFile() && path.basename(root) === taskId ? root : path.join(root, taskId);
|
|
254
|
+
if (!(await lstat(path.join(directory, "task.toml")).catch(() => null))?.isFile())
|
|
255
|
+
throw inputError("remote task input is missing task.toml");
|
|
256
|
+
return directory;
|
|
257
|
+
}
|
|
258
|
+
function safePath(value) {
|
|
259
|
+
if (typeof value !== "string" || !value || value !== value.normalize("NFC") || value.includes("\\") || path.posix.isAbsolute(value)
|
|
260
|
+
|| value.split("/").some((part) => !part || part === "." || part === ".."))
|
|
261
|
+
throw inputError("remote tree path is unsafe");
|
|
262
|
+
return value;
|
|
263
|
+
}
|
|
264
|
+
function hash(value) { return `sha256:${createHash("sha256").update(value).digest("hex")}`; }
|
|
265
|
+
function inputError(message) { return new HitchError(message, { code: "remote_input_invalid", exitCode: 12 }); }
|
|
266
|
+
//# sourceMappingURL=remote-work-inputs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"remote-work-inputs.js","sourceRoot":"","sources":["../../../src/control-plane/remote-work-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpH,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAGzF,MAAM,eAAe,GAAG,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;AAC1C,MAAM,SAAS,GAAG,OAAO,CAAC;AAC1B,MAAM,MAAM,GAAG,yBAAyB,CAAC;AAYzC,MAAM,OAAO,oBAAoB;IACd,SAAS,CAAS;IAClB,KAAK,CAAS;IAE/B,YAAY,IAAY;QACtB,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,SAAS,CAAC,oCAAoC,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,YAAY,CAAC;QACpC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,mBAAmB,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,UAAU,KAAoB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5G,KAAK,CAAC,GAAG,CAAC,IAAkC,EAAE,MAAsC,EAAE,OAAe;QACnG,IAAI,OAAO,CAAC,MAAM,GAAG,eAAe;YAAE,MAAM,UAAU,CAAC,0CAA0C,CAAC,CAAC;QACnG,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;YACnF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACpD,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;oBAAE,OAAO,IAAI,CAAC;gBACpE,MAAM,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;YACH,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC;oBAAE,MAAM,UAAU,CAAC,2CAA2C,CAAC,CAAC;gBAC1H,OAAO;YACT,CAAC;YACD,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YACpC,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;YACjF,IAAI,CAAC;gBAAC,MAAM,SAAS,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBAAC,MAAM,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YAAC,CAAC;YACxG,OAAO,KAAK,EAAE,CAAC;gBAAC,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;gBAAC,MAAM,KAAK,CAAC;YAAC,CAAC;QACtE,CAAC,EAAE,EAAE,WAAW,EAAE,qBAAqB,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAC;QAChE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,GAAyB;QACpC,WAAW,CAAC,GAAG,CAAC,CAAC;QACjB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI;YAAE,MAAM,UAAU,CAAC,wCAAwC,CAAC,CAAC;QACvJ,IAAI,IAAI,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,MAAM;YAAE,MAAM,UAAU,CAAC,8CAA8C,CAAC,CAAC;QAChH,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;IACxC,CAAC;IAED,OAAO,CAAC,MAAc;QACpB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK;YAAE,MAAM,UAAU,CAAC,qCAAqC,CAAC,CAAC;QACpE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACvD,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,KAU7C;IACC,MAAM,KAAK,GAAG,IAAI,oBAAoB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnD,MAAM,KAAK,CAAC,UAAU,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACtC,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,UAAU,CAAC,6CAA6C,CAAC,CAAC;IACjH,MAAM,aAAa,GAAG,MAAM,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAChF,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,GAAG,gBAAgB,EAAE,GAAG,KAAK,CAAC,gBAAgB,CAAC;IAC9E,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;QACzC,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI;QAC3F,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,UAAU,EAAE,KAAK,CAAC,gBAAgB;QACpD,gBAAgB,EAAE,EAAE,GAAG,gBAAgB,EAAE,SAAS,EAAE,kBAAkB,EAAE;QACxE,kBAAkB,EAAE,EAAE,UAAU,EAAE,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,oBAAoB,EAAE;QACpF,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE;QAClD,gBAAgB,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;KAC5D,CAAC,IAAI,CAAC,CAAC;IACR,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC3D,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC;QACpC,UAAU,CAAC,KAAK,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;QACjH,UAAU,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;QACzG,UAAU,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,EAAE,eAAe,EAAE,IAAI,CAAC,CAAC;KACzF,CAAC,CAAC;IACH,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,KAAc;IACpD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,UAAU,CAAC,iCAAiC,CAAC,CAAC;IACrH,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG;WACxI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;QAAE,MAAM,UAAU,CAAC,iCAAiC,CAAC,CAAC;IAC5K,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,WAAW,GAAG,CAAC,MAAM,CAAC,WAAoC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAC5F,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,QAAiC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACpF,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,KAAK,EAAE,GAAG,QAAQ,CAAC,CAAC;IACxD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,SAAS,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;WACvH,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM;QAAE,MAAM,UAAU,CAAC,kDAAkD,CAAC,CAAC;IAC/I,OAAO,EAAE,cAAc,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC/D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,KAAc,EAAE,WAAmB;IACrF,MAAM,QAAQ,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,IAAI,CAAC;QACH,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,WAAW;YAAE,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACnJ,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/D,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YACtC,MAAM,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YACjG,MAAM,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/D,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YACtC,iBAAiB,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpD,MAAM,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACrC,CAAC;QACD,KAAK,MAAM,SAAS,IAAI,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE;YAAE,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;IACjJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,sBAAsB,CAAC,WAAW,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAAC,GAAyB;IAC5C,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC;WAC5F,CAAC,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;WAC/E,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,eAAe;QAAE,MAAM,UAAU,CAAC,kCAAkC,CAAC,CAAC;AAC3I,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY;IACpC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC;IACrF,IAAI,IAAI,CAAC,MAAM,GAAG,eAAe;QAAE,MAAM,UAAU,CAAC,+CAA+C,CAAC,CAAC;IACrG,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,IAAY,EAAE,QAAQ,GAAG,EAAE;IAC7C,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAClE,MAAM,WAAW,GAA4B,EAAE,CAAC;IAChD,MAAM,KAAK,GAAuB,EAAE,CAAC;IACrC,MAAM,QAAQ,GAA0B,EAAE,CAAC;IAC3C,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3G,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QAClE,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACpD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;QACjC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACvC,WAAW,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;YAAC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1G,CAAC;aACI,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,KAAK,KAAK,CAAC,EAAE,CAAC;YACrE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAClJ,CAAC;aAAM,IAAI,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;YACjC,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC1C,iBAAiB,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;YAC5C,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QACrD,CAAC;;YAAM,MAAM,UAAU,CAAC,+CAA+C,KAAK,EAAE,CAAC,CAAC;QAChF,IAAI,WAAW,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,SAAS;YAAE,MAAM,UAAU,CAAC,wCAAwC,CAAC,CAAC;IAClI,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;AAC1F,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjH,MAAM,IAAI,GAAG,KAAgC,CAAC;IAC9C,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,gBAAgB,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAAE,MAAM,UAAU,CAAC,6BAA6B,CAAC,CAAC;IAC1J,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAK,IAAI,CAAC,IAAe,GAAG,CAAC,IAAK,IAAI,CAAC,IAAe,GAAG,eAAe;WAChI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;WAC5D,OAAO,IAAI,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,kEAAkE,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC;QAAE,MAAM,UAAU,CAAC,6BAA6B,CAAC,CAAC;IAC/L,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;IAC3D,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,MAAM;QAAE,MAAM,UAAU,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;IACtI,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,CAAC,IAAc,EAAE,IAAI,EAAE,IAAI,CAAC,IAAc,EAAE,MAAM,EAAE,IAAI,CAAC,MAAgB,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC;AACtJ,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,UAAU,CAAC,kCAAkC,CAAC,CAAC;IACtH,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC;QAAE,MAAM,UAAU,CAAC,kCAAkC,CAAC,CAAC;IACtJ,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAc,EAAE,CAAC;AACpE,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,UAAU,CAAC,gCAAgC,CAAC,CAAC;IACpH,MAAM,KAAK,GAAG,KAAgC,CAAC;IAC/C,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;WACtG,CAAC,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QACzJ,MAAM,UAAU,CAAC,gCAAgC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;AAC9D,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,IAAY,EAAE,MAAc;IACnE,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,MAAM,UAAU,CAAC,sCAAsC,CAAC,CAAC;IACrJ,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;IAC1D,IAAI,QAAQ,KAAK,YAAY,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAAE,MAAM,UAAU,CAAC,sCAAsC,CAAC,CAAC;AAChJ,CAAC;AAED,KAAK,UAAU,sBAAsB,CAAC,SAAiB;IACrD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IACtD,IAAI,CAAC,IAAI;QAAE,OAAO;IAClB,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,MAAM,sBAAsB,CAAC,MAAM,CAAC,CAAC;YACrC,SAAS;QACX,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE;YAAE,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACxD,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,CAAC;IACD,MAAM,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IAC9B,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,IAAI,GAAG,KAAK,GAAG,MAAM,CAAC;IAC5B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;QAAE,MAAM,UAAU,CAAC,kCAAkC,CAAC,CAAC;IAC3E,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,OAAO,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAK,KAAgB,IAAI,KAAK,IAAK,KAAgB,IAAI,KAAK,IAAI,CAAE,KAAgB,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;AACtI,CAAC;AAED,SAAS,SAAS,CAA6B,OAAY,IAAa,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3I,SAAS,IAAI,CAA6B,OAAY,IAAS,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEpK,KAAK,UAAU,oBAAoB,CAAC,OAAe,EAAE,MAAc;IACjE,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,IAAI;QAAE,MAAM,UAAU,CAAC,0BAA0B,CAAC,CAAC;IAChJ,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACzI,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,EAAE;QAAE,MAAM,UAAU,CAAC,wCAAwC,CAAC,CAAC;IAC9I,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAK,KAAK,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC;WAC9H,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC;QAAE,MAAM,UAAU,CAAC,4BAA4B,CAAC,CAAC;IAC7H,OAAO,KAAK,CAAC;AACf,CAAC;AACD,SAAS,IAAI,CAAC,KAAa,IAAY,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;AAC7G,SAAS,UAAU,CAAC,OAAe,IAAgB,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { HitchError } from "../foundation/index.js";
|
|
2
|
+
import { validateResourceVector } from "./resources.js";
|
|
3
|
+
const SHA256 = /^sha256:[a-f0-9]{64}$/;
|
|
4
|
+
export function parseRemoteWorkItem(value) {
|
|
5
|
+
const record = exact(value, [
|
|
6
|
+
"schema_version", "work_id", "eval_id", "backend", "logical_attempt", "task_ids", "slots", "opaque_membership",
|
|
7
|
+
"requested_parallelism", "reservation", "provider", "image_refs", "artifact_id", "runtime_contract", "scheduling",
|
|
8
|
+
], "remote work item");
|
|
9
|
+
if (record.schema_version !== "1" || typeof record.work_id !== "string" || !/^work_[a-f0-9]{32}$/.test(record.work_id)
|
|
10
|
+
|| typeof record.eval_id !== "string" || !/^eval_[a-f0-9]{32}$/.test(record.eval_id) || record.backend !== "harbor"
|
|
11
|
+
|| record.logical_attempt !== null && (!Number.isSafeInteger(record.logical_attempt) || record.logical_attempt < 1)
|
|
12
|
+
|| !stringArray(record.task_ids) || !stringArray(record.slots) || typeof record.opaque_membership !== "boolean"
|
|
13
|
+
|| !Number.isSafeInteger(record.requested_parallelism) || record.requested_parallelism < 1
|
|
14
|
+
|| typeof record.provider !== "string" || !record.provider || record.image_refs !== undefined && !Array.isArray(record.image_refs)
|
|
15
|
+
|| record.artifact_id !== undefined && (typeof record.artifact_id !== "string" || !SHA256.test(record.artifact_id))
|
|
16
|
+
|| (record.artifact_id === undefined) !== (record.runtime_contract === undefined))
|
|
17
|
+
protocolError("remote work item is invalid");
|
|
18
|
+
if (record.runtime_contract !== undefined)
|
|
19
|
+
parseRuntimeContract(record.runtime_contract);
|
|
20
|
+
const scheduling = record.scheduling === undefined ? undefined : parseSchedulingHint(record.scheduling);
|
|
21
|
+
return {
|
|
22
|
+
...record,
|
|
23
|
+
reservation: validateResourceVector(record.reservation, "remote work reservation"),
|
|
24
|
+
...(scheduling ? { scheduling } : {}),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
function parseRuntimeContract(value) {
|
|
28
|
+
const runtime = exact(value, ["docker_platform", "artifact_platform", "node_version"], "remote work runtime contract");
|
|
29
|
+
if ((runtime.docker_platform !== "linux/amd64" && runtime.docker_platform !== "linux/arm64")
|
|
30
|
+
|| (runtime.artifact_platform !== "linux-x64" && runtime.artifact_platform !== "linux-arm64")
|
|
31
|
+
|| (runtime.docker_platform === "linux/amd64") !== (runtime.artifact_platform === "linux-x64")
|
|
32
|
+
|| typeof runtime.node_version !== "string" || !/^v\d+\.\d+\.\d+$/.test(runtime.node_version))
|
|
33
|
+
protocolError("remote work runtime contract is invalid");
|
|
34
|
+
}
|
|
35
|
+
function parseSchedulingHint(value) {
|
|
36
|
+
const hint = exact(value, ["policy", "estimated_duration_ms", "remaining_path_ms", "estimate_source", "estimate_sample_count"], "remote work scheduling hint");
|
|
37
|
+
const sources = new Set(["evolution-baseline", "history-p75", "task-budget", "default"]);
|
|
38
|
+
if (hint.policy !== "critical-path-lpt-v1"
|
|
39
|
+
|| !Number.isSafeInteger(hint.estimated_duration_ms) || hint.estimated_duration_ms < 1
|
|
40
|
+
|| !Number.isSafeInteger(hint.remaining_path_ms) || hint.remaining_path_ms < hint.estimated_duration_ms
|
|
41
|
+
|| !sources.has(hint.estimate_source)
|
|
42
|
+
|| !Number.isSafeInteger(hint.estimate_sample_count) || hint.estimate_sample_count < 0
|
|
43
|
+
|| ((hint.estimate_source === "default" || hint.estimate_source === "task-budget")
|
|
44
|
+
? hint.estimate_sample_count !== 0 : hint.estimate_sample_count < 1))
|
|
45
|
+
protocolError("remote work scheduling hint is invalid");
|
|
46
|
+
return hint;
|
|
47
|
+
}
|
|
48
|
+
function stringArray(value) {
|
|
49
|
+
return Array.isArray(value) && value.every((entry) => typeof entry === "string" && entry.length > 0 && !/[\0\r\n]/.test(entry)) && new Set(value).size === value.length;
|
|
50
|
+
}
|
|
51
|
+
function exact(value, keys, label) {
|
|
52
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
53
|
+
protocolError(`${label} must be an object`);
|
|
54
|
+
const record = value;
|
|
55
|
+
if (Object.keys(record).some((key) => !keys.includes(key)))
|
|
56
|
+
protocolError(`${label} has unknown fields`);
|
|
57
|
+
return record;
|
|
58
|
+
}
|
|
59
|
+
function protocolError(message) {
|
|
60
|
+
throw new HitchError(message, { code: "worker_protocol_invalid", exitCode: 2 });
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=remote-work-item.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"remote-work-item.js","sourceRoot":"","sources":["../../../src/control-plane/remote-work-item.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAExD,MAAM,MAAM,GAAG,uBAAuB,CAAC;AAEvC,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAChD,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE;QAC1B,gBAAgB,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,iBAAiB,EAAE,UAAU,EAAE,OAAO,EAAE,mBAAmB;QAC9G,uBAAuB,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,YAAY;KAClH,EAAE,kBAAkB,CAAC,CAAC;IACvB,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;WACjH,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ;WAChH,MAAM,CAAC,eAAe,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,eAAe,CAAC,IAAK,MAAM,CAAC,eAA0B,GAAG,CAAC,CAAC;WAC5H,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,OAAO,MAAM,CAAC,iBAAiB,KAAK,SAAS;WAC5G,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,qBAAqB,CAAC,IAAK,MAAM,CAAC,qBAAgC,GAAG,CAAC;WACnG,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC;WAC/H,MAAM,CAAC,WAAW,KAAK,SAAS,IAAI,CAAC,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;WAChH,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,KAAK,SAAS,CAAC;QAAE,aAAa,CAAC,6BAA6B,CAAC,CAAC;IAClI,IAAI,MAAM,CAAC,gBAAgB,KAAK,SAAS;QAAE,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACzF,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACxG,OAAO;QACL,GAAG,MAAM;QACT,WAAW,EAAE,sBAAsB,CAAC,MAAM,CAAC,WAA+B,EAAE,yBAAyB,CAAC;QACtG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACN,CAAC;AACpC,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,iBAAiB,EAAE,mBAAmB,EAAE,cAAc,CAAC,EAAE,8BAA8B,CAAC,CAAC;IACvH,IAAI,CAAC,OAAO,CAAC,eAAe,KAAK,aAAa,IAAI,OAAO,CAAC,eAAe,KAAK,aAAa,CAAC;WACvF,CAAC,OAAO,CAAC,iBAAiB,KAAK,WAAW,IAAI,OAAO,CAAC,iBAAiB,KAAK,aAAa,CAAC;WAC1F,CAAC,OAAO,CAAC,eAAe,KAAK,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,iBAAiB,KAAK,WAAW,CAAC;WAC3F,OAAO,OAAO,CAAC,YAAY,KAAK,QAAQ,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,aAAa,CAAC,yCAAyC,CAAC,CAAC;AAC5J,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc;IACzC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,uBAAuB,CAAC,EAAE,6BAA6B,CAAC,CAAC;IAC/J,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,oBAAoB,EAAE,aAAa,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC,CAAC;IACzF,IAAI,IAAI,CAAC,MAAM,KAAK,sBAAsB;WACrC,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAK,IAAI,CAAC,qBAAgC,GAAG,CAAC;WAC/F,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAK,IAAI,CAAC,iBAA4B,GAAI,IAAI,CAAC,qBAAgC;WAC5H,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,eAAyB,CAAC;WAC5C,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,qBAAqB,CAAC,IAAK,IAAI,CAAC,qBAAgC,GAAG,CAAC;WAC/F,CAAC,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,IAAI,CAAC,eAAe,KAAK,aAAa,CAAC;YAChF,CAAC,CAAC,IAAI,CAAC,qBAAqB,KAAK,CAAC,CAAC,CAAC,CAAE,IAAI,CAAC,qBAAgC,GAAG,CAAC,CAAC;QAAE,aAAa,CAAC,wCAAwC,CAAC,CAAC;IAC9I,OAAO,IAAkD,CAAC;AAC5D,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,CAAC;AAC1K,CAAC;AAED,SAAS,KAAK,CAAC,KAAc,EAAE,IAAc,EAAE,KAAa;IAC1D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,aAAa,CAAC,GAAG,KAAK,oBAAoB,CAAC,CAAC;IAC7G,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAAE,aAAa,CAAC,GAAG,KAAK,qBAAqB,CAAC,CAAC;IACzG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CAAC,OAAe;IACpC,MAAM,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,yBAAyB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;AAClF,CAAC"}
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { HitchError, readJSON } from "../foundation/index.js";
|
|
3
|
+
import { loadEnvironmentImageManifest } from "../images/index.js";
|
|
4
|
+
import { DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS, acceptExecutionLease, classifyTrialFailure, ensurePhysicalRetryDecision, heartbeatExecutionLease, loadEvalResumeState, loadTrialEnvironmentImages, markExecutionLeaseLost, markExecutionLeaseRunning, mergeEvalProgressTrial, readEvalRetryState, replaceInvalidEvalProgressTrial, resolveRetryWork, physicalRetryAllowed, physicalRetryWorkItem, releaseExecutionLease, writeEvalProgress, transitionRetryDecision, retryBackoffMs, } from "../evals/index.js";
|
|
5
|
+
import { importRemoteResultEnvelope } from "./remote-result-transport.js";
|
|
6
|
+
export const DEFAULT_REMOTE_WORKER_RECONNECT_TIMEOUT_MS = 45_000;
|
|
7
|
+
export async function recoverRemoteWorkerEvalLeases(input) {
|
|
8
|
+
const reconnectTimeoutMs = boundedReconnectTimeout(input.reconnectTimeoutMs ?? DEFAULT_REMOTE_WORKER_RECONNECT_TIMEOUT_MS);
|
|
9
|
+
const recovered = [];
|
|
10
|
+
let failure;
|
|
11
|
+
for (const lease of input.leases.filter(activeRemoteLease)) {
|
|
12
|
+
try {
|
|
13
|
+
await recoverLease(input, lease, reconnectTimeoutMs);
|
|
14
|
+
recovered.push(lease.lease_id);
|
|
15
|
+
}
|
|
16
|
+
catch (error) {
|
|
17
|
+
const typed = error instanceof HitchError;
|
|
18
|
+
failure ??= { code: typed ? error.code : "execution_state_ambiguous", message: error?.message || String(error) };
|
|
19
|
+
await markExecutionLeaseLost({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch }).catch(() => undefined);
|
|
20
|
+
input.emit?.({ type: "eval.lease.recovery-failed", lease_id: lease.lease_id, code: failure.code });
|
|
21
|
+
input.emit?.({ type: "eval.work.lost", work_id: lease.work_id, lease_id: lease.lease_id, worker_id: lease.worker_id, code: failure.code });
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return failure
|
|
25
|
+
? { status: "ambiguous", recovered_lease_ids: recovered, ...failure }
|
|
26
|
+
: { status: "resumable", recovered_lease_ids: recovered };
|
|
27
|
+
}
|
|
28
|
+
async function recoverLease(input, lease, reconnectTimeoutMs) {
|
|
29
|
+
if (Date.parse(lease.expires_at) <= Date.now())
|
|
30
|
+
input.emit?.({ type: "lease.expired", work_id: lease.work_id, lease_id: lease.lease_id, lease_epoch: lease.epoch, worker_id: lease.worker_id });
|
|
31
|
+
let offer = await input.protocol.findOfferForLease(lease.worker_id, lease.lease_id);
|
|
32
|
+
if (!offer || !sameLease(offer, lease))
|
|
33
|
+
throw ambiguous(`remote provider has no matching durable offer for ${lease.lease_id}`);
|
|
34
|
+
if (offer.state === "offered") {
|
|
35
|
+
offer = await input.protocol.withdrawUnacceptedOffer(offer.worker_id, offer.offer_id);
|
|
36
|
+
if (offer.state === "expired") {
|
|
37
|
+
input.emit?.({ type: "lease.expired", lease_id: lease.lease_id, lease_epoch: lease.epoch, worker_id: lease.worker_id });
|
|
38
|
+
await releaseExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch });
|
|
39
|
+
input.emit?.({ type: "eval.lease.recovered", lease_id: lease.lease_id, lease_epoch: lease.epoch, state: "not-started" });
|
|
40
|
+
input.emit?.({ type: "lease.recovered", lease_id: lease.lease_id, lease_epoch: lease.epoch, state: "not-started" });
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (!acceptedOrLater(offer))
|
|
45
|
+
throw ambiguous(`remote provider classified ${lease.lease_id} as ${offer.state}`);
|
|
46
|
+
let current = lease;
|
|
47
|
+
if (current.state === "offered")
|
|
48
|
+
current = await acceptExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch });
|
|
49
|
+
if (current.state === "accepted")
|
|
50
|
+
current = await markExecutionLeaseRunning({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch });
|
|
51
|
+
input.emit?.({ type: "eval.lease.recovery-probed", lease_id: lease.lease_id, lease_epoch: lease.epoch, state: offer.state });
|
|
52
|
+
if (input.cancelRequested && offer.state === "accepted")
|
|
53
|
+
offer = await input.protocol.requestCancel(offer.worker_id, offer.offer_id);
|
|
54
|
+
if (offer.state === "accepted" || offer.state === "cancel-requested")
|
|
55
|
+
offer = await waitForTerminal(input, offer, current, reconnectTimeoutMs);
|
|
56
|
+
if (offer.state !== "completed" && offer.state !== "release-requested" && offer.state !== "released") {
|
|
57
|
+
throw ambiguous(`remote work did not reach collectable terminal state: ${offer.state}`);
|
|
58
|
+
}
|
|
59
|
+
if (offer.terminal?.status === "succeeded")
|
|
60
|
+
await collectRemoteResult(input, current, offer);
|
|
61
|
+
else if (!input.cancelRequested)
|
|
62
|
+
throw new HitchError(`recovered remote work ended as ${offer.terminal?.status ?? "unknown"}`, { code: "remote_work_failed", exitCode: 13 });
|
|
63
|
+
await finishRelease(input, current, offer);
|
|
64
|
+
input.emit?.({ type: "eval.lease.recovered", lease_id: lease.lease_id, lease_epoch: lease.epoch, state: "released" });
|
|
65
|
+
input.emit?.({ type: "lease.recovered", lease_id: lease.lease_id, lease_epoch: lease.epoch, state: "released" });
|
|
66
|
+
}
|
|
67
|
+
async function waitForTerminal(input, initial, lease, reconnectTimeoutMs) {
|
|
68
|
+
let offer = initial;
|
|
69
|
+
let heartbeatFailure;
|
|
70
|
+
let unavailableSince;
|
|
71
|
+
let reportedUnavailable = false;
|
|
72
|
+
let tail = Promise.resolve();
|
|
73
|
+
const timer = setInterval(() => {
|
|
74
|
+
tail = tail.then(() => heartbeatExecutionLease({
|
|
75
|
+
evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch,
|
|
76
|
+
}).then(() => undefined)).catch((error) => { heartbeatFailure ??= error; });
|
|
77
|
+
}, DEFAULT_EXECUTION_LEASE_HEARTBEAT_MS);
|
|
78
|
+
timer.unref();
|
|
79
|
+
try {
|
|
80
|
+
while (offer.state === "accepted" || offer.state === "cancel-requested") {
|
|
81
|
+
const worker = await input.registry.get(offer.worker_id);
|
|
82
|
+
if (worker?.revoked_at)
|
|
83
|
+
throw ambiguous(`remote worker was revoked during recovery: ${offer.worker_id}`);
|
|
84
|
+
if (workerProvesLease(worker, lease)) {
|
|
85
|
+
if (reportedUnavailable)
|
|
86
|
+
input.emit?.({ type: "worker.reconnected", worker_id: offer.worker_id, lease_id: lease.lease_id, lease_epoch: lease.epoch });
|
|
87
|
+
unavailableSince = undefined;
|
|
88
|
+
reportedUnavailable = false;
|
|
89
|
+
}
|
|
90
|
+
else {
|
|
91
|
+
unavailableSince ??= Date.now();
|
|
92
|
+
if (!reportedUnavailable)
|
|
93
|
+
input.emit?.({ type: "worker.heartbeat_missed", worker_id: offer.worker_id, lease_id: lease.lease_id, lease_epoch: lease.epoch });
|
|
94
|
+
reportedUnavailable = true;
|
|
95
|
+
if (Date.now() - unavailableSince >= reconnectTimeoutMs)
|
|
96
|
+
throw ambiguous(`remote worker did not reconnect with active lease: ${offer.worker_id}`);
|
|
97
|
+
}
|
|
98
|
+
await delay(input.pollIntervalMs);
|
|
99
|
+
offer = await input.protocol.getOffer(offer.worker_id, offer.offer_id) ?? (() => { throw ambiguous("remote work offer disappeared during recovery"); })();
|
|
100
|
+
}
|
|
101
|
+
await tail;
|
|
102
|
+
if (heartbeatFailure !== undefined)
|
|
103
|
+
throw heartbeatFailure;
|
|
104
|
+
return offer;
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
clearInterval(timer);
|
|
108
|
+
await tail;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
async function collectRemoteResult(input, lease, offer) {
|
|
112
|
+
const state = await loadEvalResumeState(input.evalDirectory);
|
|
113
|
+
const retryState = await readEvalRetryState(input.evalDirectory, input.evalId);
|
|
114
|
+
const dynamic = resolveRetryWork(state.executionPlan, retryState, lease.work_id);
|
|
115
|
+
const work = state.executionPlan.work_items.find((entry) => entry.work_id === lease.work_id) ?? dynamic?.item;
|
|
116
|
+
if (!work || work.logical_attempt === null || work.task_ids.length !== 1)
|
|
117
|
+
throw ambiguous("recovered remote lease does not match one planned task slot");
|
|
118
|
+
const existing = state.progress.trials.find((trial) => trial.task_id === work.task_ids[0] && trial.attempt === work.logical_attempt);
|
|
119
|
+
if (existing?.observation_status === "valid") {
|
|
120
|
+
if (dynamic)
|
|
121
|
+
for (const decision of dynamic.decisions)
|
|
122
|
+
await settleRemoteRetryDecision(input.evalDirectory, input.evalId, decision.decision_id, decision.state, "repaired");
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
if (existing && !dynamic)
|
|
126
|
+
return;
|
|
127
|
+
const artifacts = offer.terminal?.artifacts.filter((artifact) => artifact.kind === "result-bundle") ?? [];
|
|
128
|
+
if (artifacts.length !== 1)
|
|
129
|
+
throw ambiguous("recovered remote success requires exactly one result bundle");
|
|
130
|
+
const request = await readJSON(path.join(input.evalDirectory, "request.json"));
|
|
131
|
+
const resolution = await readJSON(path.join(input.evalDirectory, "resolution.json"));
|
|
132
|
+
const runtime = state.plan.controller_runtime;
|
|
133
|
+
const environmentImages = await loadTrialEnvironmentImages({
|
|
134
|
+
taskId: work.task_ids[0],
|
|
135
|
+
uses: work.image_refs ?? [],
|
|
136
|
+
loader: (imageId) => loadEnvironmentImageManifest(input.root, imageId),
|
|
137
|
+
});
|
|
138
|
+
const artifact = artifacts[0];
|
|
139
|
+
const imported = await importRemoteResultEnvelope({
|
|
140
|
+
root: input.root, evalDirectory: input.evalDirectory, request, resolvedRevision: resolution,
|
|
141
|
+
work, lease, artifactPath: input.protocol.artifactPath(offer.worker_id, lease.lease_id, artifact.digest),
|
|
142
|
+
...(typeof runtime?.runtime_id === "string" ? { runtimeId: runtime.runtime_id } : {}),
|
|
143
|
+
...(environmentImages ? { environmentImages } : {}),
|
|
144
|
+
...(state.executionPlan.model_capture ? { modelCapturePlan: state.executionPlan.model_capture } : {}),
|
|
145
|
+
});
|
|
146
|
+
const progress = dynamic
|
|
147
|
+
? imported.ref.observation_status === "valid" ? replaceInvalidEvalProgressTrial(state.progress, imported.ref) : state.progress
|
|
148
|
+
: mergeEvalProgressTrial(state.progress, imported.ref);
|
|
149
|
+
if (progress.generation !== state.progress.generation)
|
|
150
|
+
await writeEvalProgress(input.evalDirectory, progress);
|
|
151
|
+
if (dynamic) {
|
|
152
|
+
const retryIndex = dynamic.decisions[0]?.retry_index ?? 0;
|
|
153
|
+
if (imported.ref.observation_status === "invalid" && physicalRetryAllowed(classifyTrialFailure(imported.ref))
|
|
154
|
+
&& retryIndex < state.executionPlan.retry_policy.infrastructure_retries) {
|
|
155
|
+
const origin = state.executionPlan.work_items.find((item) => item.slots.includes(dynamic.decisions[0].slot_id));
|
|
156
|
+
if (!origin)
|
|
157
|
+
throw ambiguous("recovered retry origin is absent from execution plan");
|
|
158
|
+
const nextWork = physicalRetryWorkItem(origin, retryIndex + 1, [imported.ref]);
|
|
159
|
+
await ensurePhysicalRetryDecision({
|
|
160
|
+
evalDirectory: input.evalDirectory, evalId: input.evalId, item: origin, retryIndex: retryIndex + 1, trigger: imported.ref,
|
|
161
|
+
notBefore: new Date(Date.now() + retryBackoffMs(state.executionPlan.retry_policy.infrastructure_retry_backoff_ms, retryIndex + 1, nextWork.work_id)).toISOString(),
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
const target = imported.ref.observation_status === "valid" ? "repaired"
|
|
165
|
+
: retryIndex >= state.executionPlan.retry_policy.infrastructure_retries ? "exhausted" : "invalid";
|
|
166
|
+
for (const decision of dynamic.decisions)
|
|
167
|
+
await settleRemoteRetryDecision(input.evalDirectory, input.evalId, decision.decision_id, decision.state, target);
|
|
168
|
+
}
|
|
169
|
+
input.emit?.({ type: "eval.work-item.recovered", work_id: work.work_id, lease_id: lease.lease_id, trials: 1 });
|
|
170
|
+
}
|
|
171
|
+
async function settleRemoteRetryDecision(evalDirectory, evalId, decisionId, current, target) {
|
|
172
|
+
if (current === target)
|
|
173
|
+
return;
|
|
174
|
+
if (current === "planned") {
|
|
175
|
+
await transitionRetryDecision({ evalDirectory, evalId, decisionId, state: "running" });
|
|
176
|
+
current = "running";
|
|
177
|
+
}
|
|
178
|
+
if (current === "running")
|
|
179
|
+
await transitionRetryDecision({ evalDirectory, evalId, decisionId, state: target });
|
|
180
|
+
}
|
|
181
|
+
async function finishRelease(input, lease, terminal) {
|
|
182
|
+
input.emit?.({ type: "sandbox.cleanup.started", work_id: lease.work_id, lease_id: lease.lease_id, worker_id: lease.worker_id });
|
|
183
|
+
let offer = terminal.state === "released" ? terminal : await input.protocol.requestRelease(terminal.worker_id, terminal.offer_id);
|
|
184
|
+
const deadline = Date.now() + input.releaseTimeoutMs;
|
|
185
|
+
while (offer.state !== "released" && Date.now() < deadline) {
|
|
186
|
+
await delay(input.pollIntervalMs);
|
|
187
|
+
offer = await input.protocol.getOffer(offer.worker_id, offer.offer_id) ?? offer;
|
|
188
|
+
}
|
|
189
|
+
if (offer.state === "released") {
|
|
190
|
+
await releaseExecutionLease({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch });
|
|
191
|
+
input.emit?.({ type: "lease.released", work_id: lease.work_id, lease_id: lease.lease_id, lease_epoch: lease.epoch, worker_id: lease.worker_id });
|
|
192
|
+
input.emit?.({ type: "sandbox.cleanup.completed", work_id: lease.work_id, lease_id: lease.lease_id, worker_id: lease.worker_id, residual_resources: 0 });
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
await markExecutionLeaseLost({ evalDirectory: input.evalDirectory, leaseId: lease.lease_id, expectedEpoch: lease.epoch });
|
|
196
|
+
input.emit?.({ type: "sandbox.cleanup.failed", work_id: lease.work_id, lease_id: lease.lease_id, worker_id: lease.worker_id, code: "worker_release_timeout" });
|
|
197
|
+
}
|
|
198
|
+
function activeRemoteLease(lease) {
|
|
199
|
+
return lease.provider !== "local-docker" && new Set(["offered", "accepted", "running", "releasing"]).has(lease.state);
|
|
200
|
+
}
|
|
201
|
+
function acceptedOrLater(offer) {
|
|
202
|
+
return new Set(["accepted", "cancel-requested", "completed", "release-requested", "released"]).has(offer.state)
|
|
203
|
+
&& typeof offer.accepted_at === "string" && typeof offer.accept_receipt_digest === "string";
|
|
204
|
+
}
|
|
205
|
+
function sameLease(offer, lease) {
|
|
206
|
+
return offer.lease.lease_id === lease.lease_id && offer.lease.epoch === lease.epoch && offer.worker_id === lease.worker_id
|
|
207
|
+
&& offer.lease.eval_id === lease.eval_id && offer.lease.work_id === lease.work_id;
|
|
208
|
+
}
|
|
209
|
+
function workerProvesLease(worker, lease) {
|
|
210
|
+
return worker?.worker.status === "ready"
|
|
211
|
+
&& worker.active_leases.some((active) => active.lease_id === lease.lease_id && active.epoch === lease.epoch);
|
|
212
|
+
}
|
|
213
|
+
function boundedReconnectTimeout(value) {
|
|
214
|
+
if (!Number.isSafeInteger(value) || value < 1 || value > 5 * 60_000)
|
|
215
|
+
throw new TypeError("remote worker reconnect timeout is invalid");
|
|
216
|
+
return value;
|
|
217
|
+
}
|
|
218
|
+
function ambiguous(message) { return new HitchError(message, { code: "execution_state_ambiguous", exitCode: 12 }); }
|
|
219
|
+
function delay(milliseconds) { return new Promise((resolve) => setTimeout(resolve, milliseconds)); }
|
|
220
|
+
//# sourceMappingURL=remote-work-recovery.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"remote-work-recovery.js","sourceRoot":"","sources":["../../../src/control-plane/remote-work-recovery.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAClE,OAAO,EACL,oCAAoC,EACpC,oBAAoB,EACpB,oBAAoB,EACpB,2BAA2B,EAC3B,uBAAuB,EACvB,mBAAmB,EACnB,0BAA0B,EAC1B,sBAAsB,EACtB,yBAAyB,EACzB,sBAAsB,EACtB,kBAAkB,EAClB,+BAA+B,EAC/B,gBAAgB,EAChB,oBAAoB,EACpB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,uBAAuB,EACvB,cAAc,GACf,MAAM,mBAAmB,CAAC;AAE3B,OAAO,EAAE,0BAA0B,EAAE,MAAM,8BAA8B,CAAC;AAI1E,MAAM,CAAC,MAAM,0CAA0C,GAAG,MAAM,CAAC;AAEjE,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,KAYnD;IACC,MAAM,kBAAkB,GAAG,uBAAuB,CAAC,KAAK,CAAC,kBAAkB,IAAI,0CAA0C,CAAC,CAAC;IAC3H,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,OAAsD,CAAC;IAC3D,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,KAAK,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAC;YACrD,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,KAAK,YAAY,UAAU,CAAC;YAC1C,OAAO,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,2BAA2B,EAAE,OAAO,EAAG,KAAe,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5H,MAAM,sBAAsB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACjJ,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,4BAA4B,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YACnG,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7I,CAAC;IACH,CAAC;IACD,OAAO,OAAO;QACZ,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,SAAS,EAAE,GAAG,OAAO,EAAE;QACrE,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,SAAS,EAAE,CAAC;AAC9D,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,KAA0D,EAAE,KAAuB,EAAE,kBAA0B;IACzI,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,GAAG,EAAE;QAAE,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;IAChM,IAAI,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACpF,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,MAAM,SAAS,CAAC,qDAAqD,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC/H,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,uBAAuB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QACtF,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;YACxH,MAAM,qBAAqB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;YACzH,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;YACzH,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;YACpH,OAAO;QACT,CAAC;IACH,CAAC;IACD,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;QAAE,MAAM,SAAS,CAAC,8BAA8B,KAAK,CAAC,QAAQ,OAAO,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC/G,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,GAAG,MAAM,oBAAoB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACnK,IAAI,OAAO,CAAC,KAAK,KAAK,UAAU;QAAE,OAAO,GAAG,MAAM,yBAAyB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IACzK,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,4BAA4B,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC7H,IAAI,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU;QAAE,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACrI,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,IAAI,KAAK,CAAC,KAAK,KAAK,kBAAkB;QAAE,KAAK,GAAG,MAAM,eAAe,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,kBAAkB,CAAC,CAAC;IAC/I,IAAI,KAAK,CAAC,KAAK,KAAK,WAAW,IAAI,KAAK,CAAC,KAAK,KAAK,mBAAmB,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;QACrG,MAAM,SAAS,CAAC,yDAAyD,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,EAAE,MAAM,KAAK,WAAW;QAAE,MAAM,mBAAmB,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;SACxF,IAAI,CAAC,KAAK,CAAC,eAAe;QAAE,MAAM,IAAI,UAAU,CAAC,kCAAkC,KAAK,CAAC,QAAQ,EAAE,MAAM,IAAI,SAAS,EAAE,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IAC7K,MAAM,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAC3C,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;IACtH,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;AACnH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,KAA0D,EAC1D,OAA0B,EAC1B,KAAuB,EACvB,kBAA0B;IAE1B,IAAI,KAAK,GAAG,OAAO,CAAC;IACpB,IAAI,gBAAyB,CAAC;IAC9B,IAAI,gBAAoC,CAAC;IACzC,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAChC,IAAI,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,uBAAuB,CAAC;YAC7C,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK;SACxF,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,GAAG,gBAAgB,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,CAAC,EAAE,oCAAoC,CAAC,CAAC;IACzC,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,IAAI,CAAC;QACH,OAAO,KAAK,CAAC,KAAK,KAAK,UAAU,IAAI,KAAK,CAAC,KAAK,KAAK,kBAAkB,EAAE,CAAC;YACxE,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACzD,IAAI,MAAM,EAAE,UAAU;gBAAE,MAAM,SAAS,CAAC,8CAA8C,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;YACzG,IAAI,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;gBACrC,IAAI,mBAAmB;oBAAE,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,oBAAoB,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;gBACtJ,gBAAgB,GAAG,SAAS,CAAC;gBAC7B,mBAAmB,GAAG,KAAK,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,gBAAgB,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;gBAChC,IAAI,CAAC,mBAAmB;oBAAE,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,yBAAyB,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;gBAC5J,mBAAmB,GAAG,IAAI,CAAC;gBAC3B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,IAAI,kBAAkB;oBAAE,MAAM,SAAS,CAAC,sDAAsD,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;YACpJ,CAAC;YACD,MAAM,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YAClC,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,SAAS,CAAC,+CAA+C,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5J,CAAC;QACD,MAAM,IAAI,CAAC;QACX,IAAI,gBAAgB,KAAK,SAAS;YAAE,MAAM,gBAAgB,CAAC;QAC3D,OAAO,KAAK,CAAC;IACf,CAAC;YAAS,CAAC;QACT,aAAa,CAAC,KAAK,CAAC,CAAC;QACrB,MAAM,IAAI,CAAC;IACb,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,KAA0D,EAAE,KAAuB,EAAE,KAAwB;IAC9I,MAAM,KAAK,GAAG,MAAM,mBAAmB,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/E,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,aAAa,EAAE,UAAU,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACjF,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,CAAC,IAAI,OAAO,EAAE,IAAI,CAAC;IAC9G,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,SAAS,CAAC,6DAA6D,CAAC,CAAC;IACzJ,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,eAAe,CAAC,CAAC;IACrI,IAAI,QAAQ,EAAE,kBAAkB,KAAK,OAAO,EAAE,CAAC;QAC7C,IAAI,OAAO;YAAE,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS;gBAAE,MAAM,yBAAyB,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;QAC5K,OAAO;IACT,CAAC;IACD,IAAI,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO;IACjC,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,KAAK,eAAe,CAAC,IAAI,EAAE,CAAC;IAC1G,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,SAAS,CAAC,6DAA6D,CAAC,CAAC;IAC3G,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAc,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC,CAAC;IAC5F,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAmB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE,iBAAiB,CAAC,CAAC,CAAC;IACvG,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,kBAAyD,CAAC;IACrF,MAAM,iBAAiB,GAAG,MAAM,0BAA0B,CAAC;QACzD,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAW;QAClC,IAAI,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE;QAC3B,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,4BAA4B,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC;KACvE,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAA6B,CAAC;IAC1D,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC;QAChD,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,gBAAgB,EAAE,UAAU;QAC3F,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC;QACxG,GAAG,CAAC,OAAO,OAAO,EAAE,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrF,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,aAAa,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtG,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,OAAO;QACtB,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,+BAA+B,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ;QAC9H,CAAC,CAAC,sBAAsB,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IACzD,IAAI,QAAQ,CAAC,UAAU,KAAK,KAAK,CAAC,QAAQ,CAAC,UAAU;QAAE,MAAM,iBAAiB,CAAC,KAAK,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;IAC9G,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,IAAI,CAAC,CAAC;QAC1D,IAAI,QAAQ,CAAC,GAAG,CAAC,kBAAkB,KAAK,SAAS,IAAI,oBAAoB,CAAC,oBAAoB,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;eACxG,UAAU,GAAG,KAAK,CAAC,aAAa,CAAC,YAAY,CAAC,sBAAsB,EAAE,CAAC;YAC1E,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC,CAAC;YACjH,IAAI,CAAC,MAAM;gBAAE,MAAM,SAAS,CAAC,sDAAsD,CAAC,CAAC;YACrF,MAAM,QAAQ,GAAG,qBAAqB,CAAC,MAAM,EAAE,UAAU,GAAG,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/E,MAAM,2BAA2B,CAAC;gBAChC,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,UAAU,GAAG,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,GAAG;gBACzH,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,aAAa,CAAC,YAAY,CAAC,+BAA+B,EAAE,UAAU,GAAG,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE;aACnK,CAAC,CAAC;QACL,CAAC;QACD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,CAAC,UAAU;YACrE,CAAC,CAAC,UAAU,IAAI,KAAK,CAAC,aAAa,CAAC,YAAY,CAAC,sBAAsB,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;QACpG,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS;YAAE,MAAM,yBAAyB,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAC7J,CAAC;IACD,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;AACjH,CAAC;AAED,KAAK,UAAU,yBAAyB,CACtC,aAAqB,EACrB,MAAc,EACd,UAAkB,EAClB,OAAiF,EACjF,MAA4C;IAE5C,IAAI,OAAO,KAAK,MAAM;QAAE,OAAO;IAC/B,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,uBAAuB,CAAC,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QACvF,OAAO,GAAG,SAAS,CAAC;IACtB,CAAC;IACD,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,uBAAuB,CAAC,EAAE,aAAa,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;AACjH,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,KAA0D,EAAE,KAAuB,EAAE,QAA2B;IAC3I,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,yBAAyB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;IAChI,IAAI,KAAK,GAAG,QAAQ,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAClI,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,gBAAgB,CAAC;IACrD,OAAO,KAAK,CAAC,KAAK,KAAK,UAAU,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC3D,MAAM,KAAK,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAClC,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC;IAClF,CAAC;IACD,IAAI,KAAK,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;QAC/B,MAAM,qBAAqB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QACzH,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC;QACjJ,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,kBAAkB,EAAE,CAAC,EAAE,CAAC,CAAC;QACzJ,OAAO;IACT,CAAC;IACD,MAAM,sBAAsB,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC1H,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,wBAAwB,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;AACjK,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAuB;IAChD,OAAO,KAAK,CAAC,QAAQ,KAAK,cAAc,IAAI,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AACxH,CAAC;AACD,SAAS,eAAe,CAAC,KAAwB;IAC/C,OAAO,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,kBAAkB,EAAE,WAAW,EAAE,mBAAmB,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC;WAC1G,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,qBAAqB,KAAK,QAAQ,CAAC;AAChG,CAAC;AACD,SAAS,SAAS,CAAC,KAAwB,EAAE,KAAuB;IAClE,OAAO,KAAK,CAAC,KAAK,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS;WACrH,KAAK,CAAC,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,CAAC;AACtF,CAAC;AACD,SAAS,iBAAiB,CAAC,MAAwD,EAAE,KAAuB;IAC1G,OAAO,MAAM,EAAE,MAAM,CAAC,MAAM,KAAK,OAAO;WACnC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC;AACjH,CAAC;AACD,SAAS,uBAAuB,CAAC,KAAa;IAC5C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,GAAG,MAAM;QAAE,MAAM,IAAI,SAAS,CAAC,4CAA4C,CAAC,CAAC;IACvI,OAAO,KAAK,CAAC;AACf,CAAC;AACD,SAAS,SAAS,CAAC,OAAe,IAAgB,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,2BAA2B,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AACxI,SAAS,KAAK,CAAC,YAAoB,IAAmB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC"}
|