agent-hitch 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -3
- package/README.zh-CN.md +14 -2
- package/dist/bin/hitch.js +25 -3
- package/dist/bin/hitch.js.map +1 -1
- package/dist/scripts/canary-benchmark.js +54 -0
- package/dist/scripts/canary-benchmark.js.map +1 -0
- package/dist/scripts/canary-buildkit-secrets.js +166 -0
- package/dist/scripts/canary-buildkit-secrets.js.map +1 -0
- package/dist/scripts/canary-collision-domains.js +77 -0
- package/dist/scripts/canary-collision-domains.js.map +1 -0
- package/dist/scripts/canary-docker-image-observation.js +181 -0
- package/dist/scripts/canary-docker-image-observation.js.map +1 -0
- package/dist/scripts/canary-eval-scheduler-throughput.js +47 -0
- package/dist/scripts/canary-eval-scheduler-throughput.js.map +1 -0
- package/dist/scripts/canary-gpu-hardware.js +95 -0
- package/dist/scripts/canary-gpu-hardware.js.map +1 -0
- package/dist/scripts/canary-harbor-load.js +365 -0
- package/dist/scripts/canary-harbor-load.js.map +1 -0
- package/dist/scripts/canary-packaged-harness.js +75 -0
- package/dist/scripts/canary-packaged-harness.js.map +1 -0
- package/dist/scripts/canary-resource-load.js +188 -0
- package/dist/scripts/canary-resource-load.js.map +1 -0
- package/dist/scripts/check-architecture.js +10 -5
- package/dist/scripts/check-architecture.js.map +1 -1
- package/dist/scripts/docker-reaper-crash-child.js +21 -0
- package/dist/scripts/docker-reaper-crash-child.js.map +1 -0
- package/dist/src/adapters/catalog.js +3 -1
- package/dist/src/adapters/catalog.js.map +1 -1
- package/dist/src/adapters/discovery.js +1 -1
- package/dist/src/adapters/discovery.js.map +1 -1
- package/dist/src/adapters/providers/claude.js +6 -0
- package/dist/src/adapters/providers/claude.js.map +1 -1
- package/dist/src/adapters/providers/codex-auth.js +29 -0
- package/dist/src/adapters/providers/codex-auth.js.map +1 -0
- package/dist/src/adapters/providers/codex.js +29 -6
- package/dist/src/adapters/providers/codex.js.map +1 -1
- package/dist/src/adapters/providers/deepseek.js +6 -0
- package/dist/src/adapters/providers/deepseek.js.map +1 -1
- package/dist/src/adapters/providers/model-call.js +26 -0
- package/dist/src/adapters/providers/model-call.js.map +1 -0
- package/dist/src/adapters/providers/opencode.js +6 -0
- package/dist/src/adapters/providers/opencode.js.map +1 -1
- package/dist/src/adapters/providers/pi.js +6 -0
- package/dist/src/adapters/providers/pi.js.map +1 -1
- package/dist/src/artifacts/index.js +1 -0
- package/dist/src/artifacts/index.js.map +1 -1
- package/dist/src/artifacts/preparer.js +7 -2
- package/dist/src/artifacts/preparer.js.map +1 -1
- package/dist/src/backends/harbor/agent-budget.js +31 -0
- package/dist/src/backends/harbor/agent-budget.js.map +1 -0
- package/dist/src/backends/harbor/backend.js +59 -107
- package/dist/src/backends/harbor/backend.js.map +1 -1
- package/dist/src/backends/harbor/environment-config.js +92 -0
- package/dist/src/backends/harbor/environment-config.js.map +1 -0
- package/dist/src/backends/harbor/index.js +6 -1
- package/dist/src/backends/harbor/index.js.map +1 -1
- package/dist/src/backends/harbor/model-proxy-config.js +23 -0
- package/dist/src/backends/harbor/model-proxy-config.js.map +1 -0
- package/dist/src/backends/harbor/process.js +177 -0
- package/dist/src/backends/harbor/process.js.map +1 -0
- package/dist/src/backends/harbor/regrade.js +77 -0
- package/dist/src/backends/harbor/regrade.js.map +1 -0
- package/dist/src/backends/harbor/runtime-toolchain.js +30 -0
- package/dist/src/backends/harbor/runtime-toolchain.js.map +1 -0
- package/dist/src/backends/harbor/supervisor.js +74 -0
- package/dist/src/backends/harbor/supervisor.js.map +1 -0
- package/dist/src/backends/harbor/task-resources.js +206 -0
- package/dist/src/backends/harbor/task-resources.js.map +1 -0
- package/dist/src/backends/harbor/tools.js +6 -29
- package/dist/src/backends/harbor/tools.js.map +1 -1
- package/dist/src/backends/index.js +5 -1
- package/dist/src/backends/index.js.map +1 -1
- package/dist/src/benchmarks/index.js +3 -0
- package/dist/src/benchmarks/index.js.map +1 -0
- package/dist/src/benchmarks/loader.js +271 -0
- package/dist/src/benchmarks/loader.js.map +1 -0
- package/dist/src/benchmarks/metrics.js +16 -0
- package/dist/src/benchmarks/metrics.js.map +1 -0
- package/dist/src/benchmarks/toml.js +13 -0
- package/dist/src/benchmarks/toml.js.map +1 -0
- package/dist/src/benchmarks/validation.js +242 -0
- package/dist/src/benchmarks/validation.js.map +1 -0
- package/dist/src/cli/arguments.js +6 -4
- package/dist/src/cli/arguments.js.map +1 -1
- package/dist/src/cli/commands/benchmark.js +22 -0
- package/dist/src/cli/commands/benchmark.js.map +1 -0
- package/dist/src/cli/commands/capabilities.js +18 -0
- package/dist/src/cli/commands/capabilities.js.map +1 -0
- package/dist/src/cli/commands/daemon.js +135 -8
- package/dist/src/cli/commands/daemon.js.map +1 -1
- package/dist/src/cli/commands/eval.js +228 -6
- package/dist/src/cli/commands/eval.js.map +1 -1
- package/dist/src/cli/commands/images.js +40 -0
- package/dist/src/cli/commands/images.js.map +1 -0
- package/dist/src/cli/commands/run.js +42 -17
- package/dist/src/cli/commands/run.js.map +1 -1
- package/dist/src/cli/commands/runs.js +25 -3
- package/dist/src/cli/commands/runs.js.map +1 -1
- package/dist/src/cli/commands/trajectory.js +96 -5
- package/dist/src/cli/commands/trajectory.js.map +1 -1
- package/dist/src/cli/commands/verifier.js +36 -0
- package/dist/src/cli/commands/verifier.js.map +1 -0
- package/dist/src/cli/commands/worker.js +79 -0
- package/dist/src/cli/commands/worker.js.map +1 -0
- package/dist/src/cli/main.js +10 -0
- package/dist/src/cli/main.js.map +1 -1
- package/dist/src/cli/output.js +84 -4
- package/dist/src/cli/output.js.map +1 -1
- package/dist/src/control-plane/build-admission.js +69 -0
- package/dist/src/control-plane/build-admission.js.map +1 -0
- package/dist/src/control-plane/build-records.js +6 -0
- package/dist/src/control-plane/build-records.js.map +1 -0
- package/dist/src/control-plane/collisions.js +40 -0
- package/dist/src/control-plane/collisions.js.map +1 -0
- package/dist/src/control-plane/eval-control-state.js +22 -0
- package/dist/src/control-plane/eval-control-state.js.map +1 -0
- package/dist/src/control-plane/eval-control-work.js +44 -0
- package/dist/src/control-plane/eval-control-work.js.map +1 -0
- package/dist/src/control-plane/eval-image-resolution.js +52 -0
- package/dist/src/control-plane/eval-image-resolution.js.map +1 -0
- package/dist/src/control-plane/eval-image-services.js +30 -0
- package/dist/src/control-plane/eval-image-services.js.map +1 -0
- package/dist/src/control-plane/eval-records.js +252 -0
- package/dist/src/control-plane/eval-records.js.map +1 -0
- package/dist/src/control-plane/eval-recovery.js +181 -0
- package/dist/src/control-plane/eval-recovery.js.map +1 -0
- package/dist/src/control-plane/eval-scheduler.js +450 -0
- package/dist/src/control-plane/eval-scheduler.js.map +1 -0
- package/dist/src/control-plane/index.js +20 -0
- package/dist/src/control-plane/index.js.map +1 -0
- package/dist/src/control-plane/local-worker.js +49 -0
- package/dist/src/control-plane/local-worker.js.map +1 -0
- package/dist/src/control-plane/model-capture-planning.js +12 -0
- package/dist/src/control-plane/model-capture-planning.js.map +1 -0
- package/dist/src/control-plane/remote-result-transport.js +175 -0
- package/dist/src/control-plane/remote-result-transport.js.map +1 -0
- package/dist/src/control-plane/remote-work-coordinator.js +333 -0
- package/dist/src/control-plane/remote-work-coordinator.js.map +1 -0
- package/dist/src/control-plane/remote-work-inputs.js +266 -0
- package/dist/src/control-plane/remote-work-inputs.js.map +1 -0
- package/dist/src/control-plane/remote-work-item.js +62 -0
- package/dist/src/control-plane/remote-work-item.js.map +1 -0
- package/dist/src/control-plane/remote-work-recovery.js +220 -0
- package/dist/src/control-plane/remote-work-recovery.js.map +1 -0
- package/dist/src/control-plane/remote-worker-artifacts.js +135 -0
- package/dist/src/control-plane/remote-worker-artifacts.js.map +1 -0
- package/dist/src/control-plane/remote-worker-client.js +202 -0
- package/dist/src/control-plane/remote-worker-client.js.map +1 -0
- package/dist/src/control-plane/remote-worker-configuration.js +34 -0
- package/dist/src/control-plane/remote-worker-configuration.js.map +1 -0
- package/dist/src/control-plane/remote-worker-credentials.js +58 -0
- package/dist/src/control-plane/remote-worker-credentials.js.map +1 -0
- package/dist/src/control-plane/remote-worker-protocol.js +489 -0
- package/dist/src/control-plane/remote-worker-protocol.js.map +1 -0
- package/dist/src/control-plane/remote-worker-runner.js +214 -0
- package/dist/src/control-plane/remote-worker-runner.js.map +1 -0
- package/dist/src/control-plane/remote-workers.js +377 -0
- package/dist/src/control-plane/remote-workers.js.map +1 -0
- package/dist/src/control-plane/rerun-scheduler.js +448 -0
- package/dist/src/control-plane/rerun-scheduler.js.map +1 -0
- package/dist/src/control-plane/rerun-submission.js +62 -0
- package/dist/src/control-plane/rerun-submission.js.map +1 -0
- package/dist/src/control-plane/resources.js +182 -0
- package/dist/src/control-plane/resources.js.map +1 -0
- package/dist/src/control-plane/scheduler-eval-entry.js +32 -0
- package/dist/src/control-plane/scheduler-eval-entry.js.map +1 -0
- package/dist/src/control-plane/synthetic-result.js +21 -0
- package/dist/src/control-plane/synthetic-result.js.map +1 -0
- package/dist/src/control-plane/work-admission.js +23 -0
- package/dist/src/control-plane/work-admission.js.map +1 -0
- package/dist/src/control-plane/work-dispatcher.js +237 -0
- package/dist/src/control-plane/work-dispatcher.js.map +1 -0
- package/dist/src/controller-runtime/hash.js +19 -3
- package/dist/src/controller-runtime/hash.js.map +1 -1
- package/dist/src/controller-runtime/index.js +1 -1
- package/dist/src/controller-runtime/index.js.map +1 -1
- package/dist/src/controller-runtime/store.js +15 -6
- package/dist/src/controller-runtime/store.js.map +1 -1
- package/dist/src/daemon/auth.js +5 -0
- package/dist/src/daemon/auth.js.map +1 -1
- package/dist/src/daemon/client.js.map +1 -1
- package/dist/src/daemon/event-stream.js +46 -0
- package/dist/src/daemon/event-stream.js.map +1 -0
- package/dist/src/daemon/health.js +66 -0
- package/dist/src/daemon/health.js.map +1 -0
- package/dist/src/daemon/launcher.js +28 -1
- package/dist/src/daemon/launcher.js.map +1 -1
- package/dist/src/daemon/logging.js +23 -0
- package/dist/src/daemon/logging.js.map +1 -0
- package/dist/src/daemon/scheduler.js +26 -5
- package/dist/src/daemon/scheduler.js.map +1 -1
- package/dist/src/daemon/server.js +255 -98
- package/dist/src/daemon/server.js.map +1 -1
- package/dist/src/daemon/telemetry.js +163 -0
- package/dist/src/daemon/telemetry.js.map +1 -0
- package/dist/src/daemon/worker-routes.js +219 -0
- package/dist/src/daemon/worker-routes.js.map +1 -0
- package/dist/src/domain/benchmarks.js +2 -0
- package/dist/src/domain/benchmarks.js.map +1 -0
- package/dist/src/domain/bundles.js +2 -0
- package/dist/src/domain/bundles.js.map +1 -0
- package/dist/src/domain/control-plane.js +2 -0
- package/dist/src/domain/control-plane.js.map +1 -0
- package/dist/src/domain/execution-evidence.js +2 -0
- package/dist/src/domain/execution-evidence.js.map +1 -0
- package/dist/src/domain/execution-plan.js +2 -0
- package/dist/src/domain/execution-plan.js.map +1 -0
- package/dist/src/domain/images.js +2 -0
- package/dist/src/domain/images.js.map +1 -0
- package/dist/src/domain/index.js +13 -0
- package/dist/src/domain/index.js.map +1 -1
- package/dist/src/domain/interactions.js +2 -0
- package/dist/src/domain/interactions.js.map +1 -0
- package/dist/src/domain/providers.js +2 -0
- package/dist/src/domain/providers.js.map +1 -0
- package/dist/src/domain/resources.js +2 -0
- package/dist/src/domain/resources.js.map +1 -0
- package/dist/src/domain/runs.js.map +1 -1
- package/dist/src/domain/validation.js +11 -4
- package/dist/src/domain/validation.js.map +1 -1
- package/dist/src/domain/verifier-evidence-validation.js +214 -0
- package/dist/src/domain/verifier-evidence-validation.js.map +1 -0
- package/dist/src/domain/verifier-evidence.js +3 -0
- package/dist/src/domain/verifier-evidence.js.map +1 -0
- package/dist/src/domain/verifier-score-contract.js +232 -0
- package/dist/src/domain/verifier-score-contract.js.map +1 -0
- package/dist/src/domain/workers.js +2 -0
- package/dist/src/domain/workers.js.map +1 -0
- package/dist/src/evals/benchmark-adapter-manifest.js +219 -0
- package/dist/src/evals/benchmark-adapter-manifest.js.map +1 -0
- package/dist/src/evals/benchmark-candidate.js +36 -0
- package/dist/src/evals/benchmark-candidate.js.map +1 -0
- package/dist/src/evals/benchmark-run.js +166 -0
- package/dist/src/evals/benchmark-run.js.map +1 -0
- package/dist/src/evals/collect-only-rerun.js +183 -0
- package/dist/src/evals/collect-only-rerun.js.map +1 -0
- package/dist/src/evals/directory.js +30 -0
- package/dist/src/evals/directory.js.map +1 -0
- package/dist/src/evals/docker-engine-stats.js +89 -0
- package/dist/src/evals/docker-engine-stats.js.map +1 -0
- package/dist/src/evals/docker-ownership.js +51 -0
- package/dist/src/evals/docker-ownership.js.map +1 -0
- package/dist/src/evals/docker-reaper.js +144 -0
- package/dist/src/evals/docker-reaper.js.map +1 -0
- package/dist/src/evals/docker-resource-observer.js +207 -0
- package/dist/src/evals/docker-resource-observer.js.map +1 -0
- package/dist/src/evals/duration-estimator.js +126 -0
- package/dist/src/evals/duration-estimator.js.map +1 -0
- package/dist/src/evals/environment-image-planning.js +164 -0
- package/dist/src/evals/environment-image-planning.js.map +1 -0
- package/dist/src/evals/eval-artifact-planning.js +56 -0
- package/dist/src/evals/eval-artifact-planning.js.map +1 -0
- package/dist/src/evals/eval-finalization.js +10 -0
- package/dist/src/evals/eval-finalization.js.map +1 -0
- package/dist/src/evals/eval-lifecycle-events.js +29 -0
- package/dist/src/evals/eval-lifecycle-events.js.map +1 -0
- package/dist/src/evals/eval-logical-plan.js +51 -0
- package/dist/src/evals/eval-logical-plan.js.map +1 -0
- package/dist/src/evals/eval-result-builder.js +81 -0
- package/dist/src/evals/eval-result-builder.js.map +1 -0
- package/dist/src/evals/events.js +53 -5
- package/dist/src/evals/events.js.map +1 -1
- package/dist/src/evals/evolution-baseline.js +191 -0
- package/dist/src/evals/evolution-baseline.js.map +1 -0
- package/dist/src/evals/execution-evidence.js +116 -0
- package/dist/src/evals/execution-evidence.js.map +1 -0
- package/dist/src/evals/execution-leases.js +342 -0
- package/dist/src/evals/execution-leases.js.map +1 -0
- package/dist/src/evals/execution-plan-artifacts.js +72 -0
- package/dist/src/evals/execution-plan-artifacts.js.map +1 -0
- package/dist/src/evals/execution-plan-images.js +69 -0
- package/dist/src/evals/execution-plan-images.js.map +1 -0
- package/dist/src/evals/execution-plan-resources.js +174 -0
- package/dist/src/evals/execution-plan-resources.js.map +1 -0
- package/dist/src/evals/execution-plan.js +451 -0
- package/dist/src/evals/execution-plan.js.map +1 -0
- package/dist/src/evals/failure-classifier.js +61 -0
- package/dist/src/evals/failure-classifier.js.map +1 -0
- package/dist/src/evals/harbor-artifact-builder-image.js +99 -0
- package/dist/src/evals/harbor-artifact-builder-image.js.map +1 -0
- package/dist/src/evals/harbor-artifact-builder.js +358 -0
- package/dist/src/evals/harbor-artifact-builder.js.map +1 -0
- package/dist/src/evals/harbor-bridge-error.js +58 -3
- package/dist/src/evals/harbor-bridge-error.js.map +1 -1
- package/dist/src/evals/harbor-node-runtime.js +146 -0
- package/dist/src/evals/harbor-node-runtime.js.map +1 -0
- package/dist/src/evals/harbor-phase-timings.js +60 -0
- package/dist/src/evals/harbor-phase-timings.js.map +1 -0
- package/dist/src/evals/index.js +40 -2
- package/dist/src/evals/index.js.map +1 -1
- package/dist/src/evals/infrastructure-retry.js +208 -84
- package/dist/src/evals/infrastructure-retry.js.map +1 -1
- package/dist/src/evals/interaction-capture-import.js +49 -0
- package/dist/src/evals/interaction-capture-import.js.map +1 -0
- package/dist/src/evals/local-docker-provider.js +372 -0
- package/dist/src/evals/local-docker-provider.js.map +1 -0
- package/dist/src/evals/local-eval-planning.js +80 -0
- package/dist/src/evals/local-eval-planning.js.map +1 -0
- package/dist/src/evals/model-capture-plan.js +22 -0
- package/dist/src/evals/model-capture-plan.js.map +1 -0
- package/dist/src/evals/model-capture-runtime.js +114 -0
- package/dist/src/evals/model-capture-runtime.js.map +1 -0
- package/dist/src/evals/model-proxy-runtime-state.js +53 -0
- package/dist/src/evals/model-proxy-runtime-state.js.map +1 -0
- package/dist/src/evals/native-phase-evidence.js +440 -0
- package/dist/src/evals/native-phase-evidence.js.map +1 -0
- package/dist/src/evals/physical-retry-work.js +31 -0
- package/dist/src/evals/physical-retry-work.js.map +1 -0
- package/dist/src/evals/planned-execution-support.js +66 -0
- package/dist/src/evals/planned-execution-support.js.map +1 -0
- package/dist/src/evals/planned-execution.js +469 -0
- package/dist/src/evals/planned-execution.js.map +1 -0
- package/dist/src/evals/planned-progress-publisher.js +62 -0
- package/dist/src/evals/planned-progress-publisher.js.map +1 -0
- package/dist/src/evals/planned-remote-execution.js +40 -0
- package/dist/src/evals/planned-remote-execution.js.map +1 -0
- package/dist/src/evals/planned-retry-execution.js +174 -0
- package/dist/src/evals/planned-retry-execution.js.map +1 -0
- package/dist/src/evals/planned-retry-lifecycle.js +188 -0
- package/dist/src/evals/planned-retry-lifecycle.js.map +1 -0
- package/dist/src/evals/preparation-rerun.js +225 -0
- package/dist/src/evals/preparation-rerun.js.map +1 -0
- package/dist/src/evals/prepared-harness.js +76 -0
- package/dist/src/evals/prepared-harness.js.map +1 -0
- package/dist/src/evals/progress.js +52 -12
- package/dist/src/evals/progress.js.map +1 -1
- package/dist/src/evals/records.js +20 -3
- package/dist/src/evals/records.js.map +1 -1
- package/dist/src/evals/recovery.js +242 -0
- package/dist/src/evals/recovery.js.map +1 -0
- package/dist/src/evals/regrade-evidence.js +77 -0
- package/dist/src/evals/regrade-evidence.js.map +1 -0
- package/dist/src/evals/remote-infrastructure-retry.js +165 -0
- package/dist/src/evals/remote-infrastructure-retry.js.map +1 -0
- package/dist/src/evals/request.js +21 -3
- package/dist/src/evals/request.js.map +1 -1
- package/dist/src/evals/rerun-inputs.js +108 -0
- package/dist/src/evals/rerun-inputs.js.map +1 -0
- package/dist/src/evals/rerun-slots.js +4 -3
- package/dist/src/evals/rerun-slots.js.map +1 -1
- package/dist/src/evals/rerun-types.js +45 -0
- package/dist/src/evals/rerun-types.js.map +1 -0
- package/dist/src/evals/rerun.js +133 -88
- package/dist/src/evals/rerun.js.map +1 -1
- package/dist/src/evals/result-helpers.js +78 -0
- package/dist/src/evals/result-helpers.js.map +1 -0
- package/dist/src/evals/resume-state.js +36 -0
- package/dist/src/evals/resume-state.js.map +1 -0
- package/dist/src/evals/retry-backoff.js +20 -0
- package/dist/src/evals/retry-backoff.js.map +1 -0
- package/dist/src/evals/retry-state.js +195 -0
- package/dist/src/evals/retry-state.js.map +1 -0
- package/dist/src/evals/scheduler-metrics.js +95 -0
- package/dist/src/evals/scheduler-metrics.js.map +1 -0
- package/dist/src/evals/scheduler-trace-replay.js +107 -0
- package/dist/src/evals/scheduler-trace-replay.js.map +1 -0
- package/dist/src/evals/service-types.js +2 -0
- package/dist/src/evals/service-types.js.map +1 -0
- package/dist/src/evals/service.js +331 -347
- package/dist/src/evals/service.js.map +1 -1
- package/dist/src/evals/task-resources.js +209 -0
- package/dist/src/evals/task-resources.js.map +1 -0
- package/dist/src/evals/trial-environment-evidence.js +87 -0
- package/dist/src/evals/trial-environment-evidence.js.map +1 -0
- package/dist/src/evals/trial-execution-evidence.js +13 -0
- package/dist/src/evals/trial-execution-evidence.js.map +1 -0
- package/dist/src/evals/trial-import-identity.js +31 -0
- package/dist/src/evals/trial-import-identity.js.map +1 -0
- package/dist/src/evals/trial-import.js +183 -188
- package/dist/src/evals/trial-import.js.map +1 -1
- package/dist/src/evals/trial-publication-recovery.js +144 -0
- package/dist/src/evals/trial-publication-recovery.js.map +1 -0
- package/dist/src/evals/trial-publication.js +25 -0
- package/dist/src/evals/trial-publication.js.map +1 -0
- package/dist/src/evals/trial-reference-validation.js +49 -0
- package/dist/src/evals/trial-reference-validation.js.map +1 -0
- package/dist/src/evals/verifier-artifacts.js +317 -0
- package/dist/src/evals/verifier-artifacts.js.map +1 -0
- package/dist/src/evals/verifier-eligibility.js +25 -0
- package/dist/src/evals/verifier-eligibility.js.map +1 -0
- package/dist/src/evals/verifier-only-rerun.js +210 -0
- package/dist/src/evals/verifier-only-rerun.js.map +1 -0
- package/dist/src/evals/verifier-runtime.js +61 -0
- package/dist/src/evals/verifier-runtime.js.map +1 -0
- package/dist/src/evals/verifier-score-artifacts.js +119 -0
- package/dist/src/evals/verifier-score-artifacts.js.map +1 -0
- package/dist/src/evals/work-item-artifacts.js +23 -0
- package/dist/src/evals/work-item-artifacts.js.map +1 -0
- package/dist/src/foundation/config.js +17 -0
- package/dist/src/foundation/config.js.map +1 -1
- package/dist/src/foundation/contained-file.js +85 -0
- package/dist/src/foundation/contained-file.js.map +1 -0
- package/dist/src/foundation/credential-redaction.js +162 -0
- package/dist/src/foundation/credential-redaction.js.map +1 -0
- package/dist/src/foundation/environment-image-references.js +59 -0
- package/dist/src/foundation/environment-image-references.js.map +1 -0
- package/dist/src/foundation/executable.js +45 -5
- package/dist/src/foundation/executable.js.map +1 -1
- package/dist/src/foundation/fs.js +2 -1
- package/dist/src/foundation/fs.js.map +1 -1
- package/dist/src/foundation/index.js +6 -2
- package/dist/src/foundation/index.js.map +1 -1
- package/dist/src/foundation/process-identity.js +69 -0
- package/dist/src/foundation/process-identity.js.map +1 -0
- package/dist/src/images/context.js +47 -0
- package/dist/src/images/context.js.map +1 -0
- package/dist/src/images/docker-buildkit.js +137 -0
- package/dist/src/images/docker-buildkit.js.map +1 -0
- package/dist/src/images/dockerfile.js +67 -0
- package/dist/src/images/dockerfile.js.map +1 -0
- package/dist/src/images/gc.js +261 -0
- package/dist/src/images/gc.js.map +1 -0
- package/dist/src/images/index.js +10 -0
- package/dist/src/images/index.js.map +1 -0
- package/dist/src/images/manifest.js +109 -0
- package/dist/src/images/manifest.js.map +1 -0
- package/dist/src/images/ownership.js +5 -0
- package/dist/src/images/ownership.js.map +1 -0
- package/dist/src/images/records.js +32 -0
- package/dist/src/images/records.js.map +1 -0
- package/dist/src/images/registry.js +154 -0
- package/dist/src/images/registry.js.map +1 -0
- package/dist/src/images/service.js +224 -0
- package/dist/src/images/service.js.map +1 -0
- package/dist/src/model-access/capture.js +255 -0
- package/dist/src/model-access/capture.js.map +1 -0
- package/dist/src/model-access/index.js +5 -0
- package/dist/src/model-access/index.js.map +1 -0
- package/dist/src/model-access/policy.js +49 -0
- package/dist/src/model-access/policy.js.map +1 -0
- package/dist/src/model-access/proxy.js +336 -0
- package/dist/src/model-access/proxy.js.map +1 -0
- package/dist/src/model-access/records.js +167 -0
- package/dist/src/model-access/records.js.map +1 -0
- package/dist/src/runs/adapter-process.js +12 -0
- package/dist/src/runs/adapter-process.js.map +1 -0
- package/dist/src/runs/bundle.js +450 -0
- package/dist/src/runs/bundle.js.map +1 -0
- package/dist/src/runs/events.js +12 -1
- package/dist/src/runs/events.js.map +1 -1
- package/dist/src/runs/executor.js +46 -45
- package/dist/src/runs/executor.js.map +1 -1
- package/dist/src/runs/finalizer.js +17 -0
- package/dist/src/runs/finalizer.js.map +1 -1
- package/dist/src/runs/index.js +8 -1
- package/dist/src/runs/index.js.map +1 -1
- package/dist/src/runs/manifest.js +5 -5
- package/dist/src/runs/manifest.js.map +1 -1
- package/dist/src/runs/phase-bundle.js +86 -0
- package/dist/src/runs/phase-bundle.js.map +1 -0
- package/dist/src/runs/phase-cancellation.js +107 -0
- package/dist/src/runs/phase-cancellation.js.map +1 -0
- package/dist/src/runs/phase-group.js +113 -0
- package/dist/src/runs/phase-group.js.map +1 -0
- package/dist/src/runs/query.js +1 -1
- package/dist/src/runs/query.js.map +1 -1
- package/dist/src/runs/records.js +3 -0
- package/dist/src/runs/records.js.map +1 -1
- package/dist/src/runs/request.js +10 -3
- package/dist/src/runs/request.js.map +1 -1
- package/dist/src/runs/training-candidate.js +161 -0
- package/dist/src/runs/training-candidate.js.map +1 -0
- package/dist/src/runs/verifier-evidence-redaction.js +61 -0
- package/dist/src/runs/verifier-evidence-redaction.js.map +1 -0
- package/dist/src/runs/verifier-evidence.js +459 -0
- package/dist/src/runs/verifier-evidence.js.map +1 -0
- package/dist/src/runs/verifier-structured-evidence.js +83 -0
- package/dist/src/runs/verifier-structured-evidence.js.map +1 -0
- package/dist/src/trajectories/analysis.js +288 -0
- package/dist/src/trajectories/analysis.js.map +1 -0
- package/dist/src/trajectories/chunk-projection.js +148 -0
- package/dist/src/trajectories/chunk-projection.js.map +1 -0
- package/dist/src/trajectories/content-projection.js +389 -0
- package/dist/src/trajectories/content-projection.js.map +1 -0
- package/dist/src/trajectories/dsh-chunk-contract.js +176 -0
- package/dist/src/trajectories/dsh-chunk-contract.js.map +1 -0
- package/dist/src/trajectories/dsh-contract.js +284 -0
- package/dist/src/trajectories/dsh-contract.js.map +1 -0
- package/dist/src/trajectories/events-chunk-drill.js +95 -0
- package/dist/src/trajectories/events-chunk-drill.js.map +1 -0
- package/dist/src/trajectories/events-page.js +367 -0
- package/dist/src/trajectories/events-page.js.map +1 -0
- package/dist/src/trajectories/index.js +3 -0
- package/dist/src/trajectories/index.js.map +1 -1
- package/dist/src/trajectories/provider-capture.js +15 -24
- package/dist/src/trajectories/provider-capture.js.map +1 -1
- package/dist/src/trajectories/providers/deepseek.js +3 -3
- package/dist/src/trajectories/providers/deepseek.js.map +1 -1
- package/dist/src/trajectories/request-attempt.js +47 -0
- package/dist/src/trajectories/request-attempt.js.map +1 -0
- package/dist/src/trajectories/stream-reader.js +311 -0
- package/dist/src/trajectories/stream-reader.js.map +1 -0
- package/dist/src/trajectories/surface-fold.js +126 -0
- package/dist/src/trajectories/surface-fold.js.map +1 -0
- package/dist/src/workers/index.js +3 -0
- package/dist/src/workers/index.js.map +1 -0
- package/dist/src/workers/remote-harbor-work-spec.js +97 -0
- package/dist/src/workers/remote-harbor-work-spec.js.map +1 -0
- package/dist/src/workers/remote-harbor-worker.js +223 -0
- package/dist/src/workers/remote-harbor-worker.js.map +1 -0
- package/docs/schemas/benchmark-hook-request-v1.schema.json +65 -0
- package/docs/schemas/benchmark-hook-response-v1.schema.json +75 -0
- package/docs/schemas/benchmark-hook-v1.schema.json +33 -0
- package/docs/schemas/benchmark-lock-v1.schema.json +351 -0
- package/docs/schemas/benchmark-metric-v1.schema.json +37 -0
- package/docs/schemas/benchmark-package-v1.schema.json +167 -0
- package/docs/schemas/benchmark-phase-group.schema.json +41 -0
- package/docs/schemas/benchmark-profile-v1.schema.json +141 -0
- package/docs/schemas/benchmark-task-v1.schema.json +384 -0
- package/docs/schemas/benchmark-tool-result-v1.schema.json +30 -0
- package/docs/schemas/environment-build-index.schema.json +13 -0
- package/docs/schemas/environment-build-record.schema.json +25 -0
- package/docs/schemas/environment-image-manifest.schema.json +62 -0
- package/docs/schemas/error.schema.json +22 -0
- package/docs/schemas/eval-control.schema.json +36 -0
- package/docs/schemas/eval-execution-plan.schema.json +216 -0
- package/docs/schemas/eval-progress.schema.json +1 -15
- package/docs/schemas/eval-rerun-result.schema.json +58 -0
- package/docs/schemas/eval-rerun-state.schema.json +61 -0
- package/docs/schemas/eval-rerun-submission.schema.json +60 -0
- package/docs/schemas/eval-result.schema.json +32 -13
- package/docs/schemas/eval-submission-input.schema.json +14 -0
- package/docs/schemas/eval-submission.schema.json +68 -0
- package/docs/schemas/eval-trial-publication.schema.json +15 -0
- package/docs/schemas/eval-trial-reference.schema.json +129 -0
- package/docs/schemas/execution-evidence.schema.json +96 -0
- package/docs/schemas/execution-lease.schema.json +50 -0
- package/docs/schemas/execution-provider-status.schema.json +65 -0
- package/docs/schemas/execution-worker.schema.json +53 -0
- package/docs/schemas/harbor-process-exit.schema.json +14 -0
- package/docs/schemas/interaction-capture-ref.schema.json +45 -0
- package/docs/schemas/local-provider-execution.schema.json +33 -0
- package/docs/schemas/model-interaction.schema.json +41 -0
- package/docs/schemas/regrade-assessment-reference.schema.json +11 -0
- package/docs/schemas/remote-result-envelope.schema.json +32 -0
- package/docs/schemas/remote-tree-envelope.schema.json +25 -0
- package/docs/schemas/remote-work-offer.schema.json +58 -0
- package/docs/schemas/remote-work-receipt.schema.json +50 -0
- package/docs/schemas/remote-worker-artifact.schema.json +17 -0
- package/docs/schemas/remote-worker-event.schema.json +18 -0
- package/docs/schemas/remote-worker-heartbeat.schema.json +37 -0
- package/docs/schemas/remote-worker-registration.schema.json +52 -0
- package/docs/schemas/result-bundle-index.schema.json +133 -0
- package/docs/schemas/run-context.schema.json +15 -0
- package/docs/schemas/run-environment-images.schema.json +39 -0
- package/docs/schemas/run-manifest.schema.json +4 -0
- package/docs/schemas/run-request.schema.json +4 -0
- package/docs/schemas/training-data-candidate.schema.json +46 -0
- package/docs/schemas/trajectory-analysis.schema.json +258 -0
- package/docs/schemas/trajectory-events-page.schema.json +40 -0
- package/docs/schemas/verifier-assessment.schema.json +77 -0
- package/docs/schemas/verifier-evidence.schema.json +276 -0
- package/integrations/harbor/hitch_benchmark.py +252 -0
- package/integrations/harbor/hitch_candidate_recycle.py +262 -0
- package/integrations/harbor/hitch_harbor_agent.py +698 -568
- package/integrations/harbor/hitch_harbor_environment.py +469 -0
- package/integrations/harbor/hitch_harbor_task_resources.py +275 -0
- package/integrations/harbor/hitch_harbor_verifier.py +83 -2
- package/integrations/harbor/hitch_phase_supervisor.py +452 -0
- package/integrations/harbor/hitch_tool_client.mjs +91 -0
- package/integrations/model-call/cli.js +57 -0
- package/package.json +30 -6
|
@@ -3,36 +3,91 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import asyncio
|
|
6
|
-
import errno
|
|
7
6
|
import hashlib
|
|
8
7
|
import json
|
|
9
8
|
import os
|
|
10
9
|
import re
|
|
10
|
+
import secrets
|
|
11
11
|
import shlex
|
|
12
|
-
import shutil
|
|
13
12
|
import stat as stat_module
|
|
14
13
|
import tempfile
|
|
14
|
+
import time
|
|
15
15
|
import uuid
|
|
16
|
+
from urllib.parse import urlparse
|
|
16
17
|
from datetime import datetime, timezone
|
|
17
18
|
from pathlib import Path, PurePosixPath
|
|
18
|
-
from typing import Any
|
|
19
|
+
from typing import Any, NamedTuple
|
|
19
20
|
|
|
20
21
|
from harbor.agents.base import BaseAgent
|
|
21
22
|
from harbor.environments.base import BaseEnvironment, ExecResult
|
|
22
23
|
from harbor.models.agent.context import AgentContext
|
|
23
24
|
|
|
24
25
|
CONTROLLER_RUNTIME_MANIFEST_VERSION = "2"
|
|
25
|
-
LOCAL_GIT_TRANSPORT_MANIFEST_VERSION = "1"
|
|
26
|
-
HARNESS_ARTIFACT_MANIFEST_VERSION = "1"
|
|
27
|
-
LOCAL_GIT_TRANSPORT_MAX_BYTES = 512 * 1024 * 1024
|
|
28
|
-
LOCAL_GIT_REMOTE_ROOT = "/opt/hitch-local-source"
|
|
29
26
|
HARNESS_ARTIFACT_REMOTE_ROOT = "/opt/hitch-harness-artifact"
|
|
30
|
-
HITCH_CONTAINER_STATE_ROOT = "/tmp/hitch-state"
|
|
31
27
|
HITCH_BRIDGE_ERROR_LOG = "/logs/agent/hitch-bridge-error.json"
|
|
28
|
+
HITCH_AGENT_OUTCOME_NAME = "hitch-agent-outcome.json"
|
|
32
29
|
HITCH_DIAGNOSTIC_MAX_BYTES = 8 * 1024
|
|
33
30
|
HITCH_BRIDGE_ERROR_MAX_BYTES = 64 * 1024
|
|
34
31
|
HITCH_RESULT_MISSING_EXIT = 44
|
|
35
32
|
HITCH_RESULT_NOT_FILE_EXIT = 45
|
|
33
|
+
PHASE_EXPORT_MODULE = "dist/src/runs/phase-bundle.js"
|
|
34
|
+
PHASE_CONTROL_MODULE = "dist/src/runs/phase-cancellation.js"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def artifact_directory_integrity(directory: Path, captured_files: dict[str, bytes] | None = None) -> str:
|
|
38
|
+
"""Match artifacts/integrity.ts, including modes and internal symlinks."""
|
|
39
|
+
root = Path(os.path.abspath(directory))
|
|
40
|
+
digest = hashlib.sha256()
|
|
41
|
+
|
|
42
|
+
def visit(parent: Path) -> None:
|
|
43
|
+
# JS Array.sort compares UTF-16 code units, not Unicode code points.
|
|
44
|
+
for entry in sorted(parent.iterdir(), key=lambda item: item.name.encode("utf-16-be", "surrogatepass")):
|
|
45
|
+
if parent == root and entry.name == "artifact.json":
|
|
46
|
+
continue
|
|
47
|
+
relative = entry.relative_to(root).as_posix()
|
|
48
|
+
info = entry.lstat()
|
|
49
|
+
mode = info.st_mode & 0o7777
|
|
50
|
+
if stat_module.S_ISDIR(info.st_mode):
|
|
51
|
+
digest.update(f"d\0{relative}\0{mode}\0".encode())
|
|
52
|
+
visit(entry)
|
|
53
|
+
elif stat_module.S_ISREG(info.st_mode):
|
|
54
|
+
digest.update(f"f\0{relative}\0{mode}\0{info.st_size}\0".encode())
|
|
55
|
+
capture = captured_files is not None and relative in captured_files
|
|
56
|
+
if capture and info.st_size > 16_384:
|
|
57
|
+
raise RuntimeError("captured artifact metadata exceeds its size limit")
|
|
58
|
+
content = bytearray()
|
|
59
|
+
with entry.open("rb") as handle:
|
|
60
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
61
|
+
digest.update(chunk)
|
|
62
|
+
if capture:
|
|
63
|
+
content.extend(chunk)
|
|
64
|
+
if len(content) > 16_384:
|
|
65
|
+
raise RuntimeError("captured artifact metadata exceeds its size limit")
|
|
66
|
+
if capture and captured_files is not None:
|
|
67
|
+
captured_files[relative] = bytes(content)
|
|
68
|
+
digest.update(b"\0")
|
|
69
|
+
elif stat_module.S_ISLNK(info.st_mode):
|
|
70
|
+
target = os.readlink(entry)
|
|
71
|
+
if not Path(os.path.abspath(entry.parent / target)).is_relative_to(root):
|
|
72
|
+
raise RuntimeError("artifact symlink escapes the artifact directory")
|
|
73
|
+
digest.update(f"l\0{relative}\0{target}\0".encode())
|
|
74
|
+
else:
|
|
75
|
+
raise RuntimeError("artifact contains a special file")
|
|
76
|
+
|
|
77
|
+
visit(root)
|
|
78
|
+
return "sha256:" + digest.hexdigest()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class PreparedPhase(NamedTuple):
|
|
82
|
+
run_id: str
|
|
83
|
+
instruction: str
|
|
84
|
+
context_json: str
|
|
85
|
+
parent_json: str
|
|
86
|
+
identity: str
|
|
87
|
+
deadline_ns: int
|
|
88
|
+
|
|
89
|
+
def __repr__(self) -> str:
|
|
90
|
+
return f"PreparedPhase(run_id={self.run_id!r})"
|
|
36
91
|
|
|
37
92
|
|
|
38
93
|
class HitchBridgeError(RuntimeError):
|
|
@@ -56,16 +111,22 @@ class HitchHarborAgent(BaseAgent):
|
|
|
56
111
|
candidate_id: str = "candidate-1",
|
|
57
112
|
controller_runtime_id: str | None = None,
|
|
58
113
|
harness_artifact: dict[str, Any] | None = None,
|
|
114
|
+
node_version: str = "v22.23.0",
|
|
115
|
+
# Legacy parameters are accepted only so old persisted jobs fail at
|
|
116
|
+
# the explicit artifact handoff boundary instead of leaking unknown
|
|
117
|
+
# kwargs into Harbor's BaseAgent. New jobs pass neither value.
|
|
59
118
|
harness_artifact_cache_dir: str | None = None,
|
|
60
119
|
local_source_transport: dict[str, Any] | None = None,
|
|
61
120
|
hitch_timeout_ms: int = 900_000,
|
|
62
121
|
agent_args: list[str] | None = None,
|
|
122
|
+
credential_names: list[str] | None = None,
|
|
63
123
|
workdir: str | None = None,
|
|
64
124
|
eval_id: str | None = None,
|
|
65
125
|
benchmark_id: str | None = None,
|
|
66
126
|
benchmark_revision: str | None = None,
|
|
67
127
|
verifier_identity: str | None = None,
|
|
68
128
|
logical_attempt: int | None = None,
|
|
129
|
+
model_capture: dict[str, Any] | None = None,
|
|
69
130
|
**kwargs: Any,
|
|
70
131
|
) -> None:
|
|
71
132
|
super().__init__(logs_dir=logs_dir, **kwargs)
|
|
@@ -74,11 +135,24 @@ class HitchHarborAgent(BaseAgent):
|
|
|
74
135
|
self.hitch_runtime_dir = Path(hitch_runtime_dir)
|
|
75
136
|
self.controller_runtime_id = controller_runtime_id
|
|
76
137
|
self.harness_artifact = dict(harness_artifact) if harness_artifact else None
|
|
77
|
-
|
|
78
|
-
|
|
138
|
+
if harness_artifact_cache_dir is not None:
|
|
139
|
+
raise ValueError("trial-side harness artifact caches are no longer supported")
|
|
140
|
+
if not isinstance(node_version, str) or re.fullmatch(r"v\d+\.\d+\.\d+", node_version) is None:
|
|
141
|
+
raise ValueError("node_version must be an exact stable Node.js version")
|
|
142
|
+
self.required_node_version = node_version
|
|
143
|
+
self._node_bin_directory: str | None = None
|
|
144
|
+
if local_source_transport is not None:
|
|
145
|
+
raise ValueError("trial-side local source transports are no longer supported")
|
|
79
146
|
self.candidate_id = candidate_id
|
|
80
147
|
self.hitch_timeout_ms = int(hitch_timeout_ms)
|
|
81
148
|
self.agent_args = list(agent_args or [])
|
|
149
|
+
raw_credential_names = list(credential_names or [])
|
|
150
|
+
if (
|
|
151
|
+
any(not isinstance(name, str) or re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name) is None for name in raw_credential_names)
|
|
152
|
+
or len(set(raw_credential_names)) != len(raw_credential_names)
|
|
153
|
+
):
|
|
154
|
+
raise ValueError("credential_names must contain unique environment variable names")
|
|
155
|
+
self.credential_names = sorted(raw_credential_names)
|
|
82
156
|
if workdir is not None and (not isinstance(workdir, str) or not workdir.strip()):
|
|
83
157
|
raise ValueError("workdir must be a non-empty string when provided")
|
|
84
158
|
self.workdir = workdir.strip() if isinstance(workdir, str) else None
|
|
@@ -90,13 +164,24 @@ class HitchHarborAgent(BaseAgent):
|
|
|
90
164
|
if logical_attempt is not None and (isinstance(logical_attempt, bool) or not isinstance(logical_attempt, int) or logical_attempt < 1):
|
|
91
165
|
raise ValueError("logical_attempt must be a positive integer")
|
|
92
166
|
self.logical_attempt = logical_attempt
|
|
167
|
+
self.model_capture = _validate_model_capture(model_capture)
|
|
93
168
|
self._hitch_version: str | None = None
|
|
94
169
|
self._entrypoint: str | None = None
|
|
95
170
|
self._artifact_manifest: dict[str, Any] | None = None
|
|
96
171
|
self._artifact_host_directory: Path | None = None
|
|
97
172
|
self._artifact_uploaded = False
|
|
98
173
|
self._artifact_transport_status: str | None = None
|
|
99
|
-
self.
|
|
174
|
+
self._setup_complete = False
|
|
175
|
+
self._phase_export_available = False
|
|
176
|
+
self._phase_supervision_available = False
|
|
177
|
+
self._prepared_phase: PreparedPhase | None = None
|
|
178
|
+
self._prepared_phase_keys: set[tuple[str, int]] = set()
|
|
179
|
+
self._phase_group_contracts: dict[str, tuple[str, str, int]] = {}
|
|
180
|
+
self._phase_inflight = False
|
|
181
|
+
self._active_phase: PreparedPhase | None = None
|
|
182
|
+
self._phase_cancel_lock: asyncio.Lock | None = None
|
|
183
|
+
self._phase_cancel_receipts: dict[str, dict[str, Any]] = {}
|
|
184
|
+
self._phase_control_tokens: dict[str, str] = {}
|
|
100
185
|
|
|
101
186
|
@staticmethod
|
|
102
187
|
def name() -> str:
|
|
@@ -106,6 +191,15 @@ class HitchHarborAgent(BaseAgent):
|
|
|
106
191
|
return self._hitch_version
|
|
107
192
|
|
|
108
193
|
async def setup(self, environment: BaseEnvironment) -> None:
|
|
194
|
+
started_ns = time.monotonic_ns()
|
|
195
|
+
self._setup_complete = False
|
|
196
|
+
try:
|
|
197
|
+
await self._setup(environment)
|
|
198
|
+
self._setup_complete = True
|
|
199
|
+
finally:
|
|
200
|
+
self._write_phase_timing("setup", started_ns)
|
|
201
|
+
|
|
202
|
+
async def _setup(self, environment: BaseEnvironment) -> None:
|
|
109
203
|
if not self.hitch_runtime_dir.is_dir():
|
|
110
204
|
raise RuntimeError(f"Hitch runtime directory does not exist: {self.hitch_runtime_dir}")
|
|
111
205
|
# The manifest declares the CLI entrypoint (relative to the upload
|
|
@@ -129,22 +223,15 @@ class HitchHarborAgent(BaseAgent):
|
|
|
129
223
|
# and the actual container upload, spec §4.6).
|
|
130
224
|
self._verify_manifest_identity(manifest)
|
|
131
225
|
self._verify_payload(manifest)
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
except Exception as error:
|
|
142
|
-
raise RuntimeError(f"hitch-artifact-cache: {error}") from error
|
|
143
|
-
if self.local_source_transport is not None:
|
|
144
|
-
try:
|
|
145
|
-
self._local_manifest = self._verify_local_source_host()
|
|
146
|
-
except Exception as error:
|
|
147
|
-
raise RuntimeError(f"hitch-local-source-materialize: {error}") from error
|
|
226
|
+
self._phase_export_available = {PHASE_EXPORT_MODULE, PHASE_CONTROL_MODULE}.issubset({file.get("path") for file in manifest["files"]})
|
|
227
|
+
self._phase_supervision_available = self._phase_export_available and "integrations/harbor/hitch_phase_supervisor.py" in {file.get("path") for file in manifest["files"]}
|
|
228
|
+
if self.harness_artifact is None:
|
|
229
|
+
raise RuntimeError("hitch-artifact-materialize: Harbor requires a dedicated-builder artifact")
|
|
230
|
+
try:
|
|
231
|
+
self._artifact_manifest = self._verify_harness_artifact_host()
|
|
232
|
+
self._artifact_host_directory = Path(str(self.harness_artifact["directory"]))
|
|
233
|
+
except Exception as error:
|
|
234
|
+
raise RuntimeError(f"hitch-artifact-materialize: {error}") from error
|
|
148
235
|
payload_dir = self.hitch_runtime_dir / "payload"
|
|
149
236
|
if not payload_dir.is_dir():
|
|
150
237
|
raise RuntimeError(f"Hitch runtime bundle has no payload directory: {self.hitch_runtime_dir}")
|
|
@@ -156,46 +243,18 @@ class HitchHarborAgent(BaseAgent):
|
|
|
156
243
|
await self._ensure_node(environment)
|
|
157
244
|
platform = await self._container_platform(environment)
|
|
158
245
|
node_version = await self._container_node_version(environment)
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
await self._release_artifact_cache_lock(cache_lock)
|
|
172
|
-
cache_lock = None
|
|
173
|
-
await self._upload_harness_artifact(environment, self._artifact_host_directory)
|
|
174
|
-
self._artifact_transport_status = "host_cache_hit"
|
|
175
|
-
|
|
176
|
-
if self._local_manifest is not None and not self._artifact_uploaded:
|
|
177
|
-
try:
|
|
178
|
-
await self._upload_and_materialize_local_source(environment, self._local_manifest)
|
|
179
|
-
except Exception as error:
|
|
180
|
-
raise RuntimeError(f"hitch-local-source-materialize: {error}") from error
|
|
181
|
-
entry = self._remote_entry(entrypoint)
|
|
182
|
-
version = await self._exec(environment, f"{self._node_prefix()} node {entry} --version")
|
|
183
|
-
self._hitch_version = (version.stdout or "").strip() or None
|
|
184
|
-
if not self._artifact_uploaded:
|
|
185
|
-
prepared = await self._prepare_harness_in_container(environment, entry)
|
|
186
|
-
if cache_lock is not None:
|
|
187
|
-
try:
|
|
188
|
-
self._artifact_manifest, self._artifact_host_directory = await self._cache_container_artifact(
|
|
189
|
-
environment, prepared, platform, node_version
|
|
190
|
-
)
|
|
191
|
-
self._artifact_transport_status = "host_cache_populated"
|
|
192
|
-
except Exception:
|
|
193
|
-
self._artifact_transport_status = "host_cache_write_failed"
|
|
194
|
-
else:
|
|
195
|
-
self._artifact_transport_status = "container_prepare"
|
|
196
|
-
finally:
|
|
197
|
-
if cache_lock is not None:
|
|
198
|
-
await self._release_artifact_cache_lock(cache_lock)
|
|
246
|
+
if self._artifact_manifest is None or not self._artifact_compatible(
|
|
247
|
+
self._artifact_manifest, platform, node_version
|
|
248
|
+
):
|
|
249
|
+
raise RuntimeError(
|
|
250
|
+
"hitch-artifact-platform: dedicated-builder artifact is incompatible with "
|
|
251
|
+
f"trial runtime {platform}/{node_version}"
|
|
252
|
+
)
|
|
253
|
+
await self._upload_harness_artifact(environment, self._artifact_host_directory)
|
|
254
|
+
self._artifact_transport_status = "dedicated_builder_upload"
|
|
255
|
+
entry = self._remote_entry(entrypoint)
|
|
256
|
+
version = await self._exec(environment, f"{self._node_prefix()} node {entry} --version")
|
|
257
|
+
self._hitch_version = (version.stdout or "").strip() or None
|
|
199
258
|
|
|
200
259
|
async def _resolve_workdir(self, environment: BaseEnvironment) -> str:
|
|
201
260
|
"""Resolve Harbor's effective task directory and prove it is usable.
|
|
@@ -302,6 +361,26 @@ class HitchHarborAgent(BaseAgent):
|
|
|
302
361
|
raise RuntimeError("Hitch agent setup() must resolve the Harbor task working directory before run()")
|
|
303
362
|
return self.workdir
|
|
304
363
|
|
|
364
|
+
def _write_phase_timing(self, phase: str, started_ns: int) -> None:
|
|
365
|
+
if phase not in {"setup", "agent"}:
|
|
366
|
+
raise ValueError("invalid Harbor agent phase timing")
|
|
367
|
+
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
|
368
|
+
target = self.logs_dir / "hitch-phase-timings.json"
|
|
369
|
+
value: dict[str, Any] = {"schema_version": "1", "phases": {}}
|
|
370
|
+
try:
|
|
371
|
+
existing = json.loads(target.read_text(encoding="utf-8")) if target.is_file() else None
|
|
372
|
+
if isinstance(existing, dict) and existing.get("schema_version") == "1" and isinstance(existing.get("phases"), dict):
|
|
373
|
+
value = existing
|
|
374
|
+
except (OSError, json.JSONDecodeError):
|
|
375
|
+
pass
|
|
376
|
+
value["phases"][phase] = {
|
|
377
|
+
"duration_ms": max(0, (time.monotonic_ns() - started_ns) // 1_000_000),
|
|
378
|
+
"completed_at": datetime.now(timezone.utc).isoformat(),
|
|
379
|
+
}
|
|
380
|
+
temporary = target.with_suffix(".json.tmp")
|
|
381
|
+
temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
382
|
+
temporary.replace(target)
|
|
383
|
+
|
|
305
384
|
def _verify_harness_artifact_host(self) -> dict[str, Any]:
|
|
306
385
|
"""Pin host artifact metadata before Harbor copies the directory."""
|
|
307
386
|
transport = self.harness_artifact or {}
|
|
@@ -356,6 +435,18 @@ class HitchHarborAgent(BaseAgent):
|
|
|
356
435
|
raise RuntimeError("prepared artifact Node.js version is invalid")
|
|
357
436
|
return manifest
|
|
358
437
|
|
|
438
|
+
@staticmethod
|
|
439
|
+
def _assert_regular_host_file(path: Path, label: str, maximum: int) -> Any:
|
|
440
|
+
try:
|
|
441
|
+
info = path.lstat()
|
|
442
|
+
except OSError as error:
|
|
443
|
+
raise RuntimeError(f"{label} is missing or unreadable") from error
|
|
444
|
+
if not stat_module.S_ISREG(info.st_mode):
|
|
445
|
+
raise RuntimeError(f"{label} must be a regular file")
|
|
446
|
+
if info.st_size > maximum:
|
|
447
|
+
raise RuntimeError(f"{label} exceeds the size limit ({maximum} bytes)")
|
|
448
|
+
return info
|
|
449
|
+
|
|
359
450
|
async def _container_platform(self, environment: BaseEnvironment) -> str:
|
|
360
451
|
result = await self._exec(
|
|
361
452
|
environment,
|
|
@@ -390,448 +481,6 @@ class HitchHarborAgent(BaseAgent):
|
|
|
390
481
|
await environment.upload_dir(directory, HARNESS_ARTIFACT_REMOTE_ROOT)
|
|
391
482
|
self._artifact_uploaded = True
|
|
392
483
|
|
|
393
|
-
async def _prepare_harness_in_container(
|
|
394
|
-
self,
|
|
395
|
-
environment: BaseEnvironment,
|
|
396
|
-
entry: str,
|
|
397
|
-
) -> ExecResult:
|
|
398
|
-
prepare = " ".join(
|
|
399
|
-
[
|
|
400
|
-
self._node_prefix(),
|
|
401
|
-
f"HITCH_ROOT={HITCH_CONTAINER_STATE_ROOT}",
|
|
402
|
-
*(["HITCH_HARBOR_INTERNAL=1"] if self._local_manifest is not None else []),
|
|
403
|
-
f"node {entry} prepare",
|
|
404
|
-
shlex.quote(self.harness_ref),
|
|
405
|
-
*self._local_source_cli_args(),
|
|
406
|
-
"--json",
|
|
407
|
-
]
|
|
408
|
-
)
|
|
409
|
-
return await self._exec(environment, prepare)
|
|
410
|
-
|
|
411
|
-
def _verify_harness_artifact_cache_host(self) -> None:
|
|
412
|
-
cache = self.harness_artifact_cache_dir
|
|
413
|
-
if cache is None or not cache.is_absolute():
|
|
414
|
-
raise RuntimeError("prepared artifact cache path must be absolute")
|
|
415
|
-
try:
|
|
416
|
-
info = cache.lstat()
|
|
417
|
-
except OSError as error:
|
|
418
|
-
raise RuntimeError("prepared artifact cache is missing or unreadable") from error
|
|
419
|
-
if not stat_module.S_ISDIR(info.st_mode) or cache.is_symlink():
|
|
420
|
-
raise RuntimeError("prepared artifact cache must be a regular directory")
|
|
421
|
-
for name in ("artifacts", "locks", "tmp", "invalid"):
|
|
422
|
-
child = cache / name
|
|
423
|
-
child.mkdir(mode=0o700, exist_ok=True)
|
|
424
|
-
child_info = child.lstat()
|
|
425
|
-
if not stat_module.S_ISDIR(child_info.st_mode) or child.is_symlink():
|
|
426
|
-
raise RuntimeError(f"prepared artifact cache {name} path must be a regular directory")
|
|
427
|
-
|
|
428
|
-
def _artifact_cache_key(self, platform: str, node_version: str) -> str:
|
|
429
|
-
payload = json.dumps(
|
|
430
|
-
[
|
|
431
|
-
self.revision_identity,
|
|
432
|
-
self.harness_artifact.get("adapter_version") if self.harness_artifact else None,
|
|
433
|
-
self.harness_artifact.get("recipe_version") if self.harness_artifact else None,
|
|
434
|
-
platform,
|
|
435
|
-
node_version,
|
|
436
|
-
],
|
|
437
|
-
separators=(",", ":"),
|
|
438
|
-
).encode("utf-8")
|
|
439
|
-
return hashlib.sha256(payload).hexdigest()
|
|
440
|
-
|
|
441
|
-
async def _acquire_artifact_cache_lock(self, platform: str, node_version: str) -> Any:
|
|
442
|
-
cache = self.harness_artifact_cache_dir
|
|
443
|
-
if cache is None:
|
|
444
|
-
raise RuntimeError("prepared artifact cache is unavailable")
|
|
445
|
-
handle = (cache / "locks" / f"{self._artifact_cache_key(platform, node_version)}.lock").open("a+b")
|
|
446
|
-
try:
|
|
447
|
-
while True:
|
|
448
|
-
try:
|
|
449
|
-
self._try_lock_file(handle)
|
|
450
|
-
return handle
|
|
451
|
-
except OSError as error:
|
|
452
|
-
if error.errno not in (errno.EACCES, errno.EAGAIN):
|
|
453
|
-
raise
|
|
454
|
-
await asyncio.sleep(0.1)
|
|
455
|
-
except BaseException:
|
|
456
|
-
handle.close()
|
|
457
|
-
raise
|
|
458
|
-
|
|
459
|
-
@staticmethod
|
|
460
|
-
def _try_lock_file(handle: Any) -> None:
|
|
461
|
-
if os.name == "nt":
|
|
462
|
-
import msvcrt
|
|
463
|
-
handle.seek(0)
|
|
464
|
-
if not handle.read(1):
|
|
465
|
-
handle.write(b"\0")
|
|
466
|
-
handle.flush()
|
|
467
|
-
handle.seek(0)
|
|
468
|
-
msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
|
|
469
|
-
return
|
|
470
|
-
import fcntl
|
|
471
|
-
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
472
|
-
|
|
473
|
-
@staticmethod
|
|
474
|
-
async def _release_artifact_cache_lock(handle: Any) -> None:
|
|
475
|
-
try:
|
|
476
|
-
if os.name == "nt":
|
|
477
|
-
import msvcrt
|
|
478
|
-
handle.seek(0)
|
|
479
|
-
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
|
480
|
-
else:
|
|
481
|
-
import fcntl
|
|
482
|
-
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
|
483
|
-
finally:
|
|
484
|
-
handle.close()
|
|
485
|
-
|
|
486
|
-
def _find_cached_harness_artifact(
|
|
487
|
-
self,
|
|
488
|
-
platform: str,
|
|
489
|
-
node_version: str,
|
|
490
|
-
) -> tuple[dict[str, Any], Path] | None:
|
|
491
|
-
cache = self.harness_artifact_cache_dir
|
|
492
|
-
if cache is None:
|
|
493
|
-
return None
|
|
494
|
-
requested = self.harness_artifact or {}
|
|
495
|
-
for directory in sorted((cache / "artifacts").iterdir(), key=lambda item: item.name):
|
|
496
|
-
if not re.fullmatch(r"[0-9a-f]{64}", directory.name):
|
|
497
|
-
continue
|
|
498
|
-
try:
|
|
499
|
-
manifest = json.loads((directory / "artifact.json").read_text(encoding="utf-8"))
|
|
500
|
-
except (OSError, UnicodeError, json.JSONDecodeError):
|
|
501
|
-
continue
|
|
502
|
-
if not isinstance(manifest, dict) or (
|
|
503
|
-
manifest.get("revision_identity") != self.revision_identity
|
|
504
|
-
or manifest.get("adapter_version") != requested.get("adapter_version")
|
|
505
|
-
or manifest.get("recipe_version") != requested.get("recipe_version")
|
|
506
|
-
or not self._artifact_compatible(manifest, platform, node_version)
|
|
507
|
-
):
|
|
508
|
-
continue
|
|
509
|
-
try:
|
|
510
|
-
return self._verify_cached_harness_artifact(directory, platform, node_version), directory
|
|
511
|
-
except Exception:
|
|
512
|
-
self._quarantine_cached_artifact(directory)
|
|
513
|
-
return None
|
|
514
|
-
|
|
515
|
-
async def _cache_container_artifact(
|
|
516
|
-
self,
|
|
517
|
-
environment: BaseEnvironment,
|
|
518
|
-
prepared: ExecResult,
|
|
519
|
-
platform: str,
|
|
520
|
-
node_version: str,
|
|
521
|
-
) -> tuple[dict[str, Any], Path]:
|
|
522
|
-
try:
|
|
523
|
-
payload = json.loads(prepared.stdout or "")
|
|
524
|
-
artifact = payload["artifact"]
|
|
525
|
-
except (KeyError, TypeError, json.JSONDecodeError) as error:
|
|
526
|
-
raise RuntimeError("container prepare returned invalid artifact metadata") from error
|
|
527
|
-
artifact_id = artifact.get("artifact_id") if isinstance(artifact, dict) else None
|
|
528
|
-
if not isinstance(artifact_id, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", artifact_id):
|
|
529
|
-
raise RuntimeError("container prepare returned an invalid artifact ID")
|
|
530
|
-
if (
|
|
531
|
-
artifact.get("revision_identity") != self.revision_identity
|
|
532
|
-
or not self._artifact_compatible(artifact, platform, node_version)
|
|
533
|
-
):
|
|
534
|
-
raise RuntimeError("container prepare returned an incompatible artifact")
|
|
535
|
-
cache = self.harness_artifact_cache_dir
|
|
536
|
-
if cache is None:
|
|
537
|
-
raise RuntimeError("prepared artifact cache is unavailable")
|
|
538
|
-
staging = cache / "tmp" / f"artifact-{uuid.uuid4().hex}"
|
|
539
|
-
staging.mkdir(mode=0o700)
|
|
540
|
-
try:
|
|
541
|
-
remote = f'{HITCH_CONTAINER_STATE_ROOT}/store/artifacts/{artifact_id.removeprefix("sha256:")}'
|
|
542
|
-
await environment.download_dir(remote, staging)
|
|
543
|
-
manifest = self._verify_cached_harness_artifact(staging, platform, node_version)
|
|
544
|
-
if manifest.get("artifact_id") != artifact_id:
|
|
545
|
-
raise RuntimeError("downloaded artifact ID differs from container prepare output")
|
|
546
|
-
destination = cache / "artifacts" / artifact_id.removeprefix("sha256:")
|
|
547
|
-
if destination.exists():
|
|
548
|
-
existing = self._verify_cached_harness_artifact(destination, platform, node_version)
|
|
549
|
-
shutil.rmtree(staging)
|
|
550
|
-
return existing, destination
|
|
551
|
-
staging.rename(destination)
|
|
552
|
-
return manifest, destination
|
|
553
|
-
finally:
|
|
554
|
-
if staging.exists():
|
|
555
|
-
shutil.rmtree(staging, ignore_errors=True)
|
|
556
|
-
|
|
557
|
-
def _verify_cached_harness_artifact(
|
|
558
|
-
self,
|
|
559
|
-
directory: Path,
|
|
560
|
-
platform: str,
|
|
561
|
-
node_version: str,
|
|
562
|
-
) -> dict[str, Any]:
|
|
563
|
-
info = directory.lstat()
|
|
564
|
-
if not stat_module.S_ISDIR(info.st_mode) or directory.is_symlink():
|
|
565
|
-
raise RuntimeError("cached harness artifact is not a regular directory")
|
|
566
|
-
artifact_file = directory / "artifact.json"
|
|
567
|
-
self._assert_regular_host_file(artifact_file, "cached harness artifact manifest", 1024 * 1024)
|
|
568
|
-
manifest = json.loads(artifact_file.read_text(encoding="utf-8"))
|
|
569
|
-
if not isinstance(manifest, dict) or manifest.get("schema_version") != HARNESS_ARTIFACT_MANIFEST_VERSION:
|
|
570
|
-
raise RuntimeError("cached harness artifact manifest schema is invalid")
|
|
571
|
-
for field in ("artifact_id", "artifact_integrity", "entrypoint_integrity", "revision_identity"):
|
|
572
|
-
if not re.fullmatch(r"sha256:[0-9a-f]{64}", str(manifest.get(field, ""))):
|
|
573
|
-
raise RuntimeError(f"cached harness artifact {field} is invalid")
|
|
574
|
-
if (
|
|
575
|
-
manifest.get("artifact_id") != f"sha256:{directory.name}" and directory.parent.name == "artifacts"
|
|
576
|
-
):
|
|
577
|
-
raise RuntimeError("cached harness artifact directory does not match its ID")
|
|
578
|
-
if (
|
|
579
|
-
manifest.get("revision_identity") != self.revision_identity
|
|
580
|
-
or manifest.get("resolved_revision", {}).get("identity") != self.revision_identity
|
|
581
|
-
or manifest.get("harness_id") != self.harness_ref.split("@", 1)[0]
|
|
582
|
-
or manifest.get("adapter_version") != (self.harness_artifact or {}).get("adapter_version")
|
|
583
|
-
or manifest.get("recipe_version") != (self.harness_artifact or {}).get("recipe_version")
|
|
584
|
-
or not self._artifact_compatible(manifest, platform, node_version)
|
|
585
|
-
or manifest.get("source_type") == "installed"
|
|
586
|
-
):
|
|
587
|
-
raise RuntimeError("cached harness artifact identity is incompatible")
|
|
588
|
-
entrypoint = manifest.get("entrypoint")
|
|
589
|
-
if not isinstance(entrypoint, str) or not entrypoint or PurePosixPath(entrypoint).is_absolute():
|
|
590
|
-
raise RuntimeError("cached harness artifact entrypoint is invalid")
|
|
591
|
-
parts = PurePosixPath(entrypoint).parts
|
|
592
|
-
if any(part in ("", ".", "..") for part in parts):
|
|
593
|
-
raise RuntimeError("cached harness artifact entrypoint escapes its directory")
|
|
594
|
-
executable = directory.joinpath(*parts)
|
|
595
|
-
if not executable.is_file():
|
|
596
|
-
raise RuntimeError("cached harness artifact entrypoint is missing")
|
|
597
|
-
if self._sha256_file(executable) != manifest["entrypoint_integrity"]:
|
|
598
|
-
raise RuntimeError("cached harness artifact entrypoint integrity mismatch")
|
|
599
|
-
if self._artifact_directory_integrity(directory) != manifest["artifact_integrity"]:
|
|
600
|
-
raise RuntimeError("cached harness artifact content integrity mismatch")
|
|
601
|
-
return manifest
|
|
602
|
-
|
|
603
|
-
@classmethod
|
|
604
|
-
def _artifact_directory_integrity(cls, root: Path) -> str:
|
|
605
|
-
digest = hashlib.sha256()
|
|
606
|
-
resolved_root = root.resolve()
|
|
607
|
-
|
|
608
|
-
def walk(directory: Path, relative: PurePosixPath, top_level: bool) -> None:
|
|
609
|
-
for entry in sorted(os.scandir(directory), key=lambda item: item.name):
|
|
610
|
-
if top_level and entry.name == "artifact.json":
|
|
611
|
-
continue
|
|
612
|
-
child_relative = relative / entry.name
|
|
613
|
-
child = Path(entry.path)
|
|
614
|
-
info = child.lstat()
|
|
615
|
-
mode = info.st_mode & 0o7777
|
|
616
|
-
encoded_relative = child_relative.as_posix()
|
|
617
|
-
if stat_module.S_ISDIR(info.st_mode):
|
|
618
|
-
digest.update(f"d\0{encoded_relative}\0{mode}\0".encode("utf-8"))
|
|
619
|
-
walk(child, child_relative, False)
|
|
620
|
-
elif stat_module.S_ISREG(info.st_mode):
|
|
621
|
-
digest.update(f"f\0{encoded_relative}\0{mode}\0{info.st_size}\0".encode("utf-8"))
|
|
622
|
-
with child.open("rb") as handle:
|
|
623
|
-
while chunk := handle.read(1024 * 1024):
|
|
624
|
-
digest.update(chunk)
|
|
625
|
-
digest.update(b"\0")
|
|
626
|
-
elif stat_module.S_ISLNK(info.st_mode):
|
|
627
|
-
target = os.readlink(child)
|
|
628
|
-
resolved_target = (child.parent / target).resolve()
|
|
629
|
-
if resolved_target != resolved_root and resolved_root not in resolved_target.parents:
|
|
630
|
-
raise RuntimeError("cached harness artifact symlink escapes its directory")
|
|
631
|
-
digest.update(f"l\0{encoded_relative}\0{target}\0".encode("utf-8"))
|
|
632
|
-
else:
|
|
633
|
-
raise RuntimeError("cached harness artifact contains a special file")
|
|
634
|
-
|
|
635
|
-
walk(root, PurePosixPath(), True)
|
|
636
|
-
return "sha256:" + digest.hexdigest()
|
|
637
|
-
|
|
638
|
-
def _quarantine_cached_artifact(self, directory: Path) -> None:
|
|
639
|
-
cache = self.harness_artifact_cache_dir
|
|
640
|
-
if cache is None or not directory.exists():
|
|
641
|
-
return
|
|
642
|
-
target = cache / "invalid" / f"{directory.name}-{uuid.uuid4().hex}"
|
|
643
|
-
try:
|
|
644
|
-
directory.rename(target)
|
|
645
|
-
except OSError:
|
|
646
|
-
pass
|
|
647
|
-
|
|
648
|
-
def _verify_local_source_host(self) -> dict[str, Any]:
|
|
649
|
-
"""Validate the independent local-source handoff immediately before upload."""
|
|
650
|
-
transport = self.local_source_transport or {}
|
|
651
|
-
required = {
|
|
652
|
-
"kind", "manifest_path", "payload_path", "locked_resolution_path", "harness_id",
|
|
653
|
-
"resolution_identity", "commit", "tree", "payload_sha256", "payload_bytes",
|
|
654
|
-
"object_count", "file_count",
|
|
655
|
-
}
|
|
656
|
-
if set(transport) != required:
|
|
657
|
-
raise RuntimeError("local source transport metadata fields are invalid")
|
|
658
|
-
manifest_path = Path(str(transport["manifest_path"]))
|
|
659
|
-
payload_path = Path(str(transport["payload_path"]))
|
|
660
|
-
resolution_path = Path(str(transport["locked_resolution_path"]))
|
|
661
|
-
if not all(candidate.is_absolute() for candidate in (manifest_path, payload_path, resolution_path)):
|
|
662
|
-
raise RuntimeError("local source transport host paths must be absolute")
|
|
663
|
-
self._assert_regular_host_file(manifest_path, "local source manifest", 64 * 1024)
|
|
664
|
-
payload_stat = self._assert_regular_host_file(payload_path, "local source payload", LOCAL_GIT_TRANSPORT_MAX_BYTES)
|
|
665
|
-
self._assert_regular_host_file(resolution_path, "local source locked resolution", 1024 * 1024)
|
|
666
|
-
try:
|
|
667
|
-
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
668
|
-
resolution = json.loads(resolution_path.read_text(encoding="utf-8"))
|
|
669
|
-
except (OSError, UnicodeError, json.JSONDecodeError) as error:
|
|
670
|
-
raise RuntimeError(f"local source metadata is unreadable: {error}") from error
|
|
671
|
-
self._validate_local_manifest(manifest)
|
|
672
|
-
pinned = {
|
|
673
|
-
"kind": manifest["kind"],
|
|
674
|
-
"harness_id": manifest["harness_id"],
|
|
675
|
-
"resolution_identity": manifest["resolution_identity"],
|
|
676
|
-
"commit": manifest["commit"],
|
|
677
|
-
"tree": manifest["tree"],
|
|
678
|
-
"payload_sha256": manifest["payload_sha256"],
|
|
679
|
-
"payload_bytes": manifest["payload_bytes"],
|
|
680
|
-
"object_count": manifest["object_count"],
|
|
681
|
-
"file_count": manifest["file_count"],
|
|
682
|
-
}
|
|
683
|
-
if any(transport.get(key) != value for key, value in pinned.items()):
|
|
684
|
-
raise RuntimeError("local source transport metadata does not match its manifest")
|
|
685
|
-
if manifest["resolution_identity"] != self.revision_identity:
|
|
686
|
-
raise RuntimeError("local source resolution identity does not match the job-pinned identity")
|
|
687
|
-
if self.harness_ref != f'{manifest["harness_id"]}@commit:{manifest["commit"]}':
|
|
688
|
-
raise RuntimeError("local source commit does not match the job-pinned harness ref")
|
|
689
|
-
if payload_stat.st_size != manifest["payload_bytes"]:
|
|
690
|
-
raise RuntimeError("local source payload size does not match its manifest")
|
|
691
|
-
actual_digest = self._sha256_file(payload_path)
|
|
692
|
-
if actual_digest != manifest["payload_sha256"]:
|
|
693
|
-
raise RuntimeError("local source payload digest does not match its manifest")
|
|
694
|
-
if not isinstance(resolution, dict) or (
|
|
695
|
-
resolution.get("harness_id") != manifest["harness_id"]
|
|
696
|
-
or resolution.get("identity") != manifest["resolution_identity"]
|
|
697
|
-
or resolution.get("revision", {}).get("commit") != manifest["commit"]
|
|
698
|
-
or resolution.get("source", {}).get("type") != "git"
|
|
699
|
-
or resolution.get("source", {}).get("registered") is not False
|
|
700
|
-
):
|
|
701
|
-
raise RuntimeError("local source locked resolution does not match its manifest")
|
|
702
|
-
return manifest
|
|
703
|
-
|
|
704
|
-
@staticmethod
|
|
705
|
-
def _assert_regular_host_file(path: Path, label: str, maximum: int) -> Any:
|
|
706
|
-
try:
|
|
707
|
-
info = path.lstat()
|
|
708
|
-
except OSError as error:
|
|
709
|
-
raise RuntimeError(f"{label} is missing or unreadable") from error
|
|
710
|
-
if not stat_module.S_ISREG(info.st_mode):
|
|
711
|
-
raise RuntimeError(f"{label} must be a regular file")
|
|
712
|
-
if info.st_size > maximum:
|
|
713
|
-
raise RuntimeError(f"{label} exceeds the size limit ({maximum} bytes)")
|
|
714
|
-
return info
|
|
715
|
-
|
|
716
|
-
@staticmethod
|
|
717
|
-
def _sha256_file(path: Path) -> str:
|
|
718
|
-
digest = hashlib.sha256()
|
|
719
|
-
with path.open("rb") as handle:
|
|
720
|
-
while chunk := handle.read(1024 * 1024):
|
|
721
|
-
digest.update(chunk)
|
|
722
|
-
return "sha256:" + digest.hexdigest()
|
|
723
|
-
|
|
724
|
-
@staticmethod
|
|
725
|
-
def _validate_local_manifest(manifest: Any) -> None:
|
|
726
|
-
allowed = {
|
|
727
|
-
"schema_version", "kind", "harness_id", "resolution_identity", "commit", "tree",
|
|
728
|
-
"payload_sha256", "payload_bytes", "object_count", "file_count", "created_at",
|
|
729
|
-
}
|
|
730
|
-
if not isinstance(manifest, dict) or set(manifest) != allowed:
|
|
731
|
-
raise RuntimeError("local source manifest fields are invalid")
|
|
732
|
-
if manifest.get("schema_version") != LOCAL_GIT_TRANSPORT_MANIFEST_VERSION:
|
|
733
|
-
raise RuntimeError("unsupported local source transport manifest schema")
|
|
734
|
-
if manifest.get("kind") != "local-git-commit":
|
|
735
|
-
raise RuntimeError("unsupported local source transport kind")
|
|
736
|
-
if not isinstance(manifest.get("harness_id"), str) or not re.fullmatch(r"[a-z][a-z0-9-]*", manifest["harness_id"]):
|
|
737
|
-
raise RuntimeError("local source manifest harness id is invalid")
|
|
738
|
-
if not isinstance(manifest.get("resolution_identity"), str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", manifest["resolution_identity"]):
|
|
739
|
-
raise RuntimeError("local source manifest resolution identity is invalid")
|
|
740
|
-
commit = manifest.get("commit")
|
|
741
|
-
if not isinstance(commit, str) or not re.fullmatch(r"(?:[0-9a-f]{40}|[0-9a-f]{64})", commit):
|
|
742
|
-
raise RuntimeError("local source manifest commit is not a full lowercase OID")
|
|
743
|
-
tree = manifest.get("tree")
|
|
744
|
-
if not isinstance(tree, str) or not re.fullmatch(rf"[0-9a-f]{{{len(commit)}}}", tree):
|
|
745
|
-
raise RuntimeError("local source manifest tree is invalid")
|
|
746
|
-
if not isinstance(manifest.get("payload_sha256"), str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", manifest["payload_sha256"]):
|
|
747
|
-
raise RuntimeError("local source manifest payload digest is invalid")
|
|
748
|
-
for key, maximum in (
|
|
749
|
-
("payload_bytes", LOCAL_GIT_TRANSPORT_MAX_BYTES),
|
|
750
|
-
("object_count", 100_000),
|
|
751
|
-
("file_count", 50_000),
|
|
752
|
-
):
|
|
753
|
-
value = manifest.get(key)
|
|
754
|
-
if not isinstance(value, int) or isinstance(value, bool) or value < 0 or value > maximum:
|
|
755
|
-
raise RuntimeError(f"local source manifest {key} is invalid")
|
|
756
|
-
if not isinstance(manifest.get("created_at"), str) or not manifest["created_at"]:
|
|
757
|
-
raise RuntimeError("local source manifest created_at is invalid")
|
|
758
|
-
try:
|
|
759
|
-
datetime.fromisoformat(manifest["created_at"].replace("Z", "+00:00"))
|
|
760
|
-
except ValueError as error:
|
|
761
|
-
raise RuntimeError("local source manifest created_at is invalid") from error
|
|
762
|
-
|
|
763
|
-
async def _upload_and_materialize_local_source(
|
|
764
|
-
self,
|
|
765
|
-
environment: BaseEnvironment,
|
|
766
|
-
manifest: dict[str, Any],
|
|
767
|
-
) -> None:
|
|
768
|
-
transport = self.local_source_transport or {}
|
|
769
|
-
await self._exec(environment, f"mkdir -p {LOCAL_GIT_REMOTE_ROOT} && chmod 700 {LOCAL_GIT_REMOTE_ROOT}")
|
|
770
|
-
await environment.upload_file(Path(str(transport["manifest_path"])), f"{LOCAL_GIT_REMOTE_ROOT}/manifest.json")
|
|
771
|
-
await environment.upload_file(Path(str(transport["payload_path"])), f"{LOCAL_GIT_REMOTE_ROOT}/payload.pack")
|
|
772
|
-
await environment.upload_file(Path(str(transport["locked_resolution_path"])), f"{LOCAL_GIT_REMOTE_ROOT}/resolution.json")
|
|
773
|
-
await self._exec(environment, f"chmod 600 {LOCAL_GIT_REMOTE_ROOT}/manifest.json {LOCAL_GIT_REMOTE_ROOT}/payload.pack {LOCAL_GIT_REMOTE_ROOT}/resolution.json")
|
|
774
|
-
# Verify both the uploaded manifest fields and payload bytes inside the
|
|
775
|
-
# trial before importing any Git object. All interpolated values passed
|
|
776
|
-
# to the shell have already been restricted to digest/OID grammars.
|
|
777
|
-
verifier = """
|
|
778
|
-
const fs = require('node:fs');
|
|
779
|
-
const crypto = require('node:crypto');
|
|
780
|
-
const expected = JSON.parse(process.argv[1]);
|
|
781
|
-
const manifest = JSON.parse(fs.readFileSync(process.argv[2], 'utf8'));
|
|
782
|
-
if (JSON.stringify(Object.keys(manifest).sort()) !== JSON.stringify(Object.keys(expected).sort())) throw new Error('manifest fields mismatch');
|
|
783
|
-
for (const key of Object.keys(expected)) if (manifest[key] !== expected[key]) throw new Error(`manifest mismatch: ${key}`);
|
|
784
|
-
if (fs.statSync(process.argv[3]).size !== expected.payload_bytes) throw new Error('payload size mismatch');
|
|
785
|
-
(async () => {
|
|
786
|
-
const hash = crypto.createHash('sha256');
|
|
787
|
-
for await (const chunk of fs.createReadStream(process.argv[3])) hash.update(chunk);
|
|
788
|
-
const digest = 'sha256:' + hash.digest('hex');
|
|
789
|
-
if (digest !== expected.payload_sha256) throw new Error('payload digest mismatch');
|
|
790
|
-
})().catch((error) => { console.error(error.message); process.exit(1); });
|
|
791
|
-
""".strip()
|
|
792
|
-
expected = {
|
|
793
|
-
"schema_version": manifest["schema_version"],
|
|
794
|
-
"kind": manifest["kind"],
|
|
795
|
-
"harness_id": manifest["harness_id"],
|
|
796
|
-
"resolution_identity": manifest["resolution_identity"],
|
|
797
|
-
"commit": manifest["commit"],
|
|
798
|
-
"tree": manifest["tree"],
|
|
799
|
-
"payload_sha256": manifest["payload_sha256"],
|
|
800
|
-
"payload_bytes": manifest["payload_bytes"],
|
|
801
|
-
"object_count": manifest["object_count"],
|
|
802
|
-
"file_count": manifest["file_count"],
|
|
803
|
-
"created_at": manifest["created_at"],
|
|
804
|
-
}
|
|
805
|
-
verify_command = " ".join([
|
|
806
|
-
self._node_prefix(),
|
|
807
|
-
"node", "-e", shlex.quote(verifier),
|
|
808
|
-
shlex.quote(json.dumps(expected, separators=(",", ":"))),
|
|
809
|
-
f"{LOCAL_GIT_REMOTE_ROOT}/manifest.json",
|
|
810
|
-
f"{LOCAL_GIT_REMOTE_ROOT}/payload.pack",
|
|
811
|
-
])
|
|
812
|
-
await self._exec(environment, verify_command)
|
|
813
|
-
commit = manifest["commit"]
|
|
814
|
-
tree = manifest["tree"]
|
|
815
|
-
materialize = f"""
|
|
816
|
-
set -eu
|
|
817
|
-
git init --bare {LOCAL_GIT_REMOTE_ROOT}/repo.git >/dev/null
|
|
818
|
-
git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git index-pack --stdin < {LOCAL_GIT_REMOTE_ROOT}/payload.pack >/dev/null
|
|
819
|
-
test "$(git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git rev-parse {commit}^{{commit}})" = "{commit}"
|
|
820
|
-
test "$(git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git rev-parse {commit}^{{tree}})" = "{tree}"
|
|
821
|
-
printf '%s\n' {commit} > {LOCAL_GIT_REMOTE_ROOT}/repo.git/shallow
|
|
822
|
-
git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git update-ref refs/heads/hitch-local {commit}
|
|
823
|
-
"""
|
|
824
|
-
await self._exec(environment, materialize)
|
|
825
|
-
|
|
826
|
-
def _local_source_cli_args(self) -> list[str]:
|
|
827
|
-
if self._local_manifest is None or self._artifact_uploaded:
|
|
828
|
-
return []
|
|
829
|
-
return [
|
|
830
|
-
"--internal-locked-resolution", f"{LOCAL_GIT_REMOTE_ROOT}/resolution.json",
|
|
831
|
-
"--internal-local-git-manifest", f"{LOCAL_GIT_REMOTE_ROOT}/manifest.json",
|
|
832
|
-
"--internal-local-git-source", f"{LOCAL_GIT_REMOTE_ROOT}/repo.git",
|
|
833
|
-
]
|
|
834
|
-
|
|
835
484
|
def _artifact_cli_args(self) -> list[str]:
|
|
836
485
|
if not self._artifact_uploaded or self._artifact_manifest is None:
|
|
837
486
|
return []
|
|
@@ -942,14 +591,209 @@ git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git update-ref refs/heads/hitch-local {commi
|
|
|
942
591
|
environment: BaseEnvironment,
|
|
943
592
|
context: AgentContext,
|
|
944
593
|
) -> None:
|
|
594
|
+
session = getattr(environment, "_hitch_benchmark", None)
|
|
595
|
+
config = session.config if session else None
|
|
596
|
+
driver = config["task"]["driver"] if config else None
|
|
597
|
+
phases = driver["config"].get("native_phases") if driver and driver["kind"] == "tool-server" else None
|
|
598
|
+
if phases:
|
|
599
|
+
from hitch_phase_supervisor import NativePhaseSupervisor
|
|
600
|
+
task_timeout = int(config["agent_timeout_sec"] * 1000)
|
|
601
|
+
remaining = min(task_timeout, self.hitch_timeout_ms) if self.hitch_timeout_ms > 0 else task_timeout
|
|
602
|
+
result = await NativePhaseSupervisor(
|
|
603
|
+
self, environment, controller={"service": driver["config"]["service"], "argv": phases["argv"]},
|
|
604
|
+
binding={"endpoint": driver["config"]["endpoint"], "tools": config["tools"]},
|
|
605
|
+
task_digest=config["task_digest"], timeout_ms=remaining,
|
|
606
|
+
shutdown_timeout_ms=phases["shutdown_timeout_ms"],
|
|
607
|
+
finalization_timeout_ms=phases.get("finalization_timeout_ms"),
|
|
608
|
+
task_instruction=instruction,
|
|
609
|
+
).run()
|
|
610
|
+
context.metadata = {"candidate_id": self.candidate_id, "harness_ref": self.harness_ref,
|
|
611
|
+
"revision_identity": self.revision_identity, "controller_runtime_id": self.controller_runtime_id,
|
|
612
|
+
"hitch_context_kind": "benchmark_phase_group", "hitch_run_group_id": result["run_group_id"],
|
|
613
|
+
"hitch_phase_count": len(result["phases"]), "hitch_status": "succeeded",
|
|
614
|
+
"hitch_phase_supervision": "hitch-native-phases/supervision.json"}
|
|
615
|
+
return
|
|
616
|
+
started_ns = time.monotonic_ns()
|
|
617
|
+
try:
|
|
618
|
+
await self._run(instruction, environment, context)
|
|
619
|
+
finally:
|
|
620
|
+
self._write_phase_timing("agent", started_ns)
|
|
621
|
+
|
|
622
|
+
def _phase_identity(self) -> str:
|
|
623
|
+
identity = [self.candidate_id, self.harness_ref, self.revision_identity, self.controller_runtime_id,
|
|
624
|
+
self.model_name, self.agent_args, self.credential_names, self._require_workdir(),
|
|
625
|
+
self.eval_id, self.benchmark_id, self.benchmark_revision, self.verifier_identity, self._trial_identity()]
|
|
626
|
+
return hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
|
|
627
|
+
|
|
628
|
+
def prepare_phase(self, *, instruction: str, run_group_id: str, phase_index: int,
|
|
629
|
+
task_digest: str, remaining_timeout_ms: int) -> PreparedPhase:
|
|
630
|
+
"""Reserve identity before private tool binding; does not start a model.
|
|
631
|
+
|
|
632
|
+
The supervisor supplies one frozen task digest across all phases and a
|
|
633
|
+
remaining whole-task budget. Time spent binding/uploading consumes that
|
|
634
|
+
budget too. The returned immutable handle is single-use on this agent.
|
|
635
|
+
"""
|
|
636
|
+
if not self._setup_complete or not self._phase_export_available:
|
|
637
|
+
raise RuntimeError("phase preparation requires setup with a phase-export capable runtime")
|
|
638
|
+
if self._prepared_phase is not None or self._phase_inflight:
|
|
639
|
+
raise RuntimeError("another candidate phase is already prepared or running")
|
|
640
|
+
if (not isinstance(instruction, str) or not instruction.strip()
|
|
641
|
+
or not isinstance(run_group_id, str) or not re.fullmatch(r"run_group_[a-f0-9]{32}", run_group_id)
|
|
642
|
+
or type(phase_index) is not int or not 1 <= phase_index <= 10000
|
|
643
|
+
or not isinstance(task_digest, str) or not re.fullmatch(r"sha256:[a-f0-9]{64}", task_digest)
|
|
644
|
+
or type(remaining_timeout_ms) is not int or not 1 <= remaining_timeout_ms <= 9007199254740991):
|
|
645
|
+
raise ValueError("invalid candidate phase identity or remaining budget")
|
|
646
|
+
if not all((self.eval_id, self.benchmark_id, self.benchmark_revision, self.verifier_identity)):
|
|
647
|
+
raise ValueError("candidate phases require a complete eval and benchmark identity")
|
|
648
|
+
if (not re.fullmatch(r"eval_[a-f0-9]{32}", self.eval_id)
|
|
649
|
+
or any(not re.fullmatch(r"sha256:[a-f0-9]{64}", value) for value in (self.benchmark_revision, self.verifier_identity))):
|
|
650
|
+
raise ValueError("candidate phase eval and benchmark identities must be immutable")
|
|
651
|
+
key = (run_group_id, phase_index)
|
|
652
|
+
if key in self._prepared_phase_keys:
|
|
653
|
+
raise RuntimeError("candidate phase identity was already prepared; implicit retries are forbidden")
|
|
654
|
+
identity = self._phase_identity()
|
|
655
|
+
previous = self._phase_group_contracts.get(run_group_id)
|
|
656
|
+
if (previous is None and phase_index != 1
|
|
657
|
+
or previous is not None and previous != (task_digest, identity, phase_index - 1)):
|
|
658
|
+
raise RuntimeError("candidate phase group must retain its task/candidate identity and consecutive indices")
|
|
659
|
+
trial_id, task_id, attempt = self._trial_identity()
|
|
660
|
+
context = {"kind": "benchmark_phase", "benchmark_id": self.benchmark_id,
|
|
661
|
+
"benchmark_revision": self.benchmark_revision, "task_id": task_id,
|
|
662
|
+
"task_digest": task_digest, "verifier_identity": self.verifier_identity,
|
|
663
|
+
"run_group_id": run_group_id, "phase_index": phase_index}
|
|
664
|
+
parent = {"kind": "eval", "eval_id": self.eval_id, "trial_id": trial_id, "attempt": attempt}
|
|
665
|
+
prepared = PreparedPhase("run_" + uuid.uuid4().hex, instruction, json.dumps(context, sort_keys=True),
|
|
666
|
+
json.dumps(parent, sort_keys=True), identity,
|
|
667
|
+
time.monotonic_ns() + remaining_timeout_ms * 1_000_000)
|
|
668
|
+
self._phase_control_tokens[prepared.run_id] = secrets.token_hex(32)
|
|
669
|
+
self._prepared_phase_keys.add(key)
|
|
670
|
+
self._phase_group_contracts[run_group_id] = (task_digest, identity, phase_index)
|
|
671
|
+
self._prepared_phase = prepared
|
|
672
|
+
return prepared
|
|
673
|
+
|
|
674
|
+
async def run_phase(self, prepared: PreparedPhase, environment: BaseEnvironment, context: AgentContext) -> None:
|
|
675
|
+
if not isinstance(prepared, PreparedPhase) or self._prepared_phase is not prepared or self._phase_inflight:
|
|
676
|
+
raise RuntimeError("candidate phase handle is stale, foreign, or already consumed")
|
|
677
|
+
self._prepared_phase = None # Every outcome consumes the handle.
|
|
678
|
+
if prepared.identity != self._phase_identity():
|
|
679
|
+
self._phase_control_tokens.pop(prepared.run_id, None)
|
|
680
|
+
raise RuntimeError("candidate identity changed after phase preparation")
|
|
681
|
+
if prepared.deadline_ns <= time.monotonic_ns():
|
|
682
|
+
self._phase_control_tokens.pop(prepared.run_id, None)
|
|
683
|
+
raise RuntimeError("candidate whole-task budget expired before phase start")
|
|
684
|
+
self._phase_inflight = True
|
|
685
|
+
self._active_phase = prepared
|
|
686
|
+
started_ns = time.monotonic_ns()
|
|
687
|
+
try:
|
|
688
|
+
await self._run(prepared.instruction, environment, context, prepared_phase=prepared)
|
|
689
|
+
finally:
|
|
690
|
+
self._phase_inflight = False
|
|
691
|
+
self._active_phase = None
|
|
692
|
+
self._phase_control_tokens.pop(prepared.run_id, None)
|
|
693
|
+
self._write_phase_timing("agent", started_ns)
|
|
694
|
+
|
|
695
|
+
@staticmethod
|
|
696
|
+
def _phase_control_path(prepared: PreparedPhase) -> str:
|
|
697
|
+
return f"/tmp/hitch-phase-control-{prepared.run_id}.config.json"
|
|
698
|
+
|
|
699
|
+
@staticmethod
|
|
700
|
+
async def _upload_phase_json(environment: BaseEnvironment, target: str, value: dict[str, Any]) -> None:
|
|
701
|
+
# Keep control nonces out of both argv and the mounted agent log tree.
|
|
702
|
+
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", prefix="hitch-phase-control-", delete=False) as handle:
|
|
703
|
+
json.dump(value, handle)
|
|
704
|
+
temporary = Path(handle.name)
|
|
705
|
+
try:
|
|
706
|
+
await environment.upload_file(temporary, target)
|
|
707
|
+
result = await environment.exec(f"chmod 600 {shlex.quote(target)}")
|
|
708
|
+
if result.return_code != 0:
|
|
709
|
+
raise RuntimeError("could not protect candidate phase control input")
|
|
710
|
+
finally:
|
|
711
|
+
temporary.unlink(missing_ok=True)
|
|
712
|
+
|
|
713
|
+
async def request_phase_cancellation(self, prepared: PreparedPhase, environment: BaseEnvironment, *, reason: str) -> dict[str, Any]:
|
|
714
|
+
"""Request executor cancellation; await run_phase separately for sealed evidence.
|
|
715
|
+
|
|
716
|
+
This receipt proves only that the request was delivered, not that the
|
|
717
|
+
model stopped or that a native phase completed. Await this operation
|
|
718
|
+
before recycling the candidate. A caller must separately enforce a
|
|
719
|
+
bounded shutdown/collection allowance and whole-trial failure cleanup.
|
|
720
|
+
"""
|
|
721
|
+
if reason not in {"native_phase_reset", "native_task_finished", "task_budget_expired", "cancelled"}:
|
|
722
|
+
raise ValueError("invalid phase cancellation reason")
|
|
723
|
+
if self._phase_cancel_lock is None:
|
|
724
|
+
self._phase_cancel_lock = asyncio.Lock()
|
|
725
|
+
async with self._phase_cancel_lock:
|
|
726
|
+
if not isinstance(prepared, PreparedPhase) or self._active_phase is not prepared:
|
|
727
|
+
raise RuntimeError("candidate phase is not active")
|
|
728
|
+
existing = self._phase_cancel_receipts.get(prepared.run_id)
|
|
729
|
+
if existing is not None:
|
|
730
|
+
if existing["reason"] != reason:
|
|
731
|
+
raise RuntimeError("candidate phase cancellation reason already fixed")
|
|
732
|
+
if existing["status"] != "delivered":
|
|
733
|
+
raise RuntimeError("candidate cancellation delivery is incomplete; implicit retries are forbidden")
|
|
734
|
+
return dict(existing)
|
|
735
|
+
phase = json.loads(prepared.context_json)
|
|
736
|
+
record_dir = self.logs_dir.parent / "hitch-phase-control"
|
|
737
|
+
record_dir.mkdir(mode=0o700, exist_ok=True)
|
|
738
|
+
if record_dir.is_symlink() or record_dir.stat().st_mode & 0o077:
|
|
739
|
+
raise RuntimeError("candidate cancellation records require a private host directory")
|
|
740
|
+
record_path = record_dir / f"{prepared.run_id}.request.json"
|
|
741
|
+
receipt = {"schema_version": "hitch-phase-cancel-request@1", "scope": "request-only",
|
|
742
|
+
"status": "prepared",
|
|
743
|
+
"request_id": "phase_cancel_" + uuid.uuid4().hex, "run_id": prepared.run_id,
|
|
744
|
+
"run_group_id": phase["run_group_id"], "phase_index": phase["phase_index"],
|
|
745
|
+
"reason": reason, "requested_at": datetime.now(timezone.utc).isoformat(),
|
|
746
|
+
"record_ref": f"hitch-phase-control/{record_path.name}"}
|
|
747
|
+
with record_path.open("x", encoding="utf-8") as handle:
|
|
748
|
+
json.dump(receipt, handle, sort_keys=True)
|
|
749
|
+
handle.write("\n")
|
|
750
|
+
record_path.chmod(0o600)
|
|
751
|
+
self._phase_cancel_receipts[prepared.run_id] = receipt
|
|
752
|
+
try:
|
|
753
|
+
await self._upload_phase_json(environment, self._phase_control_path(prepared).replace(".config.json", ".request.json"), {
|
|
754
|
+
"schema_version": "hitch-phase-cancel@1", "run_id": prepared.run_id,
|
|
755
|
+
"token": self._phase_control_tokens[prepared.run_id], "reason": reason,
|
|
756
|
+
})
|
|
757
|
+
receipt["status"] = "delivered"
|
|
758
|
+
except BaseException as error:
|
|
759
|
+
receipt.update(status="delivery_failed", failure_type=type(error).__name__)
|
|
760
|
+
raise
|
|
761
|
+
finally:
|
|
762
|
+
update = record_path.with_suffix(".pending")
|
|
763
|
+
with update.open("x", encoding="utf-8") as handle:
|
|
764
|
+
json.dump(receipt, handle, sort_keys=True)
|
|
765
|
+
handle.write("\n")
|
|
766
|
+
update.chmod(0o600)
|
|
767
|
+
update.replace(record_path)
|
|
768
|
+
return dict(receipt)
|
|
769
|
+
|
|
770
|
+
async def _run(
|
|
771
|
+
self,
|
|
772
|
+
instruction: str,
|
|
773
|
+
environment: BaseEnvironment,
|
|
774
|
+
context: AgentContext,
|
|
775
|
+
*,
|
|
776
|
+
prepared_phase: PreparedPhase | None = None,
|
|
777
|
+
) -> None:
|
|
778
|
+
invocation_started_ns = time.monotonic_ns()
|
|
779
|
+
session = getattr(environment, "_hitch_benchmark", None)
|
|
780
|
+
task_budget_ms = self.hitch_timeout_ms
|
|
781
|
+
if prepared_phase is None and session:
|
|
782
|
+
from hitch_benchmark import candidate_instruction
|
|
783
|
+
instruction, task_timeout = candidate_instruction(instruction, environment)
|
|
784
|
+
if task_timeout is not None:
|
|
785
|
+
task_budget_ms = min(task_timeout, task_budget_ms) if task_budget_ms > 0 else task_timeout
|
|
945
786
|
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
|
946
787
|
workdir = self._require_workdir()
|
|
947
|
-
assigned_run_id = "run_" + uuid.uuid4().hex
|
|
788
|
+
assigned_run_id = prepared_phase.run_id if prepared_phase else "run_" + uuid.uuid4().hex
|
|
948
789
|
run_id = assigned_run_id
|
|
949
790
|
trial_id, task_id, attempt = self._trial_identity()
|
|
950
791
|
context_payload: dict[str, Any] = {"kind": "ad_hoc"}
|
|
951
792
|
parent_payload: dict[str, Any] | None = None
|
|
952
|
-
if
|
|
793
|
+
if prepared_phase is not None:
|
|
794
|
+
context_payload = json.loads(prepared_phase.context_json)
|
|
795
|
+
parent_payload = json.loads(prepared_phase.parent_json)
|
|
796
|
+
elif all((self.eval_id, self.benchmark_id, self.benchmark_revision, self.verifier_identity)):
|
|
953
797
|
workspace_digest = await self._workspace_digest(environment)
|
|
954
798
|
task_digest_input = json.dumps(
|
|
955
799
|
{
|
|
@@ -1006,17 +850,39 @@ git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git update-ref refs/heads/hitch-local {commi
|
|
|
1006
850
|
if parent_temporary is not None:
|
|
1007
851
|
parent_temporary.unlink(missing_ok=True)
|
|
1008
852
|
|
|
853
|
+
proxy_environment, proxy_health = await self._model_proxy_environment(environment, run_id)
|
|
854
|
+
if prepared_phase is not None:
|
|
855
|
+
await self._upload_phase_json(environment, self._phase_control_path(prepared_phase), {
|
|
856
|
+
"schema_version": "hitch-phase-control@1", "run_id": run_id, "token": self._phase_control_tokens[run_id],
|
|
857
|
+
})
|
|
1009
858
|
if self._entrypoint is None:
|
|
1010
859
|
raise RuntimeError("Hitch agent setup() must run before run() to resolve the runtime entrypoint")
|
|
1011
860
|
entry = self._remote_entry(self._entrypoint)
|
|
861
|
+
timeout_ms = task_budget_ms
|
|
862
|
+
if prepared_phase is not None:
|
|
863
|
+
timeout_ms = (prepared_phase.deadline_ns - time.monotonic_ns()) // 1_000_000
|
|
864
|
+
if timeout_ms <= 0:
|
|
865
|
+
raise RuntimeError("candidate whole-task budget expired during phase binding/upload")
|
|
866
|
+
elif session:
|
|
867
|
+
preparation_ms = (time.monotonic_ns() - invocation_started_ns) // 1_000_000
|
|
868
|
+
timeout_ms = task_budget_ms - preparation_ms
|
|
869
|
+
(self.logs_dir / "hitch-agent-budget.json").write_text(json.dumps({
|
|
870
|
+
"schema_version": "hitch-agent-budget@1", "run_id": run_id,
|
|
871
|
+
"task_budget_ms": task_budget_ms, "preparation_ms": preparation_ms,
|
|
872
|
+
"hitch_timeout_ms": max(0, timeout_ms),
|
|
873
|
+
"collection_timeout_ms": session.config["profile"]["budget"]["collection_timeout_ms"],
|
|
874
|
+
"scope": "invocation-budget-and-collection-allowance",
|
|
875
|
+
}))
|
|
876
|
+
if timeout_ms <= 0:
|
|
877
|
+
raise RuntimeError("candidate budget expired during input preparation; no model was launched")
|
|
1012
878
|
arguments = [
|
|
1013
879
|
self._node_prefix(),
|
|
1014
880
|
"HITCH_ROOT=/tmp/hitch-state",
|
|
1015
|
-
*
|
|
881
|
+
*proxy_environment,
|
|
882
|
+
*(["HITCH_HARBOR_INTERNAL=1"] if self._artifact_uploaded or parent_payload is not None else []),
|
|
1016
883
|
f"node {entry} run",
|
|
1017
884
|
"--harness",
|
|
1018
885
|
shlex.quote(self.harness_ref),
|
|
1019
|
-
*self._local_source_cli_args(),
|
|
1020
886
|
*self._artifact_cli_args(),
|
|
1021
887
|
"--cwd",
|
|
1022
888
|
shlex.quote(workdir),
|
|
@@ -1027,7 +893,7 @@ git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git update-ref refs/heads/hitch-local {commi
|
|
|
1027
893
|
"--context-file",
|
|
1028
894
|
"/tmp/hitch-context.json",
|
|
1029
895
|
"--timeout",
|
|
1030
|
-
str(
|
|
896
|
+
str(timeout_ms),
|
|
1031
897
|
"--output",
|
|
1032
898
|
"jsonl",
|
|
1033
899
|
]
|
|
@@ -1035,12 +901,17 @@ git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git update-ref refs/heads/hitch-local {commi
|
|
|
1035
901
|
arguments.extend([
|
|
1036
902
|
"--parent-file", "/tmp/hitch-parent.json",
|
|
1037
903
|
"--internal-run-id", run_id,
|
|
1038
|
-
"--internal-defer-benchmark-observation",
|
|
1039
904
|
])
|
|
905
|
+
if prepared_phase is None:
|
|
906
|
+
arguments.append("--internal-defer-benchmark-observation")
|
|
907
|
+
else:
|
|
908
|
+
arguments.extend(["--internal-phase-control", shlex.quote(self._phase_control_path(prepared_phase))])
|
|
1040
909
|
if self.model_name:
|
|
1041
910
|
arguments.extend(["--model", shlex.quote(self.model_name)])
|
|
1042
911
|
for value in self.agent_args:
|
|
1043
912
|
arguments.extend(["--agent-arg", shlex.quote(value)])
|
|
913
|
+
for name in self.credential_names:
|
|
914
|
+
arguments.extend(["--internal-credential-name", shlex.quote(name)])
|
|
1044
915
|
command = (
|
|
1045
916
|
"set -o pipefail; "
|
|
1046
917
|
+ " ".join(arguments)
|
|
@@ -1048,6 +919,32 @@ git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git update-ref refs/heads/hitch-local {commi
|
|
|
1048
919
|
+ " | tee /logs/agent/hitch-events.jsonl"
|
|
1049
920
|
)
|
|
1050
921
|
execution = await environment.exec(command, cwd=workdir)
|
|
922
|
+
collection = self._collect_run(
|
|
923
|
+
environment, context, execution, assigned_run_id=assigned_run_id,
|
|
924
|
+
context_payload=context_payload, parent_payload=parent_payload, prepared_phase=prepared_phase,
|
|
925
|
+
workdir=workdir, proxy_health=proxy_health,
|
|
926
|
+
)
|
|
927
|
+
if prepared_phase is not None or not session:
|
|
928
|
+
await collection
|
|
929
|
+
return
|
|
930
|
+
try:
|
|
931
|
+
await asyncio.wait_for(collection, session.config["profile"]["budget"]["collection_timeout_ms"] / 1000)
|
|
932
|
+
except asyncio.TimeoutError as error:
|
|
933
|
+
# A completed process with uncollected evidence is still invalid.
|
|
934
|
+
# Do not fabricate a terminal bundle or relabel it as model success.
|
|
935
|
+
receipt = {"code": "hitch_run_collection_timeout", "run_id": assigned_run_id,
|
|
936
|
+
"process_return_code": execution.return_code}
|
|
937
|
+
(self.logs_dir / "hitch-collection-timeout.json").write_text(json.dumps(receipt))
|
|
938
|
+
raise RuntimeError("hitch_run_collection_timeout: terminal evidence export exceeded its allowance") from error
|
|
939
|
+
|
|
940
|
+
async def _collect_run(
|
|
941
|
+
self, environment: BaseEnvironment, context: AgentContext, execution: ExecResult, *,
|
|
942
|
+
assigned_run_id: str, context_payload: dict[str, Any], parent_payload: dict[str, Any] | None,
|
|
943
|
+
prepared_phase: PreparedPhase | None,
|
|
944
|
+
workdir: str, proxy_health: str,
|
|
945
|
+
) -> None:
|
|
946
|
+
run_id = assigned_run_id
|
|
947
|
+
trial_id, task_id, attempt = self._trial_identity()
|
|
1051
948
|
events = self._events(execution.stdout or "")
|
|
1052
949
|
observed_run_id = next((
|
|
1053
950
|
value
|
|
@@ -1055,7 +952,7 @@ git -C {LOCAL_GIT_REMOTE_ROOT}/repo.git update-ref refs/heads/hitch-local {commi
|
|
|
1055
952
|
for value in [event.get("run_id")]
|
|
1056
953
|
if isinstance(value, str) and re.fullmatch(r"run_[a-f0-9]{32}", value)
|
|
1057
954
|
), None)
|
|
1058
|
-
if observed_run_id:
|
|
955
|
+
if observed_run_id and prepared_phase is None:
|
|
1059
956
|
run_id = str(observed_run_id)
|
|
1060
957
|
result_path = f"/tmp/hitch-state/runs/{run_id}/result.json"
|
|
1061
958
|
quoted_result_path = shlex.quote(result_path)
|
|
@@ -1078,7 +975,7 @@ cat -- {quoted_result_path}
|
|
|
1078
975
|
"trial_id": trial_id,
|
|
1079
976
|
"completed_at": datetime.now(timezone.utc).isoformat(),
|
|
1080
977
|
}, separators=(",", ":"))
|
|
1081
|
-
|
|
978
|
+
legacy_export = (
|
|
1082
979
|
f"""
|
|
1083
980
|
set -eu
|
|
1084
981
|
source_dir={shlex.quote(f'/tmp/hitch-state/runs/{run_id}')}
|
|
@@ -1094,10 +991,39 @@ rm -rf "$target_dir"
|
|
|
1094
991
|
mv "$stage_dir" "$target_dir"
|
|
1095
992
|
""".strip()
|
|
1096
993
|
)
|
|
994
|
+
if prepared_phase is None:
|
|
995
|
+
bundle_export = await environment.exec(legacy_export)
|
|
996
|
+
else:
|
|
997
|
+
export_input = {"sourceDirectory": f"/tmp/hitch-state/runs/{run_id}", "destinationDirectory": bundle_stage,
|
|
998
|
+
"expected": {"run_id": run_id, "context": context_payload, "parent": parent_payload,
|
|
999
|
+
"revision_identity": self.revision_identity}}
|
|
1000
|
+
# Completion is outside the bundle: adding even a marker inside a
|
|
1001
|
+
# sealed bundle changes its indexed file set and invalidates it.
|
|
1002
|
+
script = (
|
|
1003
|
+
f"import {{copySealedPhaseRunBundle}} from 'file:///opt/hitch/{PHASE_EXPORT_MODULE}';"
|
|
1004
|
+
"import {open,lstat,rename,writeFile} from 'node:fs/promises';"
|
|
1005
|
+
"const input=JSON.parse(process.argv[1]);"
|
|
1006
|
+
"const target='/logs/agent/hitch-run-bundle';"
|
|
1007
|
+
"const lock=await open('/logs/agent/.hitch-phase-export.lock','wx',0o600);await lock.close();"
|
|
1008
|
+
"try{await lstat(target);throw new Error('phase export target already exists')}catch(e){if(e.code!=='ENOENT')throw e;}"
|
|
1009
|
+
"const index=await copySealedPhaseRunBundle(input);"
|
|
1010
|
+
"await rename(input.destinationDirectory,target);"
|
|
1011
|
+
"await writeFile('/logs/agent/hitch-phase.complete.json',JSON.stringify({schema_version:'1',"
|
|
1012
|
+
"run_id:index.run_id,bundle_digest:index.bundle_digest,scope:'candidate-evidence-only'}),{flag:'wx',mode:0o600});"
|
|
1013
|
+
)
|
|
1014
|
+
bundle_export = await environment.exec(
|
|
1015
|
+
f"{self._node_prefix()} node --input-type=module -e {shlex.quote(script)} {shlex.quote(json.dumps(export_input))}"
|
|
1016
|
+
)
|
|
1097
1017
|
hitch_result, result_error_code, result_error_message = self._parse_hitch_result(result_read, run_id)
|
|
1018
|
+
if prepared_phase is None and hitch_result is not None and getattr(environment, "_hitch_benchmark", None):
|
|
1019
|
+
from hitch_benchmark import export_final_response
|
|
1020
|
+
await export_final_response(environment, hitch_result)
|
|
1098
1021
|
primary_code: str | None = None
|
|
1099
1022
|
primary_message: str | None = None
|
|
1100
|
-
if
|
|
1023
|
+
if prepared_phase is not None and observed_run_id and observed_run_id != assigned_run_id:
|
|
1024
|
+
primary_code = "hitch_phase_run_identity_mismatch"
|
|
1025
|
+
primary_message = "Hitch phase emitted a different run ID from its prepared tool binding"
|
|
1026
|
+
elif execution.return_code != 0:
|
|
1101
1027
|
primary_code = "hitch_process_failed"
|
|
1102
1028
|
diagnostic = self._exec_diagnostic(execution)
|
|
1103
1029
|
if hitch_result and isinstance(hitch_result.get("error"), dict):
|
|
@@ -1125,6 +1051,13 @@ mv "$stage_dir" "$target_dir"
|
|
|
1125
1051
|
primary_code = "hitch_result_artifact_copy_failed"
|
|
1126
1052
|
primary_message = f"Hitch result artifact copy failed (run_id={run_id}, trial_id={trial_id})"
|
|
1127
1053
|
|
|
1054
|
+
self._write_trusted_agent_outcome(
|
|
1055
|
+
run_id=run_id,
|
|
1056
|
+
hitch_result=hitch_result,
|
|
1057
|
+
bundle_export=bundle_export,
|
|
1058
|
+
reason_code=primary_code,
|
|
1059
|
+
)
|
|
1060
|
+
|
|
1128
1061
|
context.metadata = {
|
|
1129
1062
|
"candidate_id": self.candidate_id,
|
|
1130
1063
|
"harness_ref": self.harness_ref,
|
|
@@ -1138,26 +1071,22 @@ mv "$stage_dir" "$target_dir"
|
|
|
1138
1071
|
"trial_id": trial_id,
|
|
1139
1072
|
"task_id": task_id,
|
|
1140
1073
|
"attempt": attempt,
|
|
1074
|
+
"model_capture_health": proxy_health,
|
|
1141
1075
|
"hitch_status": hitch_result.get("status") if hitch_result else None,
|
|
1142
1076
|
"hitch_artifact_id": hitch_result.get("artifact_id") if hitch_result else None,
|
|
1143
1077
|
}
|
|
1144
|
-
if
|
|
1145
|
-
context.metadata["
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
"resolution_identity": self._local_manifest["resolution_identity"],
|
|
1150
|
-
"payload_sha256": self._local_manifest["payload_sha256"],
|
|
1151
|
-
"payload_bytes": self._local_manifest["payload_bytes"],
|
|
1152
|
-
"status": "verified",
|
|
1153
|
-
}
|
|
1078
|
+
if prepared_phase is not None:
|
|
1079
|
+
context.metadata.update(hitch_context_kind="benchmark_phase", hitch_run_group_id=context_payload["run_group_id"],
|
|
1080
|
+
hitch_phase_index=context_payload["phase_index"],
|
|
1081
|
+
hitch_phase_bundle_exported=bundle_export.return_code == 0,
|
|
1082
|
+
hitch_phase_completion="hitch-phase.complete.json")
|
|
1154
1083
|
if self._artifact_manifest is not None:
|
|
1155
1084
|
context.metadata["harness_artifact_transport"] = {
|
|
1156
1085
|
"artifact_id": self._artifact_manifest["artifact_id"],
|
|
1157
1086
|
"artifact_integrity": self._artifact_manifest["artifact_integrity"],
|
|
1158
1087
|
"platform": self._artifact_manifest["platform"],
|
|
1159
1088
|
"node_version": self._artifact_manifest.get("toolchain", {}).get("node"),
|
|
1160
|
-
"status": self._artifact_transport_status or "
|
|
1089
|
+
"status": self._artifact_transport_status or "dedicated_builder_upload",
|
|
1161
1090
|
}
|
|
1162
1091
|
if primary_code is not None:
|
|
1163
1092
|
context.metadata["hitch_bridge_error_code"] = primary_code
|
|
@@ -1181,6 +1110,32 @@ mv "$stage_dir" "$target_dir"
|
|
|
1181
1110
|
await self._write_bridge_error(environment, evidence)
|
|
1182
1111
|
raise HitchBridgeError(primary_code, primary_message or primary_code, evidence)
|
|
1183
1112
|
|
|
1113
|
+
async def _model_proxy_environment(
|
|
1114
|
+
self,
|
|
1115
|
+
environment: BaseEnvironment,
|
|
1116
|
+
run_id: str,
|
|
1117
|
+
) -> tuple[list[str], str]:
|
|
1118
|
+
if self.model_capture is None:
|
|
1119
|
+
return [], "not-configured"
|
|
1120
|
+
health = self.model_capture["health_url_template"].replace("{run_id}", run_id)
|
|
1121
|
+
script = (
|
|
1122
|
+
"fetch(process.argv[1],{signal:AbortSignal.timeout(5000)})"
|
|
1123
|
+
".then(r=>{if(!r.ok)throw new Error('status '+r.status)})"
|
|
1124
|
+
".catch(()=>{console.error('model proxy health check failed');process.exit(1)})"
|
|
1125
|
+
)
|
|
1126
|
+
probe = await environment.exec(
|
|
1127
|
+
f"{self._node_prefix()} node -e {shlex.quote(script)} {shlex.quote(health)}"
|
|
1128
|
+
)
|
|
1129
|
+
if probe.return_code != 0:
|
|
1130
|
+
if self.model_capture["required"]:
|
|
1131
|
+
raise RuntimeError("hitch-model-proxy-health: required model proxy is unreachable")
|
|
1132
|
+
return [], "degraded-unreachable"
|
|
1133
|
+
base = self.model_capture["base_url_template"].replace("{run_id}", run_id)
|
|
1134
|
+
return [
|
|
1135
|
+
f"OPENAI_BASE_URL={shlex.quote(base.replace('{provider}', 'openai'))}",
|
|
1136
|
+
f"ANTHROPIC_BASE_URL={shlex.quote(base.replace('{provider}', 'anthropic'))}",
|
|
1137
|
+
], "healthy"
|
|
1138
|
+
|
|
1184
1139
|
@staticmethod
|
|
1185
1140
|
def _parse_hitch_result(
|
|
1186
1141
|
result: ExecResult,
|
|
@@ -1342,6 +1297,37 @@ mv "$stage_dir" "$target_dir"
|
|
|
1342
1297
|
cwd="/",
|
|
1343
1298
|
)
|
|
1344
1299
|
|
|
1300
|
+
def _write_trusted_agent_outcome(
|
|
1301
|
+
self,
|
|
1302
|
+
*,
|
|
1303
|
+
run_id: str,
|
|
1304
|
+
hitch_result: dict[str, Any] | None,
|
|
1305
|
+
bundle_export: ExecResult,
|
|
1306
|
+
reason_code: str | None,
|
|
1307
|
+
) -> None:
|
|
1308
|
+
result_status = hitch_result.get("status") if hitch_result else "failed"
|
|
1309
|
+
if result_status not in {"succeeded", "failed", "timed_out", "cancelled"}:
|
|
1310
|
+
result_status = "failed"
|
|
1311
|
+
bundle = "complete" if hitch_result is not None and bundle_export.return_code == 0 else (
|
|
1312
|
+
"missing" if bundle_export.return_code != 0 else "invalid"
|
|
1313
|
+
)
|
|
1314
|
+
outcome = {
|
|
1315
|
+
"schema_version": "1",
|
|
1316
|
+
"run_id": run_id,
|
|
1317
|
+
"status": result_status,
|
|
1318
|
+
"candidate_bundle": bundle,
|
|
1319
|
+
"submission_snapshot": "not-required",
|
|
1320
|
+
"gradeability": "gradeable" if bundle == "complete" else "ungradeable",
|
|
1321
|
+
}
|
|
1322
|
+
if bundle != "complete":
|
|
1323
|
+
outcome["reason_code"] = reason_code or "candidate_evidence_unavailable"
|
|
1324
|
+
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
|
1325
|
+
target = self.logs_dir / HITCH_AGENT_OUTCOME_NAME
|
|
1326
|
+
temporary = target.with_name(f".{target.name}.{uuid.uuid4().hex}.tmp")
|
|
1327
|
+
temporary.write_text(json.dumps(outcome, separators=(",", ":"), sort_keys=True) + "\n", encoding="utf-8")
|
|
1328
|
+
temporary.chmod(0o600)
|
|
1329
|
+
temporary.replace(target)
|
|
1330
|
+
|
|
1345
1331
|
def _trial_identity(self) -> tuple[str, str, int]:
|
|
1346
1332
|
"""Read Harbor's stable trial/task identity from the persisted trial state."""
|
|
1347
1333
|
trial_dir = self.logs_dir.parent if self.logs_dir.name == "agent" else self.logs_dir
|
|
@@ -1421,47 +1407,158 @@ process.stdout.write('sha256:' + hash.digest('hex'));
|
|
|
1421
1407
|
return "sha256:" + hashlib.sha256(b"workspace-unavailable").hexdigest()
|
|
1422
1408
|
|
|
1423
1409
|
async def _ensure_node(self, environment: BaseEnvironment) -> None:
|
|
1424
|
-
|
|
1425
|
-
|
|
1410
|
+
"""Select a compatible system Node or an authenticated offline runtime.
|
|
1411
|
+
|
|
1412
|
+
No network, package manager, shell profile, or existing system binary
|
|
1413
|
+
is modified here. The archive travels inside the job-pinned artifact,
|
|
1414
|
+
including on remote workers and reruns.
|
|
1415
|
+
"""
|
|
1416
|
+
platform = str((self._artifact_manifest or {}).get("platform", ""))
|
|
1417
|
+
self._node_bin_directory = None
|
|
1418
|
+
check = (
|
|
1419
|
+
f"process.exit(process.version === {json.dumps(self.required_node_version)} && "
|
|
1420
|
+
f"process.platform + '-' + process.arch === {json.dumps(platform)} ? 0 : 1)"
|
|
1426
1421
|
)
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
"""
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1422
|
+
staging: str | None = None
|
|
1423
|
+
try:
|
|
1424
|
+
probe = await asyncio.wait_for(environment.exec(f"node -e {shlex.quote(check)}"), timeout=30)
|
|
1425
|
+
if probe.return_code == 0:
|
|
1426
|
+
self._node_bin_directory = None
|
|
1427
|
+
self._record_node_runtime({"source": "system", "node_version": self.required_node_version, "platform": platform})
|
|
1428
|
+
return
|
|
1429
|
+
archive, runtime = self._offline_node_runtime()
|
|
1430
|
+
native = await self._node_setup_exec(environment, """set -eu
|
|
1431
|
+
test "$(uname -s)" = Linux
|
|
1432
|
+
case "$(uname -m)" in
|
|
1433
|
+
x86_64|amd64) echo linux-x64 ;;
|
|
1434
|
+
aarch64|arm64) echo linux-arm64 ;;
|
|
1435
|
+
*) exit 1 ;;
|
|
1436
|
+
esac""", "hitch_node_runtime_incompatible")
|
|
1437
|
+
if (native.stdout or "").strip() != runtime["platform"]:
|
|
1438
|
+
raise self._node_runtime_error("hitch_node_runtime_incompatible", "offline Node architecture does not match the task container")
|
|
1439
|
+
libc = await self._node_setup_exec(environment, "getconf GNU_LIBC_VERSION", "hitch_node_runtime_incompatible")
|
|
1440
|
+
if not re.fullmatch(r"glibc \d+\.\d+", (libc.stdout or "").strip()):
|
|
1441
|
+
raise self._node_runtime_error("hitch_node_runtime_incompatible", "offline Node requires a glibc task image; musl is not supported")
|
|
1442
|
+
await self._node_setup_exec(
|
|
1443
|
+
environment, "command -v sha256sum && command -v tar && command -v gzip", "hitch_node_runtime_prerequisite_missing",
|
|
1444
|
+
)
|
|
1445
|
+
# Unique per setup; never extract over a system Node or an old,
|
|
1446
|
+
# partially installed runtime. The ready directory appears only
|
|
1447
|
+
# after checksum, extraction and executable compatibility checks.
|
|
1448
|
+
target = f"/opt/hitch-node-runtime-{uuid.uuid4().hex}"
|
|
1449
|
+
await self._node_setup_exec(environment, f"mkdir -m 755 {target}", "hitch_node_runtime_install_failed")
|
|
1450
|
+
staging = target
|
|
1451
|
+
await asyncio.wait_for(environment.upload_file(archive, f"{staging}/node-runtime.tar.gz"), timeout=120)
|
|
1452
|
+
checksum = runtime["archive_sha256"].removeprefix("sha256:")
|
|
1453
|
+
await self._node_setup_exec(
|
|
1454
|
+
environment,
|
|
1455
|
+
f"cd {staging} && printf '%s\\n' '{checksum} node-runtime.tar.gz' | sha256sum -c -",
|
|
1456
|
+
"hitch_node_runtime_integrity_mismatch",
|
|
1457
|
+
)
|
|
1458
|
+
await self._node_setup_exec(
|
|
1459
|
+
environment,
|
|
1460
|
+
f"umask 022; mkdir -m 755 {staging}/unpacked && tar --no-same-owner -xzf {staging}/node-runtime.tar.gz -C {staging}/unpacked",
|
|
1461
|
+
"hitch_node_runtime_install_failed",
|
|
1462
|
+
)
|
|
1463
|
+
await self._node_setup_exec(
|
|
1464
|
+
environment, f"{staging}/unpacked/bin/node -e {shlex.quote(check)}", "hitch_node_runtime_incompatible",
|
|
1465
|
+
)
|
|
1466
|
+
await self._node_setup_exec(
|
|
1467
|
+
environment, f"mv {staging}/unpacked {staging}/ready && rm {staging}/node-runtime.tar.gz", "hitch_node_runtime_install_failed",
|
|
1468
|
+
)
|
|
1469
|
+
self._node_bin_directory = f"{staging}/ready/bin"
|
|
1470
|
+
self._record_node_runtime({"source": "offline-artifact", **runtime, "bin_directory": self._node_bin_directory})
|
|
1471
|
+
staging = None
|
|
1472
|
+
except Exception as error:
|
|
1473
|
+
failure = error if isinstance(error, HitchBridgeError) else self._node_runtime_error(
|
|
1474
|
+
"hitch_node_runtime_setup_failed", f"{type(error).__name__}: {error}",
|
|
1475
|
+
)
|
|
1476
|
+
self._record_node_runtime({"source": "failed", **failure.evidence})
|
|
1477
|
+
try:
|
|
1478
|
+
await self._write_bridge_error(environment, failure.evidence)
|
|
1479
|
+
except Exception:
|
|
1480
|
+
pass # Preserve the original Node failure even if log export fails.
|
|
1481
|
+
if failure is error:
|
|
1482
|
+
raise
|
|
1483
|
+
raise failure from error
|
|
1484
|
+
finally:
|
|
1485
|
+
if staging is not None:
|
|
1486
|
+
# Only the random directory created by this setup is removed.
|
|
1487
|
+
try:
|
|
1488
|
+
await asyncio.wait_for(environment.exec(f"rm -rf -- {staging}", user=0), timeout=30)
|
|
1489
|
+
except Exception:
|
|
1490
|
+
pass
|
|
1491
|
+
|
|
1492
|
+
def _offline_node_runtime(self) -> tuple[Path, dict[str, Any]]:
|
|
1493
|
+
directory = self._artifact_host_directory
|
|
1494
|
+
if directory is None or not (directory / ".hitch-node-runtime").exists():
|
|
1495
|
+
raise self._node_runtime_error(
|
|
1496
|
+
"hitch_node_runtime_missing",
|
|
1497
|
+
"task has no matching Node and the pinned artifact has no offline runtime; prepare a new eval with the updated controller (no online fallback)",
|
|
1498
|
+
)
|
|
1499
|
+
try:
|
|
1500
|
+
# Authenticate the runtime metadata against the job's CONTENT pin,
|
|
1501
|
+
# not a self-reported checksum in a mutable sidecar. This is needed
|
|
1502
|
+
# before Node can run Hitch's normal in-container artifact check.
|
|
1503
|
+
captured = {".hitch-node-runtime/node-runtime.json": b""}
|
|
1504
|
+
if artifact_directory_integrity(directory, captured) != (self.harness_artifact or {}).get("artifact_integrity"):
|
|
1505
|
+
raise RuntimeError("job-pinned harness content digest mismatch before Node bootstrap")
|
|
1506
|
+
bundle = directory / ".hitch-node-runtime"
|
|
1507
|
+
if bundle.is_symlink() or not bundle.is_dir():
|
|
1508
|
+
raise RuntimeError("offline Node bundle must be a regular directory")
|
|
1509
|
+
manifest_path = bundle / "node-runtime.json"
|
|
1510
|
+
self._assert_regular_host_file(manifest_path, "offline Node manifest", 16_384)
|
|
1511
|
+
# Parse exactly the bytes hashed above, not a second sidecar read
|
|
1512
|
+
# which could race a cache mutation after content authentication.
|
|
1513
|
+
runtime = json.loads(captured[".hitch-node-runtime/node-runtime.json"].decode("utf-8"))
|
|
1514
|
+
fields = {"schema_version", "recipe_version", "runtime_id", "node_version", "platform", "libc", "builder_image_id", "archive_sha256", "archive_bytes"}
|
|
1515
|
+
if not isinstance(runtime, dict) or set(runtime) != fields:
|
|
1516
|
+
raise RuntimeError("offline Node manifest fields are invalid")
|
|
1517
|
+
payload = {key: value for key, value in runtime.items() if key != "runtime_id"}
|
|
1518
|
+
identity = "sha256:" + hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
|
1519
|
+
if runtime["schema_version"] != "1" or runtime["recipe_version"] != "1" or runtime["runtime_id"] != identity:
|
|
1520
|
+
raise RuntimeError("offline Node runtime identity is invalid")
|
|
1521
|
+
if runtime["node_version"] != self.required_node_version or runtime["platform"] != (self._artifact_manifest or {}).get("platform") or runtime["libc"] != "glibc":
|
|
1522
|
+
raise RuntimeError("offline Node manifest does not match the job runtime contract")
|
|
1523
|
+
for field in ("archive_sha256", "builder_image_id"):
|
|
1524
|
+
if not isinstance(runtime[field], str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", runtime[field]):
|
|
1525
|
+
raise RuntimeError(f"offline Node {field} is invalid")
|
|
1526
|
+
archive = bundle / "node-runtime.tar.gz"
|
|
1527
|
+
info = self._assert_regular_host_file(archive, "offline Node archive", 128 * 1024 * 1024)
|
|
1528
|
+
if type(runtime["archive_bytes"]) is not int or info.st_size != runtime["archive_bytes"] or info.st_size <= 0:
|
|
1529
|
+
raise RuntimeError("offline Node archive size mismatch")
|
|
1530
|
+
archive_digest = hashlib.sha256()
|
|
1531
|
+
with archive.open("rb") as handle:
|
|
1532
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
1533
|
+
archive_digest.update(chunk)
|
|
1534
|
+
if "sha256:" + archive_digest.hexdigest() != runtime["archive_sha256"]:
|
|
1535
|
+
raise RuntimeError("offline Node archive checksum mismatch")
|
|
1536
|
+
return archive, runtime
|
|
1537
|
+
except Exception as error:
|
|
1538
|
+
raise self._node_runtime_error("hitch_node_runtime_integrity_mismatch", str(error)) from error
|
|
1539
|
+
|
|
1540
|
+
async def _node_setup_exec(self, environment: BaseEnvironment, command: str, code: str) -> ExecResult:
|
|
1541
|
+
result = await asyncio.wait_for(environment.exec(command, user=0), timeout=60)
|
|
1542
|
+
if result.return_code != 0:
|
|
1543
|
+
raise self._node_runtime_error(code, f"exit={result.return_code}: {self._exec_diagnostic(result)}")
|
|
1544
|
+
return result
|
|
1461
1545
|
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1546
|
+
def _node_runtime_error(self, code: str, message: str) -> HitchBridgeError:
|
|
1547
|
+
return HitchBridgeError(code, self._bounded_tail(message, 2048), {
|
|
1548
|
+
"schema_version": "1", "code": code, "message": self._bounded_tail(message, 2048),
|
|
1549
|
+
"eval_id": self.eval_id, "node_version": self.required_node_version,
|
|
1550
|
+
"platform": (self._artifact_manifest or {}).get("platform"),
|
|
1551
|
+
"artifact_id": (self.harness_artifact or {}).get("artifact_id"),
|
|
1552
|
+
})
|
|
1553
|
+
|
|
1554
|
+
def _record_node_runtime(self, evidence: dict[str, Any]) -> None:
|
|
1555
|
+
self.logs_dir.mkdir(parents=True, exist_ok=True)
|
|
1556
|
+
(self.logs_dir / "hitch-node-runtime.json").write_text(
|
|
1557
|
+
json.dumps({"schema_version": "1", **evidence}, indent=2, sort_keys=True) + "\n", encoding="utf-8",
|
|
1558
|
+
)
|
|
1559
|
+
|
|
1560
|
+
def _node_prefix(self) -> str:
|
|
1561
|
+
return f"export PATH={shlex.quote(self._node_bin_directory)}:\"$PATH\";" if self._node_bin_directory else ""
|
|
1465
1562
|
|
|
1466
1563
|
@staticmethod
|
|
1467
1564
|
def _events(output: str) -> list[dict[str, Any]]:
|
|
@@ -1495,6 +1592,39 @@ node -e 'process.exit(Number(process.versions.node.split(".")[0]) >= 22 ? 0 : 1)
|
|
|
1495
1592
|
return "no diagnostic output"
|
|
1496
1593
|
|
|
1497
1594
|
|
|
1595
|
+
def _validate_model_capture(value: dict[str, Any] | None) -> dict[str, Any] | None:
|
|
1596
|
+
if value is None:
|
|
1597
|
+
return None
|
|
1598
|
+
if not isinstance(value, dict) or set(value) != {
|
|
1599
|
+
"schema_version", "mode", "required", "topology", "base_url_template", "health_url_template"
|
|
1600
|
+
}:
|
|
1601
|
+
raise ValueError("model_capture fields are invalid")
|
|
1602
|
+
if (
|
|
1603
|
+
value.get("schema_version") != "1"
|
|
1604
|
+
or value.get("mode") not in {"proxy", "hybrid"}
|
|
1605
|
+
or not isinstance(value.get("required"), bool)
|
|
1606
|
+
or value.get("topology") != "host-side"
|
|
1607
|
+
):
|
|
1608
|
+
raise ValueError("model_capture identity is invalid")
|
|
1609
|
+
for field, provider_count in (("base_url_template", 1), ("health_url_template", 0)):
|
|
1610
|
+
template = value.get(field)
|
|
1611
|
+
if (
|
|
1612
|
+
not isinstance(template, str)
|
|
1613
|
+
or not template
|
|
1614
|
+
or len(template) > 2048
|
|
1615
|
+
or any(character in template for character in ("\x00", "\r", "\n"))
|
|
1616
|
+
or template.count("{run_id}") != 1
|
|
1617
|
+
or template.count("{provider}") != provider_count
|
|
1618
|
+
):
|
|
1619
|
+
raise ValueError(f"model_capture {field} is invalid")
|
|
1620
|
+
parsed = urlparse(
|
|
1621
|
+
template.replace("{run_id}", "run_" + "a" * 32).replace("{provider}", "openai")
|
|
1622
|
+
)
|
|
1623
|
+
if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
|
|
1624
|
+
raise ValueError(f"model_capture {field} URL is invalid")
|
|
1625
|
+
return dict(value)
|
|
1626
|
+
|
|
1627
|
+
|
|
1498
1628
|
def canonical_manifest_json(manifest: dict[str, Any]) -> str:
|
|
1499
1629
|
"""Canonically encode the runtime identity `{ schema_version, node_range,
|
|
1500
1630
|
entrypoints, files }` with sorted object keys and no insignificant
|