agent-hitch 0.2.9 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +206 -137
- package/README.zh-CN.md +193 -119
- package/dist/scripts/canary-native-training.js +221 -0
- package/dist/scripts/canary-native-training.js.map +1 -0
- package/dist/scripts/canary-remote-training.js +246 -0
- package/dist/scripts/canary-remote-training.js.map +1 -0
- package/dist/scripts/canary-remote-verifier.js +124 -0
- package/dist/scripts/canary-remote-verifier.js.map +1 -0
- package/dist/scripts/canary-worker-host.js +113 -0
- package/dist/scripts/canary-worker-host.js.map +1 -0
- package/dist/scripts/canary-worker-peer.js +188 -0
- package/dist/scripts/canary-worker-peer.js.map +1 -0
- package/dist/scripts/check-architecture.js +4 -3
- package/dist/scripts/check-architecture.js.map +1 -1
- package/dist/src/adapters/catalog.js +2 -1
- package/dist/src/adapters/catalog.js.map +1 -1
- package/dist/src/adapters/providers/codex.js +65 -2
- package/dist/src/adapters/providers/codex.js.map +1 -1
- package/dist/src/adapters/providers/model-call.js +17 -1
- package/dist/src/adapters/providers/model-call.js.map +1 -1
- package/dist/src/adapters/providers/training-tool.js +29 -0
- package/dist/src/adapters/providers/training-tool.js.map +1 -0
- package/dist/src/artifacts/index.js +1 -1
- package/dist/src/artifacts/index.js.map +1 -1
- package/dist/src/artifacts/store.js +15 -10
- package/dist/src/artifacts/store.js.map +1 -1
- package/dist/src/backends/harbor/backend.js +24 -15
- package/dist/src/backends/harbor/backend.js.map +1 -1
- package/dist/src/backends/harbor/bridge-environment.js +12 -0
- package/dist/src/backends/harbor/bridge-environment.js.map +1 -0
- package/dist/src/backends/harbor/environment-config.js +2 -2
- package/dist/src/backends/harbor/environment-config.js.map +1 -1
- package/dist/src/backends/harbor/host-credential-helper.js +81 -0
- package/dist/src/backends/harbor/host-credential-helper.js.map +1 -0
- package/dist/src/backends/harbor/index.js +3 -0
- package/dist/src/backends/harbor/index.js.map +1 -1
- package/dist/src/backends/harbor/model-proxy-config.js +18 -1
- package/dist/src/backends/harbor/model-proxy-config.js.map +1 -1
- package/dist/src/backends/harbor/observation.js +61 -0
- package/dist/src/backends/harbor/observation.js.map +1 -0
- package/dist/src/backends/harbor/portable-regrade.js +150 -0
- package/dist/src/backends/harbor/portable-regrade.js.map +1 -0
- package/dist/src/backends/harbor/process.js +26 -5
- package/dist/src/backends/harbor/process.js.map +1 -1
- package/dist/src/backends/harbor/regrade.js +4 -1
- package/dist/src/backends/harbor/regrade.js.map +1 -1
- package/dist/src/backends/harbor/supervisor.js +22 -0
- package/dist/src/backends/harbor/supervisor.js.map +1 -1
- package/dist/src/backends/harbor/tools.js +2 -0
- package/dist/src/backends/harbor/tools.js.map +1 -1
- package/dist/src/backends/index.js +3 -0
- package/dist/src/backends/index.js.map +1 -1
- package/dist/src/cli/arguments.js +33 -0
- package/dist/src/cli/arguments.js.map +1 -1
- package/dist/src/cli/commands/capabilities.js +14 -1
- package/dist/src/cli/commands/capabilities.js.map +1 -1
- package/dist/src/cli/commands/daemon.js +19 -0
- package/dist/src/cli/commands/daemon.js.map +1 -1
- package/dist/src/cli/commands/eval.js +38 -10
- package/dist/src/cli/commands/eval.js.map +1 -1
- package/dist/src/cli/commands/local.js +130 -0
- package/dist/src/cli/commands/local.js.map +1 -0
- package/dist/src/cli/commands/model-node.js +26 -0
- package/dist/src/cli/commands/model-node.js.map +1 -0
- package/dist/src/cli/commands/models.js +72 -0
- package/dist/src/cli/commands/models.js.map +1 -0
- package/dist/src/cli/commands/run.js +36 -1
- package/dist/src/cli/commands/run.js.map +1 -1
- package/dist/src/cli/commands/training.js +32 -0
- package/dist/src/cli/commands/training.js.map +1 -0
- package/dist/src/cli/commands/worker.js +30 -2
- package/dist/src/cli/commands/worker.js.map +1 -1
- package/dist/src/cli/main.js +8 -0
- package/dist/src/cli/main.js.map +1 -1
- package/dist/src/cli/output.js +24 -2
- package/dist/src/cli/output.js.map +1 -1
- package/dist/src/control-plane/eval-records.js +2 -0
- package/dist/src/control-plane/eval-records.js.map +1 -1
- package/dist/src/control-plane/eval-recovery.js +7 -1
- package/dist/src/control-plane/eval-recovery.js.map +1 -1
- package/dist/src/control-plane/eval-scheduler.js +24 -50
- package/dist/src/control-plane/eval-scheduler.js.map +1 -1
- package/dist/src/control-plane/eval-submission-persist.js +30 -0
- package/dist/src/control-plane/eval-submission-persist.js.map +1 -0
- package/dist/src/control-plane/execution-observation-contract.js +60 -0
- package/dist/src/control-plane/execution-observation-contract.js.map +1 -0
- package/dist/src/control-plane/index.js +10 -0
- package/dist/src/control-plane/index.js.map +1 -1
- package/dist/src/control-plane/inference-manager.js +357 -0
- package/dist/src/control-plane/inference-manager.js.map +1 -0
- package/dist/src/control-plane/managed-gateway-receipt.js +26 -0
- package/dist/src/control-plane/managed-gateway-receipt.js.map +1 -0
- package/dist/src/control-plane/managed-service-recovery.js +158 -0
- package/dist/src/control-plane/managed-service-recovery.js.map +1 -0
- package/dist/src/control-plane/model-capture-planning.js +15 -1
- package/dist/src/control-plane/model-capture-planning.js.map +1 -1
- package/dist/src/control-plane/ordered-eval-control.js +221 -0
- package/dist/src/control-plane/ordered-eval-control.js.map +1 -0
- package/dist/src/control-plane/remote-backend-result.js +18 -0
- package/dist/src/control-plane/remote-backend-result.js.map +1 -0
- package/dist/src/control-plane/remote-execution-lease.js +15 -0
- package/dist/src/control-plane/remote-execution-lease.js.map +1 -0
- package/dist/src/control-plane/remote-execution-observation.js +108 -0
- package/dist/src/control-plane/remote-execution-observation.js.map +1 -0
- package/dist/src/control-plane/remote-model-routes.js +107 -0
- package/dist/src/control-plane/remote-model-routes.js.map +1 -0
- package/dist/src/control-plane/remote-observation-reporter.js +37 -0
- package/dist/src/control-plane/remote-observation-reporter.js.map +1 -0
- package/dist/src/control-plane/remote-offer-contract.js +48 -0
- package/dist/src/control-plane/remote-offer-contract.js.map +1 -0
- package/dist/src/control-plane/remote-rerun-journal.js +61 -0
- package/dist/src/control-plane/remote-rerun-journal.js.map +1 -0
- package/dist/src/control-plane/remote-result-transport.js +55 -9
- package/dist/src/control-plane/remote-result-transport.js.map +1 -1
- package/dist/src/control-plane/remote-verifier-import.js +52 -0
- package/dist/src/control-plane/remote-verifier-import.js.map +1 -0
- package/dist/src/control-plane/remote-verifier-result.js +61 -0
- package/dist/src/control-plane/remote-verifier-result.js.map +1 -0
- package/dist/src/control-plane/remote-verifier-source-transport.js +68 -0
- package/dist/src/control-plane/remote-verifier-source-transport.js.map +1 -0
- package/dist/src/control-plane/remote-work-coordinator.js +103 -47
- package/dist/src/control-plane/remote-work-coordinator.js.map +1 -1
- package/dist/src/control-plane/remote-work-inputs.js +26 -5
- package/dist/src/control-plane/remote-work-inputs.js.map +1 -1
- package/dist/src/control-plane/remote-work-recovery.js +123 -34
- package/dist/src/control-plane/remote-work-recovery.js.map +1 -1
- package/dist/src/control-plane/remote-worker-artifacts.js +10 -7
- package/dist/src/control-plane/remote-worker-artifacts.js.map +1 -1
- package/dist/src/control-plane/remote-worker-cleanup.js +104 -0
- package/dist/src/control-plane/remote-worker-cleanup.js.map +1 -0
- package/dist/src/control-plane/remote-worker-client.js +156 -14
- package/dist/src/control-plane/remote-worker-client.js.map +1 -1
- package/dist/src/control-plane/remote-worker-codecs.js +160 -0
- package/dist/src/control-plane/remote-worker-codecs.js.map +1 -0
- package/dist/src/control-plane/remote-worker-generation-recovery.js +67 -0
- package/dist/src/control-plane/remote-worker-generation-recovery.js.map +1 -0
- package/dist/src/control-plane/remote-worker-ownership.js +51 -0
- package/dist/src/control-plane/remote-worker-ownership.js.map +1 -0
- package/dist/src/control-plane/remote-worker-protocol.js +85 -119
- package/dist/src/control-plane/remote-worker-protocol.js.map +1 -1
- package/dist/src/control-plane/remote-worker-runner.js +95 -18
- package/dist/src/control-plane/remote-worker-runner.js.map +1 -1
- package/dist/src/control-plane/remote-workers.js +33 -5
- package/dist/src/control-plane/remote-workers.js.map +1 -1
- package/dist/src/control-plane/rerun-recovery.js +89 -0
- package/dist/src/control-plane/rerun-recovery.js.map +1 -0
- package/dist/src/control-plane/rerun-scheduler.js +70 -96
- package/dist/src/control-plane/rerun-scheduler.js.map +1 -1
- package/dist/src/control-plane/rerun-source.js +57 -0
- package/dist/src/control-plane/rerun-source.js.map +1 -0
- package/dist/src/control-plane/worker-model-relay.js +76 -0
- package/dist/src/control-plane/worker-model-relay.js.map +1 -0
- package/dist/src/controller-runtime/hash.js +3 -0
- package/dist/src/controller-runtime/hash.js.map +1 -1
- package/dist/src/controller-runtime/index.js +2 -0
- package/dist/src/controller-runtime/index.js.map +1 -1
- package/dist/src/controller-runtime/observation.js +29 -0
- package/dist/src/controller-runtime/observation.js.map +1 -0
- package/dist/src/controller-runtime/resident.js +22 -0
- package/dist/src/controller-runtime/resident.js.map +1 -0
- package/dist/src/daemon/client.js +34 -1
- package/dist/src/daemon/client.js.map +1 -1
- package/dist/src/daemon/execution-observation.js +38 -0
- package/dist/src/daemon/execution-observation.js.map +1 -0
- package/dist/src/daemon/http.js +57 -0
- package/dist/src/daemon/http.js.map +1 -0
- package/dist/src/daemon/inference-prepare.js +47 -0
- package/dist/src/daemon/inference-prepare.js.map +1 -0
- package/dist/src/daemon/scheduler.js +4 -1
- package/dist/src/daemon/scheduler.js.map +1 -1
- package/dist/src/daemon/server.js +80 -82
- package/dist/src/daemon/server.js.map +1 -1
- package/dist/src/daemon/worker-observation.js +34 -0
- package/dist/src/daemon/worker-observation.js.map +1 -0
- package/dist/src/daemon/worker-ownership.js +71 -0
- package/dist/src/daemon/worker-ownership.js.map +1 -0
- package/dist/src/daemon/worker-routes.js +83 -15
- package/dist/src/daemon/worker-routes.js.map +1 -1
- package/dist/src/domain/execution-observation.js +2 -0
- package/dist/src/domain/execution-observation.js.map +1 -0
- package/dist/src/domain/index.js +1 -0
- package/dist/src/domain/index.js.map +1 -1
- package/dist/src/domain/inference.js +2 -0
- package/dist/src/domain/inference.js.map +1 -0
- package/dist/src/domain/runs.js.map +1 -1
- package/dist/src/domain/training.js +2 -0
- package/dist/src/domain/training.js.map +1 -0
- package/dist/src/domain/validation.js +11 -0
- package/dist/src/domain/validation.js.map +1 -1
- package/dist/src/domain/verifier-source.js +2 -0
- package/dist/src/domain/verifier-source.js.map +1 -0
- package/dist/src/domain/worker-recovery.js +2 -0
- package/dist/src/domain/worker-recovery.js.map +1 -0
- package/dist/src/evals/eval-logical-plan.js +8 -1
- package/dist/src/evals/eval-logical-plan.js.map +1 -1
- package/dist/src/evals/execution-leases.js +15 -2
- package/dist/src/evals/execution-leases.js.map +1 -1
- package/dist/src/evals/execution-plan.js +11 -1
- package/dist/src/evals/execution-plan.js.map +1 -1
- package/dist/src/evals/harbor-bridge-error.js +8 -0
- package/dist/src/evals/harbor-bridge-error.js.map +1 -1
- package/dist/src/evals/index.js +19 -0
- package/dist/src/evals/index.js.map +1 -1
- package/dist/src/evals/model-capture-plan.js +12 -0
- package/dist/src/evals/model-capture-plan.js.map +1 -1
- package/dist/src/evals/model-capture-runtime.js +25 -0
- package/dist/src/evals/model-capture-runtime.js.map +1 -1
- package/dist/src/evals/physical-work-plan.js +57 -0
- package/dist/src/evals/physical-work-plan.js.map +1 -0
- package/dist/src/evals/preparation-rerun.js +6 -0
- package/dist/src/evals/preparation-rerun.js.map +1 -1
- package/dist/src/evals/recovery.js +8 -1
- package/dist/src/evals/recovery.js.map +1 -1
- package/dist/src/evals/regrade-evidence.js +4 -4
- package/dist/src/evals/regrade-evidence.js.map +1 -1
- package/dist/src/evals/remote-execution-ownership.js +50 -0
- package/dist/src/evals/remote-execution-ownership.js.map +1 -0
- package/dist/src/evals/remote-infrastructure-retry.js +2 -0
- package/dist/src/evals/remote-infrastructure-retry.js.map +1 -1
- package/dist/src/evals/remote-lease-release.js +58 -0
- package/dist/src/evals/remote-lease-release.js.map +1 -0
- package/dist/src/evals/remote-rerun-execution.js +85 -0
- package/dist/src/evals/remote-rerun-execution.js.map +1 -0
- package/dist/src/evals/remote-trial.js +6 -0
- package/dist/src/evals/remote-trial.js.map +1 -0
- package/dist/src/evals/remote-verifier-assessment.js +85 -0
- package/dist/src/evals/remote-verifier-assessment.js.map +1 -0
- package/dist/src/evals/remote-verifier-contract.js +43 -0
- package/dist/src/evals/remote-verifier-contract.js.map +1 -0
- package/dist/src/evals/remote-verifier-execution.js +105 -0
- package/dist/src/evals/remote-verifier-execution.js.map +1 -0
- package/dist/src/evals/remote-verifier-journal.js +21 -0
- package/dist/src/evals/remote-verifier-journal.js.map +1 -0
- package/dist/src/evals/remote-verifier-outcome.js +51 -0
- package/dist/src/evals/remote-verifier-outcome.js.map +1 -0
- package/dist/src/evals/remote-verifier-rerun.js +134 -0
- package/dist/src/evals/remote-verifier-rerun.js.map +1 -0
- package/dist/src/evals/remote-verifier-selection.js +61 -0
- package/dist/src/evals/remote-verifier-selection.js.map +1 -0
- package/dist/src/evals/remote-verifier-source.js +33 -0
- package/dist/src/evals/remote-verifier-source.js.map +1 -0
- package/dist/src/evals/remote-worker-cleanup.js +46 -0
- package/dist/src/evals/remote-worker-cleanup.js.map +1 -0
- package/dist/src/evals/request.js +58 -1
- package/dist/src/evals/request.js.map +1 -1
- package/dist/src/evals/rerun-completion.js +39 -0
- package/dist/src/evals/rerun-completion.js.map +1 -0
- package/dist/src/evals/rerun-resume.js +70 -0
- package/dist/src/evals/rerun-resume.js.map +1 -0
- package/dist/src/evals/rerun-state.js +25 -0
- package/dist/src/evals/rerun-state.js.map +1 -0
- package/dist/src/evals/rerun-types.js.map +1 -1
- package/dist/src/evals/rerun.js +90 -46
- package/dist/src/evals/rerun.js.map +1 -1
- package/dist/src/evals/service.js +59 -5
- package/dist/src/evals/service.js.map +1 -1
- package/dist/src/evals/verifier-only-rerun.js +2 -16
- package/dist/src/evals/verifier-only-rerun.js.map +1 -1
- package/dist/src/evals/verifier-rerun-result.js +24 -0
- package/dist/src/evals/verifier-rerun-result.js.map +1 -0
- package/dist/src/evals/verifier-source-import.js +48 -0
- package/dist/src/evals/verifier-source-import.js.map +1 -0
- package/dist/src/evals/verifier-source.js +139 -0
- package/dist/src/evals/verifier-source.js.map +1 -0
- package/dist/src/foundation/base64.js +19 -0
- package/dist/src/foundation/base64.js.map +1 -0
- package/dist/src/foundation/config.js +10 -0
- package/dist/src/foundation/config.js.map +1 -1
- package/dist/src/foundation/credential-redaction.js +22 -9
- package/dist/src/foundation/credential-redaction.js.map +1 -1
- package/dist/src/foundation/index.js +1 -0
- package/dist/src/foundation/index.js.map +1 -1
- package/dist/src/foundation/process-identity.js +13 -9
- package/dist/src/foundation/process-identity.js.map +1 -1
- package/dist/src/inference/daemon-capacity.js +41 -0
- package/dist/src/inference/daemon-capacity.js.map +1 -0
- package/dist/src/inference/device-reservation.js +58 -0
- package/dist/src/inference/device-reservation.js.map +1 -0
- package/dist/src/inference/doctor.js +95 -0
- package/dist/src/inference/doctor.js.map +1 -0
- package/dist/src/inference/generation-recovery.js +83 -0
- package/dist/src/inference/generation-recovery.js.map +1 -0
- package/dist/src/inference/index.js +24 -0
- package/dist/src/inference/index.js.map +1 -0
- package/dist/src/inference/lock.js +331 -0
- package/dist/src/inference/lock.js.map +1 -0
- package/dist/src/inference/managed-launcher.js +35 -0
- package/dist/src/inference/managed-launcher.js.map +1 -0
- package/dist/src/inference/manifest.js +262 -0
- package/dist/src/inference/manifest.js.map +1 -0
- package/dist/src/inference/materialize.js +66 -0
- package/dist/src/inference/materialize.js.map +1 -0
- package/dist/src/inference/model-location.js +34 -0
- package/dist/src/inference/model-location.js.map +1 -0
- package/dist/src/inference/model-store.js +439 -0
- package/dist/src/inference/model-store.js.map +1 -0
- package/dist/src/inference/node-client.js +167 -0
- package/dist/src/inference/node-client.js.map +1 -0
- package/dist/src/inference/node-registry.js +79 -0
- package/dist/src/inference/node-registry.js.map +1 -0
- package/dist/src/inference/observation.js +79 -0
- package/dist/src/inference/observation.js.map +1 -0
- package/dist/src/inference/preflight.js +132 -0
- package/dist/src/inference/preflight.js.map +1 -0
- package/dist/src/inference/process-sglang.js +169 -0
- package/dist/src/inference/process-sglang.js.map +1 -0
- package/dist/src/inference/protocol-probe.js +29 -0
- package/dist/src/inference/protocol-probe.js.map +1 -0
- package/dist/src/inference/remote-model-store.js +24 -0
- package/dist/src/inference/remote-model-store.js.map +1 -0
- package/dist/src/inference/runtime-catalog.js +41 -0
- package/dist/src/inference/runtime-catalog.js.map +1 -0
- package/dist/src/inference/runtime-store.js +93 -0
- package/dist/src/inference/runtime-store.js.map +1 -0
- package/dist/src/inference/service-attachment.js +46 -0
- package/dist/src/inference/service-attachment.js.map +1 -0
- package/dist/src/inference/service-inspection.js +38 -0
- package/dist/src/inference/service-inspection.js.map +1 -0
- package/dist/src/inference/sglang.js +279 -0
- package/dist/src/inference/sglang.js.map +1 -0
- package/dist/src/inference/supervisor.js +460 -0
- package/dist/src/inference/supervisor.js.map +1 -0
- package/dist/src/model-access/environment.js +28 -0
- package/dist/src/model-access/environment.js.map +1 -0
- package/dist/src/model-access/index.js +4 -0
- package/dist/src/model-access/index.js.map +1 -1
- package/dist/src/model-access/local-gateway-request.js +139 -0
- package/dist/src/model-access/local-gateway-request.js.map +1 -0
- package/dist/src/model-access/local-gateway.js +323 -0
- package/dist/src/model-access/local-gateway.js.map +1 -0
- package/dist/src/model-access/proxy.js +90 -5
- package/dist/src/model-access/proxy.js.map +1 -1
- package/dist/src/model-access/remote-model.js +58 -0
- package/dist/src/model-access/remote-model.js.map +1 -0
- package/dist/src/model-access/training.js +115 -0
- package/dist/src/model-access/training.js.map +1 -0
- package/dist/src/runs/adapter-process.js +2 -2
- package/dist/src/runs/adapter-process.js.map +1 -1
- package/dist/src/runs/executor-types.js +2 -0
- package/dist/src/runs/executor-types.js.map +1 -0
- package/dist/src/runs/executor.js +54 -34
- package/dist/src/runs/executor.js.map +1 -1
- package/dist/src/runs/index.js +1 -0
- package/dist/src/runs/index.js.map +1 -1
- package/dist/src/runs/local-inference-environment.js +59 -0
- package/dist/src/runs/local-inference-environment.js.map +1 -0
- package/dist/src/runs/local-inference-run.js +53 -0
- package/dist/src/runs/local-inference-run.js.map +1 -0
- package/dist/src/runs/outcome.js +22 -0
- package/dist/src/runs/outcome.js.map +1 -1
- package/dist/src/runs/records.js +9 -0
- package/dist/src/runs/records.js.map +1 -1
- package/dist/src/runs/request.js +58 -3
- package/dist/src/runs/request.js.map +1 -1
- package/dist/src/runs/training-evidence.js +22 -0
- package/dist/src/runs/training-evidence.js.map +1 -0
- package/dist/src/runs/training-runtime.js +20 -0
- package/dist/src/runs/training-runtime.js.map +1 -0
- package/dist/src/workers/execution-observation.js +32 -0
- package/dist/src/workers/execution-observation.js.map +1 -0
- package/dist/src/workers/index.js +2 -0
- package/dist/src/workers/index.js.map +1 -1
- package/dist/src/workers/remote-harbor-ownership.js +266 -0
- package/dist/src/workers/remote-harbor-ownership.js.map +1 -0
- package/dist/src/workers/remote-harbor-verifier.js +76 -0
- package/dist/src/workers/remote-harbor-verifier.js.map +1 -0
- package/dist/src/workers/remote-harbor-work-spec.js +66 -10
- package/dist/src/workers/remote-harbor-work-spec.js.map +1 -1
- package/dist/src/workers/remote-harbor-worker.js +56 -49
- package/dist/src/workers/remote-harbor-worker.js.map +1 -1
- package/dist/src/workers/remote-worker-host.js +33 -0
- package/dist/src/workers/remote-worker-host.js.map +1 -0
- package/docs/local-model-inference-spec.zh-CN.md +666 -0
- package/docs/schemas/controller-runtime-observation.schema.json +21 -0
- package/docs/schemas/eval-execution-plan.schema.json +1 -0
- package/docs/schemas/eval-request.schema.json +3 -1
- package/docs/schemas/execution-lease.schema.json +95 -1
- package/docs/schemas/execution-provider-status.schema.json +162 -28
- package/docs/schemas/inference-execution-evidence.schema.json +487 -0
- package/docs/schemas/inference-lock.schema.json +487 -0
- package/docs/schemas/inference-runtime-manifest.schema.json +156 -0
- package/docs/schemas/inference-service-record.schema.json +227 -0
- package/docs/schemas/local-inference-selection.schema.json +98 -0
- package/docs/schemas/local-model-manifest.schema.json +43 -0
- package/docs/schemas/model-node-binding.schema.json +34 -0
- package/docs/schemas/model-node-generation-release.schema.json +46 -0
- package/docs/schemas/ordered-eval-control.schema.json +39 -0
- package/docs/schemas/remote-execution-lease-grant.schema.json +45 -0
- package/docs/schemas/remote-harbor-work-spec.schema.json +325 -0
- package/docs/schemas/remote-model-binding.schema.json +21 -0
- package/docs/schemas/remote-physical-execution.schema.json +108 -0
- package/docs/schemas/remote-result-envelope.schema.json +9 -3
- package/docs/schemas/remote-tree-envelope.schema.json +17 -3
- package/docs/schemas/remote-verifier-result.schema.json +203 -0
- package/docs/schemas/remote-verifier-source.schema.json +52 -0
- package/docs/schemas/remote-verifier-work.schema.json +125 -0
- package/docs/schemas/remote-work-offer.schema.json +170 -34
- package/docs/schemas/remote-worker-execution-admission.schema.json +214 -0
- package/docs/schemas/remote-worker-generation-cleanup.schema.json +199 -0
- package/docs/schemas/remote-worker-registration.schema.json +226 -25
- package/docs/schemas/run-manifest.schema.json +211 -44
- package/docs/schemas/run-request.schema.json +2 -1
- package/docs/schemas/training-external-binding.schema.json +26 -0
- package/integrations/harbor/hitch_harbor_agent.py +241 -13
- package/integrations/harbor/hitch_harbor_environment.py +22 -1
- package/integrations/harbor/hitch_host_credentials.py +354 -0
- package/integrations/harbor/hitch_private_exec.py +150 -0
- package/integrations/model-call/cli.js +13 -5
- package/integrations/training-tool/cli.js +72 -0
- package/package.json +8 -2
|
@@ -31,6 +31,7 @@ _REQUIRED_LABELS = {
|
|
|
31
31
|
_ALLOWED_LABELS = _REQUIRED_LABELS | {_LABEL_TASK}
|
|
32
32
|
_COMPOSE_RESET_MARKER = "__HITCH_COMPOSE_RESET_NULL__"
|
|
33
33
|
_GPU_OVERRIDE_PREFIX = "__HITCH_GPU_OVERRIDE_"
|
|
34
|
+
_HOST_CREDENTIAL_HELPER_ENV = "HITCH_HOST_CREDENTIAL_HELPER_JSON"
|
|
34
35
|
|
|
35
36
|
|
|
36
37
|
class HitchHarborDockerEnvironment(DockerEnvironment):
|
|
@@ -128,6 +129,25 @@ class HitchHarborDockerEnvironment(DockerEnvironment):
|
|
|
128
129
|
return result.model_copy(update={"disable_internet": True, "dynamic_network_policy": False})
|
|
129
130
|
return result
|
|
130
131
|
|
|
132
|
+
def _compose_env_vars(self, include_os_env=True):
|
|
133
|
+
result = super()._compose_env_vars(include_os_env=include_os_env)
|
|
134
|
+
# The host-only helper command is not a task Compose interpolation
|
|
135
|
+
# input. The agent reads it directly from its trusted host process.
|
|
136
|
+
result.pop(_HOST_CREDENTIAL_HELPER_ENV, None)
|
|
137
|
+
return result
|
|
138
|
+
|
|
139
|
+
async def hitch_exec_with_private_env(self, command, *, env, cwd=None, timeout_sec=None, user=None):
|
|
140
|
+
"""Execute one Target without placing per-exec credential values in argv."""
|
|
141
|
+
from hitch_private_exec import exec_with_private_environment
|
|
142
|
+
return await exec_with_private_environment(
|
|
143
|
+
self,
|
|
144
|
+
command,
|
|
145
|
+
env=env,
|
|
146
|
+
cwd=cwd,
|
|
147
|
+
timeout_sec=timeout_sec,
|
|
148
|
+
user=user,
|
|
149
|
+
)
|
|
150
|
+
|
|
131
151
|
async def _apply_network_policy(self, network_policy):
|
|
132
152
|
if self._hitch_static_no_network:
|
|
133
153
|
if network_policy != self.network_policy:
|
|
@@ -375,7 +395,8 @@ def _validate_labels(value: Mapping[str, str] | None) -> dict[str, str]:
|
|
|
375
395
|
labels = dict(value)
|
|
376
396
|
if (
|
|
377
397
|
not re.fullmatch(r"[a-f0-9]{24}", labels.get(_LABEL_ROOT, ""))
|
|
378
|
-
or labels.get(_LABEL_PROVIDER)
|
|
398
|
+
or not isinstance(labels.get(_LABEL_PROVIDER), str)
|
|
399
|
+
or not re.fullmatch(r"[a-z0-9][a-z0-9._-]{0,127}", labels[_LABEL_PROVIDER])
|
|
379
400
|
or not re.fullmatch(r"eval_[a-f0-9]{32}", labels.get(_LABEL_EVAL, ""))
|
|
380
401
|
or not re.fullmatch(r"work_[a-f0-9]{32}", labels.get(_LABEL_WORK, ""))
|
|
381
402
|
or not re.fullmatch(r"lease_[a-f0-9]{32}", labels.get(_LABEL_LEASE, ""))
|
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
"""Prepare short-lived credentials on the Harbor host at Target start."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import signal
|
|
10
|
+
import subprocess
|
|
11
|
+
import sys
|
|
12
|
+
import time
|
|
13
|
+
from typing import Any, NamedTuple
|
|
14
|
+
|
|
15
|
+
HOST_CREDENTIAL_HELPER_ENV = "HITCH_HOST_CREDENTIAL_HELPER_JSON"
|
|
16
|
+
HOST_CREDENTIAL_HELPER_CAPABILITY = "host-task-credential-helper-v1"
|
|
17
|
+
HOST_CREDENTIAL_VALIDITY_MARGIN_MS = 5 * 60 * 1000
|
|
18
|
+
|
|
19
|
+
_MAX_CONFIG_BYTES = 64 * 1024
|
|
20
|
+
_MAX_OUTPUT_BYTES = 256 * 1024
|
|
21
|
+
_MAX_ARGUMENTS = 32
|
|
22
|
+
_MAX_ARGUMENT_LENGTH = 4_096
|
|
23
|
+
_MAX_CREDENTIAL_BYTES = 256 * 1024
|
|
24
|
+
_ENVIRONMENT_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
25
|
+
_WINDOWS_JOB_LAUNCHER = "--hitch-host-credential-helper-job"
|
|
26
|
+
_WINDOWS_JOB_HANDLE = None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class _HelperOutputTooLarge(RuntimeError):
|
|
30
|
+
pass
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class HostCredentialHelperError(RuntimeError):
|
|
34
|
+
"""A stable failure that never includes helper output or credential values."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, code: str, message: str) -> None:
|
|
37
|
+
super().__init__(f"{code}: {message}")
|
|
38
|
+
self.code = code
|
|
39
|
+
self.message = message
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class HostCredentialHelperConfig(NamedTuple):
|
|
43
|
+
argv: tuple[str, ...]
|
|
44
|
+
credential_names: tuple[str, ...]
|
|
45
|
+
timeout_ms: int
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def load_host_credential_helper(available_names: list[str]) -> HostCredentialHelperConfig | None:
|
|
49
|
+
encoded = os.environ.get(HOST_CREDENTIAL_HELPER_ENV)
|
|
50
|
+
if encoded is None:
|
|
51
|
+
return None
|
|
52
|
+
try:
|
|
53
|
+
if not encoded or len(encoded.encode("utf-8")) > _MAX_CONFIG_BYTES:
|
|
54
|
+
raise ValueError
|
|
55
|
+
value = json.loads(encoded)
|
|
56
|
+
if not isinstance(value, dict) or set(value) != {"version", "argv", "credentialNames", "timeoutMs"}:
|
|
57
|
+
raise ValueError
|
|
58
|
+
if isinstance(value["version"], bool) or value["version"] != 1:
|
|
59
|
+
raise ValueError
|
|
60
|
+
argv = value["argv"]
|
|
61
|
+
if (
|
|
62
|
+
not isinstance(argv, list)
|
|
63
|
+
or not 1 <= len(argv) <= _MAX_ARGUMENTS
|
|
64
|
+
or any(not isinstance(item, str) or not item or len(item) > _MAX_ARGUMENT_LENGTH or re.search(r"[\0\r\n]", item) for item in argv)
|
|
65
|
+
or not os.path.isabs(argv[0])
|
|
66
|
+
):
|
|
67
|
+
raise ValueError
|
|
68
|
+
names = value["credentialNames"]
|
|
69
|
+
if (
|
|
70
|
+
not isinstance(names, list)
|
|
71
|
+
or not names
|
|
72
|
+
or any(not isinstance(name, str) or _ENVIRONMENT_NAME.fullmatch(name) is None for name in names)
|
|
73
|
+
or len(set(names)) != len(names)
|
|
74
|
+
or any(name not in available_names for name in names)
|
|
75
|
+
):
|
|
76
|
+
raise ValueError
|
|
77
|
+
timeout_ms = value["timeoutMs"]
|
|
78
|
+
if isinstance(timeout_ms, bool) or not isinstance(timeout_ms, int) or not 100 <= timeout_ms <= 60_000:
|
|
79
|
+
raise ValueError
|
|
80
|
+
except (KeyError, TypeError, ValueError, json.JSONDecodeError):
|
|
81
|
+
raise HostCredentialHelperError(
|
|
82
|
+
"host_credential_helper_config_invalid",
|
|
83
|
+
"host credential helper configuration is invalid",
|
|
84
|
+
) from None
|
|
85
|
+
return HostCredentialHelperConfig(tuple(argv), tuple(sorted(names)), timeout_ms)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
async def prepare_host_credentials(
|
|
89
|
+
config: HostCredentialHelperConfig,
|
|
90
|
+
minimum_validity_ms: int,
|
|
91
|
+
) -> dict[str, str]:
|
|
92
|
+
if (
|
|
93
|
+
isinstance(minimum_validity_ms, bool)
|
|
94
|
+
or not isinstance(minimum_validity_ms, int)
|
|
95
|
+
or not 1 <= minimum_validity_ms <= 9_007_199_254_740_991
|
|
96
|
+
):
|
|
97
|
+
raise HostCredentialHelperError(
|
|
98
|
+
"host_credential_helper_request_invalid",
|
|
99
|
+
"host credential helper request is invalid",
|
|
100
|
+
)
|
|
101
|
+
request = json.dumps(
|
|
102
|
+
{
|
|
103
|
+
"version": 1,
|
|
104
|
+
"credentialNames": list(config.credential_names),
|
|
105
|
+
"minimumValidityMs": minimum_validity_ms,
|
|
106
|
+
},
|
|
107
|
+
separators=(",", ":"),
|
|
108
|
+
sort_keys=True,
|
|
109
|
+
).encode("utf-8") + b"\n"
|
|
110
|
+
|
|
111
|
+
spawn_options: dict[str, Any]
|
|
112
|
+
argv: tuple[str, ...]
|
|
113
|
+
if os.name == "nt":
|
|
114
|
+
argv = (sys.executable, os.path.abspath(__file__), _WINDOWS_JOB_LAUNCHER, *config.argv)
|
|
115
|
+
spawn_options = {"creationflags": subprocess.CREATE_NEW_PROCESS_GROUP}
|
|
116
|
+
else:
|
|
117
|
+
argv = config.argv
|
|
118
|
+
spawn_options = {"start_new_session": True}
|
|
119
|
+
|
|
120
|
+
try:
|
|
121
|
+
process = await asyncio.create_subprocess_exec(
|
|
122
|
+
*argv,
|
|
123
|
+
stdin=asyncio.subprocess.PIPE,
|
|
124
|
+
stdout=asyncio.subprocess.PIPE,
|
|
125
|
+
stderr=asyncio.subprocess.PIPE,
|
|
126
|
+
env=dict(os.environ),
|
|
127
|
+
**spawn_options,
|
|
128
|
+
)
|
|
129
|
+
except (OSError, ValueError):
|
|
130
|
+
raise HostCredentialHelperError(
|
|
131
|
+
"host_credential_helper_unavailable",
|
|
132
|
+
"host credential helper could not be started",
|
|
133
|
+
) from None
|
|
134
|
+
|
|
135
|
+
try:
|
|
136
|
+
stdout, _stderr = await asyncio.wait_for(
|
|
137
|
+
_communicate_limited(process, request),
|
|
138
|
+
timeout=config.timeout_ms / 1000,
|
|
139
|
+
)
|
|
140
|
+
except asyncio.TimeoutError:
|
|
141
|
+
await _terminate_helper(process)
|
|
142
|
+
raise HostCredentialHelperError(
|
|
143
|
+
"host_credential_helper_timed_out",
|
|
144
|
+
"host credential helper exceeded its time limit",
|
|
145
|
+
) from None
|
|
146
|
+
except _HelperOutputTooLarge:
|
|
147
|
+
await _terminate_helper(process)
|
|
148
|
+
raise invalid_response() from None
|
|
149
|
+
except asyncio.CancelledError:
|
|
150
|
+
await _terminate_helper(process)
|
|
151
|
+
raise
|
|
152
|
+
|
|
153
|
+
if process.returncode != 0:
|
|
154
|
+
raise HostCredentialHelperError(
|
|
155
|
+
"host_credential_helper_failed",
|
|
156
|
+
"host credential helper did not provide credentials",
|
|
157
|
+
)
|
|
158
|
+
try:
|
|
159
|
+
response: Any = json.loads(stdout.decode("utf-8"))
|
|
160
|
+
if (
|
|
161
|
+
not isinstance(response, dict)
|
|
162
|
+
or set(response) != {"version", "env", "expiresAtMs"}
|
|
163
|
+
or isinstance(response["version"], bool)
|
|
164
|
+
or response["version"] != 1
|
|
165
|
+
):
|
|
166
|
+
raise ValueError
|
|
167
|
+
expires_at_ms = response["expiresAtMs"]
|
|
168
|
+
if isinstance(expires_at_ms, bool) or not isinstance(expires_at_ms, int):
|
|
169
|
+
raise ValueError
|
|
170
|
+
values = response["env"]
|
|
171
|
+
if not isinstance(values, dict) or set(values) != set(config.credential_names):
|
|
172
|
+
raise ValueError
|
|
173
|
+
if any(
|
|
174
|
+
not isinstance(name, str)
|
|
175
|
+
or not isinstance(value, str)
|
|
176
|
+
or not value
|
|
177
|
+
or "\0" in value
|
|
178
|
+
for name, value in values.items()
|
|
179
|
+
):
|
|
180
|
+
raise ValueError
|
|
181
|
+
if sum(len(value.encode("utf-8")) for value in values.values()) > _MAX_CREDENTIAL_BYTES:
|
|
182
|
+
raise ValueError
|
|
183
|
+
except (TypeError, ValueError, UnicodeDecodeError, json.JSONDecodeError):
|
|
184
|
+
raise invalid_response() from None
|
|
185
|
+
|
|
186
|
+
if expires_at_ms <= int(time.time() * 1000) + minimum_validity_ms:
|
|
187
|
+
raise HostCredentialHelperError(
|
|
188
|
+
"host_credential_helper_insufficient_validity",
|
|
189
|
+
"host credential helper returned credentials with insufficient validity",
|
|
190
|
+
)
|
|
191
|
+
return dict(values)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
async def _communicate_limited(process, request: bytes) -> tuple[bytes, bytes]:
|
|
195
|
+
async def read(stream) -> bytes:
|
|
196
|
+
chunks: list[bytes] = []
|
|
197
|
+
total = 0
|
|
198
|
+
while True:
|
|
199
|
+
chunk = await stream.read(min(64 * 1024, _MAX_OUTPUT_BYTES + 1 - total))
|
|
200
|
+
if not chunk:
|
|
201
|
+
return b"".join(chunks)
|
|
202
|
+
chunks.append(chunk)
|
|
203
|
+
total += len(chunk)
|
|
204
|
+
if total > _MAX_OUTPUT_BYTES:
|
|
205
|
+
raise _HelperOutputTooLarge
|
|
206
|
+
|
|
207
|
+
stdout_task = asyncio.create_task(read(process.stdout))
|
|
208
|
+
stderr_task = asyncio.create_task(read(process.stderr))
|
|
209
|
+
wait_task = asyncio.create_task(process.wait())
|
|
210
|
+
try:
|
|
211
|
+
process.stdin.write(request)
|
|
212
|
+
try:
|
|
213
|
+
await process.stdin.drain()
|
|
214
|
+
except (BrokenPipeError, ConnectionResetError):
|
|
215
|
+
pass
|
|
216
|
+
process.stdin.close()
|
|
217
|
+
stdout, stderr, _return_code = await asyncio.gather(stdout_task, stderr_task, wait_task)
|
|
218
|
+
return stdout, stderr
|
|
219
|
+
finally:
|
|
220
|
+
for task in (stdout_task, stderr_task, wait_task):
|
|
221
|
+
if not task.done():
|
|
222
|
+
task.cancel()
|
|
223
|
+
await asyncio.gather(stdout_task, stderr_task, wait_task, return_exceptions=True)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
async def _terminate_helper(process) -> None:
|
|
227
|
+
"""Kill a failed helper and drain both pipes without retaining their contents."""
|
|
228
|
+
if os.name == "nt":
|
|
229
|
+
# The launcher owns a kill-on-close Job, so killing it closes the only
|
|
230
|
+
# Job handle and terminates the helper tree even if its leader exited.
|
|
231
|
+
try:
|
|
232
|
+
process.kill()
|
|
233
|
+
except ProcessLookupError:
|
|
234
|
+
pass
|
|
235
|
+
else:
|
|
236
|
+
# The group may outlive its leader while a descendant keeps a pipe open.
|
|
237
|
+
try:
|
|
238
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
239
|
+
except ProcessLookupError:
|
|
240
|
+
pass
|
|
241
|
+
|
|
242
|
+
async def discard(stream) -> None:
|
|
243
|
+
while await stream.read(64 * 1024):
|
|
244
|
+
pass
|
|
245
|
+
|
|
246
|
+
cleanup = asyncio.gather(discard(process.stdout), discard(process.stderr), process.wait())
|
|
247
|
+
try:
|
|
248
|
+
await asyncio.wait_for(cleanup, timeout=5)
|
|
249
|
+
except asyncio.TimeoutError:
|
|
250
|
+
# Closing the subprocess transport is a last-resort, bounded cleanup.
|
|
251
|
+
# The helper is already SIGKILLed and never runs inside the Target.
|
|
252
|
+
transport = getattr(process, "_transport", None)
|
|
253
|
+
if transport is not None:
|
|
254
|
+
transport.close()
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def invalid_response() -> HostCredentialHelperError:
|
|
258
|
+
return HostCredentialHelperError(
|
|
259
|
+
"host_credential_helper_response_invalid",
|
|
260
|
+
"host credential helper returned an invalid response",
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _run_windows_job_launcher(argv: list[str]) -> int:
|
|
265
|
+
"""Run one helper in a Job that owns every non-breakaway descendant."""
|
|
266
|
+
if os.name != "nt" or not argv:
|
|
267
|
+
return 125
|
|
268
|
+
try:
|
|
269
|
+
_create_windows_kill_on_close_job()
|
|
270
|
+
return subprocess.call(argv)
|
|
271
|
+
except (OSError, ValueError):
|
|
272
|
+
return 125
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _create_windows_kill_on_close_job() -> None:
|
|
276
|
+
# Importing ctypes only inside the Windows launcher keeps the Harbor runtime
|
|
277
|
+
# importable on every supported POSIX Python.
|
|
278
|
+
import ctypes
|
|
279
|
+
from ctypes import wintypes
|
|
280
|
+
|
|
281
|
+
class BasicLimitInformation(ctypes.Structure):
|
|
282
|
+
_fields_ = [
|
|
283
|
+
("PerProcessUserTimeLimit", ctypes.c_longlong),
|
|
284
|
+
("PerJobUserTimeLimit", ctypes.c_longlong),
|
|
285
|
+
("LimitFlags", wintypes.DWORD),
|
|
286
|
+
("MinimumWorkingSetSize", ctypes.c_size_t),
|
|
287
|
+
("MaximumWorkingSetSize", ctypes.c_size_t),
|
|
288
|
+
("ActiveProcessLimit", wintypes.DWORD),
|
|
289
|
+
("Affinity", ctypes.c_size_t),
|
|
290
|
+
("PriorityClass", wintypes.DWORD),
|
|
291
|
+
("SchedulingClass", wintypes.DWORD),
|
|
292
|
+
]
|
|
293
|
+
|
|
294
|
+
class IoCounters(ctypes.Structure):
|
|
295
|
+
_fields_ = [
|
|
296
|
+
("ReadOperationCount", ctypes.c_ulonglong),
|
|
297
|
+
("WriteOperationCount", ctypes.c_ulonglong),
|
|
298
|
+
("OtherOperationCount", ctypes.c_ulonglong),
|
|
299
|
+
("ReadTransferCount", ctypes.c_ulonglong),
|
|
300
|
+
("WriteTransferCount", ctypes.c_ulonglong),
|
|
301
|
+
("OtherTransferCount", ctypes.c_ulonglong),
|
|
302
|
+
]
|
|
303
|
+
|
|
304
|
+
class ExtendedLimitInformation(ctypes.Structure):
|
|
305
|
+
_fields_ = [
|
|
306
|
+
("BasicLimitInformation", BasicLimitInformation),
|
|
307
|
+
("IoInfo", IoCounters),
|
|
308
|
+
("ProcessMemoryLimit", ctypes.c_size_t),
|
|
309
|
+
("JobMemoryLimit", ctypes.c_size_t),
|
|
310
|
+
("PeakProcessMemoryUsed", ctypes.c_size_t),
|
|
311
|
+
("PeakJobMemoryUsed", ctypes.c_size_t),
|
|
312
|
+
]
|
|
313
|
+
|
|
314
|
+
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
315
|
+
kernel32.CreateJobObjectW.argtypes = (ctypes.c_void_p, wintypes.LPCWSTR)
|
|
316
|
+
kernel32.CreateJobObjectW.restype = wintypes.HANDLE
|
|
317
|
+
kernel32.SetInformationJobObject.argtypes = (
|
|
318
|
+
wintypes.HANDLE,
|
|
319
|
+
ctypes.c_int,
|
|
320
|
+
ctypes.c_void_p,
|
|
321
|
+
wintypes.DWORD,
|
|
322
|
+
)
|
|
323
|
+
kernel32.SetInformationJobObject.restype = wintypes.BOOL
|
|
324
|
+
kernel32.GetCurrentProcess.argtypes = ()
|
|
325
|
+
kernel32.GetCurrentProcess.restype = wintypes.HANDLE
|
|
326
|
+
kernel32.AssignProcessToJobObject.argtypes = (wintypes.HANDLE, wintypes.HANDLE)
|
|
327
|
+
kernel32.AssignProcessToJobObject.restype = wintypes.BOOL
|
|
328
|
+
kernel32.CloseHandle.argtypes = (wintypes.HANDLE,)
|
|
329
|
+
kernel32.CloseHandle.restype = wintypes.BOOL
|
|
330
|
+
|
|
331
|
+
handle = kernel32.CreateJobObjectW(None, None)
|
|
332
|
+
if not handle:
|
|
333
|
+
raise ctypes.WinError(ctypes.get_last_error())
|
|
334
|
+
information = ExtendedLimitInformation()
|
|
335
|
+
information.BasicLimitInformation.LimitFlags = 0x00002000 # JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
|
|
336
|
+
if not kernel32.SetInformationJobObject(handle, 9, ctypes.byref(information), ctypes.sizeof(information)):
|
|
337
|
+
error = ctypes.get_last_error()
|
|
338
|
+
kernel32.CloseHandle(handle)
|
|
339
|
+
raise ctypes.WinError(error)
|
|
340
|
+
if not kernel32.AssignProcessToJobObject(handle, kernel32.GetCurrentProcess()):
|
|
341
|
+
error = ctypes.get_last_error()
|
|
342
|
+
kernel32.CloseHandle(handle)
|
|
343
|
+
raise ctypes.WinError(error)
|
|
344
|
+
|
|
345
|
+
# Deliberately retain the sole non-inheritable Job handle until ExitProcess.
|
|
346
|
+
# Its automatic close then terminates any helper descendants still running.
|
|
347
|
+
global _WINDOWS_JOB_HANDLE
|
|
348
|
+
_WINDOWS_JOB_HANDLE = handle
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
if __name__ == "__main__":
|
|
352
|
+
if len(sys.argv) >= 3 and sys.argv[1] == _WINDOWS_JOB_LAUNCHER:
|
|
353
|
+
raise SystemExit(_run_windows_job_launcher(sys.argv[2:]))
|
|
354
|
+
raise SystemExit(125)
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
"""Run one Docker Target with per-exec secrets absent from process argv."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import base64
|
|
7
|
+
import json
|
|
8
|
+
import re
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from harbor.environments.base import ExecResult
|
|
12
|
+
from harbor.environments.docker.docker import _sanitize_docker_compose_project_name
|
|
13
|
+
|
|
14
|
+
_ENVIRONMENT_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
async def exec_with_private_environment(
|
|
18
|
+
environment: Any,
|
|
19
|
+
command: str,
|
|
20
|
+
*,
|
|
21
|
+
env: dict[str, str],
|
|
22
|
+
cwd: str | None = None,
|
|
23
|
+
timeout_sec: int | None = None,
|
|
24
|
+
user: str | int | None = None,
|
|
25
|
+
) -> ExecResult:
|
|
26
|
+
"""Pass values through the compose client's environment and only names in argv."""
|
|
27
|
+
if (
|
|
28
|
+
not isinstance(env, dict)
|
|
29
|
+
or not env
|
|
30
|
+
or any(
|
|
31
|
+
not isinstance(name, str)
|
|
32
|
+
or _ENVIRONMENT_NAME.fullmatch(name) is None
|
|
33
|
+
or not isinstance(value, str)
|
|
34
|
+
or not value
|
|
35
|
+
or "\0" in value
|
|
36
|
+
for name, value in env.items()
|
|
37
|
+
)
|
|
38
|
+
):
|
|
39
|
+
raise RuntimeError("private Target environment is invalid")
|
|
40
|
+
|
|
41
|
+
resolved_user = environment._resolve_user(user)
|
|
42
|
+
# Preserve Harbor's persistent and task-scoped agent environment, then let
|
|
43
|
+
# the fresh helper values replace the deliberately blank startup values.
|
|
44
|
+
target_environment = dict(environment._merge_env(None) or {})
|
|
45
|
+
target_environment.update(env)
|
|
46
|
+
exec_command = ["exec"]
|
|
47
|
+
effective_cwd = cwd or environment.task_env_config.workdir
|
|
48
|
+
if effective_cwd:
|
|
49
|
+
exec_command.extend(["-w", effective_cwd])
|
|
50
|
+
for name in sorted(target_environment):
|
|
51
|
+
# Docker Compose resolves a name without `=value` from its own process
|
|
52
|
+
# environment. The secret therefore never appears in host process argv.
|
|
53
|
+
exec_command.extend(["-e", name])
|
|
54
|
+
if resolved_user is not None:
|
|
55
|
+
exec_command.extend(["-u", str(resolved_user)])
|
|
56
|
+
exec_command.append("main")
|
|
57
|
+
exec_command.extend(environment._platform.exec_shell_args(command))
|
|
58
|
+
|
|
59
|
+
full_command = [
|
|
60
|
+
"docker",
|
|
61
|
+
"compose",
|
|
62
|
+
"--project-name",
|
|
63
|
+
_sanitize_docker_compose_project_name(environment.session_id),
|
|
64
|
+
"--project-directory",
|
|
65
|
+
str(environment.environment_dir.resolve().absolute()),
|
|
66
|
+
]
|
|
67
|
+
for compose_path in environment._docker_compose_paths:
|
|
68
|
+
full_command.extend(["-f", str(compose_path.resolve().absolute())])
|
|
69
|
+
full_command.extend(exec_command)
|
|
70
|
+
|
|
71
|
+
host_environment = environment._compose_env_vars(include_os_env=True)
|
|
72
|
+
host_environment.update(target_environment)
|
|
73
|
+
process = await asyncio.create_subprocess_exec(
|
|
74
|
+
*full_command,
|
|
75
|
+
env=host_environment,
|
|
76
|
+
stdin=asyncio.subprocess.DEVNULL,
|
|
77
|
+
stdout=asyncio.subprocess.PIPE,
|
|
78
|
+
stderr=asyncio.subprocess.STDOUT,
|
|
79
|
+
)
|
|
80
|
+
try:
|
|
81
|
+
if timeout_sec is None:
|
|
82
|
+
stdout, stderr = await process.communicate()
|
|
83
|
+
else:
|
|
84
|
+
stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout_sec)
|
|
85
|
+
except asyncio.TimeoutError:
|
|
86
|
+
await _terminate_compose_client(process)
|
|
87
|
+
raise RuntimeError("private Target execution timed out") from None
|
|
88
|
+
except BaseException:
|
|
89
|
+
# Match Harbor 0.21's compose-exec cleanup. The surrounding trial still
|
|
90
|
+
# owns Target teardown; stopping this client alone is not treated as
|
|
91
|
+
# proof that the command or container has stopped.
|
|
92
|
+
await _terminate_compose_client(process)
|
|
93
|
+
raise
|
|
94
|
+
secrets = sorted(
|
|
95
|
+
{secret for name, value in env.items() for secret in _credential_secrets(name, value)},
|
|
96
|
+
key=len,
|
|
97
|
+
reverse=True,
|
|
98
|
+
)
|
|
99
|
+
stdout_text = _redact(stdout.decode(errors="replace") if stdout else None, secrets)
|
|
100
|
+
stderr_text = _redact(stderr.decode(errors="replace") if stderr else None, secrets)
|
|
101
|
+
return ExecResult(
|
|
102
|
+
stdout=stdout_text,
|
|
103
|
+
stderr=stderr_text,
|
|
104
|
+
return_code=process.returncode or 0,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
async def _terminate_compose_client(process) -> None:
|
|
109
|
+
if process.returncode is not None:
|
|
110
|
+
return
|
|
111
|
+
process.terminate()
|
|
112
|
+
try:
|
|
113
|
+
await asyncio.wait_for(process.wait(), timeout=5)
|
|
114
|
+
except asyncio.TimeoutError:
|
|
115
|
+
process.kill()
|
|
116
|
+
await process.wait()
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _redact(value: str | None, secrets: list[str]) -> str | None:
|
|
120
|
+
if value is None:
|
|
121
|
+
return None
|
|
122
|
+
for secret in secrets:
|
|
123
|
+
value = value.replace(secret, "[REDACTED]")
|
|
124
|
+
return value
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _credential_secrets(name: str, value: str) -> list[str]:
|
|
128
|
+
secrets = [value]
|
|
129
|
+
if not name.endswith("_B64") or len(value) > 512 * 1024:
|
|
130
|
+
return secrets
|
|
131
|
+
try:
|
|
132
|
+
decoded = base64.b64decode(value, validate=True)
|
|
133
|
+
if len(decoded) > 256 * 1024:
|
|
134
|
+
return secrets
|
|
135
|
+
parsed = json.loads(decoded.decode("utf-8"))
|
|
136
|
+
except (ValueError, UnicodeDecodeError, json.JSONDecodeError):
|
|
137
|
+
return secrets
|
|
138
|
+
|
|
139
|
+
def collect(entry):
|
|
140
|
+
if isinstance(entry, str) and len(entry) >= 16:
|
|
141
|
+
secrets.append(entry)
|
|
142
|
+
elif isinstance(entry, dict):
|
|
143
|
+
for nested in entry.values():
|
|
144
|
+
collect(nested)
|
|
145
|
+
elif isinstance(entry, list):
|
|
146
|
+
for nested in entry:
|
|
147
|
+
collect(nested)
|
|
148
|
+
|
|
149
|
+
collect(parsed)
|
|
150
|
+
return secrets
|
|
@@ -10,10 +10,17 @@ if (process.argv.includes('--version')) {
|
|
|
10
10
|
const args = process.argv.slice(2);
|
|
11
11
|
if (args.length !== 2 || args[0] !== '--model' || !args[1]) throw new Error('Usage: model-call --model MODEL');
|
|
12
12
|
const model = args[1];
|
|
13
|
-
const key = process.env.OPENAI_API_KEY;
|
|
14
|
-
if (!key) throw new Error('
|
|
15
|
-
const base = new URL(process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1/');
|
|
16
|
-
|
|
13
|
+
const key = process.env.HITCH_LOCAL_MODEL_TOKEN || process.env.OPENAI_API_KEY;
|
|
14
|
+
if (!key) throw new Error('a model API credential is required for the model-call harness');
|
|
15
|
+
const base = new URL(process.env.HITCH_LOCAL_MODEL_BASE_URL || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1/');
|
|
16
|
+
const loopback = base.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(base.hostname);
|
|
17
|
+
const managedRunId = process.env.HITCH_MANAGED_RUN_ID || '';
|
|
18
|
+
const managed = process.env.HITCH_MANAGED_LOCAL_INFERENCE === '1'
|
|
19
|
+
&& key === 'hitch-managed-local'
|
|
20
|
+
&& /^run_[a-f0-9]{32}$/.test(managedRunId)
|
|
21
|
+
&& base.protocol === 'http:'
|
|
22
|
+
&& new RegExp(`^/[a-f0-9]{48}/${managedRunId}/openai/?$`).test(base.pathname);
|
|
23
|
+
if (base.username || base.password || base.search || base.hash || (base.protocol !== 'https:' && !loopback && !managed)) throw new Error('invalid model endpoint');
|
|
17
24
|
if (!base.pathname.endsWith('/')) base.pathname += '/';
|
|
18
25
|
let raw = '';
|
|
19
26
|
for await (const chunk of process.stdin) {
|
|
@@ -33,7 +40,8 @@ for (const message of input.messages) {
|
|
|
33
40
|
} else throw new Error('unsupported input modality');
|
|
34
41
|
}
|
|
35
42
|
}
|
|
36
|
-
const
|
|
43
|
+
const localDefault = process.env.HITCH_LOCAL_MAX_OUTPUT_TOKENS === undefined ? 8192 : Number(process.env.HITCH_LOCAL_MAX_OUTPUT_TOKENS);
|
|
44
|
+
const maxTokens = input.max_output_tokens ?? localDefault;
|
|
37
45
|
if (!Number.isSafeInteger(maxTokens) || maxTokens < 1 || maxTokens > 131072) throw new Error('invalid max_output_tokens');
|
|
38
46
|
if (input.reasoning_effort !== undefined && !['none', 'minimal', 'low', 'medium', 'high', 'xhigh'].includes(input.reasoning_effort)) throw new Error('invalid reasoning effort');
|
|
39
47
|
const emit = event => console.log(JSON.stringify(event));
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Fixed linear tool loop, restricted to a managed Harbor task container.
|
|
3
|
+
import { randomUUID } from 'node:crypto';
|
|
4
|
+
import { spawn } from 'node:child_process';
|
|
5
|
+
|
|
6
|
+
if (process.argv.includes('--version')) { console.log('hitch-training-tool 1.0.0'); process.exit(0); }
|
|
7
|
+
const args = process.argv.slice(2);
|
|
8
|
+
if (args.length !== 2 || args[0] !== '--model' || !args[1]) throw new Error('Usage: training-tool --model MODEL');
|
|
9
|
+
if (process.env.HITCH_HARBOR_INTERNAL !== '1') throw new Error('training-tool requires a Harbor task container');
|
|
10
|
+
const binding = JSON.parse(process.env.HITCH_TRAINING_BINDING || 'null');
|
|
11
|
+
const training = process.env.HITCH_TRAINING_EXTERNAL === '1';
|
|
12
|
+
if (!training && process.env.HITCH_MANAGED_LOCAL_INFERENCE !== '1') throw new Error('explicit Hitch model binding required');
|
|
13
|
+
const runId = training ? process.env.HITCH_TRAINING_RUN_ID : process.env.HITCH_MANAGED_RUN_ID;
|
|
14
|
+
const base = new URL(process.env.OPENAI_BASE_URL || '');
|
|
15
|
+
if (!/^run_[a-f0-9]{32}$/.test(runId || '') || base.protocol !== 'http:' || base.username || base.password || base.search || base.hash
|
|
16
|
+
|| !new RegExp(`^/[a-f0-9]{48}/${runId}/openai/?$`).test(base.pathname)) throw new Error('invalid run-scoped endpoint');
|
|
17
|
+
if (training && args[1] !== `training/${binding?.binding_id}`) throw new Error('wrong training binding');
|
|
18
|
+
if (!base.pathname.endsWith('/')) base.pathname += '/';
|
|
19
|
+
const maxTokens = training ? binding.max_output_tokens : Number(process.env.HITCH_LOCAL_MAX_OUTPUT_TOKENS || 2048);
|
|
20
|
+
const maxSteps = training ? binding.max_episode_steps : Number(process.env.HITCH_TRAINING_EVAL_MAX_STEPS || 16);
|
|
21
|
+
if (![maxTokens, maxSteps].every(n => Number.isSafeInteger(n) && n > 0)) throw new Error('invalid harness budget');
|
|
22
|
+
let prompt = '';
|
|
23
|
+
for await (const chunk of process.stdin) { prompt += chunk; if (Buffer.byteLength(prompt) > 8 * 1024 * 1024) throw new Error('prompt exceeds 8 MiB'); }
|
|
24
|
+
const messages = [{ role: 'user', content: prompt }];
|
|
25
|
+
const tools = [{ type: 'function', function: { name: 'bash', description: 'Run a command in the task container.', parameters: { type: 'object', properties: { command: { type: 'string' } }, required: ['command'], additionalProperties: false } } }];
|
|
26
|
+
const emit = event => console.log(JSON.stringify(event));
|
|
27
|
+
let child;
|
|
28
|
+
const abort = new AbortController();
|
|
29
|
+
const killTool = () => { if (child?.pid) { try { process.kill(-child.pid, 'SIGKILL'); } catch {} } };
|
|
30
|
+
const cancel = () => { abort.abort(); killTool(); };
|
|
31
|
+
process.once('SIGTERM', cancel); process.once('SIGINT', cancel);
|
|
32
|
+
emit({ type: 'session.created', session_id: randomUUID(), policy_version: binding?.policy_version });
|
|
33
|
+
for (let step = 0; step < maxSteps; step++) {
|
|
34
|
+
abort.signal.throwIfAborted();
|
|
35
|
+
const response = await fetch(new URL('chat/completions', base), {
|
|
36
|
+
method: 'POST', redirect: 'error', headers: { Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, 'Content-Type': 'application/json', 'Idempotency-Key': `${runId}-${step}` },
|
|
37
|
+
body: JSON.stringify({ model: args[1], messages, tools, stream: false, max_tokens: maxTokens }), signal: abort.signal,
|
|
38
|
+
});
|
|
39
|
+
if (!response.ok) throw new Error(`training gateway returned HTTP ${response.status}`);
|
|
40
|
+
const body = await response.json(); abort.signal.throwIfAborted(); const choice = body.choices?.[0];
|
|
41
|
+
if (!choice?.message || body.choices.length !== 1) throw new Error('invalid completion');
|
|
42
|
+
emit({ type: 'provider.response', response: body, receipt_id: response.headers.get('x-gear-receipt-id') });
|
|
43
|
+
emit({ type: 'usage.updated', usage: body.usage || {} });
|
|
44
|
+
if (choice.finish_reason === 'length') { emit({ type: 'training.terminated', termination: 'truncated' }); process.exitCode = 8; break; }
|
|
45
|
+
const message = choice.message; messages.push(message);
|
|
46
|
+
const calls = message.tool_calls || [];
|
|
47
|
+
if (!calls.length) {
|
|
48
|
+
if (choice.finish_reason !== 'stop') throw new Error('missing terminal state');
|
|
49
|
+
emit({ type: 'message.completed', text: message.content || '' }); emit({ type: 'training.terminated', termination: 'terminated' }); break;
|
|
50
|
+
}
|
|
51
|
+
if (choice.finish_reason !== 'tool_calls') throw new Error('missing tool terminal state');
|
|
52
|
+
for (const call of calls) {
|
|
53
|
+
abort.signal.throwIfAborted();
|
|
54
|
+
if (call.function?.name !== 'bash' || typeof call.id !== 'string') throw new Error('unsupported tool');
|
|
55
|
+
const input = JSON.parse(call.function.arguments);
|
|
56
|
+
if (typeof input.command !== 'string' || Object.keys(input).join(',') !== 'command') throw new Error('invalid bash arguments');
|
|
57
|
+
emit({ type: 'tool.started', call_id: call.id, name: 'bash', arguments: input });
|
|
58
|
+
const result = await new Promise((resolve, reject) => {
|
|
59
|
+
let output = ''; let truncated = false;
|
|
60
|
+
child = spawn('/bin/bash', ['-lc', input.command], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
61
|
+
const append = data => { const remaining = 32768 - output.length; if (remaining > 0) output += data.toString().slice(0, remaining); if (data.length > remaining) truncated = true; };
|
|
62
|
+
child.stdout.on('data', append); child.stderr.on('data', append);
|
|
63
|
+
const timer = setTimeout(killTool, 30000);
|
|
64
|
+
child.once('error', error => { clearTimeout(timer); reject(error); });
|
|
65
|
+
child.once('close', code => { clearTimeout(timer); child = undefined; resolve(`exit=${code}\n${output}${truncated ? '\n[tool output truncated]' : ''}`); });
|
|
66
|
+
});
|
|
67
|
+
messages.push({ role: 'tool', tool_call_id: call.id, content: result });
|
|
68
|
+
emit({ type: 'tool.completed', call_id: call.id, output: result });
|
|
69
|
+
abort.signal.throwIfAborted();
|
|
70
|
+
}
|
|
71
|
+
if (step === maxSteps - 1) { emit({ type: 'training.terminated', termination: 'truncated' }); process.exitCode = 8; }
|
|
72
|
+
}
|