agent-hitch 0.2.8 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +206 -137
- package/README.zh-CN.md +193 -119
- package/dist/scripts/canary-native-training.js +221 -0
- package/dist/scripts/canary-native-training.js.map +1 -0
- package/dist/scripts/canary-remote-training.js +246 -0
- package/dist/scripts/canary-remote-training.js.map +1 -0
- package/dist/scripts/canary-remote-verifier.js +124 -0
- package/dist/scripts/canary-remote-verifier.js.map +1 -0
- package/dist/scripts/canary-worker-host.js +113 -0
- package/dist/scripts/canary-worker-host.js.map +1 -0
- package/dist/scripts/canary-worker-peer.js +188 -0
- package/dist/scripts/canary-worker-peer.js.map +1 -0
- package/dist/scripts/check-architecture.js +4 -3
- package/dist/scripts/check-architecture.js.map +1 -1
- package/dist/src/adapters/catalog.js +2 -1
- package/dist/src/adapters/catalog.js.map +1 -1
- package/dist/src/adapters/providers/codex.js +65 -2
- package/dist/src/adapters/providers/codex.js.map +1 -1
- package/dist/src/adapters/providers/model-call.js +17 -1
- package/dist/src/adapters/providers/model-call.js.map +1 -1
- package/dist/src/adapters/providers/training-tool.js +29 -0
- package/dist/src/adapters/providers/training-tool.js.map +1 -0
- package/dist/src/artifacts/index.js +1 -1
- package/dist/src/artifacts/index.js.map +1 -1
- package/dist/src/artifacts/store.js +15 -10
- package/dist/src/artifacts/store.js.map +1 -1
- package/dist/src/backends/harbor/backend.js +24 -15
- package/dist/src/backends/harbor/backend.js.map +1 -1
- package/dist/src/backends/harbor/bridge-environment.js +12 -0
- package/dist/src/backends/harbor/bridge-environment.js.map +1 -0
- package/dist/src/backends/harbor/environment-config.js +2 -2
- package/dist/src/backends/harbor/environment-config.js.map +1 -1
- package/dist/src/backends/harbor/host-credential-helper.js +81 -0
- package/dist/src/backends/harbor/host-credential-helper.js.map +1 -0
- package/dist/src/backends/harbor/index.js +3 -0
- package/dist/src/backends/harbor/index.js.map +1 -1
- package/dist/src/backends/harbor/model-proxy-config.js +18 -1
- package/dist/src/backends/harbor/model-proxy-config.js.map +1 -1
- package/dist/src/backends/harbor/observation.js +61 -0
- package/dist/src/backends/harbor/observation.js.map +1 -0
- package/dist/src/backends/harbor/portable-regrade.js +150 -0
- package/dist/src/backends/harbor/portable-regrade.js.map +1 -0
- package/dist/src/backends/harbor/process.js +26 -5
- package/dist/src/backends/harbor/process.js.map +1 -1
- package/dist/src/backends/harbor/regrade.js +4 -1
- package/dist/src/backends/harbor/regrade.js.map +1 -1
- package/dist/src/backends/harbor/supervisor.js +22 -0
- package/dist/src/backends/harbor/supervisor.js.map +1 -1
- package/dist/src/backends/harbor/tools.js +2 -0
- package/dist/src/backends/harbor/tools.js.map +1 -1
- package/dist/src/backends/index.js +3 -0
- package/dist/src/backends/index.js.map +1 -1
- package/dist/src/cli/arguments.js +33 -0
- package/dist/src/cli/arguments.js.map +1 -1
- package/dist/src/cli/commands/capabilities.js +14 -1
- package/dist/src/cli/commands/capabilities.js.map +1 -1
- package/dist/src/cli/commands/daemon.js +19 -0
- package/dist/src/cli/commands/daemon.js.map +1 -1
- package/dist/src/cli/commands/eval.js +38 -10
- package/dist/src/cli/commands/eval.js.map +1 -1
- package/dist/src/cli/commands/local.js +130 -0
- package/dist/src/cli/commands/local.js.map +1 -0
- package/dist/src/cli/commands/model-node.js +26 -0
- package/dist/src/cli/commands/model-node.js.map +1 -0
- package/dist/src/cli/commands/models.js +72 -0
- package/dist/src/cli/commands/models.js.map +1 -0
- package/dist/src/cli/commands/run.js +36 -1
- package/dist/src/cli/commands/run.js.map +1 -1
- package/dist/src/cli/commands/training.js +32 -0
- package/dist/src/cli/commands/training.js.map +1 -0
- package/dist/src/cli/commands/worker.js +30 -2
- package/dist/src/cli/commands/worker.js.map +1 -1
- package/dist/src/cli/main.js +8 -0
- package/dist/src/cli/main.js.map +1 -1
- package/dist/src/cli/output.js +24 -2
- package/dist/src/cli/output.js.map +1 -1
- package/dist/src/control-plane/eval-records.js +2 -0
- package/dist/src/control-plane/eval-records.js.map +1 -1
- package/dist/src/control-plane/eval-recovery.js +7 -1
- package/dist/src/control-plane/eval-recovery.js.map +1 -1
- package/dist/src/control-plane/eval-scheduler.js +24 -50
- package/dist/src/control-plane/eval-scheduler.js.map +1 -1
- package/dist/src/control-plane/eval-submission-persist.js +30 -0
- package/dist/src/control-plane/eval-submission-persist.js.map +1 -0
- package/dist/src/control-plane/execution-observation-contract.js +60 -0
- package/dist/src/control-plane/execution-observation-contract.js.map +1 -0
- package/dist/src/control-plane/index.js +10 -0
- package/dist/src/control-plane/index.js.map +1 -1
- package/dist/src/control-plane/inference-manager.js +357 -0
- package/dist/src/control-plane/inference-manager.js.map +1 -0
- package/dist/src/control-plane/managed-gateway-receipt.js +26 -0
- package/dist/src/control-plane/managed-gateway-receipt.js.map +1 -0
- package/dist/src/control-plane/managed-service-recovery.js +158 -0
- package/dist/src/control-plane/managed-service-recovery.js.map +1 -0
- package/dist/src/control-plane/model-capture-planning.js +15 -1
- package/dist/src/control-plane/model-capture-planning.js.map +1 -1
- package/dist/src/control-plane/ordered-eval-control.js +221 -0
- package/dist/src/control-plane/ordered-eval-control.js.map +1 -0
- package/dist/src/control-plane/remote-backend-result.js +18 -0
- package/dist/src/control-plane/remote-backend-result.js.map +1 -0
- package/dist/src/control-plane/remote-execution-lease.js +15 -0
- package/dist/src/control-plane/remote-execution-lease.js.map +1 -0
- package/dist/src/control-plane/remote-execution-observation.js +108 -0
- package/dist/src/control-plane/remote-execution-observation.js.map +1 -0
- package/dist/src/control-plane/remote-model-routes.js +107 -0
- package/dist/src/control-plane/remote-model-routes.js.map +1 -0
- package/dist/src/control-plane/remote-observation-reporter.js +37 -0
- package/dist/src/control-plane/remote-observation-reporter.js.map +1 -0
- package/dist/src/control-plane/remote-offer-contract.js +48 -0
- package/dist/src/control-plane/remote-offer-contract.js.map +1 -0
- package/dist/src/control-plane/remote-rerun-journal.js +61 -0
- package/dist/src/control-plane/remote-rerun-journal.js.map +1 -0
- package/dist/src/control-plane/remote-result-transport.js +55 -9
- package/dist/src/control-plane/remote-result-transport.js.map +1 -1
- package/dist/src/control-plane/remote-verifier-import.js +52 -0
- package/dist/src/control-plane/remote-verifier-import.js.map +1 -0
- package/dist/src/control-plane/remote-verifier-result.js +61 -0
- package/dist/src/control-plane/remote-verifier-result.js.map +1 -0
- package/dist/src/control-plane/remote-verifier-source-transport.js +68 -0
- package/dist/src/control-plane/remote-verifier-source-transport.js.map +1 -0
- package/dist/src/control-plane/remote-work-coordinator.js +103 -47
- package/dist/src/control-plane/remote-work-coordinator.js.map +1 -1
- package/dist/src/control-plane/remote-work-inputs.js +26 -5
- package/dist/src/control-plane/remote-work-inputs.js.map +1 -1
- package/dist/src/control-plane/remote-work-recovery.js +123 -34
- package/dist/src/control-plane/remote-work-recovery.js.map +1 -1
- package/dist/src/control-plane/remote-worker-artifacts.js +10 -7
- package/dist/src/control-plane/remote-worker-artifacts.js.map +1 -1
- package/dist/src/control-plane/remote-worker-cleanup.js +104 -0
- package/dist/src/control-plane/remote-worker-cleanup.js.map +1 -0
- package/dist/src/control-plane/remote-worker-client.js +156 -14
- package/dist/src/control-plane/remote-worker-client.js.map +1 -1
- package/dist/src/control-plane/remote-worker-codecs.js +160 -0
- package/dist/src/control-plane/remote-worker-codecs.js.map +1 -0
- package/dist/src/control-plane/remote-worker-generation-recovery.js +67 -0
- package/dist/src/control-plane/remote-worker-generation-recovery.js.map +1 -0
- package/dist/src/control-plane/remote-worker-ownership.js +51 -0
- package/dist/src/control-plane/remote-worker-ownership.js.map +1 -0
- package/dist/src/control-plane/remote-worker-protocol.js +85 -119
- package/dist/src/control-plane/remote-worker-protocol.js.map +1 -1
- package/dist/src/control-plane/remote-worker-runner.js +95 -18
- package/dist/src/control-plane/remote-worker-runner.js.map +1 -1
- package/dist/src/control-plane/remote-workers.js +33 -5
- package/dist/src/control-plane/remote-workers.js.map +1 -1
- package/dist/src/control-plane/rerun-recovery.js +89 -0
- package/dist/src/control-plane/rerun-recovery.js.map +1 -0
- package/dist/src/control-plane/rerun-scheduler.js +70 -96
- package/dist/src/control-plane/rerun-scheduler.js.map +1 -1
- package/dist/src/control-plane/rerun-source.js +57 -0
- package/dist/src/control-plane/rerun-source.js.map +1 -0
- package/dist/src/control-plane/worker-model-relay.js +76 -0
- package/dist/src/control-plane/worker-model-relay.js.map +1 -0
- package/dist/src/controller-runtime/hash.js +3 -0
- package/dist/src/controller-runtime/hash.js.map +1 -1
- package/dist/src/controller-runtime/index.js +2 -0
- package/dist/src/controller-runtime/index.js.map +1 -1
- package/dist/src/controller-runtime/observation.js +29 -0
- package/dist/src/controller-runtime/observation.js.map +1 -0
- package/dist/src/controller-runtime/resident.js +22 -0
- package/dist/src/controller-runtime/resident.js.map +1 -0
- package/dist/src/daemon/auth.js +7 -2
- package/dist/src/daemon/auth.js.map +1 -1
- package/dist/src/daemon/client.js +34 -1
- package/dist/src/daemon/client.js.map +1 -1
- package/dist/src/daemon/execution-observation.js +38 -0
- package/dist/src/daemon/execution-observation.js.map +1 -0
- package/dist/src/daemon/http.js +57 -0
- package/dist/src/daemon/http.js.map +1 -0
- package/dist/src/daemon/inference-prepare.js +47 -0
- package/dist/src/daemon/inference-prepare.js.map +1 -0
- package/dist/src/daemon/scheduler.js +4 -1
- package/dist/src/daemon/scheduler.js.map +1 -1
- package/dist/src/daemon/server.js +80 -82
- package/dist/src/daemon/server.js.map +1 -1
- package/dist/src/daemon/worker-observation.js +34 -0
- package/dist/src/daemon/worker-observation.js.map +1 -0
- package/dist/src/daemon/worker-ownership.js +71 -0
- package/dist/src/daemon/worker-ownership.js.map +1 -0
- package/dist/src/daemon/worker-routes.js +83 -15
- package/dist/src/daemon/worker-routes.js.map +1 -1
- package/dist/src/domain/execution-observation.js +2 -0
- package/dist/src/domain/execution-observation.js.map +1 -0
- package/dist/src/domain/index.js +1 -0
- package/dist/src/domain/index.js.map +1 -1
- package/dist/src/domain/inference.js +2 -0
- package/dist/src/domain/inference.js.map +1 -0
- package/dist/src/domain/runs.js.map +1 -1
- package/dist/src/domain/training.js +2 -0
- package/dist/src/domain/training.js.map +1 -0
- package/dist/src/domain/validation.js +11 -0
- package/dist/src/domain/validation.js.map +1 -1
- package/dist/src/domain/verifier-source.js +2 -0
- package/dist/src/domain/verifier-source.js.map +1 -0
- package/dist/src/domain/worker-recovery.js +2 -0
- package/dist/src/domain/worker-recovery.js.map +1 -0
- package/dist/src/evals/eval-logical-plan.js +8 -1
- package/dist/src/evals/eval-logical-plan.js.map +1 -1
- package/dist/src/evals/execution-leases.js +15 -2
- package/dist/src/evals/execution-leases.js.map +1 -1
- package/dist/src/evals/execution-plan.js +11 -1
- package/dist/src/evals/execution-plan.js.map +1 -1
- package/dist/src/evals/harbor-bridge-error.js +9 -0
- package/dist/src/evals/harbor-bridge-error.js.map +1 -1
- package/dist/src/evals/index.js +19 -0
- package/dist/src/evals/index.js.map +1 -1
- package/dist/src/evals/model-capture-plan.js +12 -0
- package/dist/src/evals/model-capture-plan.js.map +1 -1
- package/dist/src/evals/model-capture-runtime.js +25 -0
- package/dist/src/evals/model-capture-runtime.js.map +1 -1
- package/dist/src/evals/physical-work-plan.js +57 -0
- package/dist/src/evals/physical-work-plan.js.map +1 -0
- package/dist/src/evals/preparation-rerun.js +6 -0
- package/dist/src/evals/preparation-rerun.js.map +1 -1
- package/dist/src/evals/recovery.js +8 -1
- package/dist/src/evals/recovery.js.map +1 -1
- package/dist/src/evals/regrade-evidence.js +4 -4
- package/dist/src/evals/regrade-evidence.js.map +1 -1
- package/dist/src/evals/remote-execution-ownership.js +50 -0
- package/dist/src/evals/remote-execution-ownership.js.map +1 -0
- package/dist/src/evals/remote-infrastructure-retry.js +2 -0
- package/dist/src/evals/remote-infrastructure-retry.js.map +1 -1
- package/dist/src/evals/remote-lease-release.js +58 -0
- package/dist/src/evals/remote-lease-release.js.map +1 -0
- package/dist/src/evals/remote-rerun-execution.js +85 -0
- package/dist/src/evals/remote-rerun-execution.js.map +1 -0
- package/dist/src/evals/remote-trial.js +6 -0
- package/dist/src/evals/remote-trial.js.map +1 -0
- package/dist/src/evals/remote-verifier-assessment.js +85 -0
- package/dist/src/evals/remote-verifier-assessment.js.map +1 -0
- package/dist/src/evals/remote-verifier-contract.js +43 -0
- package/dist/src/evals/remote-verifier-contract.js.map +1 -0
- package/dist/src/evals/remote-verifier-execution.js +105 -0
- package/dist/src/evals/remote-verifier-execution.js.map +1 -0
- package/dist/src/evals/remote-verifier-journal.js +21 -0
- package/dist/src/evals/remote-verifier-journal.js.map +1 -0
- package/dist/src/evals/remote-verifier-outcome.js +51 -0
- package/dist/src/evals/remote-verifier-outcome.js.map +1 -0
- package/dist/src/evals/remote-verifier-rerun.js +134 -0
- package/dist/src/evals/remote-verifier-rerun.js.map +1 -0
- package/dist/src/evals/remote-verifier-selection.js +61 -0
- package/dist/src/evals/remote-verifier-selection.js.map +1 -0
- package/dist/src/evals/remote-verifier-source.js +33 -0
- package/dist/src/evals/remote-verifier-source.js.map +1 -0
- package/dist/src/evals/remote-worker-cleanup.js +46 -0
- package/dist/src/evals/remote-worker-cleanup.js.map +1 -0
- package/dist/src/evals/request.js +58 -1
- package/dist/src/evals/request.js.map +1 -1
- package/dist/src/evals/rerun-completion.js +39 -0
- package/dist/src/evals/rerun-completion.js.map +1 -0
- package/dist/src/evals/rerun-inputs.js +54 -0
- package/dist/src/evals/rerun-inputs.js.map +1 -1
- package/dist/src/evals/rerun-resume.js +70 -0
- package/dist/src/evals/rerun-resume.js.map +1 -0
- package/dist/src/evals/rerun-state.js +25 -0
- package/dist/src/evals/rerun-state.js.map +1 -0
- package/dist/src/evals/rerun-types.js.map +1 -1
- package/dist/src/evals/rerun.js +94 -101
- package/dist/src/evals/rerun.js.map +1 -1
- package/dist/src/evals/service.js +59 -5
- package/dist/src/evals/service.js.map +1 -1
- package/dist/src/evals/verifier-only-rerun.js +2 -16
- package/dist/src/evals/verifier-only-rerun.js.map +1 -1
- package/dist/src/evals/verifier-rerun-result.js +24 -0
- package/dist/src/evals/verifier-rerun-result.js.map +1 -0
- package/dist/src/evals/verifier-source-import.js +48 -0
- package/dist/src/evals/verifier-source-import.js.map +1 -0
- package/dist/src/evals/verifier-source.js +139 -0
- package/dist/src/evals/verifier-source.js.map +1 -0
- package/dist/src/foundation/base64.js +19 -0
- package/dist/src/foundation/base64.js.map +1 -0
- package/dist/src/foundation/config.js +10 -0
- package/dist/src/foundation/config.js.map +1 -1
- package/dist/src/foundation/credential-redaction.js +22 -9
- package/dist/src/foundation/credential-redaction.js.map +1 -1
- package/dist/src/foundation/index.js +1 -0
- package/dist/src/foundation/index.js.map +1 -1
- package/dist/src/foundation/process-identity.js +13 -9
- package/dist/src/foundation/process-identity.js.map +1 -1
- package/dist/src/inference/daemon-capacity.js +41 -0
- package/dist/src/inference/daemon-capacity.js.map +1 -0
- package/dist/src/inference/device-reservation.js +58 -0
- package/dist/src/inference/device-reservation.js.map +1 -0
- package/dist/src/inference/doctor.js +95 -0
- package/dist/src/inference/doctor.js.map +1 -0
- package/dist/src/inference/generation-recovery.js +83 -0
- package/dist/src/inference/generation-recovery.js.map +1 -0
- package/dist/src/inference/index.js +24 -0
- package/dist/src/inference/index.js.map +1 -0
- package/dist/src/inference/lock.js +331 -0
- package/dist/src/inference/lock.js.map +1 -0
- package/dist/src/inference/managed-launcher.js +35 -0
- package/dist/src/inference/managed-launcher.js.map +1 -0
- package/dist/src/inference/manifest.js +262 -0
- package/dist/src/inference/manifest.js.map +1 -0
- package/dist/src/inference/materialize.js +66 -0
- package/dist/src/inference/materialize.js.map +1 -0
- package/dist/src/inference/model-location.js +34 -0
- package/dist/src/inference/model-location.js.map +1 -0
- package/dist/src/inference/model-store.js +439 -0
- package/dist/src/inference/model-store.js.map +1 -0
- package/dist/src/inference/node-client.js +167 -0
- package/dist/src/inference/node-client.js.map +1 -0
- package/dist/src/inference/node-registry.js +79 -0
- package/dist/src/inference/node-registry.js.map +1 -0
- package/dist/src/inference/observation.js +79 -0
- package/dist/src/inference/observation.js.map +1 -0
- package/dist/src/inference/preflight.js +132 -0
- package/dist/src/inference/preflight.js.map +1 -0
- package/dist/src/inference/process-sglang.js +169 -0
- package/dist/src/inference/process-sglang.js.map +1 -0
- package/dist/src/inference/protocol-probe.js +29 -0
- package/dist/src/inference/protocol-probe.js.map +1 -0
- package/dist/src/inference/remote-model-store.js +24 -0
- package/dist/src/inference/remote-model-store.js.map +1 -0
- package/dist/src/inference/runtime-catalog.js +41 -0
- package/dist/src/inference/runtime-catalog.js.map +1 -0
- package/dist/src/inference/runtime-store.js +93 -0
- package/dist/src/inference/runtime-store.js.map +1 -0
- package/dist/src/inference/service-attachment.js +46 -0
- package/dist/src/inference/service-attachment.js.map +1 -0
- package/dist/src/inference/service-inspection.js +38 -0
- package/dist/src/inference/service-inspection.js.map +1 -0
- package/dist/src/inference/sglang.js +279 -0
- package/dist/src/inference/sglang.js.map +1 -0
- package/dist/src/inference/supervisor.js +460 -0
- package/dist/src/inference/supervisor.js.map +1 -0
- package/dist/src/model-access/environment.js +28 -0
- package/dist/src/model-access/environment.js.map +1 -0
- package/dist/src/model-access/index.js +4 -0
- package/dist/src/model-access/index.js.map +1 -1
- package/dist/src/model-access/local-gateway-request.js +139 -0
- package/dist/src/model-access/local-gateway-request.js.map +1 -0
- package/dist/src/model-access/local-gateway.js +323 -0
- package/dist/src/model-access/local-gateway.js.map +1 -0
- package/dist/src/model-access/proxy.js +90 -5
- package/dist/src/model-access/proxy.js.map +1 -1
- package/dist/src/model-access/remote-model.js +58 -0
- package/dist/src/model-access/remote-model.js.map +1 -0
- package/dist/src/model-access/training.js +115 -0
- package/dist/src/model-access/training.js.map +1 -0
- package/dist/src/runs/adapter-process.js +2 -2
- package/dist/src/runs/adapter-process.js.map +1 -1
- package/dist/src/runs/executor-types.js +2 -0
- package/dist/src/runs/executor-types.js.map +1 -0
- package/dist/src/runs/executor.js +54 -34
- package/dist/src/runs/executor.js.map +1 -1
- package/dist/src/runs/index.js +1 -0
- package/dist/src/runs/index.js.map +1 -1
- package/dist/src/runs/local-inference-environment.js +59 -0
- package/dist/src/runs/local-inference-environment.js.map +1 -0
- package/dist/src/runs/local-inference-run.js +53 -0
- package/dist/src/runs/local-inference-run.js.map +1 -0
- package/dist/src/runs/outcome.js +22 -0
- package/dist/src/runs/outcome.js.map +1 -1
- package/dist/src/runs/records.js +9 -0
- package/dist/src/runs/records.js.map +1 -1
- package/dist/src/runs/request.js +58 -3
- package/dist/src/runs/request.js.map +1 -1
- package/dist/src/runs/training-evidence.js +22 -0
- package/dist/src/runs/training-evidence.js.map +1 -0
- package/dist/src/runs/training-runtime.js +20 -0
- package/dist/src/runs/training-runtime.js.map +1 -0
- package/dist/src/workers/execution-observation.js +32 -0
- package/dist/src/workers/execution-observation.js.map +1 -0
- package/dist/src/workers/index.js +2 -0
- package/dist/src/workers/index.js.map +1 -1
- package/dist/src/workers/remote-harbor-ownership.js +266 -0
- package/dist/src/workers/remote-harbor-ownership.js.map +1 -0
- package/dist/src/workers/remote-harbor-verifier.js +76 -0
- package/dist/src/workers/remote-harbor-verifier.js.map +1 -0
- package/dist/src/workers/remote-harbor-work-spec.js +66 -10
- package/dist/src/workers/remote-harbor-work-spec.js.map +1 -1
- package/dist/src/workers/remote-harbor-worker.js +56 -49
- package/dist/src/workers/remote-harbor-worker.js.map +1 -1
- package/dist/src/workers/remote-worker-host.js +33 -0
- package/dist/src/workers/remote-worker-host.js.map +1 -0
- package/docs/local-model-inference-spec.zh-CN.md +666 -0
- package/docs/schemas/controller-runtime-observation.schema.json +21 -0
- package/docs/schemas/eval-execution-plan.schema.json +1 -0
- package/docs/schemas/eval-request.schema.json +3 -1
- package/docs/schemas/execution-lease.schema.json +95 -1
- package/docs/schemas/execution-provider-status.schema.json +162 -28
- package/docs/schemas/inference-execution-evidence.schema.json +487 -0
- package/docs/schemas/inference-lock.schema.json +487 -0
- package/docs/schemas/inference-runtime-manifest.schema.json +156 -0
- package/docs/schemas/inference-service-record.schema.json +227 -0
- package/docs/schemas/local-inference-selection.schema.json +98 -0
- package/docs/schemas/local-model-manifest.schema.json +43 -0
- package/docs/schemas/model-node-binding.schema.json +34 -0
- package/docs/schemas/model-node-generation-release.schema.json +46 -0
- package/docs/schemas/ordered-eval-control.schema.json +39 -0
- package/docs/schemas/remote-execution-lease-grant.schema.json +45 -0
- package/docs/schemas/remote-harbor-work-spec.schema.json +325 -0
- package/docs/schemas/remote-model-binding.schema.json +21 -0
- package/docs/schemas/remote-physical-execution.schema.json +108 -0
- package/docs/schemas/remote-result-envelope.schema.json +9 -3
- package/docs/schemas/remote-tree-envelope.schema.json +17 -3
- package/docs/schemas/remote-verifier-result.schema.json +203 -0
- package/docs/schemas/remote-verifier-source.schema.json +52 -0
- package/docs/schemas/remote-verifier-work.schema.json +125 -0
- package/docs/schemas/remote-work-offer.schema.json +170 -34
- package/docs/schemas/remote-worker-execution-admission.schema.json +214 -0
- package/docs/schemas/remote-worker-generation-cleanup.schema.json +199 -0
- package/docs/schemas/remote-worker-registration.schema.json +226 -25
- package/docs/schemas/run-manifest.schema.json +211 -44
- package/docs/schemas/run-request.schema.json +2 -1
- package/docs/schemas/training-external-binding.schema.json +26 -0
- package/integrations/harbor/hitch_harbor_agent.py +366 -38
- package/integrations/harbor/hitch_harbor_environment.py +22 -1
- package/integrations/harbor/hitch_host_credentials.py +354 -0
- package/integrations/harbor/hitch_private_exec.py +150 -0
- package/integrations/model-call/cli.js +13 -5
- package/integrations/training-tool/cli.js +72 -0
- package/package.json +8 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"canary-native-training.js","sourceRoot":"","sources":["../../scripts/canary-native-training.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAChF,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC/G,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAEjG,2FAA2F;AAC3F,+FAA+F;AAC/F,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;AACpD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC;AACtD,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,KAAK,EAAE,yFAAyF,CAAC,CAAC;AACtH,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;AACzD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;AAC9C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,+BAA+B,CAAC,CAAC,CAAC;AACjF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC;AAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;AACtD,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;AACjE,MAAM,CAAC,EAAE,CAAC,aAAa,EAAE,oCAAoC,CAAC,CAAC;AAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAA4D,CAAC;AAC/H,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,8BAA8B,CAAC,CAAC;AAC7D,MAAM,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AACrD,MAAM,CAAC,EAAE,CAAC,YAAY,IAAI,MAAM,EAAE,gEAAgE,CAAC,CAAC;AACpG,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAA+D,CAAC;AAChI,MAAM,OAAO,GAAG,oBAAoB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AACvD,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC;AACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC5C,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AACvF,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;AAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;AAC/E,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7F,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;AACrF,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAC3D,MAAM,QAAQ,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,IAAI,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AACnG,MAAM,IAAI,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AACpF,MAAM,GAAG,GAAsB,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,wBAAwB,EAAE,MAAM,EAAE,CAAC;AACpF,OAAO,GAAG,CAAC,gCAAgC,CAAC;AAC5C,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;AAC3H,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAqD,CAAC;AACnG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,EAAE,uBAAuB,CAAC,CAAC;AACpD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC/C,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC,EAAE,IAAI,SAAS,CAAC,YAAY,EAAE,EAAE,aAAa,CAAC,CAAC;AACzE,MAAM,OAAO,GAAG,CAAC,MAAM,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,wEAAwE,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AAClK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAChC,MAAM,WAAW,GAAG,CAAC,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AACvH,MAAM,YAAY,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;AAChE,MAAM,iBAAiB,GAAG,MAAM,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;AACjF,MAAM,MAAM,GAAG,iBAAiB,EAAE,MAAM,IAAI,WAAW,CAAC;AACxD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC;IAC/E,gBAAgB,EAAE,EAAE,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE,kBAAkB,EAAE,QAAQ;IAC/E,YAAY,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC,CAAC;AACvG,IAAI,MAA4C,CAAC;AACjD,IAAI,UAA8C,CAAC;AACnD,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;AACnE,IAAI,MAAqD,CAAC;AAC1D,IAAI,MAAM,GAAG,KAAK,CAAC;AACnB,MAAM,OAAO,GAA4B,EAAE,IAAI,EAAE,6CAA6C,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,oBAAoB,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;AACrQ,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AACrF,IAAI,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,MAAM,YAAY,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,SAAS,EAAE,sBAAsB,EAAE,QAAQ,EAAE,eAAe;QACtG,mBAAmB,EAAE,iBAAiB,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QAC/G,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,yBAAyB,EAAE,GAAG,EAAE;QACjI,eAAe,EAAE,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,CAAC;IAChH,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,mBAAmB,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC;IACxI,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC3D,MAAM,eAAe,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;IACtD,MAAM,eAAe,CAAC,WAAW,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IACtH,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC/I,MAAM,OAAO,GAAG,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAAC;IAClD,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB;QACpH,oBAAoB,EAAE,UAAU,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,mBAAmB,EAAE,cAAc,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5H,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB;QAC5N,mBAAmB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,uBAAuB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,OAAO,EAAE,sBAAsB,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IACjN,MAAM,CAAC,MAAO,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IAAC,MAAM,CAAC,MAAO,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/F,UAAU,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,MAAO,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,MAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/H,KAAK,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACxF,OAAO,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,OAAO,CAAC,SAAS,EAAE,EAAE,gBAAgB,EAAE,OAAO;gBACzG,cAAc,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE;YAC7G,SAAS,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,aAAa,EAAE,QAAQ,EAAE;gBAChG,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE;SACjF,CAAC,EAAE,CAAC,CAAC;IACN,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC;IACzC,IAAI,OAAO,GAA4B,EAAE,EAAE,QAAQ,GAAG,EAAE,CAAC;IACzD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YAAC,QAAQ,GAAG,KAAK,CAAC;YAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAAC,CAAC;QAC/H,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM;QAC1B,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC3H,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,sBAAsB,CAAC,EAAE,OAAO,CAAC,CAAC;IACxE,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,iCAAiC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,MAA4G,CAAC;IACpI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACjE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACnG,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;IAC5C,MAAM,CAAC,EAAE,CAAC,UAAU,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC,EAAE,qDAAqD,CAAC,CAAC;IACvG,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;IAC1C,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,QAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9J,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAqH,CAAC;IACvK,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IACvD,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,iBAAiB,CAAC,cAAc,EAAE,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAC7F,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;IAC7D,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACzF,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC;IACvG,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4E,CAAC,CAAC;IACtJ,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,aAAa,KAAK,gBAAgB,CAAC,CAAC,MAAM,CAAC;IAChG,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,aAAa,KAAK,mBAAmB,CAAC,CAAC;IAC5F,MAAM,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,qEAAqE,CAAC,CAAC;IAC1H,MAAM,CAAC,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACzG,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAK,CAAC,MAAO,CAAC,UAAW,CAAC,CAAC;IAC3E,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC1D,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,EAAE,UAAU,CAAC,CAAC;IACnE,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,wBAAwB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9E,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5J,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,EAAE,OAAO,CAAC,CAAC;IAE1E,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACrF,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3E,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,oBAAoB,CAAC,CAAC;IAC3C,MAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxH,MAAM,YAAY,EAAE,OAAO,EAAE,CAAC;IAC9B,KAAK,MAAM,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;QAC5G,KAAK,MAAM,IAAI,IAAI,MAAM,YAAY,CAAC,SAAS,CAAC;YAAE,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,gCAAgC,IAAI,EAAE,CAAC,CAAC;IACtK,CAAC;IACD,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,KAAK,CAAC;QAClG,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,EAAE,wCAAwC,EAAE,KAAK,EAAE,CAAC,CAAC;IACtM,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5G,MAAM,KAAK,CAAC;AACd,CAAC;QAAS,CAAC;IACT,IAAI,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACxE,MAAM,KAAK,EAAE,OAAO,CAAC,aAAa,MAAM,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrG,CAAC;IACD,IAAI,CAAC;QAAC,MAAM,YAAY,EAAE,IAAI,EAAE,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,OAAO,CAAC,iBAAiB,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;IACzI,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI;QAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/D,IAAI,UAAU;QAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACxH,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAAC,MAAM,UAAU,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAAC,CAAC;IACxG,SAAS,CAAC,GAAG,EAAE,CAAC;IAChB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnH,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAChC,wBAAwB,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;QACjE,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YACvG,CAAC,CAAC,wBAAwB,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;KAClE,CAAC,CAAC;IACH,IAAI,YAAY,IAAI,CAAC,MAAM;QAAE,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC,uBAAuB,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/H,OAAO,CAAC,cAAc,GAAG,CAAC,OAAO,CAAC,iBAAiB,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;IAC3I,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC;IAChE,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AACvC,CAAC;AACD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,EAAE,8CAA8C,CAAC,CAAC;AAE3F,KAAK,UAAU,aAAa;IAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IACpF,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IACxG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,gBAAgB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;IAC1E,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,EAAE,QAAQ,SAAS,CAAC,EAAE,IAAI,CAAC,CAAC;IACrF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,gFAAgF,SAAS,CAAC,EAAE,yDAAyD,CAAC,CAAC;IACrM,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,oPAAoP,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzT,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,4BAA4B,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,mCAAmC,CAAC,EAAE,MAAM,QAAQ,CAAC,mCAAmC,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9I,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,mBAAmB,CAAC,CAAC;IACxE,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,iCAAiC,EAAE,QAAQ,EAAE,IAAI,EAAE,+BAA+B,CAAC,CAAC;QAAE,MAAM,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3O,MAAM,MAAM,GAAG,CAAC,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAClH,OAAO,qBAAqB,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,SAAiB;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,KAAK,CAAC,WAAW,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9G,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { remoteCanaryWorker } from "./canary-worker-host.js";
|
|
3
|
+
import { randomBytes } from "node:crypto";
|
|
4
|
+
import { spawn } from "node:child_process";
|
|
5
|
+
import http from "node:http";
|
|
6
|
+
import { once } from "node:events";
|
|
7
|
+
import { createWriteStream } from "node:fs";
|
|
8
|
+
import { mkdir, mkdtemp, readFile, readdir, writeFile } from "node:fs/promises";
|
|
9
|
+
import { tmpdir } from "node:os";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { pathToFileURL } from "node:url";
|
|
12
|
+
import { DaemonServer, daemonClient } from "../src/daemon/index.js";
|
|
13
|
+
import { readExecutionLeases, reapOwnedDockerResources, runEval, validateEvalId } from "../src/evals/index.js";
|
|
14
|
+
import { parseTrainingBinding } from "../src/model-access/index.js";
|
|
15
|
+
import { atomicWriteJSON, runCommand, sha256JSON, statePaths } from "../src/foundation/index.js";
|
|
16
|
+
// Real Harbor/Docker and public worker CLI, with a deterministic model fixture.
|
|
17
|
+
// This cannot certify native token facts, Slime training, or a GPU runtime.
|
|
18
|
+
const python = process.env.HITCH_HARBOR_TEST_PYTHON;
|
|
19
|
+
const image = process.env.HITCH_TRAINING_CANARY_IMAGE;
|
|
20
|
+
assert.ok(python && image, "Set HITCH_HARBOR_TEST_PYTHON and HITCH_TRAINING_CANARY_IMAGE (local bash-capable image)");
|
|
21
|
+
const harbor = path.join(path.dirname(python), "harbor");
|
|
22
|
+
const cli = path.resolve("dist/bin/hitch.js");
|
|
23
|
+
const root = await mkdtemp(path.join(tmpdir(), "hitch-real-remote-training-"));
|
|
24
|
+
const controllerRoot = path.join(root, "controller"), workerRoot = path.join(root, "worker");
|
|
25
|
+
const privateRoot = path.join(root, "private"), dataset = path.join(root, "dataset");
|
|
26
|
+
await mkdir(privateRoot, { recursive: true, mode: 0o700 });
|
|
27
|
+
const resource = { cpu_millis: 1000, memory_bytes: 1024 ** 3, container_slots: 1, build_slots: 0 };
|
|
28
|
+
const zero = { cpu_millis: 0, memory_bytes: 0, container_slots: 0, build_slots: 0 };
|
|
29
|
+
const env = { ...process.env, HITCH_HARBOR_PYTHON_PATH: python };
|
|
30
|
+
delete env.HITCH_TEST_HOST_ARTIFACT_BUILDER;
|
|
31
|
+
const imageInfo = await runCommand("docker", ["image", "inspect", image, "--format", "{{json .}}"], { timeoutMs: 10_000 });
|
|
32
|
+
const inspected = JSON.parse(imageInfo.stdout);
|
|
33
|
+
assert.match(inspected.Id, /^sha256:[a-f0-9]{64}$/);
|
|
34
|
+
assert.equal(`${inspected.Os}/${inspected.Architecture}`, "linux/amd64");
|
|
35
|
+
const version = (await runCommand(python, ["-c", 'import importlib.metadata; print(importlib.metadata.version("harbor"))'], { timeoutMs: 10_000 })).stdout.trim();
|
|
36
|
+
assert.equal(version, "0.21.0");
|
|
37
|
+
const localEngine = (await runCommand("docker", ["info", "--format", "{{.ID}}"], { timeoutMs: 10_000 })).stdout.trim();
|
|
38
|
+
const remoteWorker = await remoteCanaryWorker(root, workerRoot);
|
|
39
|
+
const workerObservation = await remoteWorker?.observe(inspected.Id, localEngine);
|
|
40
|
+
const engine = workerObservation?.engine ?? localEngine;
|
|
41
|
+
const hash = sha256JSON({ canary: "remote-training", image: inspected.Id });
|
|
42
|
+
const secret = randomBytes(32).toString("hex");
|
|
43
|
+
const binding = parseTrainingBinding({ kind: "training-external", bindingId: "canary_training", trainingRunId: "train_canary",
|
|
44
|
+
policyLeaseRef: { uri: `cas:${hash}`, digest: hash, mediaType: "application/json" }, expectedPolicyVersion: "canary/update-0",
|
|
45
|
+
fencingToken: randomBytes(16).toString("hex"), expiresAt: new Date(Date.now() + 10 * 60_000).toISOString(),
|
|
46
|
+
endpointRef: "hitch-training:canary_training", credentialRef: "hitch-training:canary_training",
|
|
47
|
+
generationContractDigest: hash, requiredCapture: "exact-policy-tokens-v1", api: "chat-completions", maxOutputTokens: 64, maxEpisodeSteps: 4 });
|
|
48
|
+
let boundRun, generations = 0;
|
|
49
|
+
const protocolErrors = [];
|
|
50
|
+
const upstream = http.createServer((request, response) => {
|
|
51
|
+
const chunks = [];
|
|
52
|
+
request.on("data", chunk => chunks.push(Buffer.from(chunk)));
|
|
53
|
+
request.on("end", () => {
|
|
54
|
+
try {
|
|
55
|
+
assert.ok(request.headers.authorization === `Bearer ${secret}`, "missing private model credential");
|
|
56
|
+
response.setHeader("content-type", "application/json");
|
|
57
|
+
if (request.url === "/v1/lease") {
|
|
58
|
+
response.end(JSON.stringify({ schemaVersion: 1, trainingRunId: binding.trainingRunId, policyVersion: binding.expectedPolicyVersion,
|
|
59
|
+
fencingToken: binding.fencingToken, state: "serving", generationContractDigest: hash, capture: binding.requiredCapture }));
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
const body = JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
63
|
+
if (request.url === "/v1/hitch/run") {
|
|
64
|
+
assert.ok(!boundRun || boundRun === body.runId, "binding acquired another canonical run");
|
|
65
|
+
boundRun = body.runId;
|
|
66
|
+
response.end(JSON.stringify({ runId: boundRun, policyVersion: binding.expectedPolicyVersion }));
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
assert.equal(request.url, "/v1/chat/completions");
|
|
70
|
+
assert.ok(boundRun);
|
|
71
|
+
assert.equal(request.headers["idempotency-key"], `${boundRun}-${generations}`);
|
|
72
|
+
assert.equal(body.model, binding.expectedPolicyVersion);
|
|
73
|
+
if (generations === 1)
|
|
74
|
+
assert.match(body.messages.at(-1).content, /first-tool/);
|
|
75
|
+
if (generations === 2)
|
|
76
|
+
assert.match(body.messages.at(-1).content, /first-toolsecond-tool/);
|
|
77
|
+
generations++;
|
|
78
|
+
assert.ok(generations <= 3, "unexpected extra model generation");
|
|
79
|
+
const command = generations === 1
|
|
80
|
+
? "printf first-tool > /tmp/gear-canary.txt; cat /tmp/gear-canary.txt"
|
|
81
|
+
: "printf second-tool >> /tmp/gear-canary.txt; cat /tmp/gear-canary.txt";
|
|
82
|
+
response.setHeader("x-gear-receipt-id", `fixture-receipt-${generations}`);
|
|
83
|
+
response.end(JSON.stringify({ model: binding.expectedPolicyVersion, choices: [{ index: 0,
|
|
84
|
+
finish_reason: generations < 3 ? "tool_calls" : "stop", message: generations < 3
|
|
85
|
+
? { role: "assistant", content: null, tool_calls: [{ id: `call-${generations}`, type: "function", function: { name: "bash", arguments: JSON.stringify({ command }) } }] }
|
|
86
|
+
: { role: "assistant", content: "done" } }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }));
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
protocolErrors.push(error instanceof Error ? error.message : String(error));
|
|
90
|
+
response.statusCode = 500;
|
|
91
|
+
response.end(JSON.stringify({ error: "canary model protocol mismatch" }));
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
upstream.listen(0, "127.0.0.1");
|
|
96
|
+
await once(upstream, "listening");
|
|
97
|
+
const upstreamUrl = `http://127.0.0.1:${upstream.address().port}`;
|
|
98
|
+
const server = new DaemonServer({ root: controllerRoot, port: 0, maxConcurrent: 1,
|
|
99
|
+
resourceCapacity: { ...resource, build_slots: 1 }, evalTrialResources: resource,
|
|
100
|
+
evalExecutor: options => runEval({ ...options, harborExecutable: harbor, env }), logger: () => { } });
|
|
101
|
+
let worker;
|
|
102
|
+
let workerDone;
|
|
103
|
+
const workerLog = createWriteStream(path.join(root, "worker.log"));
|
|
104
|
+
let evalId;
|
|
105
|
+
let passed = false;
|
|
106
|
+
const summary = { kind: "real-harbor-training-transport-canary", validated_gpu: false, worker_host: remoteWorker ? "ssh" : "local-process", root, harbor: version, image_id: inspected.Id, docker_engine_digest: sha256JSON(engine) };
|
|
107
|
+
console.log(JSON.stringify({ event: "canary-start", root, image_id: inspected.Id }));
|
|
108
|
+
try {
|
|
109
|
+
const harness = await prepareInputs();
|
|
110
|
+
await server.start();
|
|
111
|
+
const registration = { schema_version: "1", worker_id: "worker_real_training", provider: "remote-docker",
|
|
112
|
+
collision_domain_id: `docker-engine:${engine}`, platforms: ["linux-x64"], backends: [{ id: "harbor", version }],
|
|
113
|
+
features: { docker: true, buildkit: true, model_proxy: true, isolated_same_task_attempts: false, training_external_binding: "2" },
|
|
114
|
+
task_membership: ["known"], capacity: { total: resource, reserved_for_system: zero, allocatable: resource } };
|
|
115
|
+
const registrationFile = path.join(privateRoot, "registration.json"), credentialFile = path.join(privateRoot, "worker-credential.json");
|
|
116
|
+
const bindingFile = path.join(privateRoot, "binding.json");
|
|
117
|
+
await atomicWriteJSON(registrationFile, registration);
|
|
118
|
+
await atomicWriteJSON(bindingFile, { schema_version: "1", binding, base_url: `${upstreamUrl}/v1`, credential: secret });
|
|
119
|
+
await runCommand(process.execPath, [cli, "--root", controllerRoot, "training", "register", "--file", bindingFile], { env, timeoutMs: 10_000 });
|
|
120
|
+
const baseUrl = `http://127.0.0.1:${server.port}`;
|
|
121
|
+
await runCommand(process.execPath, [cli, "worker", "register", "--server", baseUrl, "--registration", registrationFile,
|
|
122
|
+
"--admin-token-file", statePaths(controllerRoot).token, "--credential-file", credentialFile], { env, timeoutMs: 10_000 });
|
|
123
|
+
worker = remoteWorker ? await remoteWorker.start(server.port, registrationFile, credentialFile) : spawn(process.execPath, [cli, "--root", workerRoot, "worker", "run", "--server", baseUrl, "--registration", registrationFile,
|
|
124
|
+
"--credential-file", credentialFile, "--harbor", harbor, "--docker", "/usr/local/bin/docker", "--once", "--poll-interval", "100ms", "--heartbeat-interval", "1s"], { env, stdio: ["ignore", "pipe", "pipe"] });
|
|
125
|
+
worker.stdout.pipe(workerLog, { end: false });
|
|
126
|
+
worker.stderr.pipe(workerLog, { end: false });
|
|
127
|
+
workerDone = new Promise((resolve, reject) => { worker.once("error", reject); worker.once("exit", code => resolve(code)); });
|
|
128
|
+
void workerDone.catch(() => { });
|
|
129
|
+
const admin = await daemonClient(controllerRoot);
|
|
130
|
+
const submitted = await admin.request("/v1/evals", { method: "POST", body: JSON.stringify({
|
|
131
|
+
request: { dataset, harness_ref: harness, model: `training/${binding.bindingId}`, training_binding: binding,
|
|
132
|
+
max_concurrent: 1, attempts: 1, infrastructure_retries: 0, timeout_ms: 120_000, setup_timeout_ms: 120_000 },
|
|
133
|
+
execution: { provider: "remote-docker", max_parallelism: 1, resources: { default_trial: resource },
|
|
134
|
+
build: { mode: "backend" }, model_capture: { mode: "proxy", required: true } },
|
|
135
|
+
}) });
|
|
136
|
+
evalId = validateEvalId(String(submitted.eval_id));
|
|
137
|
+
console.log(JSON.stringify({ event: "eval-submitted", eval_id: evalId }));
|
|
138
|
+
const deadline = Date.now() + 5 * 60_000;
|
|
139
|
+
let current = {}, previous = "";
|
|
140
|
+
while (Date.now() < deadline) {
|
|
141
|
+
current = await admin.request(`/v1/evals/${evalId}`);
|
|
142
|
+
const phase = JSON.stringify(current.control);
|
|
143
|
+
if (phase !== previous) {
|
|
144
|
+
previous = phase;
|
|
145
|
+
console.log(JSON.stringify({ event: "eval-control", control: current.control }));
|
|
146
|
+
}
|
|
147
|
+
if (current.result)
|
|
148
|
+
break;
|
|
149
|
+
if (worker.exitCode !== null && worker.exitCode !== 0)
|
|
150
|
+
throw new Error(`worker exited before result (${worker.exitCode})`);
|
|
151
|
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
152
|
+
}
|
|
153
|
+
await atomicWriteJSON(path.join(root, "eval-inspection.json"), current);
|
|
154
|
+
assert.ok(current.result, "real Harbor candidate timed out");
|
|
155
|
+
const result = current.result;
|
|
156
|
+
assert.equal(result.status, "succeeded", JSON.stringify(result));
|
|
157
|
+
assert.equal(result.trials.length, 1);
|
|
158
|
+
assert.equal(result.trials[0].observation_status, "valid");
|
|
159
|
+
assert.equal(result.trials[0].reward, 1);
|
|
160
|
+
assert.equal(result.trials[0].run_id, boundRun);
|
|
161
|
+
assert.equal(generations, 3);
|
|
162
|
+
assert.deepEqual(protocolErrors, []);
|
|
163
|
+
const evidence = await runCommand(process.execPath, [cli, "--root", controllerRoot, "training", "evidence", boundRun, "--json"], { env, timeoutMs: 10_000 });
|
|
164
|
+
await writeFile(path.join(root, "training-evidence.json"), evidence.stdout);
|
|
165
|
+
const leases = await readExecutionLeases(path.join(controllerRoot, "evals", evalId));
|
|
166
|
+
assert.equal(leases.length, 1);
|
|
167
|
+
assert.equal(leases[0].state, "released");
|
|
168
|
+
assert.ok(leases[0].release_confirmation);
|
|
169
|
+
assert.equal(await Promise.race([workerDone, new Promise(resolve => setTimeout(() => resolve("timeout"), 10_000))]), 0);
|
|
170
|
+
await remoteWorker?.collect();
|
|
171
|
+
for (const directory of [path.join(controllerRoot, "runs"), path.join(controllerRoot, "evals"), workerRoot]) {
|
|
172
|
+
for (const file of await regularFiles(directory))
|
|
173
|
+
assert.equal((await readFile(file)).includes(Buffer.from(secret)), false, `model credential leaked into ${file}`);
|
|
174
|
+
}
|
|
175
|
+
Object.assign(summary, { status: "passed", eval_id: evalId, run_id: boundRun, model_calls: generations, tool_calls: 2, lease_id: leases[0].lease_id, private_model_credential_in_public_files: false });
|
|
176
|
+
passed = true;
|
|
177
|
+
}
|
|
178
|
+
catch (error) {
|
|
179
|
+
Object.assign(summary, { status: "failed", error: error instanceof Error ? error.message : String(error), protocol_errors: protocolErrors });
|
|
180
|
+
throw error;
|
|
181
|
+
}
|
|
182
|
+
finally {
|
|
183
|
+
if (evalId && !passed) {
|
|
184
|
+
const admin = await daemonClient(controllerRoot).catch(() => undefined);
|
|
185
|
+
await admin?.request(`/v1/evals/${evalId}/cancel`, { method: "POST", body: "{}" }).catch(() => { });
|
|
186
|
+
}
|
|
187
|
+
try {
|
|
188
|
+
await remoteWorker?.stop();
|
|
189
|
+
}
|
|
190
|
+
catch (error) {
|
|
191
|
+
summary.remote_stop_error = error instanceof Error ? error.message : String(error);
|
|
192
|
+
}
|
|
193
|
+
if (worker && worker.exitCode === null)
|
|
194
|
+
worker.kill("SIGTERM");
|
|
195
|
+
if (workerDone)
|
|
196
|
+
await Promise.race([workerDone.catch(() => null), new Promise(resolve => setTimeout(resolve, 10_000))]);
|
|
197
|
+
if (worker && worker.exitCode === null) {
|
|
198
|
+
worker.kill("SIGKILL");
|
|
199
|
+
await workerDone?.catch(() => null);
|
|
200
|
+
}
|
|
201
|
+
workerLog.end();
|
|
202
|
+
await server.close();
|
|
203
|
+
upstream.closeAllConnections();
|
|
204
|
+
await new Promise(resolve => upstream.close(() => resolve()));
|
|
205
|
+
const leases = evalId ? await readExecutionLeases(path.join(controllerRoot, "evals", evalId)).catch(() => []) : [];
|
|
206
|
+
const leaseIds = leases.map(lease => lease.lease_id);
|
|
207
|
+
const cleanup = await Promise.all([
|
|
208
|
+
reapOwnedDockerResources({ root: controllerRoot, leaseIds, env }),
|
|
209
|
+
remoteWorker ? remoteWorker.cleanup(leaseIds).catch(error => ({ issues: [String(error)], retained: [] }))
|
|
210
|
+
: reapOwnedDockerResources({ root: workerRoot, leaseIds, env }),
|
|
211
|
+
]);
|
|
212
|
+
if (remoteWorker && !passed)
|
|
213
|
+
await remoteWorker.collect().catch(error => { summary.remote_collection_error = String(error); });
|
|
214
|
+
summary.cleanup_proven = !summary.remote_stop_error && cleanup.every(report => report.issues.length === 0 && report.retained.length === 0);
|
|
215
|
+
await atomicWriteJSON(path.join(root, "summary.json"), summary);
|
|
216
|
+
console.log(JSON.stringify(summary));
|
|
217
|
+
}
|
|
218
|
+
assert.equal(summary.cleanup_proven, true, "owned Docker resources were not all released");
|
|
219
|
+
async function prepareInputs() {
|
|
220
|
+
const task = path.join(dataset, "one"), source = path.join(root, "training-source");
|
|
221
|
+
await mkdir(path.join(task, "environment"), { recursive: true });
|
|
222
|
+
await mkdir(path.join(task, "tests"));
|
|
223
|
+
await writeFile(path.join(task, "instruction.md"), "Use two bash tool calls to write first-toolsecond-tool into /tmp/gear-canary.txt.\n");
|
|
224
|
+
await writeFile(path.join(task, "environment/Dockerfile"), `FROM ${inspected.Id}\n`);
|
|
225
|
+
await writeFile(path.join(task, "task.toml"), `schema_version="1.4"\n[agent]\ntimeout_sec=120\n[environment]\ndocker_image="${inspected.Id}"\ncpus=1\nmemory_mb=1024\n[verifier]\ntimeout_sec=30\n`);
|
|
226
|
+
await writeFile(path.join(task, "tests/test.sh"), '#!/bin/bash\nset -eu\nmkdir -p /logs/verifier\nif test -f /tmp/gear-canary.txt && test "$(cat /tmp/gear-canary.txt)" = first-toolsecond-tool; then\n printf "1" > /logs/verifier/reward.txt\nelse\n printf "0" > /logs/verifier/reward.txt\nfi\n', { mode: 0o755 });
|
|
227
|
+
await mkdir(path.join(source, "integrations/training-tool"), { recursive: true });
|
|
228
|
+
await writeFile(path.join(source, "integrations/training-tool/cli.js"), await readFile("integrations/training-tool/cli.js"), { mode: 0o755 });
|
|
229
|
+
await writeFile(path.join(source, "package.json"), '{"type":"module"}');
|
|
230
|
+
for (const args of [["init"], ["add", "."], ["-c", "user.name=Hitch Canary", "-c", "user.email=canary@hitch.invalid", "commit", "-m", "fixed training harness canary"]])
|
|
231
|
+
await runCommand("git", args, { cwd: source, timeoutMs: 10_000 });
|
|
232
|
+
const commit = (await runCommand("git", ["rev-parse", "HEAD"], { cwd: source, timeoutMs: 10_000 })).stdout.trim();
|
|
233
|
+
return `training-tool@git+${pathToFileURL(source).href}#${commit}`;
|
|
234
|
+
}
|
|
235
|
+
async function regularFiles(directory) {
|
|
236
|
+
const files = [];
|
|
237
|
+
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
|
238
|
+
const file = path.join(directory, entry.name);
|
|
239
|
+
if (entry.isDirectory())
|
|
240
|
+
files.push(...await regularFiles(file));
|
|
241
|
+
else if (entry.isFile())
|
|
242
|
+
files.push(file);
|
|
243
|
+
}
|
|
244
|
+
return files;
|
|
245
|
+
}
|
|
246
|
+
//# sourceMappingURL=canary-remote-training.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"canary-remote-training.js","sourceRoot":"","sources":["../../scripts/canary-remote-training.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAChF,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC/G,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAEjG,gFAAgF;AAChF,4EAA4E;AAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;AACpD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC;AACtD,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,KAAK,EAAE,yFAAyF,CAAC,CAAC;AACtH,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;AACzD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;AAC9C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,6BAA6B,CAAC,CAAC,CAAC;AAC/E,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7F,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;AACrF,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAC3D,MAAM,QAAQ,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,IAAI,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AACnG,MAAM,IAAI,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AACpF,MAAM,GAAG,GAAsB,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,wBAAwB,EAAE,MAAM,EAAE,CAAC;AACpF,OAAO,GAAG,CAAC,gCAAgC,CAAC;AAC5C,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;AAC3H,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAqD,CAAC;AACnG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,EAAE,uBAAuB,CAAC,CAAC;AACpD,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC,EAAE,IAAI,SAAS,CAAC,YAAY,EAAE,EAAE,aAAa,CAAC,CAAC;AACzE,MAAM,OAAO,GAAG,CAAC,MAAM,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,wEAAwE,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AAClK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAChC,MAAM,WAAW,GAAG,CAAC,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AACvH,MAAM,YAAY,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;AAChE,MAAM,iBAAiB,GAAG,MAAM,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;AACjF,MAAM,MAAM,GAAG,iBAAiB,EAAE,MAAM,IAAI,WAAW,CAAC;AACxD,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,MAAM,EAAE,iBAAiB,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5E,MAAM,MAAM,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAC/C,MAAM,OAAO,GAAG,oBAAoB,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,SAAS,EAAE,iBAAiB,EAAE,aAAa,EAAE,cAAc;IAC3H,cAAc,EAAE,EAAE,GAAG,EAAE,OAAO,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,kBAAkB,EAAE,EAAE,qBAAqB,EAAE,iBAAiB;IAC7H,YAAY,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,WAAW,EAAE;IAC1G,WAAW,EAAE,gCAAgC,EAAE,aAAa,EAAE,gCAAgC;IAC9F,wBAAwB,EAAE,IAAI,EAAE,eAAe,EAAE,wBAAwB,EAAE,GAAG,EAAE,kBAAkB,EAAE,eAAe,EAAE,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;AACjJ,IAAI,QAA4B,EAAE,WAAW,GAAG,CAAC,CAAC;AAClD,MAAM,cAAc,GAAa,EAAE,CAAC;AACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE;IACvD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACrB,IAAI,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,KAAK,UAAU,MAAM,EAAE,EAAE,kCAAkC,CAAC,CAAC;YACpG,QAAQ,CAAC,SAAS,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;YACvD,IAAI,OAAO,CAAC,GAAG,KAAK,WAAW,EAAE,CAAC;gBAChC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,aAAa,EAAE,OAAO,CAAC,qBAAqB;oBAChI,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,wBAAwB,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;gBAAC,OAAO;YACvI,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;YAChE,IAAI,OAAO,CAAC,GAAG,KAAK,eAAe,EAAE,CAAC;gBACpC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,KAAK,EAAE,wCAAwC,CAAC,CAAC;gBAC1F,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC;gBAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;gBAAC,OAAO;YACjI,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,sBAAsB,CAAC,CAAC;YAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;YACvE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,GAAG,QAAQ,IAAI,WAAW,EAAE,CAAC,CAAC;YAC/E,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,qBAAqB,CAAC,CAAC;YACxD,IAAI,WAAW,KAAK,CAAC;gBAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;YAChF,IAAI,WAAW,KAAK,CAAC;gBAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,uBAAuB,CAAC,CAAC;YAC3F,WAAW,EAAE,CAAC;YACd,MAAM,CAAC,EAAE,CAAC,WAAW,IAAI,CAAC,EAAE,mCAAmC,CAAC,CAAC;YACjE,MAAM,OAAO,GAAG,WAAW,KAAK,CAAC;gBAC/B,CAAC,CAAC,oEAAoE;gBACtE,CAAC,CAAC,sEAAsE,CAAC;YAC3E,QAAQ,CAAC,SAAS,CAAC,mBAAmB,EAAE,mBAAmB,WAAW,EAAE,CAAC,CAAC;YAC1E,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,qBAAqB,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;wBACtF,aAAa,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,CAAC;4BAC9E,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,WAAW,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE;4BACzK,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1H,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5E,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC;YAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC,CAAC,CAAC;QACvG,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AACH,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;AAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AACnE,MAAM,WAAW,GAAG,oBAAqB,QAAQ,CAAC,OAAO,EAAqC,CAAC,IAAI,EAAE,CAAC;AACtG,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC;IAC/E,gBAAgB,EAAE,EAAE,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE,kBAAkB,EAAE,QAAQ;IAC/E,YAAY,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC,CAAC;AACvG,IAAI,MAA4C,CAAC;AACjD,IAAI,UAA8C,CAAC;AACnD,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;AACnE,IAAI,MAAqD,CAAC;AAC1D,IAAI,MAAM,GAAG,KAAK,CAAC;AACnB,MAAM,OAAO,GAA4B,EAAE,IAAI,EAAE,uCAAuC,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,oBAAoB,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;AAC/P,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AACrF,IAAI,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,MAAM,YAAY,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,SAAS,EAAE,sBAAsB,EAAE,QAAQ,EAAE,eAAe;QACtG,mBAAmB,EAAE,iBAAiB,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QAC/G,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,yBAAyB,EAAE,GAAG,EAAE;QACjI,eAAe,EAAE,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,CAAC;IAChH,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,mBAAmB,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC;IACxI,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC3D,MAAM,eAAe,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;IACtD,MAAM,eAAe,CAAC,WAAW,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,WAAW,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IACxH,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC/I,MAAM,OAAO,GAAG,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAAC;IAClD,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB;QACpH,oBAAoB,EAAE,UAAU,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,mBAAmB,EAAE,cAAc,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5H,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB;QAC5N,mBAAmB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,uBAAuB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,OAAO,EAAE,sBAAsB,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IACjN,MAAM,CAAC,MAAO,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IAAC,MAAM,CAAC,MAAO,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/F,UAAU,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,MAAO,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,MAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/H,KAAK,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACxF,OAAO,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,OAAO,CAAC,SAAS,EAAE,EAAE,gBAAgB,EAAE,OAAO;gBACzG,cAAc,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE;YAC7G,SAAS,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,aAAa,EAAE,QAAQ,EAAE;gBAChG,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE;SACjF,CAAC,EAAE,CAAC,CAAC;IACN,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC;IACzC,IAAI,OAAO,GAA4B,EAAE,EAAE,QAAQ,GAAG,EAAE,CAAC;IACzD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YAAC,QAAQ,GAAG,KAAK,CAAC;YAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAAC,CAAC;QAC/H,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM;QAC1B,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC3H,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,sBAAsB,CAAC,EAAE,OAAO,CAAC,CAAC;IACxE,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,iCAAiC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,MAA4G,CAAC;IACpI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACjE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACnG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjD,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,QAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9J,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACrF,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3E,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,oBAAoB,CAAC,CAAC;IAC3C,MAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxH,MAAM,YAAY,EAAE,OAAO,EAAE,CAAC;IAC9B,KAAK,MAAM,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;QAC5G,KAAK,MAAM,IAAI,IAAI,MAAM,YAAY,CAAC,SAAS,CAAC;YAAE,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,gCAAgC,IAAI,EAAE,CAAC,CAAC;IACtK,CAAC;IACD,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,EAAE,wCAAwC,EAAE,KAAK,EAAE,CAAC,CAAC;IACzM,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC;IAC7I,MAAM,KAAK,CAAC;AACd,CAAC;QAAS,CAAC;IACT,IAAI,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACxE,MAAM,KAAK,EAAE,OAAO,CAAC,aAAa,MAAM,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrG,CAAC;IACD,IAAI,CAAC;QAAC,MAAM,YAAY,EAAE,IAAI,EAAE,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,OAAO,CAAC,iBAAiB,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;IACzI,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI;QAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/D,IAAI,UAAU;QAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACxH,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAAC,MAAM,UAAU,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAAC,CAAC;IACxG,SAAS,CAAC,GAAG,EAAE,CAAC;IAChB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,QAAQ,CAAC,mBAAmB,EAAE,CAAC;IAAC,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACpG,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnH,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAChC,wBAAwB,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;QACjE,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YACvG,CAAC,CAAC,wBAAwB,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;KAClE,CAAC,CAAC;IACH,IAAI,YAAY,IAAI,CAAC,MAAM;QAAE,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC,uBAAuB,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/H,OAAO,CAAC,cAAc,GAAG,CAAC,OAAO,CAAC,iBAAiB,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;IAC3I,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AACvC,CAAC;AACD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,EAAE,8CAA8C,CAAC,CAAC;AAE3F,KAAK,UAAU,aAAa;IAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IACpF,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IACxG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,gBAAgB,CAAC,EAAE,qFAAqF,CAAC,CAAC;IAC1I,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,EAAE,QAAQ,SAAS,CAAC,EAAE,IAAI,CAAC,CAAC;IACrF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,gFAAgF,SAAS,CAAC,EAAE,yDAAyD,CAAC,CAAC;IACrM,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,oPAAoP,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzT,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,4BAA4B,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,mCAAmC,CAAC,EAAE,MAAM,QAAQ,CAAC,mCAAmC,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9I,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,mBAAmB,CAAC,CAAC;IACxE,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,iCAAiC,EAAE,QAAQ,EAAE,IAAI,EAAE,+BAA+B,CAAC,CAAC;QAAE,MAAM,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3O,MAAM,MAAM,GAAG,CAAC,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAClH,OAAO,qBAAqB,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,SAAiB;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,KAAK,CAAC,WAAW,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9G,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { buildEvalExecutionPlan, captureRemoteVerifierSource, createExecutionLease, portableRemoteTrial, reapOwnedDockerResources, runRemoteVerifierWork, validateEvalId } from "../src/evals/index.js";
|
|
6
|
+
import { ensureControllerRuntime } from "../src/controller-runtime/index.js";
|
|
7
|
+
import { atomicWriteJSON, readJSON, runCommand, sha256JSON } from "../src/foundation/index.js";
|
|
8
|
+
import { writeResultBundleIndex } from "../src/runs/index.js";
|
|
9
|
+
// Real Harbor/Docker CPU canary. Requires a locally available image with bash; never pulls or provisions GPUs.
|
|
10
|
+
const python = process.env.HITCH_HARBOR_TEST_PYTHON, image = process.env.HITCH_VERIFIER_IMAGE;
|
|
11
|
+
if (!python || !image)
|
|
12
|
+
throw Error("Set HITCH_HARBOR_TEST_PYTHON and HITCH_VERIFIER_IMAGE to the Harbor 0.21.0 Python and a local bash-capable Docker image");
|
|
13
|
+
const inspected = await runCommand("docker", ["image", "inspect", image, "--format", "{{.Id}}"], { timeoutMs: 10_000 });
|
|
14
|
+
const imageId = inspected.stdout.trim();
|
|
15
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(imageId))
|
|
16
|
+
throw Error("A local immutable Docker image ID is required");
|
|
17
|
+
const root = await mkdtemp(path.join(tmpdir(), "hitch-real-verifier-"));
|
|
18
|
+
const evalId = validateEvalId(`eval_${"b".repeat(32)}`), runId = `run_${"a".repeat(32)}`;
|
|
19
|
+
const evalDirectory = path.join(root, "evals", evalId), taskDirectory = path.join(root, "one"), trialDirectory = path.join(root, "original");
|
|
20
|
+
const reservation = { cpu_millis: 1000, memory_bytes: 512 * 1024 ** 2, container_slots: 1, build_slots: 0 };
|
|
21
|
+
const lease = await createExecutionLease({ evalId, evalDirectory, workId: `work_${"c".repeat(32)}`,
|
|
22
|
+
worker: { workerId: "verifier-canary", provider: "remote-docker", collisionDomainId: "verifier-canary-docker" }, reservation, ttlMs: 45_000 });
|
|
23
|
+
const controller = new AbortController();
|
|
24
|
+
const heartbeat = setInterval(() => { void lease.heartbeat().catch(error => controller.abort(error)); }, 10_000);
|
|
25
|
+
heartbeat.unref();
|
|
26
|
+
let cleanupProven = false, passed = false;
|
|
27
|
+
let summary = {};
|
|
28
|
+
try {
|
|
29
|
+
const runtime = await ensureControllerRuntime({ root });
|
|
30
|
+
await mkdir(path.join(taskDirectory, "tests"), { recursive: true });
|
|
31
|
+
await mkdir(path.join(taskDirectory, "environment"));
|
|
32
|
+
await writeFile(path.join(taskDirectory, "instruction.md"), "The candidate has already produced candidate.txt. Regrade its recorded output.\n");
|
|
33
|
+
await writeFile(path.join(taskDirectory, "environment/Dockerfile"), `FROM ${imageId}\n`);
|
|
34
|
+
await writeFile(path.join(taskDirectory, "task.toml"), `schema_version="1.4"
|
|
35
|
+
[agent]
|
|
36
|
+
timeout_sec=10
|
|
37
|
+
[environment]
|
|
38
|
+
docker_image="${imageId}"
|
|
39
|
+
cpus=1
|
|
40
|
+
memory_mb=512
|
|
41
|
+
[verifier]
|
|
42
|
+
timeout_sec=30
|
|
43
|
+
environment_mode="separate"
|
|
44
|
+
[verifier.environment]
|
|
45
|
+
docker_image="${imageId}"
|
|
46
|
+
cpus=1
|
|
47
|
+
memory_mb=512
|
|
48
|
+
`);
|
|
49
|
+
await writeFile(path.join(taskDirectory, "tests/test.sh"), '#!/bin/bash\nset -eu\ntest "$(cat /logs/artifacts/candidate.txt)" = "recorded candidate"\nmkdir -p /logs/verifier\nprintf "1" > /logs/verifier/reward.txt\n');
|
|
50
|
+
await mkdir(path.join(trialDirectory, "artifacts/logs/artifacts"), { recursive: true });
|
|
51
|
+
await writeFile(path.join(trialDirectory, "artifacts/logs/artifacts/candidate.txt"), "recorded candidate");
|
|
52
|
+
await atomicWriteJSON(path.join(trialDirectory, "artifacts/manifest.json"), [{ source: "/logs/artifacts", destination: "artifacts/logs/artifacts", type: "directory", status: "ok", service: null }]);
|
|
53
|
+
await atomicWriteJSON(path.join(trialDirectory, "benchmark-lifecycle.json"), { schema_version: "1", failure: null, phases: {} });
|
|
54
|
+
await atomicWriteJSON(path.join(trialDirectory, "config.json"), { task: { path: taskDirectory }, trial_name: "one__original", trials_dir: root,
|
|
55
|
+
agent: { import_path: "hitch_harbor_agent:HitchHarborAgent", kwargs: { private_canary_marker: "never-transfer-agent-config" } },
|
|
56
|
+
environment: { type: "docker", import_path: "hitch_harbor_environment:HitchHarborDockerEnvironment", delete: false,
|
|
57
|
+
cpu_enforcement_policy: "limit", memory_enforcement_policy: "limit", override_cpus: 1, override_memory_mb: 512 },
|
|
58
|
+
verifier: { import_path: "hitch_harbor_verifier:HitchRetryingVerifier", override_timeout_sec: 30,
|
|
59
|
+
kwargs: { infrastructure_retries: 0, infrastructure_retry_backoff_ms: 0 } } });
|
|
60
|
+
await runCommand(python, ["-c", `import json,sys
|
|
61
|
+
from pathlib import Path
|
|
62
|
+
from harbor.models.trial.config import TrialConfig
|
|
63
|
+
from harbor.models.trial.result import TrialResult
|
|
64
|
+
from harbor.models.task.task import Task
|
|
65
|
+
p=Path(sys.argv[1]); c=TrialConfig.model_validate_json((p/'config.json').read_text()); t=Task(task_dir=c.task.path)
|
|
66
|
+
(p/'config.json').write_text(c.model_dump_json())
|
|
67
|
+
r=TrialResult(task_name=t.name,trial_name=c.trial_name,trial_uri=p.as_uri(),task_id={'path':c.task.path},task_checksum=t.checksum,config=c,
|
|
68
|
+
agent_info={'name':'HitchHarborAgent','version':'canary'},agent_result={'metadata':{'hitch_run_id':sys.argv[2],'controller_runtime_id':sys.argv[3]}})
|
|
69
|
+
(p/'result.json').write_text(r.model_dump_json())
|
|
70
|
+
`, trialDirectory, runId, runtime.runtime_id], { timeoutMs: 10_000 });
|
|
71
|
+
const trial = await readJSON(path.join(trialDirectory, "result.json"));
|
|
72
|
+
const candidateResult = { run_id: runId, status: "succeeded", output: "recorded candidate" }, bundleDirectory = path.join(root, "bundle");
|
|
73
|
+
await atomicWriteJSON(path.join(bundleDirectory, "manifest.json"), { schema_version: "1", run_id: runId, sealed: true });
|
|
74
|
+
await atomicWriteJSON(path.join(bundleDirectory, "result.json"), candidateResult);
|
|
75
|
+
await writeResultBundleIndex(bundleDirectory);
|
|
76
|
+
const capture = await captureRemoteVerifierSource({ taskId: "one", runId, runtimeId: runtime.runtime_id, taskDirectory, trialDirectory, trial,
|
|
77
|
+
bundleDirectory, destination: path.join(root, "snapshot") });
|
|
78
|
+
assert.equal(capture.manifest.status, "available");
|
|
79
|
+
await lease.markRunning();
|
|
80
|
+
const directory = path.join(root, "regrade");
|
|
81
|
+
const plan = buildEvalExecutionPlan({ evalId, tasks: ["one"], maxParallelism: 1, provider: lease.current().provider, trialResources: reservation,
|
|
82
|
+
candidate: { revisionIdentity: sha256JSON("recorded-canary-revision"), artifactId: sha256JSON("recorded-canary-artifact") },
|
|
83
|
+
request: { schema_version: "1", backend: "harbor", dataset: taskDirectory, harness_ref: "canary@version:1", model: "canary/unused",
|
|
84
|
+
attempts: 1, max_concurrent: 1, infrastructure_retries: 0, infrastructure_retry_backoff_ms: 0, timeout_ms: 10_000, setup_timeout_ms: 10_000,
|
|
85
|
+
agent_args: [], pass_env: [], benchmark_id: "verifier-canary", benchmark_revision: sha256JSON("canary") } });
|
|
86
|
+
const outcome = await runRemoteVerifierWork({ root, directory, sourceSnapshotDirectory: capture.directory, taskDirectory,
|
|
87
|
+
sourceManifest: capture.manifest, sourceTrial: portableRemoteTrial(trial), candidateResult, candidateResultDigest: sha256JSON(candidateResult),
|
|
88
|
+
runtimeDirectory: runtime.directory, verifierRuntimeDirectory: runtime.directory, verifierRuntimeId: runtime.runtime_id,
|
|
89
|
+
trialName: "one__regrade", lease: lease.current(), plan,
|
|
90
|
+
env: process.env, harborExecutable: path.join(path.dirname(python), "harbor"), signal: controller.signal });
|
|
91
|
+
assert.equal(outcome.backend.process_exit_code, 0);
|
|
92
|
+
assert.equal(outcome.trial.agent_setup, null);
|
|
93
|
+
assert.equal(outcome.trial.agent_execution, null);
|
|
94
|
+
assert.deepEqual(outcome.trial.agent_result, trial.agent_result);
|
|
95
|
+
assert.equal(outcome.trial.verifier_result?.rewards?.reward, 1);
|
|
96
|
+
assert.equal((await readFile(path.join(directory, "regrade.json"), "utf8")).includes("never-transfer-agent-config"), false);
|
|
97
|
+
passed = true;
|
|
98
|
+
summary = { status: "passed", harbor: outcome.backend.version, image_id: imageId, candidate_executes: false,
|
|
99
|
+
reward: 1, agent_result_preserved: true, private_config_transferred: false };
|
|
100
|
+
}
|
|
101
|
+
finally {
|
|
102
|
+
clearInterval(heartbeat);
|
|
103
|
+
await lease.release();
|
|
104
|
+
const report = await reapOwnedDockerResources({ root, leaseIds: [lease.leaseId], env: process.env });
|
|
105
|
+
cleanupProven = report.issues.length === 0 && report.retained.length === 0;
|
|
106
|
+
if (cleanupProven && passed) {
|
|
107
|
+
await writable(root);
|
|
108
|
+
await rm(root, { recursive: true, force: true });
|
|
109
|
+
}
|
|
110
|
+
else
|
|
111
|
+
console.error(`Verifier canary retained evidence: ${root}; cleanup_proven=${cleanupProven}`);
|
|
112
|
+
}
|
|
113
|
+
assert.equal(cleanupProven, true);
|
|
114
|
+
console.log(JSON.stringify({ ...summary, cleanup_proven: true }));
|
|
115
|
+
async function writable(directory) {
|
|
116
|
+
const info = await lstat(directory);
|
|
117
|
+
if (info.isSymbolicLink())
|
|
118
|
+
return;
|
|
119
|
+
await chmod(directory, info.isDirectory() ? 0o700 : 0o600);
|
|
120
|
+
if (info.isDirectory())
|
|
121
|
+
for (const entry of await readdir(directory))
|
|
122
|
+
await writable(path.join(directory, entry));
|
|
123
|
+
}
|
|
124
|
+
//# sourceMappingURL=canary-remote-verifier.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"canary-remote-verifier.js","sourceRoot":"","sources":["../../scripts/canary-remote-verifier.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClG,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,sBAAsB,EAAE,2BAA2B,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACxM,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC/F,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAE9D,+GAA+G;AAC/G,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;AAC9F,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK;IAAE,MAAM,KAAK,CAAC,yHAAyH,CAAC,CAAC;AAC9J,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;AACxH,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AACxC,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC;IAAE,MAAM,KAAK,CAAC,+CAA+C,CAAC,CAAC;AACzG,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAAC;AACxE,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,GAAG,OAAO,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;AACzF,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;AAC7I,MAAM,WAAW,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,GAAG,IAAI,IAAI,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AAC5G,MAAM,KAAK,GAAG,MAAM,oBAAoB,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE;IAChG,MAAM,EAAE,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,eAAe,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;AACjJ,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;AACzC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AACjH,SAAS,CAAC,KAAK,EAAE,CAAC;AAClB,IAAI,aAAa,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;AAC1C,IAAI,OAAO,GAA4B,EAAE,CAAC;AAC1C,IAAI,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC,CAAC;IAC1H,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,gBAAgB,CAAC,EAAE,kFAAkF,CAAC,CAAC;IAChJ,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,wBAAwB,CAAC,EAAE,QAAQ,OAAO,IAAI,CAAC,CAAC;IACzF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE;;;;gBAIzC,OAAO;;;;;;;gBAOP,OAAO;;;CAGtB,CAAC,CAAC;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,6JAA6J,CAAC,CAAC;IAC1N,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,0BAA0B,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,wCAAwC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAC3G,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,yBAAyB,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,iBAAiB,EAAE,WAAW,EAAE,0BAA0B,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtM,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,0BAA0B,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;IACjI,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,UAAU,EAAE,eAAe,EAAE,UAAU,EAAE,IAAI;QAC5I,KAAK,EAAE,EAAE,WAAW,EAAE,qCAAqC,EAAE,MAAM,EAAE,EAAE,qBAAqB,EAAE,6BAA6B,EAAE,EAAE;QAC/H,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,uDAAuD,EAAE,MAAM,EAAE,KAAK;YAChH,sBAAsB,EAAE,OAAO,EAAE,yBAAyB,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,kBAAkB,EAAE,GAAG,EAAE;QAClH,QAAQ,EAAE,EAAE,WAAW,EAAE,6CAA6C,EAAE,oBAAoB,EAAE,EAAE;YAC9F,MAAM,EAAE,EAAE,sBAAsB,EAAE,CAAC,EAAE,+BAA+B,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACnF,MAAM,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE;;;;;;;;;;CAUjC,EAAE,cAAc,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAA0B,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC,CAAC;IAChG,MAAM,eAAe,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,oBAAoB,EAAE,EAAE,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC1I,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACzH,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,aAAa,CAAC,EAAE,eAAe,CAAC,CAAC;IAAC,MAAM,sBAAsB,CAAC,eAAe,CAAC,CAAC;IACjI,MAAM,OAAO,GAAG,MAAM,2BAA2B,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE,aAAa,EAAE,cAAc,EAAE,KAAK;QAC3I,eAAe,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC,CAAC;IAC/D,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;IAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,sBAAsB,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,cAAc,EAAE,WAAW;QAC9I,SAAS,EAAE,EAAE,gBAAgB,EAAE,UAAU,CAAC,0BAA0B,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,0BAA0B,CAAC,EAAE;QAC3H,OAAO,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,kBAAkB,EAAE,KAAK,EAAE,eAAe;YAChI,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,+BAA+B,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM;YAC3I,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;IACjH,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,uBAAuB,EAAE,OAAO,CAAC,SAAU,EAAE,aAAa;QACvH,cAAc,EAAE,OAAO,CAAC,QAAQ,EAAE,WAAW,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,eAAe,EAAE,qBAAqB,EAAE,UAAU,CAAC,eAAe,CAAC;QAC9I,gBAAgB,EAAE,OAAO,CAAC,SAAS,EAAE,wBAAwB,EAAE,OAAO,CAAC,SAAS,EAAE,iBAAiB,EAAE,OAAO,CAAC,UAAU;QACvH,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,IAAI;QACvD,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9G,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;IACnD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC9C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IACjE,MAAM,CAAC,KAAK,CAAE,OAAO,CAAC,KAAK,CAAC,eAAqD,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IACvG,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE,KAAK,CAAC,CAAC;IAC5H,MAAM,GAAG,IAAI,CAAC;IACd,OAAO,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,kBAAkB,EAAE,KAAK;QACzG,MAAM,EAAE,CAAC,EAAE,sBAAsB,EAAE,IAAI,EAAE,0BAA0B,EAAE,KAAK,EAAE,CAAC;AACjF,CAAC;QAAS,CAAC;IACT,aAAa,CAAC,SAAS,CAAC,CAAC;IACzB,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACrG,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC;IAC3E,IAAI,aAAa,IAAI,MAAM,EAAE,CAAC;QAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;QAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,CAAC;;QACnG,OAAO,CAAC,KAAK,CAAC,sCAAsC,IAAI,oBAAoB,aAAa,EAAE,CAAC,CAAC;AACpG,CAAC;AACD,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;AAClC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAElE,KAAK,UAAU,QAAQ,CAAC,SAAiB;IACvC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,IAAI,CAAC,cAAc,EAAE;QAAE,OAAO;IAClC,MAAM,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC3D,IAAI,IAAI,CAAC,WAAW,EAAE;QAAE,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,CAAC;YAAE,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;AACpH,CAAC"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
3
|
+
import { spawn } from "node:child_process";
|
|
4
|
+
import { mkdir, readFile } from "node:fs/promises";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { observeControllerRuntime } from "../src/controller-runtime/index.js";
|
|
7
|
+
import { atomicWriteJSON, hitchRootId, runCommand } from "../src/foundation/index.js";
|
|
8
|
+
import { parseCanaryWorkerConfig } from "./canary-worker-peer.js";
|
|
9
|
+
export function shellQuote(value) { return `'${value.replaceAll("'", `'"'"'`)}'`; }
|
|
10
|
+
export class RemoteCanaryWorker {
|
|
11
|
+
config;
|
|
12
|
+
localRoot;
|
|
13
|
+
localWorkerRoot;
|
|
14
|
+
id = `hitch-canary-${randomBytes(16).toString("hex")}`;
|
|
15
|
+
prepared = false;
|
|
16
|
+
constructor(config, localRoot, localWorkerRoot) {
|
|
17
|
+
this.config = config;
|
|
18
|
+
this.localRoot = localRoot;
|
|
19
|
+
this.localWorkerRoot = localWorkerRoot;
|
|
20
|
+
}
|
|
21
|
+
get workerRoot() { return path.posix.join(this.config.runs_directory, this.id, "worker"); }
|
|
22
|
+
sshOptions() {
|
|
23
|
+
return ["-F", this.config.ssh_config, "-o", "BatchMode=yes", "-o", "ConnectTimeout=10",
|
|
24
|
+
"-o", "ExitOnForwardFailure=yes", "-o", "ServerAliveInterval=10", "-o", "ServerAliveCountMax=3"];
|
|
25
|
+
}
|
|
26
|
+
launch(operation, details = {}, controllerPort) {
|
|
27
|
+
const forward = controllerPort === undefined ? [] : ["-R", `127.0.0.1:${this.config.remote_port}:127.0.0.1:${controllerPort}`];
|
|
28
|
+
const command = [this.config.node, path.posix.join(this.config.hitch, "dist/scripts/canary-worker-peer.js")].map(shellQuote).join(" ");
|
|
29
|
+
const child = spawn("ssh", [...this.sshOptions(), ...forward, this.config.ssh_host, command], { stdio: ["pipe", "pipe", "pipe"] });
|
|
30
|
+
child.stdin.on("error", () => { });
|
|
31
|
+
child.stdin.end(JSON.stringify({ operation, config: this.config, id: this.id, ...details }));
|
|
32
|
+
return child;
|
|
33
|
+
}
|
|
34
|
+
async request(operation, details = {}) {
|
|
35
|
+
const child = this.launch(operation, details);
|
|
36
|
+
let stdout = "", stderr = "", oversized = false, timedOut = false;
|
|
37
|
+
child.stdout.on("data", chunk => {
|
|
38
|
+
stdout += chunk.toString();
|
|
39
|
+
if (stdout.length > 1024 * 1024) {
|
|
40
|
+
oversized = true;
|
|
41
|
+
child.kill("SIGTERM");
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
child.stderr.on("data", chunk => { stderr = (stderr + chunk.toString()).slice(-32_768); });
|
|
45
|
+
const timer = setTimeout(() => { timedOut = true; child.kill("SIGTERM"); }, 60_000);
|
|
46
|
+
try {
|
|
47
|
+
const code = await new Promise((resolve, reject) => { child.once("error", reject); child.once("close", resolve); });
|
|
48
|
+
assert.equal(timedOut, false, "SSH worker observation timed out; remote state remains unconfirmed");
|
|
49
|
+
assert.equal(oversized, false, "SSH worker response exceeded the diagnostic bound");
|
|
50
|
+
assert.equal(code, 0, `SSH canary ${operation} failed: ${stderr}`);
|
|
51
|
+
return JSON.parse(stdout);
|
|
52
|
+
}
|
|
53
|
+
finally {
|
|
54
|
+
clearTimeout(timer);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
async observe(imageId, localEngine) {
|
|
58
|
+
await atomicWriteJSON(path.join(this.localRoot, "remote-worker-location.json"), {
|
|
59
|
+
diagnosticId: this.id, sshHost: this.config.ssh_host, workerRoot: this.workerRoot,
|
|
60
|
+
});
|
|
61
|
+
const observed = await this.request("observe", { imageId });
|
|
62
|
+
assert.equal(observed.imageId, imageId);
|
|
63
|
+
assert.equal(observed.harbor, "0.21.0");
|
|
64
|
+
assert.ok(typeof observed.engine === "string" && observed.engine.length > 0);
|
|
65
|
+
assert.notEqual(observed.engine, localEngine, "cross-host canary requires a different Docker engine");
|
|
66
|
+
assert.equal(typeof observed.bootId, "string");
|
|
67
|
+
assert.match(String(observed.bootId), /^[a-f0-9-]{36}$/);
|
|
68
|
+
assert.deepEqual(observed.runtime, await observeControllerRuntime(), "remote worker runtime differs from controller");
|
|
69
|
+
await atomicWriteJSON(path.join(this.localRoot, "remote-worker-observation.json"), observed);
|
|
70
|
+
return { engine: observed.engine, harbor: "0.21.0" };
|
|
71
|
+
}
|
|
72
|
+
async start(controllerPort, registrationFile, credentialFile) {
|
|
73
|
+
assert.ok(Number.isSafeInteger(controllerPort) && controllerPort > 0 && controllerPort <= 65535);
|
|
74
|
+
this.prepared = true; // An uncertain prepare must still be followed by a remote stop.
|
|
75
|
+
const prepared = await this.request("prepare", {
|
|
76
|
+
registration: JSON.parse(await readFile(registrationFile, "utf8")),
|
|
77
|
+
credential: JSON.parse(await readFile(credentialFile, "utf8")),
|
|
78
|
+
});
|
|
79
|
+
assert.equal(prepared.prepared, true, "remote worker did not confirm preparation");
|
|
80
|
+
return this.launch("run", {}, controllerPort);
|
|
81
|
+
}
|
|
82
|
+
async stop() {
|
|
83
|
+
if (!this.prepared)
|
|
84
|
+
return;
|
|
85
|
+
const result = await this.request("stop");
|
|
86
|
+
assert.equal(result.workerProcessTerminal, true);
|
|
87
|
+
await atomicWriteJSON(path.join(this.localRoot, "remote-worker-process-stop.json"), result);
|
|
88
|
+
}
|
|
89
|
+
async collect() {
|
|
90
|
+
if (!this.prepared)
|
|
91
|
+
return;
|
|
92
|
+
await mkdir(this.localWorkerRoot, { recursive: true, mode: 0o700 });
|
|
93
|
+
const sshCommand = ["ssh", ...this.sshOptions()].map(shellQuote).join(" ");
|
|
94
|
+
// Copy only this canary's worker state. Registration credentials are in a
|
|
95
|
+
// sibling private directory and never included. Do not follow symlinks.
|
|
96
|
+
await runCommand("rsync", ["-r", "--safe-links", "--checksum", "--partial", "--timeout=60", "-e", sshCommand,
|
|
97
|
+
`${this.config.ssh_host}:${shellQuote(`${this.workerRoot}/`)}`, `${this.localWorkerRoot}/`], { timeoutMs: 90_000 });
|
|
98
|
+
}
|
|
99
|
+
async cleanup(leaseIds) {
|
|
100
|
+
if (!this.prepared)
|
|
101
|
+
return { issues: [], retained: [] };
|
|
102
|
+
const result = await this.request("cleanup", { leaseIds });
|
|
103
|
+
assert.equal(result.root_id, hitchRootId(this.workerRoot));
|
|
104
|
+
assert.ok(Array.isArray(result.issues) && Array.isArray(result.retained));
|
|
105
|
+
await atomicWriteJSON(path.join(this.localRoot, "remote-worker-docker-cleanup.json"), result);
|
|
106
|
+
return { issues: result.issues, retained: result.retained };
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
export async function remoteCanaryWorker(localRoot, localWorkerRoot) {
|
|
110
|
+
const file = process.env.HITCH_CANARY_WORKER_SSH_CONFIG;
|
|
111
|
+
return file ? new RemoteCanaryWorker(parseCanaryWorkerConfig(JSON.parse(await readFile(file, "utf8"))), localRoot, localWorkerRoot) : undefined;
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=canary-worker-host.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"canary-worker-host.js","sourceRoot":"","sources":["../../scripts/canary-worker-host.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAqB,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,wBAAwB,EAAE,MAAM,oCAAoC,CAAC;AAC9E,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACtF,OAAO,EAAE,uBAAuB,EAA2B,MAAM,yBAAyB,CAAC;AAE3F,MAAM,UAAU,UAAU,CAAC,KAAa,IAAY,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;AAEnG,MAAM,OAAO,kBAAkB;IAGR;IAAqC;IAA4B;IAF7E,EAAE,GAAG,gBAAgB,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;IACxD,QAAQ,GAAG,KAAK,CAAC;IACzB,YAAqB,MAA0B,EAAW,SAAiB,EAAW,eAAuB;QAAxF,WAAM,GAAN,MAAM,CAAoB;QAAW,cAAS,GAAT,SAAS,CAAQ;QAAW,oBAAe,GAAf,eAAe,CAAQ;IAAG,CAAC;IAEjH,IAAI,UAAU,KAAa,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC3F,UAAU;QAChB,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,mBAAmB;YACpF,IAAI,EAAE,0BAA0B,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,uBAAuB,CAAC,CAAC;IACrG,CAAC;IACO,MAAM,CAAC,SAAiB,EAAE,UAAmC,EAAE,EAAE,cAAuB;QAC9F,MAAM,OAAO,GAAG,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,aAAa,IAAI,CAAC,MAAM,CAAC,WAAW,cAAc,cAAc,EAAE,CAAC,CAAC;QAC/H,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,oCAAoC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvI,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACnI,KAAK,CAAC,KAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACnC,KAAK,CAAC,KAAM,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9F,OAAO,KAAK,CAAC;IACf,CAAC;IACO,KAAK,CAAC,OAAO,CAAC,SAAiB,EAAE,UAAmC,EAAE;QAC5E,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,MAAM,GAAG,EAAE,EAAE,MAAM,GAAG,EAAE,EAAE,SAAS,GAAG,KAAK,EAAE,QAAQ,GAAG,KAAK,CAAC;QAClE,KAAK,CAAC,MAAO,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC3B,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;gBAAC,SAAS,GAAG,IAAI,CAAC;gBAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,MAAO,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,GAAG,MAAM,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5F,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACpF,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnI,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,oEAAoE,CAAC,CAAC;YACpG,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,mDAAmD,CAAC,CAAC;YACpF,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,cAAc,SAAS,YAAY,MAAM,EAAE,CAAC,CAAC;YACnE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;gBAAS,CAAC;YAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAe,EAAE,WAAmB;QAChD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,6BAA6B,CAAC,EAAE;YAC9E,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU;SAClF,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAC5D,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACxC,MAAM,CAAC,EAAE,CAAC,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,sDAAsD,CAAC,CAAC;QACtG,MAAM,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,iBAAiB,CAAC,CAAC;QACzD,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,wBAAwB,EAAE,EAAE,+CAA+C,CAAC,CAAC;QACtH,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gCAAgC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC7F,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,cAAsB,EAAE,gBAAwB,EAAE,cAAsB;QAClF,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,IAAI,cAAc,IAAI,KAAK,CAAC,CAAC;QACjG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,gEAAgE;QACtF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC7C,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;YAClE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;SAC/D,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,2CAA2C,CAAC,CAAC;QACnF,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iCAAiC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC9F,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC3B,MAAM,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACpE,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3E,0EAA0E;QAC1E,wEAAwE;QACxE,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,IAAI,EAAE,UAAU;YAC1G,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,eAAe,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IACxH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,QAAkB;QAC9B,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QACxD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC3D,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAC3D,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1E,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mCAAmC,CAAC,EAAE,MAAM,CAAC,CAAC;QAC9F,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC9D,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,SAAiB,EAAE,eAAuB;IACjF,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;IACxD,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,kBAAkB,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAClJ,CAAC"}
|