agent-hitch 0.2.9 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +206 -137
- package/README.zh-CN.md +193 -119
- package/dist/scripts/canary-native-training.js +221 -0
- package/dist/scripts/canary-native-training.js.map +1 -0
- package/dist/scripts/canary-remote-training.js +246 -0
- package/dist/scripts/canary-remote-training.js.map +1 -0
- package/dist/scripts/canary-remote-verifier.js +124 -0
- package/dist/scripts/canary-remote-verifier.js.map +1 -0
- package/dist/scripts/canary-worker-host.js +113 -0
- package/dist/scripts/canary-worker-host.js.map +1 -0
- package/dist/scripts/canary-worker-peer.js +188 -0
- package/dist/scripts/canary-worker-peer.js.map +1 -0
- package/dist/scripts/check-architecture.js +4 -3
- package/dist/scripts/check-architecture.js.map +1 -1
- package/dist/src/adapters/catalog.js +2 -1
- package/dist/src/adapters/catalog.js.map +1 -1
- package/dist/src/adapters/providers/codex.js +65 -2
- package/dist/src/adapters/providers/codex.js.map +1 -1
- package/dist/src/adapters/providers/model-call.js +17 -1
- package/dist/src/adapters/providers/model-call.js.map +1 -1
- package/dist/src/adapters/providers/training-tool.js +29 -0
- package/dist/src/adapters/providers/training-tool.js.map +1 -0
- package/dist/src/artifacts/index.js +1 -1
- package/dist/src/artifacts/index.js.map +1 -1
- package/dist/src/artifacts/store.js +15 -10
- package/dist/src/artifacts/store.js.map +1 -1
- package/dist/src/backends/harbor/backend.js +24 -15
- package/dist/src/backends/harbor/backend.js.map +1 -1
- package/dist/src/backends/harbor/bridge-environment.js +12 -0
- package/dist/src/backends/harbor/bridge-environment.js.map +1 -0
- package/dist/src/backends/harbor/environment-config.js +2 -2
- package/dist/src/backends/harbor/environment-config.js.map +1 -1
- package/dist/src/backends/harbor/host-credential-helper.js +81 -0
- package/dist/src/backends/harbor/host-credential-helper.js.map +1 -0
- package/dist/src/backends/harbor/index.js +3 -0
- package/dist/src/backends/harbor/index.js.map +1 -1
- package/dist/src/backends/harbor/model-proxy-config.js +18 -1
- package/dist/src/backends/harbor/model-proxy-config.js.map +1 -1
- package/dist/src/backends/harbor/observation.js +61 -0
- package/dist/src/backends/harbor/observation.js.map +1 -0
- package/dist/src/backends/harbor/portable-regrade.js +150 -0
- package/dist/src/backends/harbor/portable-regrade.js.map +1 -0
- package/dist/src/backends/harbor/process.js +26 -5
- package/dist/src/backends/harbor/process.js.map +1 -1
- package/dist/src/backends/harbor/regrade.js +4 -1
- package/dist/src/backends/harbor/regrade.js.map +1 -1
- package/dist/src/backends/harbor/supervisor.js +22 -0
- package/dist/src/backends/harbor/supervisor.js.map +1 -1
- package/dist/src/backends/harbor/tools.js +2 -0
- package/dist/src/backends/harbor/tools.js.map +1 -1
- package/dist/src/backends/index.js +3 -0
- package/dist/src/backends/index.js.map +1 -1
- package/dist/src/cli/arguments.js +33 -0
- package/dist/src/cli/arguments.js.map +1 -1
- package/dist/src/cli/commands/capabilities.js +14 -1
- package/dist/src/cli/commands/capabilities.js.map +1 -1
- package/dist/src/cli/commands/daemon.js +19 -0
- package/dist/src/cli/commands/daemon.js.map +1 -1
- package/dist/src/cli/commands/eval.js +38 -10
- package/dist/src/cli/commands/eval.js.map +1 -1
- package/dist/src/cli/commands/local.js +130 -0
- package/dist/src/cli/commands/local.js.map +1 -0
- package/dist/src/cli/commands/model-node.js +26 -0
- package/dist/src/cli/commands/model-node.js.map +1 -0
- package/dist/src/cli/commands/models.js +72 -0
- package/dist/src/cli/commands/models.js.map +1 -0
- package/dist/src/cli/commands/run.js +36 -1
- package/dist/src/cli/commands/run.js.map +1 -1
- package/dist/src/cli/commands/training.js +32 -0
- package/dist/src/cli/commands/training.js.map +1 -0
- package/dist/src/cli/commands/worker.js +30 -2
- package/dist/src/cli/commands/worker.js.map +1 -1
- package/dist/src/cli/main.js +8 -0
- package/dist/src/cli/main.js.map +1 -1
- package/dist/src/cli/output.js +24 -2
- package/dist/src/cli/output.js.map +1 -1
- package/dist/src/control-plane/eval-records.js +2 -0
- package/dist/src/control-plane/eval-records.js.map +1 -1
- package/dist/src/control-plane/eval-recovery.js +7 -1
- package/dist/src/control-plane/eval-recovery.js.map +1 -1
- package/dist/src/control-plane/eval-scheduler.js +24 -50
- package/dist/src/control-plane/eval-scheduler.js.map +1 -1
- package/dist/src/control-plane/eval-submission-persist.js +30 -0
- package/dist/src/control-plane/eval-submission-persist.js.map +1 -0
- package/dist/src/control-plane/execution-observation-contract.js +60 -0
- package/dist/src/control-plane/execution-observation-contract.js.map +1 -0
- package/dist/src/control-plane/index.js +10 -0
- package/dist/src/control-plane/index.js.map +1 -1
- package/dist/src/control-plane/inference-manager.js +357 -0
- package/dist/src/control-plane/inference-manager.js.map +1 -0
- package/dist/src/control-plane/managed-gateway-receipt.js +26 -0
- package/dist/src/control-plane/managed-gateway-receipt.js.map +1 -0
- package/dist/src/control-plane/managed-service-recovery.js +158 -0
- package/dist/src/control-plane/managed-service-recovery.js.map +1 -0
- package/dist/src/control-plane/model-capture-planning.js +15 -1
- package/dist/src/control-plane/model-capture-planning.js.map +1 -1
- package/dist/src/control-plane/ordered-eval-control.js +221 -0
- package/dist/src/control-plane/ordered-eval-control.js.map +1 -0
- package/dist/src/control-plane/remote-backend-result.js +18 -0
- package/dist/src/control-plane/remote-backend-result.js.map +1 -0
- package/dist/src/control-plane/remote-execution-lease.js +15 -0
- package/dist/src/control-plane/remote-execution-lease.js.map +1 -0
- package/dist/src/control-plane/remote-execution-observation.js +108 -0
- package/dist/src/control-plane/remote-execution-observation.js.map +1 -0
- package/dist/src/control-plane/remote-model-routes.js +107 -0
- package/dist/src/control-plane/remote-model-routes.js.map +1 -0
- package/dist/src/control-plane/remote-observation-reporter.js +37 -0
- package/dist/src/control-plane/remote-observation-reporter.js.map +1 -0
- package/dist/src/control-plane/remote-offer-contract.js +48 -0
- package/dist/src/control-plane/remote-offer-contract.js.map +1 -0
- package/dist/src/control-plane/remote-rerun-journal.js +61 -0
- package/dist/src/control-plane/remote-rerun-journal.js.map +1 -0
- package/dist/src/control-plane/remote-result-transport.js +55 -9
- package/dist/src/control-plane/remote-result-transport.js.map +1 -1
- package/dist/src/control-plane/remote-verifier-import.js +52 -0
- package/dist/src/control-plane/remote-verifier-import.js.map +1 -0
- package/dist/src/control-plane/remote-verifier-result.js +61 -0
- package/dist/src/control-plane/remote-verifier-result.js.map +1 -0
- package/dist/src/control-plane/remote-verifier-source-transport.js +68 -0
- package/dist/src/control-plane/remote-verifier-source-transport.js.map +1 -0
- package/dist/src/control-plane/remote-work-coordinator.js +103 -47
- package/dist/src/control-plane/remote-work-coordinator.js.map +1 -1
- package/dist/src/control-plane/remote-work-inputs.js +26 -5
- package/dist/src/control-plane/remote-work-inputs.js.map +1 -1
- package/dist/src/control-plane/remote-work-recovery.js +123 -34
- package/dist/src/control-plane/remote-work-recovery.js.map +1 -1
- package/dist/src/control-plane/remote-worker-artifacts.js +10 -7
- package/dist/src/control-plane/remote-worker-artifacts.js.map +1 -1
- package/dist/src/control-plane/remote-worker-cleanup.js +104 -0
- package/dist/src/control-plane/remote-worker-cleanup.js.map +1 -0
- package/dist/src/control-plane/remote-worker-client.js +156 -14
- package/dist/src/control-plane/remote-worker-client.js.map +1 -1
- package/dist/src/control-plane/remote-worker-codecs.js +160 -0
- package/dist/src/control-plane/remote-worker-codecs.js.map +1 -0
- package/dist/src/control-plane/remote-worker-generation-recovery.js +67 -0
- package/dist/src/control-plane/remote-worker-generation-recovery.js.map +1 -0
- package/dist/src/control-plane/remote-worker-ownership.js +51 -0
- package/dist/src/control-plane/remote-worker-ownership.js.map +1 -0
- package/dist/src/control-plane/remote-worker-protocol.js +85 -119
- package/dist/src/control-plane/remote-worker-protocol.js.map +1 -1
- package/dist/src/control-plane/remote-worker-runner.js +95 -18
- package/dist/src/control-plane/remote-worker-runner.js.map +1 -1
- package/dist/src/control-plane/remote-workers.js +33 -5
- package/dist/src/control-plane/remote-workers.js.map +1 -1
- package/dist/src/control-plane/rerun-recovery.js +89 -0
- package/dist/src/control-plane/rerun-recovery.js.map +1 -0
- package/dist/src/control-plane/rerun-scheduler.js +70 -96
- package/dist/src/control-plane/rerun-scheduler.js.map +1 -1
- package/dist/src/control-plane/rerun-source.js +57 -0
- package/dist/src/control-plane/rerun-source.js.map +1 -0
- package/dist/src/control-plane/worker-model-relay.js +76 -0
- package/dist/src/control-plane/worker-model-relay.js.map +1 -0
- package/dist/src/controller-runtime/hash.js +3 -0
- package/dist/src/controller-runtime/hash.js.map +1 -1
- package/dist/src/controller-runtime/index.js +2 -0
- package/dist/src/controller-runtime/index.js.map +1 -1
- package/dist/src/controller-runtime/observation.js +29 -0
- package/dist/src/controller-runtime/observation.js.map +1 -0
- package/dist/src/controller-runtime/resident.js +22 -0
- package/dist/src/controller-runtime/resident.js.map +1 -0
- package/dist/src/daemon/client.js +34 -1
- package/dist/src/daemon/client.js.map +1 -1
- package/dist/src/daemon/execution-observation.js +38 -0
- package/dist/src/daemon/execution-observation.js.map +1 -0
- package/dist/src/daemon/http.js +57 -0
- package/dist/src/daemon/http.js.map +1 -0
- package/dist/src/daemon/inference-prepare.js +47 -0
- package/dist/src/daemon/inference-prepare.js.map +1 -0
- package/dist/src/daemon/scheduler.js +4 -1
- package/dist/src/daemon/scheduler.js.map +1 -1
- package/dist/src/daemon/server.js +80 -82
- package/dist/src/daemon/server.js.map +1 -1
- package/dist/src/daemon/worker-observation.js +34 -0
- package/dist/src/daemon/worker-observation.js.map +1 -0
- package/dist/src/daemon/worker-ownership.js +71 -0
- package/dist/src/daemon/worker-ownership.js.map +1 -0
- package/dist/src/daemon/worker-routes.js +83 -15
- package/dist/src/daemon/worker-routes.js.map +1 -1
- package/dist/src/domain/execution-observation.js +2 -0
- package/dist/src/domain/execution-observation.js.map +1 -0
- package/dist/src/domain/index.js +1 -0
- package/dist/src/domain/index.js.map +1 -1
- package/dist/src/domain/inference.js +2 -0
- package/dist/src/domain/inference.js.map +1 -0
- package/dist/src/domain/runs.js.map +1 -1
- package/dist/src/domain/training.js +2 -0
- package/dist/src/domain/training.js.map +1 -0
- package/dist/src/domain/validation.js +11 -0
- package/dist/src/domain/validation.js.map +1 -1
- package/dist/src/domain/verifier-source.js +2 -0
- package/dist/src/domain/verifier-source.js.map +1 -0
- package/dist/src/domain/worker-recovery.js +2 -0
- package/dist/src/domain/worker-recovery.js.map +1 -0
- package/dist/src/evals/eval-logical-plan.js +8 -1
- package/dist/src/evals/eval-logical-plan.js.map +1 -1
- package/dist/src/evals/execution-leases.js +15 -2
- package/dist/src/evals/execution-leases.js.map +1 -1
- package/dist/src/evals/execution-plan.js +11 -1
- package/dist/src/evals/execution-plan.js.map +1 -1
- package/dist/src/evals/harbor-bridge-error.js +8 -0
- package/dist/src/evals/harbor-bridge-error.js.map +1 -1
- package/dist/src/evals/index.js +19 -0
- package/dist/src/evals/index.js.map +1 -1
- package/dist/src/evals/model-capture-plan.js +12 -0
- package/dist/src/evals/model-capture-plan.js.map +1 -1
- package/dist/src/evals/model-capture-runtime.js +25 -0
- package/dist/src/evals/model-capture-runtime.js.map +1 -1
- package/dist/src/evals/physical-work-plan.js +57 -0
- package/dist/src/evals/physical-work-plan.js.map +1 -0
- package/dist/src/evals/preparation-rerun.js +6 -0
- package/dist/src/evals/preparation-rerun.js.map +1 -1
- package/dist/src/evals/recovery.js +8 -1
- package/dist/src/evals/recovery.js.map +1 -1
- package/dist/src/evals/regrade-evidence.js +4 -4
- package/dist/src/evals/regrade-evidence.js.map +1 -1
- package/dist/src/evals/remote-execution-ownership.js +50 -0
- package/dist/src/evals/remote-execution-ownership.js.map +1 -0
- package/dist/src/evals/remote-infrastructure-retry.js +2 -0
- package/dist/src/evals/remote-infrastructure-retry.js.map +1 -1
- package/dist/src/evals/remote-lease-release.js +58 -0
- package/dist/src/evals/remote-lease-release.js.map +1 -0
- package/dist/src/evals/remote-rerun-execution.js +85 -0
- package/dist/src/evals/remote-rerun-execution.js.map +1 -0
- package/dist/src/evals/remote-trial.js +6 -0
- package/dist/src/evals/remote-trial.js.map +1 -0
- package/dist/src/evals/remote-verifier-assessment.js +85 -0
- package/dist/src/evals/remote-verifier-assessment.js.map +1 -0
- package/dist/src/evals/remote-verifier-contract.js +43 -0
- package/dist/src/evals/remote-verifier-contract.js.map +1 -0
- package/dist/src/evals/remote-verifier-execution.js +105 -0
- package/dist/src/evals/remote-verifier-execution.js.map +1 -0
- package/dist/src/evals/remote-verifier-journal.js +21 -0
- package/dist/src/evals/remote-verifier-journal.js.map +1 -0
- package/dist/src/evals/remote-verifier-outcome.js +51 -0
- package/dist/src/evals/remote-verifier-outcome.js.map +1 -0
- package/dist/src/evals/remote-verifier-rerun.js +134 -0
- package/dist/src/evals/remote-verifier-rerun.js.map +1 -0
- package/dist/src/evals/remote-verifier-selection.js +61 -0
- package/dist/src/evals/remote-verifier-selection.js.map +1 -0
- package/dist/src/evals/remote-verifier-source.js +33 -0
- package/dist/src/evals/remote-verifier-source.js.map +1 -0
- package/dist/src/evals/remote-worker-cleanup.js +46 -0
- package/dist/src/evals/remote-worker-cleanup.js.map +1 -0
- package/dist/src/evals/request.js +58 -1
- package/dist/src/evals/request.js.map +1 -1
- package/dist/src/evals/rerun-completion.js +39 -0
- package/dist/src/evals/rerun-completion.js.map +1 -0
- package/dist/src/evals/rerun-resume.js +70 -0
- package/dist/src/evals/rerun-resume.js.map +1 -0
- package/dist/src/evals/rerun-state.js +25 -0
- package/dist/src/evals/rerun-state.js.map +1 -0
- package/dist/src/evals/rerun-types.js.map +1 -1
- package/dist/src/evals/rerun.js +90 -46
- package/dist/src/evals/rerun.js.map +1 -1
- package/dist/src/evals/service.js +59 -5
- package/dist/src/evals/service.js.map +1 -1
- package/dist/src/evals/verifier-only-rerun.js +2 -16
- package/dist/src/evals/verifier-only-rerun.js.map +1 -1
- package/dist/src/evals/verifier-rerun-result.js +24 -0
- package/dist/src/evals/verifier-rerun-result.js.map +1 -0
- package/dist/src/evals/verifier-source-import.js +48 -0
- package/dist/src/evals/verifier-source-import.js.map +1 -0
- package/dist/src/evals/verifier-source.js +139 -0
- package/dist/src/evals/verifier-source.js.map +1 -0
- package/dist/src/foundation/base64.js +19 -0
- package/dist/src/foundation/base64.js.map +1 -0
- package/dist/src/foundation/config.js +10 -0
- package/dist/src/foundation/config.js.map +1 -1
- package/dist/src/foundation/credential-redaction.js +22 -9
- package/dist/src/foundation/credential-redaction.js.map +1 -1
- package/dist/src/foundation/index.js +1 -0
- package/dist/src/foundation/index.js.map +1 -1
- package/dist/src/foundation/process-identity.js +13 -9
- package/dist/src/foundation/process-identity.js.map +1 -1
- package/dist/src/inference/daemon-capacity.js +41 -0
- package/dist/src/inference/daemon-capacity.js.map +1 -0
- package/dist/src/inference/device-reservation.js +58 -0
- package/dist/src/inference/device-reservation.js.map +1 -0
- package/dist/src/inference/doctor.js +95 -0
- package/dist/src/inference/doctor.js.map +1 -0
- package/dist/src/inference/generation-recovery.js +83 -0
- package/dist/src/inference/generation-recovery.js.map +1 -0
- package/dist/src/inference/index.js +24 -0
- package/dist/src/inference/index.js.map +1 -0
- package/dist/src/inference/lock.js +331 -0
- package/dist/src/inference/lock.js.map +1 -0
- package/dist/src/inference/managed-launcher.js +35 -0
- package/dist/src/inference/managed-launcher.js.map +1 -0
- package/dist/src/inference/manifest.js +262 -0
- package/dist/src/inference/manifest.js.map +1 -0
- package/dist/src/inference/materialize.js +66 -0
- package/dist/src/inference/materialize.js.map +1 -0
- package/dist/src/inference/model-location.js +34 -0
- package/dist/src/inference/model-location.js.map +1 -0
- package/dist/src/inference/model-store.js +439 -0
- package/dist/src/inference/model-store.js.map +1 -0
- package/dist/src/inference/node-client.js +167 -0
- package/dist/src/inference/node-client.js.map +1 -0
- package/dist/src/inference/node-registry.js +79 -0
- package/dist/src/inference/node-registry.js.map +1 -0
- package/dist/src/inference/observation.js +79 -0
- package/dist/src/inference/observation.js.map +1 -0
- package/dist/src/inference/preflight.js +132 -0
- package/dist/src/inference/preflight.js.map +1 -0
- package/dist/src/inference/process-sglang.js +169 -0
- package/dist/src/inference/process-sglang.js.map +1 -0
- package/dist/src/inference/protocol-probe.js +29 -0
- package/dist/src/inference/protocol-probe.js.map +1 -0
- package/dist/src/inference/remote-model-store.js +24 -0
- package/dist/src/inference/remote-model-store.js.map +1 -0
- package/dist/src/inference/runtime-catalog.js +41 -0
- package/dist/src/inference/runtime-catalog.js.map +1 -0
- package/dist/src/inference/runtime-store.js +93 -0
- package/dist/src/inference/runtime-store.js.map +1 -0
- package/dist/src/inference/service-attachment.js +46 -0
- package/dist/src/inference/service-attachment.js.map +1 -0
- package/dist/src/inference/service-inspection.js +38 -0
- package/dist/src/inference/service-inspection.js.map +1 -0
- package/dist/src/inference/sglang.js +279 -0
- package/dist/src/inference/sglang.js.map +1 -0
- package/dist/src/inference/supervisor.js +460 -0
- package/dist/src/inference/supervisor.js.map +1 -0
- package/dist/src/model-access/environment.js +28 -0
- package/dist/src/model-access/environment.js.map +1 -0
- package/dist/src/model-access/index.js +4 -0
- package/dist/src/model-access/index.js.map +1 -1
- package/dist/src/model-access/local-gateway-request.js +139 -0
- package/dist/src/model-access/local-gateway-request.js.map +1 -0
- package/dist/src/model-access/local-gateway.js +323 -0
- package/dist/src/model-access/local-gateway.js.map +1 -0
- package/dist/src/model-access/proxy.js +90 -5
- package/dist/src/model-access/proxy.js.map +1 -1
- package/dist/src/model-access/remote-model.js +58 -0
- package/dist/src/model-access/remote-model.js.map +1 -0
- package/dist/src/model-access/training.js +115 -0
- package/dist/src/model-access/training.js.map +1 -0
- package/dist/src/runs/adapter-process.js +2 -2
- package/dist/src/runs/adapter-process.js.map +1 -1
- package/dist/src/runs/executor-types.js +2 -0
- package/dist/src/runs/executor-types.js.map +1 -0
- package/dist/src/runs/executor.js +54 -34
- package/dist/src/runs/executor.js.map +1 -1
- package/dist/src/runs/index.js +1 -0
- package/dist/src/runs/index.js.map +1 -1
- package/dist/src/runs/local-inference-environment.js +59 -0
- package/dist/src/runs/local-inference-environment.js.map +1 -0
- package/dist/src/runs/local-inference-run.js +53 -0
- package/dist/src/runs/local-inference-run.js.map +1 -0
- package/dist/src/runs/outcome.js +22 -0
- package/dist/src/runs/outcome.js.map +1 -1
- package/dist/src/runs/records.js +9 -0
- package/dist/src/runs/records.js.map +1 -1
- package/dist/src/runs/request.js +58 -3
- package/dist/src/runs/request.js.map +1 -1
- package/dist/src/runs/training-evidence.js +22 -0
- package/dist/src/runs/training-evidence.js.map +1 -0
- package/dist/src/runs/training-runtime.js +20 -0
- package/dist/src/runs/training-runtime.js.map +1 -0
- package/dist/src/workers/execution-observation.js +32 -0
- package/dist/src/workers/execution-observation.js.map +1 -0
- package/dist/src/workers/index.js +2 -0
- package/dist/src/workers/index.js.map +1 -1
- package/dist/src/workers/remote-harbor-ownership.js +266 -0
- package/dist/src/workers/remote-harbor-ownership.js.map +1 -0
- package/dist/src/workers/remote-harbor-verifier.js +76 -0
- package/dist/src/workers/remote-harbor-verifier.js.map +1 -0
- package/dist/src/workers/remote-harbor-work-spec.js +66 -10
- package/dist/src/workers/remote-harbor-work-spec.js.map +1 -1
- package/dist/src/workers/remote-harbor-worker.js +56 -49
- package/dist/src/workers/remote-harbor-worker.js.map +1 -1
- package/dist/src/workers/remote-worker-host.js +33 -0
- package/dist/src/workers/remote-worker-host.js.map +1 -0
- package/docs/local-model-inference-spec.zh-CN.md +666 -0
- package/docs/schemas/controller-runtime-observation.schema.json +21 -0
- package/docs/schemas/eval-execution-plan.schema.json +1 -0
- package/docs/schemas/eval-request.schema.json +3 -1
- package/docs/schemas/execution-lease.schema.json +95 -1
- package/docs/schemas/execution-provider-status.schema.json +162 -28
- package/docs/schemas/inference-execution-evidence.schema.json +487 -0
- package/docs/schemas/inference-lock.schema.json +487 -0
- package/docs/schemas/inference-runtime-manifest.schema.json +156 -0
- package/docs/schemas/inference-service-record.schema.json +227 -0
- package/docs/schemas/local-inference-selection.schema.json +98 -0
- package/docs/schemas/local-model-manifest.schema.json +43 -0
- package/docs/schemas/model-node-binding.schema.json +34 -0
- package/docs/schemas/model-node-generation-release.schema.json +46 -0
- package/docs/schemas/ordered-eval-control.schema.json +39 -0
- package/docs/schemas/remote-execution-lease-grant.schema.json +45 -0
- package/docs/schemas/remote-harbor-work-spec.schema.json +325 -0
- package/docs/schemas/remote-model-binding.schema.json +21 -0
- package/docs/schemas/remote-physical-execution.schema.json +108 -0
- package/docs/schemas/remote-result-envelope.schema.json +9 -3
- package/docs/schemas/remote-tree-envelope.schema.json +17 -3
- package/docs/schemas/remote-verifier-result.schema.json +203 -0
- package/docs/schemas/remote-verifier-source.schema.json +52 -0
- package/docs/schemas/remote-verifier-work.schema.json +125 -0
- package/docs/schemas/remote-work-offer.schema.json +170 -34
- package/docs/schemas/remote-worker-execution-admission.schema.json +214 -0
- package/docs/schemas/remote-worker-generation-cleanup.schema.json +199 -0
- package/docs/schemas/remote-worker-registration.schema.json +226 -25
- package/docs/schemas/run-manifest.schema.json +211 -44
- package/docs/schemas/run-request.schema.json +2 -1
- package/docs/schemas/training-external-binding.schema.json +26 -0
- package/integrations/harbor/hitch_harbor_agent.py +241 -13
- package/integrations/harbor/hitch_harbor_environment.py +22 -1
- package/integrations/harbor/hitch_host_credentials.py +354 -0
- package/integrations/harbor/hitch_private_exec.py +150 -0
- package/integrations/model-call/cli.js +13 -5
- package/integrations/training-tool/cli.js +72 -0
- package/package.json +8 -2
|
@@ -0,0 +1,439 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { createReadStream } from "node:fs";
|
|
3
|
+
import { chmod, copyFile, lstat, open, readdir, readFile, rename, rm, stat } from "node:fs/promises";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { HitchError, atomicWriteJSON, ensureDir, invalidInput, readJSON, sha256Bytes, sha256JSON, statePaths, withFileLock, } from "../foundation/index.js";
|
|
6
|
+
import { localModelIdentity, parseLocalModelManifest } from "./manifest.js";
|
|
7
|
+
const ALIAS = /^[a-z0-9][a-z0-9._-]{0,63}$/;
|
|
8
|
+
const FORBIDDEN = /(?:^|\/)(?:[^/]+\.(?:bin|pt|pth|pkl|pickle|py|so|dylib|dll))$/i;
|
|
9
|
+
const MAX_FILES = 100_000;
|
|
10
|
+
export async function addLocalModel(options) {
|
|
11
|
+
return withFileLock(statePaths(options.root).inferenceOperationLocks, "model-store", () => addLocalModelLocked(options));
|
|
12
|
+
}
|
|
13
|
+
/** Persist a manifest only after the caller has verified its content on an explicit model node. */
|
|
14
|
+
export async function addModelManifest(options) {
|
|
15
|
+
if (!ALIAS.test(options.name))
|
|
16
|
+
throw invalidInput("model name must use lowercase letters, digits, '.', '_' or '-'");
|
|
17
|
+
const manifest = parseLocalModelManifest(options.manifest), paths = statePaths(options.root);
|
|
18
|
+
return withFileLock(paths.inferenceOperationLocks, "model-store", async () => {
|
|
19
|
+
const directory = path.join(paths.models, manifest.model_id.slice(7));
|
|
20
|
+
let existing;
|
|
21
|
+
try {
|
|
22
|
+
existing = parseLocalModelManifest(await readJSON(path.join(directory, "manifest.json")));
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
if (error.code !== "ENOENT")
|
|
26
|
+
throw error;
|
|
27
|
+
}
|
|
28
|
+
if (existing && existing.model_id !== manifest.model_id)
|
|
29
|
+
throw integrityError("stored model identity differs");
|
|
30
|
+
await ensureDir(directory);
|
|
31
|
+
await atomicWriteJSON(path.join(directory, "manifest.json"), existing ?? manifest);
|
|
32
|
+
await writeAlias(paths.modelAliases, paths.inferenceOperationLocks, options.name, manifest.model_id, options.force === true);
|
|
33
|
+
return existing ?? manifest;
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
async function addLocalModelLocked(options) {
|
|
37
|
+
if (!ALIAS.test(options.name))
|
|
38
|
+
throw invalidInput("model name must use lowercase letters, digits, '.', '_' or '-'");
|
|
39
|
+
const source = path.resolve(options.directory);
|
|
40
|
+
const paths = statePaths(options.root);
|
|
41
|
+
const rootRelative = path.relative(paths.root, source);
|
|
42
|
+
if (rootRelative === "" || (!rootRelative.startsWith("..") && !path.isAbsolute(rootRelative))) {
|
|
43
|
+
throw invalidInput("model source directory must be outside HITCH_ROOT");
|
|
44
|
+
}
|
|
45
|
+
let sourceStat;
|
|
46
|
+
try {
|
|
47
|
+
sourceStat = await stat(source);
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
throw new HitchError(`local model directory does not exist: ${source}`, { code: "local_model_not_found", exitCode: 2, cause: error });
|
|
51
|
+
}
|
|
52
|
+
if (!sourceStat.isDirectory())
|
|
53
|
+
throw invalidInput("local model source must be a directory");
|
|
54
|
+
const relativeFiles = await walkModelFiles(source);
|
|
55
|
+
validateRequiredFiles(relativeFiles);
|
|
56
|
+
const config = await readModelJSON(path.join(source, "config.json"), "config.json");
|
|
57
|
+
rejectDynamicModelCode(config);
|
|
58
|
+
validateSafetensorsIndex(source, relativeFiles, await optionalModelJSON(path.join(source, "model.safetensors.index.json")));
|
|
59
|
+
for (const relative of relativeFiles.filter((file) => file.endsWith(".safetensors"))) {
|
|
60
|
+
await validateSafetensorsFile(path.join(source, ...relative.split("/")), relative);
|
|
61
|
+
}
|
|
62
|
+
await ensureDir(paths.modelFiles);
|
|
63
|
+
const files = [];
|
|
64
|
+
for (const relative of relativeFiles)
|
|
65
|
+
files.push(await importFile(paths.modelFiles, source, relative));
|
|
66
|
+
const tokenizerFiles = files.filter((file) => /(?:^|\/)(?:tokenizer|special_tokens_map|added_tokens|tokenizer_config)(?:\.|$)/.test(file.path));
|
|
67
|
+
const template = await discoverTemplate(source, relativeFiles);
|
|
68
|
+
const quantization = modelQuantization(config);
|
|
69
|
+
const identityInput = {
|
|
70
|
+
schema_version: "1",
|
|
71
|
+
format: "hf-safetensors",
|
|
72
|
+
files,
|
|
73
|
+
architecture: firstString(config.architectures) || "unknown",
|
|
74
|
+
model_type: stringValue(config.model_type) || "unknown",
|
|
75
|
+
dtype: stringValue(config.torch_dtype) || stringValue(config.dtype) || "unknown",
|
|
76
|
+
quantization,
|
|
77
|
+
context_tokens: positiveIntegerValue(config.max_position_embeddings),
|
|
78
|
+
tokenizer_digest: sha256JSON(tokenizerFiles.map(({ path: filePath, sha256 }) => ({ path: filePath, sha256 }))),
|
|
79
|
+
template_digest: template === null ? null : sha256Bytes(template),
|
|
80
|
+
};
|
|
81
|
+
const manifest = {
|
|
82
|
+
...identityInput,
|
|
83
|
+
model_id: localModelIdentity(identityInput),
|
|
84
|
+
source: {
|
|
85
|
+
kind: "local-directory",
|
|
86
|
+
label: path.basename(source),
|
|
87
|
+
license: stringValue(config.license),
|
|
88
|
+
},
|
|
89
|
+
created_at: new Date().toISOString(),
|
|
90
|
+
};
|
|
91
|
+
const modelDirectory = path.join(paths.models, manifest.model_id.slice("sha256:".length));
|
|
92
|
+
await ensureDir(modelDirectory);
|
|
93
|
+
await atomicWriteJSON(path.join(modelDirectory, "manifest.json"), manifest);
|
|
94
|
+
await writeAlias(paths.modelAliases, paths.inferenceOperationLocks, options.name, manifest.model_id, options.force === true);
|
|
95
|
+
return manifest;
|
|
96
|
+
}
|
|
97
|
+
export async function gcLocalModels(root, apply = false) {
|
|
98
|
+
const paths = statePaths(root);
|
|
99
|
+
return withFileLock(paths.inferenceOperationLocks, "model-store", async () => {
|
|
100
|
+
const aliases = await loadAliasIndex(paths.modelAliases);
|
|
101
|
+
const referenced = new Set(Object.values(aliases.aliases));
|
|
102
|
+
await collectLockedModelIds(paths.inferenceLocks, referenced);
|
|
103
|
+
const manifests = await listStoredModels(paths.models);
|
|
104
|
+
const removed = manifests.filter((manifest) => !referenced.has(manifest.model_id));
|
|
105
|
+
const keptFiles = new Set(manifests.filter((manifest) => referenced.has(manifest.model_id))
|
|
106
|
+
.flatMap((manifest) => manifest.files.map((file) => file.sha256)));
|
|
107
|
+
const files = new Map();
|
|
108
|
+
for (const manifest of removed)
|
|
109
|
+
for (const file of manifest.files) {
|
|
110
|
+
if (!keptFiles.has(file.sha256))
|
|
111
|
+
files.set(file.sha256, file.size);
|
|
112
|
+
}
|
|
113
|
+
if (apply) {
|
|
114
|
+
for (const manifest of removed) {
|
|
115
|
+
await rm(path.join(paths.models, manifest.model_id.slice("sha256:".length)), { recursive: true, force: true });
|
|
116
|
+
}
|
|
117
|
+
for (const digest of files.keys())
|
|
118
|
+
await rm(path.join(paths.modelFiles, digest.slice("sha256:".length)), { force: true });
|
|
119
|
+
}
|
|
120
|
+
return {
|
|
121
|
+
schema_version: "1",
|
|
122
|
+
applied: apply,
|
|
123
|
+
models: removed.map((manifest) => manifest.model_id).sort(),
|
|
124
|
+
files: [...files.keys()].sort(),
|
|
125
|
+
reclaimed_bytes: [...files.values()].reduce((total, size) => total + size, 0),
|
|
126
|
+
};
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
async function collectLockedModelIds(directory, referenced) {
|
|
130
|
+
let entries;
|
|
131
|
+
try {
|
|
132
|
+
entries = await readdir(directory, { withFileTypes: true });
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
if (error.code === "ENOENT")
|
|
136
|
+
return;
|
|
137
|
+
throw error;
|
|
138
|
+
}
|
|
139
|
+
for (const entry of entries) {
|
|
140
|
+
if (!entry.isDirectory() || !/^[a-f0-9]{64}$/.test(entry.name))
|
|
141
|
+
continue;
|
|
142
|
+
const value = await readJSON(path.join(directory, entry.name, "lock.json"));
|
|
143
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
144
|
+
throw integrityError("inference lock index is invalid");
|
|
145
|
+
const record = value;
|
|
146
|
+
if (record.inference_id !== `sha256:${entry.name}` || typeof record.model_id !== "string" || !/^sha256:[a-f0-9]{64}$/.test(record.model_id)) {
|
|
147
|
+
throw integrityError("inference lock index is invalid");
|
|
148
|
+
}
|
|
149
|
+
referenced.add(record.model_id);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
async function listStoredModels(directory) {
|
|
153
|
+
let entries;
|
|
154
|
+
try {
|
|
155
|
+
entries = await readdir(directory, { withFileTypes: true });
|
|
156
|
+
}
|
|
157
|
+
catch (error) {
|
|
158
|
+
if (error.code === "ENOENT")
|
|
159
|
+
return [];
|
|
160
|
+
throw error;
|
|
161
|
+
}
|
|
162
|
+
const manifests = [];
|
|
163
|
+
for (const entry of entries) {
|
|
164
|
+
if (!entry.isDirectory() || !/^[a-f0-9]{64}$/.test(entry.name))
|
|
165
|
+
continue;
|
|
166
|
+
const manifest = parseLocalModelManifest(await readJSON(path.join(directory, entry.name, "manifest.json")));
|
|
167
|
+
if (manifest.model_id !== `sha256:${entry.name}`)
|
|
168
|
+
throw integrityError("model directory identity is invalid");
|
|
169
|
+
manifests.push(manifest);
|
|
170
|
+
}
|
|
171
|
+
return manifests;
|
|
172
|
+
}
|
|
173
|
+
export async function resolveLocalModel(root, reference) {
|
|
174
|
+
const paths = statePaths(root);
|
|
175
|
+
let modelId;
|
|
176
|
+
if (/^local\/sha256:[a-f0-9]{64}$/.test(reference)) {
|
|
177
|
+
modelId = reference.slice("local/".length);
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
180
|
+
const match = reference.match(/^local\/([a-z0-9][a-z0-9._-]{0,63})$/);
|
|
181
|
+
if (!match)
|
|
182
|
+
throw invalidInput("local model must be local/<name> or local/sha256:<digest>");
|
|
183
|
+
const index = await loadAliasIndex(paths.modelAliases);
|
|
184
|
+
const resolved = index.aliases[match[1]];
|
|
185
|
+
if (!resolved)
|
|
186
|
+
throw new HitchError(`local model is not registered: ${reference}`, { code: "local_model_not_found", exitCode: 2 });
|
|
187
|
+
modelId = resolved;
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
return parseLocalModelManifest(await readJSON(path.join(paths.models, modelId.slice("sha256:".length), "manifest.json")));
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
if (error instanceof HitchError)
|
|
194
|
+
throw error;
|
|
195
|
+
throw new HitchError(`local model manifest is unavailable or invalid: ${modelId}`, {
|
|
196
|
+
code: "local_model_integrity_failed", exitCode: 5, cause: error,
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
export async function verifyLocalModel(root, manifest) {
|
|
201
|
+
const paths = statePaths(root);
|
|
202
|
+
for (const file of manifest.files) {
|
|
203
|
+
const stored = path.join(paths.modelFiles, file.sha256.slice("sha256:".length));
|
|
204
|
+
let fileStat;
|
|
205
|
+
try {
|
|
206
|
+
fileStat = await stat(stored);
|
|
207
|
+
}
|
|
208
|
+
catch (error) {
|
|
209
|
+
throw integrityError(`model content is missing: ${file.path}`, error);
|
|
210
|
+
}
|
|
211
|
+
if (!fileStat.isFile() || fileStat.size !== file.size || await hashFile(stored) !== file.sha256) {
|
|
212
|
+
throw integrityError(`model content failed verification: ${file.path}`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
async function walkModelFiles(root, relative = "") {
|
|
217
|
+
const directory = path.join(root, relative);
|
|
218
|
+
const entries = await readdir(directory, { withFileTypes: true });
|
|
219
|
+
const result = [];
|
|
220
|
+
for (const entry of entries.sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) {
|
|
221
|
+
const candidate = relative ? `${relative}/${entry.name}` : entry.name;
|
|
222
|
+
if (entry.isSymbolicLink())
|
|
223
|
+
throw invalidInput(`model directory contains a symbolic link: ${candidate}`);
|
|
224
|
+
if (entry.isDirectory())
|
|
225
|
+
result.push(...await walkModelFiles(root, candidate));
|
|
226
|
+
else if (entry.isFile()) {
|
|
227
|
+
if (FORBIDDEN.test(candidate))
|
|
228
|
+
throw invalidInput(`model directory contains an unsafe executable or pickle file: ${candidate}`);
|
|
229
|
+
result.push(candidate);
|
|
230
|
+
}
|
|
231
|
+
else
|
|
232
|
+
throw invalidInput(`model directory contains a non-regular entry: ${candidate}`);
|
|
233
|
+
if (result.length > MAX_FILES)
|
|
234
|
+
throw invalidInput(`model directory exceeds ${MAX_FILES} files`);
|
|
235
|
+
}
|
|
236
|
+
return result;
|
|
237
|
+
}
|
|
238
|
+
function validateRequiredFiles(files) {
|
|
239
|
+
if (!files.includes("config.json"))
|
|
240
|
+
throw invalidInput("model directory must contain config.json");
|
|
241
|
+
if (!files.some((file) => file.endsWith(".safetensors")))
|
|
242
|
+
throw invalidInput("model directory must contain safetensors weights");
|
|
243
|
+
if (!files.some((file) => /^(?:.*\/)?tokenizer(?:\.json|\.model)$/.test(file))) {
|
|
244
|
+
throw invalidInput("model directory must contain tokenizer.json or tokenizer.model");
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
async function importFile(store, sourceRoot, relative) {
|
|
248
|
+
const source = path.join(sourceRoot, ...relative.split("/"));
|
|
249
|
+
const before = await lstat(source);
|
|
250
|
+
if (!before.isFile())
|
|
251
|
+
throw invalidInput(`model entry changed during import: ${relative}`);
|
|
252
|
+
const temporary = path.join(store, `.import-${process.pid}-${Date.now()}-${createHash("sha256").update(relative).digest("hex").slice(0, 12)}`);
|
|
253
|
+
try {
|
|
254
|
+
await copyFile(source, temporary);
|
|
255
|
+
const copied = await stat(temporary);
|
|
256
|
+
const fileDigest = await hashFile(temporary);
|
|
257
|
+
const after = await lstat(source);
|
|
258
|
+
if (!after.isFile() || before.dev !== after.dev || before.ino !== after.ino || before.size !== after.size
|
|
259
|
+
|| before.mtimeMs !== after.mtimeMs || copied.size !== after.size) {
|
|
260
|
+
throw new HitchError(`model source changed during import: ${relative}`, { code: "local_model_integrity_failed", exitCode: 5 });
|
|
261
|
+
}
|
|
262
|
+
const target = path.join(store, fileDigest.slice("sha256:".length));
|
|
263
|
+
try {
|
|
264
|
+
const existing = await stat(target);
|
|
265
|
+
if (!existing.isFile() || existing.size !== copied.size || await hashFile(target) !== fileDigest) {
|
|
266
|
+
throw integrityError(`existing model CAS object is corrupt: ${fileDigest}`);
|
|
267
|
+
}
|
|
268
|
+
await rm(temporary, { force: true });
|
|
269
|
+
}
|
|
270
|
+
catch (error) {
|
|
271
|
+
if (error?.code !== "ENOENT")
|
|
272
|
+
throw error;
|
|
273
|
+
await rename(temporary, target);
|
|
274
|
+
await chmod(target, 0o444);
|
|
275
|
+
}
|
|
276
|
+
return { path: relative, size: copied.size, sha256: fileDigest };
|
|
277
|
+
}
|
|
278
|
+
finally {
|
|
279
|
+
await rm(temporary, { force: true }).catch(() => { });
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
async function hashFile(file) {
|
|
283
|
+
const hash = createHash("sha256");
|
|
284
|
+
await new Promise((resolve, reject) => {
|
|
285
|
+
const stream = createReadStream(file);
|
|
286
|
+
stream.on("data", (chunk) => hash.update(chunk));
|
|
287
|
+
stream.once("error", reject);
|
|
288
|
+
stream.once("end", resolve);
|
|
289
|
+
});
|
|
290
|
+
return `sha256:${hash.digest("hex")}`;
|
|
291
|
+
}
|
|
292
|
+
async function writeAlias(file, locks, name, modelId, force) {
|
|
293
|
+
await withFileLock(locks, "model-alias-index", async () => {
|
|
294
|
+
const index = await loadAliasIndex(file);
|
|
295
|
+
const current = index.aliases[name];
|
|
296
|
+
if (current && current !== modelId && !force)
|
|
297
|
+
throw invalidInput(`model name already refers to ${current}; use --force to replace it`);
|
|
298
|
+
await atomicWriteJSON(file, { schema_version: "1", aliases: { ...index.aliases, [name]: modelId } });
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
async function loadAliasIndex(file) {
|
|
302
|
+
const value = await readJSON(file, { schema_version: "1", aliases: {} });
|
|
303
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
304
|
+
throw integrityError("model alias index is invalid");
|
|
305
|
+
const record = value;
|
|
306
|
+
if (record.schema_version !== "1" || !record.aliases || typeof record.aliases !== "object" || Array.isArray(record.aliases)) {
|
|
307
|
+
throw integrityError("model alias index is invalid");
|
|
308
|
+
}
|
|
309
|
+
const aliases = {};
|
|
310
|
+
for (const [name, modelId] of Object.entries(record.aliases)) {
|
|
311
|
+
if (!ALIAS.test(name) || typeof modelId !== "string" || !/^sha256:[a-f0-9]{64}$/.test(modelId)) {
|
|
312
|
+
throw integrityError("model alias index contains an invalid entry");
|
|
313
|
+
}
|
|
314
|
+
aliases[name] = modelId;
|
|
315
|
+
}
|
|
316
|
+
return { schema_version: "1", aliases };
|
|
317
|
+
}
|
|
318
|
+
async function readModelJSON(file, label) {
|
|
319
|
+
let value;
|
|
320
|
+
try {
|
|
321
|
+
value = JSON.parse(await readFile(file, "utf8"));
|
|
322
|
+
}
|
|
323
|
+
catch (error) {
|
|
324
|
+
throw invalidInput(`${label} is not valid JSON`, { cause: error });
|
|
325
|
+
}
|
|
326
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
327
|
+
throw invalidInput(`${label} must contain a JSON object`);
|
|
328
|
+
return value;
|
|
329
|
+
}
|
|
330
|
+
async function optionalModelJSON(file) {
|
|
331
|
+
try {
|
|
332
|
+
return await readModelJSON(file, path.basename(file));
|
|
333
|
+
}
|
|
334
|
+
catch (error) {
|
|
335
|
+
if (error?.code === "ENOENT" || error.cause && error.cause.code === "ENOENT")
|
|
336
|
+
return null;
|
|
337
|
+
throw error;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
function rejectDynamicModelCode(config) {
|
|
341
|
+
if (config.auto_map !== undefined || config.trust_remote_code === true) {
|
|
342
|
+
throw invalidInput("models requiring trust_remote_code or auto_map are not supported");
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
function validateSafetensorsIndex(source, files, index) {
|
|
346
|
+
if (!index)
|
|
347
|
+
return;
|
|
348
|
+
const weights = index.weight_map;
|
|
349
|
+
if (!weights || typeof weights !== "object" || Array.isArray(weights))
|
|
350
|
+
throw invalidInput("safetensors index weight_map is invalid");
|
|
351
|
+
for (const target of Object.values(weights)) {
|
|
352
|
+
if (typeof target !== "string" || !target.endsWith(".safetensors") || target.includes("\\") || path.isAbsolute(target)
|
|
353
|
+
|| target.split("/").some((part) => !part || part === "." || part === "..") || !files.includes(target)) {
|
|
354
|
+
throw invalidInput("safetensors index references a missing or unsafe shard");
|
|
355
|
+
}
|
|
356
|
+
if (!path.resolve(source, target).startsWith(`${source}${path.sep}`))
|
|
357
|
+
throw invalidInput("safetensors index escapes the model directory");
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
async function validateSafetensorsFile(file, label) {
|
|
361
|
+
const handle = await open(file, "r");
|
|
362
|
+
try {
|
|
363
|
+
const fileStat = await handle.stat();
|
|
364
|
+
if (!fileStat.isFile() || fileStat.size < 11)
|
|
365
|
+
throw invalidInput(`safetensors file is truncated: ${label}`);
|
|
366
|
+
const prefix = Buffer.alloc(8);
|
|
367
|
+
if ((await handle.read(prefix, 0, prefix.length, 0)).bytesRead !== prefix.length)
|
|
368
|
+
throw invalidInput(`safetensors header is truncated: ${label}`);
|
|
369
|
+
const headerLength = prefix.readBigUInt64LE();
|
|
370
|
+
if (headerLength < 2n || headerLength > 100n * 1024n * 1024n || headerLength > BigInt(fileStat.size - 8)) {
|
|
371
|
+
throw invalidInput(`safetensors header length is invalid: ${label}`);
|
|
372
|
+
}
|
|
373
|
+
const header = Buffer.alloc(Number(headerLength));
|
|
374
|
+
if ((await handle.read(header, 0, header.length, 8)).bytesRead !== header.length)
|
|
375
|
+
throw invalidInput(`safetensors header is truncated: ${label}`);
|
|
376
|
+
let value;
|
|
377
|
+
try {
|
|
378
|
+
value = JSON.parse(header.toString("utf8").trimEnd());
|
|
379
|
+
}
|
|
380
|
+
catch (error) {
|
|
381
|
+
throw invalidInput(`safetensors header is not valid JSON: ${label}`, { cause: error });
|
|
382
|
+
}
|
|
383
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
384
|
+
throw invalidInput(`safetensors header must be an object: ${label}`);
|
|
385
|
+
const dataBytes = fileStat.size - 8 - header.length;
|
|
386
|
+
let tensors = 0;
|
|
387
|
+
for (const [name, descriptor] of Object.entries(value)) {
|
|
388
|
+
if (name === "__metadata__") {
|
|
389
|
+
if (!descriptor || typeof descriptor !== "object" || Array.isArray(descriptor))
|
|
390
|
+
throw invalidInput(`safetensors metadata is invalid: ${label}`);
|
|
391
|
+
continue;
|
|
392
|
+
}
|
|
393
|
+
tensors += 1;
|
|
394
|
+
if (!descriptor || typeof descriptor !== "object" || Array.isArray(descriptor))
|
|
395
|
+
throw invalidInput(`safetensors tensor descriptor is invalid: ${label}`);
|
|
396
|
+
const record = descriptor;
|
|
397
|
+
if (typeof record.dtype !== "string" || !record.dtype || !Array.isArray(record.shape)
|
|
398
|
+
|| record.shape.some((dimension) => !Number.isSafeInteger(dimension) || dimension < 0)
|
|
399
|
+
|| !Array.isArray(record.data_offsets) || record.data_offsets.length !== 2
|
|
400
|
+
|| record.data_offsets.some((offset) => !Number.isSafeInteger(offset) || offset < 0)
|
|
401
|
+
|| record.data_offsets[0] > record.data_offsets[1]
|
|
402
|
+
|| record.data_offsets[1] > dataBytes) {
|
|
403
|
+
throw invalidInput(`safetensors tensor descriptor is invalid: ${label}`);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
if (tensors === 0)
|
|
407
|
+
throw invalidInput(`safetensors file contains no tensors: ${label}`);
|
|
408
|
+
}
|
|
409
|
+
finally {
|
|
410
|
+
await handle.close();
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
async function discoverTemplate(source, files) {
|
|
414
|
+
if (files.includes("chat_template.jinja"))
|
|
415
|
+
return readFile(path.join(source, "chat_template.jinja"), "utf8");
|
|
416
|
+
if (!files.includes("tokenizer_config.json"))
|
|
417
|
+
return null;
|
|
418
|
+
const config = await readModelJSON(path.join(source, "tokenizer_config.json"), "tokenizer_config.json");
|
|
419
|
+
return typeof config.chat_template === "string" ? config.chat_template : null;
|
|
420
|
+
}
|
|
421
|
+
function modelQuantization(config) {
|
|
422
|
+
const quantization = config.quantization_config;
|
|
423
|
+
if (!quantization || typeof quantization !== "object" || Array.isArray(quantization))
|
|
424
|
+
return null;
|
|
425
|
+
return stringValue(quantization.quant_method);
|
|
426
|
+
}
|
|
427
|
+
function firstString(value) {
|
|
428
|
+
return Array.isArray(value) && typeof value[0] === "string" && value[0] ? value[0] : null;
|
|
429
|
+
}
|
|
430
|
+
function stringValue(value) {
|
|
431
|
+
return typeof value === "string" && value.trim() ? value : null;
|
|
432
|
+
}
|
|
433
|
+
function positiveIntegerValue(value) {
|
|
434
|
+
return Number.isSafeInteger(value) && value > 0 ? value : null;
|
|
435
|
+
}
|
|
436
|
+
function integrityError(message, cause) {
|
|
437
|
+
return new HitchError(message, { code: "local_model_integrity_failed", exitCode: 5, cause });
|
|
438
|
+
}
|
|
439
|
+
//# sourceMappingURL=model-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-store.js","sourceRoot":"","sources":["../../../src/inference/model-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAS,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC5G,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EACL,UAAU,EACV,eAAe,EACf,SAAS,EACT,YAAY,EACZ,QAAQ,EACR,WAAW,EACX,UAAU,EACV,UAAU,EACV,YAAY,GACb,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAE5E,MAAM,KAAK,GAAG,6BAA6B,CAAC;AAC5C,MAAM,SAAS,GAAG,gEAAgE,CAAC;AACnF,MAAM,SAAS,GAAG,OAAO,CAAC;AAc1B,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,OAA6B;IAC/D,OAAO,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,uBAAuB,EAAE,aAAa,EAAE,GAAG,EAAE,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC,CAAC;AAC3H,CAAC;AAED,mGAAmG;AACnG,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAwF;IAC7H,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,gEAAgE,CAAC,CAAC;IACpH,MAAM,QAAQ,GAAG,uBAAuB,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7F,OAAO,YAAY,CAAC,KAAK,CAAC,uBAAuB,EAAE,aAAa,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,IAAI,QAA0C,CAAC;QAC/C,IAAI,CAAC;YAAC,QAAQ,GAAG,uBAAuB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;QAAC,CAAC;QAClG,OAAO,KAAK,EAAE,CAAC;YAAC,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAC;QAAC,CAAC;QACtF,IAAI,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ;YAAE,MAAM,cAAc,CAAC,+BAA+B,CAAC,CAAC;QAC/G,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;QAC3B,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,QAAQ,IAAI,QAAQ,CAAC,CAAC;QACnF,MAAM,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,uBAAuB,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC;QAC7H,OAAO,QAAQ,IAAI,QAAQ,CAAC;IAC9B,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,OAA6B;IAC9D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,gEAAgE,CAAC,CAAC;IACpH,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACvD,IAAI,YAAY,KAAK,EAAE,IAAI,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,YAAY,CAAC,mDAAmD,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,UAAU,CAAC;IACf,IAAI,CAAC;QAAC,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACtD,MAAM,IAAI,UAAU,CAAC,yCAAyC,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACxI,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE;QAAE,MAAM,YAAY,CAAC,wCAAwC,CAAC,CAAC;IAE5F,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,CAAC;IACnD,qBAAqB,CAAC,aAAa,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,aAAa,CAAC,CAAC;IACpF,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/B,wBAAwB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,8BAA8B,CAAC,CAAC,CAAC,CAAC;IAC5H,KAAK,MAAM,QAAQ,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC;QACrF,MAAM,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACrF,CAAC;IAED,MAAM,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAClC,MAAM,KAAK,GAAuB,EAAE,CAAC;IACrC,KAAK,MAAM,QAAQ,IAAI,aAAa;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACvG,MAAM,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,gFAAgF,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAChJ,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC/C,MAAM,aAAa,GAAG;QACpB,cAAc,EAAE,GAAY;QAC5B,MAAM,EAAE,gBAAyB;QACjC,KAAK;QACL,YAAY,EAAE,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,SAAS;QAC5D,UAAU,EAAE,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,SAAS;QACvD,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,SAAS;QAChF,YAAY;QACZ,cAAc,EAAE,oBAAoB,CAAC,MAAM,CAAC,uBAAuB,CAAC;QACpE,gBAAgB,EAAE,UAAU,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAC9G,eAAe,EAAE,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC;KAClE,CAAC;IACF,MAAM,QAAQ,GAAyB;QACrC,GAAG,aAAa;QAChB,QAAQ,EAAE,kBAAkB,CAAC,aAAa,CAAC;QAC3C,MAAM,EAAE;YACN,IAAI,EAAE,iBAAiB;YACvB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC5B,OAAO,EAAE,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC;SACrC;QACD,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACrC,CAAC;IACF,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1F,MAAM,SAAS,CAAC,cAAc,CAAC,CAAC;IAChC,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAC;IAC5E,MAAM,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,uBAAuB,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC;IAC7H,OAAO,QAAQ,CAAC;AAClB,CAAC;AAUD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,KAAK,GAAG,KAAK;IAC7D,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,OAAO,YAAY,CAAC,KAAK,CAAC,uBAAuB,EAAE,aAAa,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACnE,MAAM,qBAAqB,CAAC,KAAK,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;QACnF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;aACxF,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACrE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;QACxC,KAAK,MAAM,QAAQ,IAAI,OAAO;YAAE,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAClE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC;oBAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YACrE,CAAC;QACD,IAAI,KAAK,EAAE,CAAC;YACV,KAAK,MAAM,QAAQ,IAAI,OAAO,EAAE,CAAC;gBAC/B,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACjH,CAAC;YACD,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE;gBAAE,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5H,CAAC;QACD,OAAO;YACL,cAAc,EAAE,GAAG;YACnB,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE;YAC3D,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;YAC/B,eAAe,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,EAAE,CAAC,CAAC;SAC9E,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,SAAiB,EAAE,UAAuB;IAC7E,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QAAC,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAClF,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO;QAC/D,MAAM,KAAK,CAAC;IACd,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QACzE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAU,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC,CAAC;QACrF,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,MAAM,cAAc,CAAC,iCAAiC,CAAC,CAAC;QACzH,MAAM,MAAM,GAAG,KAAgC,CAAC;QAChD,IAAI,MAAM,CAAC,YAAY,KAAK,UAAU,KAAK,CAAC,IAAI,EAAE,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5I,MAAM,cAAc,CAAC,iCAAiC,CAAC,CAAC;QAC1D,CAAC;QACD,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,QAAkB,CAAC,CAAC;IAC5C,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,SAAiB;IAC/C,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QAAC,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAClF,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAC;QAClE,MAAM,KAAK,CAAC;IACd,CAAC;IACD,MAAM,SAAS,GAA2B,EAAE,CAAC;IAC7C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAAE,SAAS;QACzE,MAAM,QAAQ,GAAG,uBAAuB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;QAC5G,IAAI,QAAQ,CAAC,QAAQ,KAAK,UAAU,KAAK,CAAC,IAAI,EAAE;YAAE,MAAM,cAAc,CAAC,qCAAqC,CAAC,CAAC;QAC9G,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY,EAAE,SAAiB;IACrE,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,OAAe,CAAC;IACpB,IAAI,8BAA8B,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACnD,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAW,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACtE,IAAI,CAAC,KAAK;YAAE,MAAM,YAAY,CAAC,2DAA2D,CAAC,CAAC;QAC5F,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAW,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,UAAU,CAAC,kCAAkC,SAAS,EAAE,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QACnI,OAAO,GAAG,QAAQ,CAAC;IACrB,CAAC;IACD,IAAI,CAAC;QACH,OAAO,uBAAuB,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC;IAC5H,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,UAAU;YAAE,MAAM,KAAK,CAAC;QAC7C,MAAM,IAAI,UAAU,CAAC,mDAAmD,OAAO,EAAE,EAAE;YACjF,IAAI,EAAE,8BAA8B,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK;SAChE,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAY,EAAE,QAA8B;IACjF,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;QAChF,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACpD,MAAM,cAAc,CAAC,6BAA6B,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;QACxE,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,IAAI,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YAChG,MAAM,cAAc,CAAC,sCAAsC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,QAAQ,GAAG,EAAE;IACvD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAClE,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QACtE,IAAI,KAAK,CAAC,cAAc,EAAE;YAAE,MAAM,YAAY,CAAC,6CAA6C,SAAS,EAAE,CAAC,CAAC;QACzG,IAAI,KAAK,CAAC,WAAW,EAAE;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;aAC1E,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACxB,IAAI,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC;gBAAE,MAAM,YAAY,CAAC,iEAAiE,SAAS,EAAE,CAAC,CAAC;YAChI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzB,CAAC;;YAAM,MAAM,YAAY,CAAC,iDAAiD,SAAS,EAAE,CAAC,CAAC;QACxF,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS;YAAE,MAAM,YAAY,CAAC,2BAA2B,SAAS,QAAQ,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAe;IAC5C,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa,CAAC;QAAE,MAAM,YAAY,CAAC,0CAA0C,CAAC,CAAC;IACnG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QAAE,MAAM,YAAY,CAAC,kDAAkD,CAAC,CAAC;IACjI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,wCAAwC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/E,MAAM,YAAY,CAAC,gEAAgE,CAAC,CAAC;IACvF,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,KAAa,EAAE,UAAkB,EAAE,QAAgB;IAC3E,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;IACnC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;QAAE,MAAM,YAAY,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;IAC3F,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,WAAW,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;IAC/I,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC;QACrC,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,MAAM,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,KAAK,KAAK,CAAC,GAAG,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI;eACpG,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC;YACpE,MAAM,IAAI,UAAU,CAAC,uCAAuC,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,8BAA8B,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QACjI,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;QACpE,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;YACpC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,IAAI,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,UAAU,EAAE,CAAC;gBACjG,MAAM,cAAc,CAAC,yCAAyC,UAAU,EAAE,CAAC,CAAC;YAC9E,CAAC;YACD,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,EAAE,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAC;YACrE,MAAM,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAChC,MAAM,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IACnE,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACvD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC9B,CAAC,CAAC,CAAC;IACH,OAAO,UAAU,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AACxC,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY,EAAE,KAAa,EAAE,IAAY,EAAE,OAAe,EAAE,KAAc;IAClG,MAAM,YAAY,CAAC,KAAK,EAAE,mBAAmB,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,OAAO,IAAI,OAAO,KAAK,OAAO,IAAI,CAAC,KAAK;YAAE,MAAM,YAAY,CAAC,gCAAgC,OAAO,6BAA6B,CAAC,CAAC;QACvI,MAAM,eAAe,CAAC,IAAI,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC;IACvG,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAY;IACxC,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAU,IAAI,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;IAClF,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,cAAc,CAAC,8BAA8B,CAAC,CAAC;IACtH,MAAM,MAAM,GAAG,KAAgC,CAAC;IAChD,IAAI,MAAM,CAAC,cAAc,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5H,MAAM,cAAc,CAAC,8BAA8B,CAAC,CAAC;IACvD,CAAC;IACD,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAkC,CAAC,EAAE,CAAC;QACxF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/F,MAAM,cAAc,CAAC,6CAA6C,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,OAAiB,CAAC;IACpC,CAAC;IACD,OAAO,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,KAAa;IACtD,IAAI,KAAc,CAAC;IACnB,IAAI,CAAC;QAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACvE,MAAM,YAAY,CAAC,GAAG,KAAK,oBAAoB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,YAAY,CAAC,GAAG,KAAK,6BAA6B,CAAC,CAAC;IAC3H,OAAO,KAAgC,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,IAAY;IAC3C,IAAI,CAAC;QAAC,OAAO,MAAM,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAC5E,IAAK,KAA+B,EAAE,IAAI,KAAK,QAAQ,IAAK,KAAe,CAAC,KAAK,IAAM,KAAe,CAAC,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QACtK,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,sBAAsB,CAAC,MAA+B;IAC7D,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,iBAAiB,KAAK,IAAI,EAAE,CAAC;QACvE,MAAM,YAAY,CAAC,kEAAkE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAAC,MAAc,EAAE,KAAe,EAAE,KAAqC;IACtG,IAAI,CAAC,KAAK;QAAE,OAAO;IACnB,MAAM,OAAO,GAAG,KAAK,CAAC,UAAU,CAAC;IACjC,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,MAAM,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACrI,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,OAAkC,CAAC,EAAE,CAAC;QACvE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;eACjH,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACzG,MAAM,YAAY,CAAC,wDAAwD,CAAC,CAAC;QAC/E,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAAE,MAAM,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC5I,CAAC;AACH,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,IAAY,EAAE,KAAa;IAChE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACrC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,QAAQ,CAAC,IAAI,GAAG,EAAE;YAAE,MAAM,YAAY,CAAC,kCAAkC,KAAK,EAAE,CAAC,CAAC;QAC5G,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,KAAK,MAAM,CAAC,MAAM;YAAE,MAAM,YAAY,CAAC,oCAAoC,KAAK,EAAE,CAAC,CAAC;QAClJ,MAAM,YAAY,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QAC9C,IAAI,YAAY,GAAG,EAAE,IAAI,YAAY,GAAG,IAAI,GAAG,KAAK,GAAG,KAAK,IAAI,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC;YACzG,MAAM,YAAY,CAAC,yCAAyC,KAAK,EAAE,CAAC,CAAC;QACvE,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,KAAK,MAAM,CAAC,MAAM;YAAE,MAAM,YAAY,CAAC,oCAAoC,KAAK,EAAE,CAAC,CAAC;QAClJ,IAAI,KAAc,CAAC;QACnB,IAAI,CAAC;YAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAAC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAC5E,MAAM,YAAY,CAAC,yCAAyC,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YAAE,MAAM,YAAY,CAAC,yCAAyC,KAAK,EAAE,CAAC,CAAC;QACtI,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACpD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;YAClF,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;gBAC5B,IAAI,CAAC,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;oBAAE,MAAM,YAAY,CAAC,oCAAoC,KAAK,EAAE,CAAC,CAAC;gBAChJ,SAAS;YACX,CAAC;YACD,OAAO,IAAI,CAAC,CAAC;YACb,IAAI,CAAC,UAAU,IAAI,OAAO,UAAU,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;gBAAE,MAAM,YAAY,CAAC,6CAA6C,KAAK,EAAE,CAAC,CAAC;YACzJ,MAAM,MAAM,GAAG,UAAqC,CAAC;YACrD,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;mBAChF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,IAAK,SAAoB,GAAG,CAAC,CAAC;mBAC/F,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;mBACvE,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAK,MAAiB,GAAG,CAAC,CAAC;mBAC5F,MAAM,CAAC,YAAY,CAAC,CAAC,CAAY,GAAI,MAAM,CAAC,YAAY,CAAC,CAAC,CAAY;mBACtE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAY,GAAG,SAAS,EAAE,CAAC;gBACpD,MAAM,YAAY,CAAC,6CAA6C,KAAK,EAAE,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;QACD,IAAI,OAAO,KAAK,CAAC;YAAE,MAAM,YAAY,CAAC,yCAAyC,KAAK,EAAE,CAAC,CAAC;IAC1F,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,MAAc,EAAE,KAAe;IAC7D,IAAI,KAAK,CAAC,QAAQ,CAAC,qBAAqB,CAAC;QAAE,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,qBAAqB,CAAC,EAAE,MAAM,CAAC,CAAC;IAC7G,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,uBAAuB,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1D,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,uBAAuB,CAAC,EAAE,uBAAuB,CAAC,CAAC;IACxG,OAAO,OAAO,MAAM,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC;AAChF,CAAC;AAED,SAAS,iBAAiB,CAAC,MAA+B;IACxD,MAAM,YAAY,GAAG,MAAM,CAAC,mBAAmB,CAAC;IAChD,IAAI,CAAC,YAAY,IAAI,OAAO,YAAY,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC;IAClG,OAAO,WAAW,CAAE,YAAwC,CAAC,YAAY,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5F,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAClE,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,OAAO,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAK,KAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,KAAe,CAAC,CAAC,CAAC,IAAI,CAAC;AACvF,CAAC;AAED,SAAS,cAAc,CAAC,OAAe,EAAE,KAAe;IACtD,OAAO,IAAI,UAAU,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,8BAA8B,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;AAC/F,CAAC"}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
import { createReadStream } from "node:fs";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { lstat, mkdir } from "node:fs/promises";
|
|
6
|
+
import { Readable } from "node:stream";
|
|
7
|
+
import { pipeline } from "node:stream/promises";
|
|
8
|
+
import { HitchError, atomicWriteJSON, hitchRootId, readJSON, runCommand, sha256JSON, statePaths, withFileLock } from "../foundation/index.js";
|
|
9
|
+
import { verifyLocalModel } from "./model-store.js";
|
|
10
|
+
import { modelNodeSnapshot } from "./model-location.js";
|
|
11
|
+
import { parseLocalModelManifest } from "./manifest.js";
|
|
12
|
+
const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`;
|
|
13
|
+
/** Gear node protocol v2. Model bytes use separate bounded-header binary CAS. */
|
|
14
|
+
export class PythonInferenceNodeClient {
|
|
15
|
+
connection;
|
|
16
|
+
node;
|
|
17
|
+
env;
|
|
18
|
+
constructor(connection, node, env = process.env) {
|
|
19
|
+
this.connection = connection;
|
|
20
|
+
this.node = node;
|
|
21
|
+
this.env = env;
|
|
22
|
+
if (!connection.python.length || connection.python.some(arg => typeof arg !== "string" || !arg || arg.includes("\0"))
|
|
23
|
+
|| !path.posix.isAbsolute(connection.configPath) || !node.nodeId || !node.generation
|
|
24
|
+
|| [connection.gateway.localPort, connection.gateway.nodePort].some(port => !Number.isInteger(port) || port < 1 || port > 65535)
|
|
25
|
+
|| (connection.transport.type === "ssh" && !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/.test(connection.transport.host))) {
|
|
26
|
+
throw new TypeError("invalid explicit model-node connection");
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
command(action) {
|
|
30
|
+
const command = [...this.connection.python, "-m", "gear_training.node", action, "--config", this.connection.configPath];
|
|
31
|
+
return this.connection.transport.type === "local" ? command
|
|
32
|
+
: ["ssh", "-T", "-o", "BatchMode=yes", "-o", "ConnectTimeout=15", "--", this.connection.transport.host, command.map(quote).join(" ")];
|
|
33
|
+
}
|
|
34
|
+
envelope(operation, payload) {
|
|
35
|
+
return { schemaVersion: 2, requestId: randomUUID(), node: this.node, operation, inputDigest: sha256JSON(payload), payload };
|
|
36
|
+
}
|
|
37
|
+
response(value, envelope) {
|
|
38
|
+
const data = value;
|
|
39
|
+
if (!data || data.schemaVersion !== 2 || data.requestId !== envelope.requestId || data.inputDigest !== envelope.inputDigest
|
|
40
|
+
|| sha256JSON(data.node ?? null) !== sha256JSON(this.node) || !("result" in data)) {
|
|
41
|
+
throw new HitchError("model-node reply identity or generation changed", { code: "inference_runtime_mismatch", exitCode: 12 });
|
|
42
|
+
}
|
|
43
|
+
return data.result;
|
|
44
|
+
}
|
|
45
|
+
async exchange(action, envelope, file) {
|
|
46
|
+
const command = this.command(action);
|
|
47
|
+
const child = spawn(command[0], command.slice(1), { env: this.env, stdio: ["pipe", "pipe", "pipe"] });
|
|
48
|
+
child.stderr.resume();
|
|
49
|
+
child.stdin.on("error", () => { });
|
|
50
|
+
const timer = setTimeout(() => child.kill("SIGKILL"), file ? 3_600_000 : 120_000);
|
|
51
|
+
const completed = new Promise((resolve, reject) => {
|
|
52
|
+
child.once("error", reject);
|
|
53
|
+
child.once("close", resolve);
|
|
54
|
+
});
|
|
55
|
+
void completed.catch(() => { });
|
|
56
|
+
let output = "";
|
|
57
|
+
let length = 0;
|
|
58
|
+
const read = async () => {
|
|
59
|
+
for await (const chunk of child.stdout) {
|
|
60
|
+
length += chunk.byteLength;
|
|
61
|
+
if (length > 4 * 1024 * 1024)
|
|
62
|
+
throw new TypeError("model-node response exceeds control protocol limit");
|
|
63
|
+
output += chunk.toString();
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
async function* input() { yield Buffer.from(JSON.stringify(envelope) + "\n"); if (file)
|
|
67
|
+
yield* createReadStream(file); }
|
|
68
|
+
try {
|
|
69
|
+
const [, , code] = await Promise.all([pipeline(Readable.from(input()), child.stdin), read(), completed]);
|
|
70
|
+
if (code !== 0) {
|
|
71
|
+
let reason = "transport-failed";
|
|
72
|
+
try {
|
|
73
|
+
const parsed = JSON.parse(output);
|
|
74
|
+
if (/^[a-z][a-z0-9_-]{0,100}$/.test(parsed.error?.code))
|
|
75
|
+
reason = parsed.error.code;
|
|
76
|
+
}
|
|
77
|
+
catch { }
|
|
78
|
+
throw new HitchError(`model-node ${envelope.operation} failed (${reason}); resource ownership remains unresolved`, { code: "inference_route_unavailable", exitCode: 12 });
|
|
79
|
+
}
|
|
80
|
+
return this.response(JSON.parse(output), envelope);
|
|
81
|
+
}
|
|
82
|
+
finally {
|
|
83
|
+
clearTimeout(timer);
|
|
84
|
+
if (child.exitCode === null)
|
|
85
|
+
child.kill("SIGKILL");
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
async call(operation, payload) { return this.exchange("rpc", this.envelope(operation, payload)); }
|
|
89
|
+
payload(input) {
|
|
90
|
+
return { serviceId: input.serviceId, ownerId: hitchRootId(input.root), model: input.model, runtime: input.runtime, lock: input.lock };
|
|
91
|
+
}
|
|
92
|
+
async prepare(input) {
|
|
93
|
+
const snapshotRef = input.lock.model_node ? await modelNodeSnapshot(input.root, input.model, input.lock.model_node) : null;
|
|
94
|
+
if (snapshotRef) {
|
|
95
|
+
const observed = parseLocalModelManifest(await this.call("cas.hfManifest", { snapshotRef }));
|
|
96
|
+
if (observed.model_id !== input.model.model_id)
|
|
97
|
+
throw new TypeError("model-node content differs from its registered model");
|
|
98
|
+
await this.call("inference.prepare", this.payload(input));
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
await verifyLocalModel(input.root, input.model);
|
|
102
|
+
for (const file of input.model.files) {
|
|
103
|
+
const state = await this.call("cas.stat", { digest: file.sha256 });
|
|
104
|
+
if (state.present && state.size === file.size)
|
|
105
|
+
continue;
|
|
106
|
+
if (state.present)
|
|
107
|
+
throw new TypeError("model-node object size differs from the model manifest");
|
|
108
|
+
const result = await this.exchange("cas-import", this.envelope("cas.import", { digest: file.sha256, size: file.size }), path.join(statePaths(input.root).modelFiles, file.sha256.slice(7)));
|
|
109
|
+
if (!result.present || result.size !== file.size)
|
|
110
|
+
throw new TypeError("node did not acknowledge durable model import");
|
|
111
|
+
}
|
|
112
|
+
await this.call("inference.prepare", this.payload(input));
|
|
113
|
+
}
|
|
114
|
+
start(input) { return this.call("inference.start", this.payload(input)); }
|
|
115
|
+
inspect(owner) { return this.call("inference.inspect", owner); }
|
|
116
|
+
attach(input) {
|
|
117
|
+
const original = { serviceId: input.record.service_id, ownerId: hitchRootId(input.root), model: input.model, runtime: input.runtime, lock: input.lock };
|
|
118
|
+
return this.call("inference.attach", { serviceId: original.serviceId, ownerId: original.ownerId,
|
|
119
|
+
inferenceId: input.lock.inference_id, inputDigest: sha256JSON(original), expectedHandle: input.record.service_handle });
|
|
120
|
+
}
|
|
121
|
+
stop(owner) { return this.call("inference.stop", owner); }
|
|
122
|
+
async route(root, nodePort) {
|
|
123
|
+
const { gateway, transport } = this.connection;
|
|
124
|
+
if (nodePort !== gateway.nodePort || (transport.type === "local" && gateway.localPort !== gateway.nodePort)) {
|
|
125
|
+
throw new TypeError("model-node service port differs from its configured route");
|
|
126
|
+
}
|
|
127
|
+
if (transport.type === "ssh") {
|
|
128
|
+
const directory = statePaths(root).inferenceOperationLocks;
|
|
129
|
+
const identity = sha256JSON({ node: this.node, connection: this.connection });
|
|
130
|
+
// Short absolute socket paths fit the Unix-domain limit on macOS/Linux.
|
|
131
|
+
const socketDirectory = path.join("/tmp", `hitch-node-${process.getuid?.() ?? "user"}-${hitchRootId(root).slice(-12)}`);
|
|
132
|
+
await mkdir(socketDirectory, { mode: 0o700, recursive: true });
|
|
133
|
+
const permissions = await lstat(socketDirectory);
|
|
134
|
+
if (!permissions.isDirectory() || permissions.isSymbolicLink() || (permissions.mode & 0o077) !== 0
|
|
135
|
+
|| (process.getuid && permissions.uid !== process.getuid()))
|
|
136
|
+
throw new TypeError("model-node SSH socket directory is not private");
|
|
137
|
+
const socket = path.join(socketDirectory, `${identity.slice(-20)}.sock`);
|
|
138
|
+
const recordPath = path.join(directory, identity.slice(7) + ".tunnel.json");
|
|
139
|
+
await withFileLock(directory, identity.slice(7), async () => {
|
|
140
|
+
let prior;
|
|
141
|
+
try {
|
|
142
|
+
prior = await readJSON(recordPath);
|
|
143
|
+
}
|
|
144
|
+
catch (error) {
|
|
145
|
+
if (error.code !== "ENOENT")
|
|
146
|
+
throw error;
|
|
147
|
+
}
|
|
148
|
+
if (prior && prior.identity !== identity)
|
|
149
|
+
throw new TypeError("model-node tunnel ownership changed");
|
|
150
|
+
const invoke = (args) => runCommand("ssh", args, { env: this.env, timeoutMs: 30_000, failureCode: "inference_route_unavailable", failureExitCode: 12 });
|
|
151
|
+
const common = ["-S", socket, "-o", "BatchMode=yes", "-o", "ConnectTimeout=15"];
|
|
152
|
+
if (prior) {
|
|
153
|
+
try {
|
|
154
|
+
await invoke([...common, "-O", "check", "--", transport.host]);
|
|
155
|
+
return;
|
|
156
|
+
}
|
|
157
|
+
catch { }
|
|
158
|
+
}
|
|
159
|
+
await atomicWriteJSON(recordPath, { identity, node: this.node, connection: this.connection });
|
|
160
|
+
await invoke([...common, "-M", "-fNT", "-o", "ExitOnForwardFailure=yes", "-o", "ServerAliveInterval=15", "-o", "ServerAliveCountMax=3",
|
|
161
|
+
"-L", `127.0.0.1:${gateway.localPort}:127.0.0.1:${gateway.nodePort}`, "--", transport.host]);
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
return `http://127.0.0.1:${gateway.localPort}`;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=node-client.js.map
|