@kici-dev/orchestrator 0.16.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (592) hide show
  1. package/dist/agent/agent-version.d.ts +12 -21
  2. package/dist/agent/dispatcher.d.ts +283 -437
  3. package/dist/agent/host-roster-reaper.d.ts +8 -10
  4. package/dist/agent/host-roster.d.ts +77 -100
  5. package/dist/agent/ownership-refusal.d.ts +3 -5
  6. package/dist/agent/ownership-tracker.d.ts +42 -76
  7. package/dist/agent/registry.d.ts +121 -233
  8. package/dist/agent/token-store.d.ts +34 -77
  9. package/dist/agent-packaging/availability.d.ts +6 -9
  10. package/dist/agent-packaging/download.d.ts +6 -9
  11. package/dist/agent-packaging/install-closure.d.ts +9 -12
  12. package/dist/agent-packaging/node-binary.d.ts +6 -10
  13. package/dist/agent-packaging/node-runtime-members.d.ts +3 -5
  14. package/dist/agent-packaging/refresh.d.ts +12 -16
  15. package/dist/agent-packaging/store.d.ts +5 -6
  16. package/dist/agent-packaging/upload.d.ts +3 -4
  17. package/dist/app.d.ts +0 -3
  18. package/dist/approvals/apply-decision.d.ts +39 -45
  19. package/dist/approvals/approval-resolver.d.ts +11 -23
  20. package/dist/approvals/held-run-release-wiring.d.ts +14 -21
  21. package/dist/approvals/independent-wiring.d.ts +13 -20
  22. package/dist/approvals/resume-router.d.ts +1 -1
  23. package/dist/approvals/step-approval-bridge.d.ts +11 -21
  24. package/dist/approvals/triggerer-subject.d.ts +8 -10
  25. package/dist/artifacts/artifact-store.d.ts +55 -95
  26. package/dist/artifacts/failure-messages.d.ts +15 -20
  27. package/dist/audit/access-log.d.ts +27 -45
  28. package/dist/audit/sampling-rate-limiter.d.ts +11 -23
  29. package/dist/cache/agent-job-failed-error.d.ts +2 -3
  30. package/dist/cache/build-coordinator.d.ts +5 -28
  31. package/dist/cache/cache-upload-settle.d.ts +11 -11
  32. package/dist/cache/dep-cache.d.ts +32 -60
  33. package/dist/cache/dispatch-cache-ref-tracker.d.ts +15 -23
  34. package/dist/cache/global-eval-round-cache.d.ts +33 -41
  35. package/dist/cache/pending-builds.d.ts +2 -6
  36. package/dist/cache/pending-dynamics.d.ts +2 -7
  37. package/dist/cache/pending-global-evals.d.ts +9 -12
  38. package/dist/cache/pending-inits.d.ts +11 -22
  39. package/dist/cache/pending-precursor-db-watcher.d.ts +24 -32
  40. package/dist/cache/pending-tracker.d.ts +17 -45
  41. package/dist/cache/precursor-result.d.ts +17 -30
  42. package/dist/cache/source-cache.d.ts +36 -65
  43. package/dist/cache/user-cache.d.ts +21 -25
  44. package/dist/cancel/cancel-held-run.d.ts +13 -16
  45. package/dist/cancel/cancel-route-answer.d.ts +1 -2
  46. package/dist/cancel/cancel-run.d.ts +26 -50
  47. package/dist/cancel/dashboard-cancel-handler.d.ts +2 -2
  48. package/dist/cancel/peer-job-cancel.d.ts +11 -14
  49. package/dist/cancel/sweep-stuck-cancelling.d.ts +9 -13
  50. package/dist/cli/api-client.d.ts +33 -76
  51. package/dist/cli/commands/access-log.d.ts +4 -24
  52. package/dist/cli/commands/agent-package.d.ts +7 -15
  53. package/dist/cli/commands/agent-service/agent-env.d.ts +3 -6
  54. package/dist/cli/commands/agent-service/index.d.ts +4 -10
  55. package/dist/cli/commands/agent-service/install.d.ts +4 -11
  56. package/dist/cli/commands/agent-service/status.d.ts +4 -9
  57. package/dist/cli/commands/agent-service/upgrade.d.ts +4 -16
  58. package/dist/cli/commands/agent.d.ts +12 -25
  59. package/dist/cli/commands/api-key.d.ts +7 -7
  60. package/dist/cli/commands/attestations-list.d.ts +7 -17
  61. package/dist/cli/commands/attestations-retry.d.ts +5 -11
  62. package/dist/cli/commands/attestations-reverify.d.ts +4 -5
  63. package/dist/cli/commands/attestations.d.ts +3 -8
  64. package/dist/cli/commands/audit.d.ts +1 -5
  65. package/dist/cli/commands/backend.d.ts +2 -11
  66. package/dist/cli/commands/check-run.d.ts +7 -18
  67. package/dist/cli/commands/cluster-name.d.ts +5 -12
  68. package/dist/cli/commands/cluster-settings.d.ts +28 -53
  69. package/dist/cli/commands/cold-store.d.ts +7 -15
  70. package/dist/cli/commands/config.d.ts +9 -20
  71. package/dist/cli/commands/context.d.ts +8 -20
  72. package/dist/cli/commands/dashboard-encryption-key.d.ts +9 -14
  73. package/dist/cli/commands/db-backup.d.ts +17 -32
  74. package/dist/cli/commands/db.d.ts +3 -4
  75. package/dist/cli/commands/debug-bundle.d.ts +3 -10
  76. package/dist/cli/commands/diagnose.d.ts +3 -14
  77. package/dist/cli/commands/event-dlq.d.ts +4 -12
  78. package/dist/cli/commands/event-log.d.ts +2 -20
  79. package/dist/cli/commands/execution.d.ts +3 -9
  80. package/dist/cli/commands/firecracker/index.d.ts +3 -6
  81. package/dist/cli/commands/held-run.d.ts +23 -37
  82. package/dist/cli/commands/host.d.ts +4 -11
  83. package/dist/cli/commands/inspect-bundle.d.ts +3 -11
  84. package/dist/cli/commands/local-hook.d.ts +2 -10
  85. package/dist/cli/commands/local-trigger.d.ts +2 -2
  86. package/dist/cli/commands/orchestrator-service/drain.d.ts +3 -6
  87. package/dist/cli/commands/orchestrator-service/index.d.ts +6 -14
  88. package/dist/cli/commands/orchestrator-service/install-env.d.ts +25 -49
  89. package/dist/cli/commands/orchestrator-service/install.d.ts +2 -5
  90. package/dist/cli/commands/orchestrator-service/status.d.ts +10 -22
  91. package/dist/cli/commands/orchestrator-service/upgrade.d.ts +6 -15
  92. package/dist/cli/commands/org-settings.d.ts +5 -11
  93. package/dist/cli/commands/peer.d.ts +5 -10
  94. package/dist/cli/commands/queue.d.ts +3 -9
  95. package/dist/cli/commands/registration.d.ts +4 -8
  96. package/dist/cli/commands/remote-source.d.ts +6 -12
  97. package/dist/cli/commands/rotate.d.ts +1 -5
  98. package/dist/cli/commands/runs.d.ts +1 -30
  99. package/dist/cli/commands/scaler.d.ts +5 -12
  100. package/dist/cli/commands/secret.d.ts +1 -9
  101. package/dist/cli/commands/service-health.d.ts +5 -10
  102. package/dist/cli/commands/shared/admin-cli-access-log.d.ts +10 -19
  103. package/dist/cli/commands/shared/bare-metal-binary-check.d.ts +86 -0
  104. package/dist/cli/commands/shared/cli-action.d.ts +1 -4
  105. package/dist/cli/commands/shared/confirm.d.ts +5 -10
  106. package/dist/cli/commands/shared/db.d.ts +1 -12
  107. package/dist/cli/commands/shared/env-file-mode.d.ts +6 -9
  108. package/dist/cli/commands/shared/secret-input.d.ts +7 -21
  109. package/dist/cli/commands/shared/service-verbs.d.ts +6 -12
  110. package/dist/cli/commands/shared/unbound-context-warning.d.ts +2 -4
  111. package/dist/cli/commands/shared/upgrade-hooks.d.ts +7 -16
  112. package/dist/cli/commands/shared/upgrade-safety.d.ts +20 -31
  113. package/dist/cli/commands/shared/versioned-upgrade.d.ts +83 -161
  114. package/dist/cli/commands/signing-key.d.ts +4 -9
  115. package/dist/cli/commands/source-manifest.d.ts +11 -14
  116. package/dist/cli/commands/source.d.ts +7 -32
  117. package/dist/cli/commands/token.d.ts +4 -15
  118. package/dist/cli/commands/trust-policy.d.ts +35 -88
  119. package/dist/cli/commands/variable.d.ts +3 -5
  120. package/dist/cli/commands/workflow.d.ts +3 -24
  121. package/dist/cli/join.d.ts +3 -6
  122. package/dist/cli/kici-admin.d.ts +11 -29
  123. package/dist/cli/lazy-deps/cache.d.ts +1 -14
  124. package/dist/cli/lazy-deps/downloader.d.ts +4 -27
  125. package/dist/cli/lazy-deps/registry.d.ts +5 -25
  126. package/dist/cli/local-github-ingress-url.d.ts +5 -7
  127. package/dist/cli/loopback-callback.d.ts +6 -7
  128. package/dist/cli/open-browser.d.ts +4 -8
  129. package/dist/cli/root-option-routing.d.ts +17 -29
  130. package/dist/cli/service/backup-timer.d.ts +16 -39
  131. package/dist/cli/service/compose-path.d.ts +3 -7
  132. package/dist/cli/service/compose.d.ts +12 -30
  133. package/dist/cli/service/deploy-env.d.ts +6 -18
  134. package/dist/cli/service/entrypoint.d.ts +14 -25
  135. package/dist/cli/service/env-assignment.d.ts +6 -17
  136. package/dist/cli/service/image-digests.d.ts +6 -15
  137. package/dist/cli/service/index.d.ts +2 -18
  138. package/dist/cli/service/instance/index-file.d.ts +9 -16
  139. package/dist/cli/service/instance/manifest.d.ts +33 -59
  140. package/dist/cli/service/instance/resolve.d.ts +32 -68
  141. package/dist/cli/service/instance/types.d.ts +12 -29
  142. package/dist/cli/service/launchd.d.ts +29 -60
  143. package/dist/cli/service/platform-detect.d.ts +13 -68
  144. package/dist/cli/service/privilege.d.ts +12 -25
  145. package/dist/cli/service/shutdown-grace.d.ts +11 -20
  146. package/dist/cli/service/systemd.d.ts +7 -20
  147. package/dist/cli/service/types.d.ts +63 -114
  148. package/dist/cli/service/windows-acl.d.ts +14 -21
  149. package/dist/cli/service/windows-env-file.d.ts +10 -28
  150. package/dist/cli/service/windows.d.ts +15 -31
  151. package/dist/cli/webhook-url-notes.d.ts +4 -6
  152. package/dist/cli/wizard/agent-wizard.d.ts +3 -14
  153. package/dist/cli/wizard/orchestrator-wizard.d.ts +12 -37
  154. package/dist/cli/wizard/prompts.d.ts +6 -16
  155. package/dist/cli.js +4553 -6556
  156. package/dist/cluster/cluster-identity.d.ts +20 -32
  157. package/dist/cluster/cluster-settings-reader.d.ts +50 -77
  158. package/dist/cluster/coordinator-credential.d.ts +4 -5
  159. package/dist/cluster/coordinator.d.ts +76 -148
  160. package/dist/cluster/health-api.d.ts +2 -10
  161. package/dist/cluster/index.d.ts +1 -6
  162. package/dist/cluster/instance-heartbeat.d.ts +26 -42
  163. package/dist/cluster/join-client.d.ts +4 -7
  164. package/dist/cluster/join-handler.d.ts +3 -7
  165. package/dist/cluster/join-token.d.ts +13 -27
  166. package/dist/cluster/leader-gated-scheduler.d.ts +8 -17
  167. package/dist/cluster/orphan-recovery.d.ts +13 -35
  168. package/dist/cluster/peer-auth-coordinator.d.ts +2 -8
  169. package/dist/cluster/peer-client.d.ts +46 -76
  170. package/dist/cluster/peer-credentials.d.ts +24 -70
  171. package/dist/cluster/peer-crypto.d.ts +5 -26
  172. package/dist/cluster/peer-forget.d.ts +3 -3
  173. package/dist/cluster/peer-handler.d.ts +47 -71
  174. package/dist/cluster/peer-registry.d.ts +37 -106
  175. package/dist/cluster/plan-headroom-store.d.ts +4 -6
  176. package/dist/cluster/raft-state.d.ts +4 -13
  177. package/dist/cluster/raft.d.ts +18 -53
  178. package/dist/cluster/reconcile-identity.d.ts +8 -11
  179. package/dist/cluster/rerouted-job-guard.d.ts +7 -11
  180. package/dist/cluster/verified-issuer.d.ts +2 -3
  181. package/dist/cluster/worker-cluster-settings.d.ts +7 -10
  182. package/dist/cluster/worker-eviction.d.ts +16 -26
  183. package/dist/cold-store/load-access-log-range.d.ts +15 -31
  184. package/dist/cold-store/load-event-log-range.d.ts +13 -27
  185. package/dist/cold-store/load-secret-audit-log-range.d.ts +8 -12
  186. package/dist/cold-store/orchestrator-cold-store.d.ts +8 -28
  187. package/dist/cold-store/tables/access-log.d.ts +12 -14
  188. package/dist/cold-store/tables/event-log.d.ts +4 -8
  189. package/dist/cold-store/tables/execution-runs.d.ts +6 -11
  190. package/dist/cold-store/tables/secret-audit-log.d.ts +5 -12
  191. package/dist/concurrency/dispatch-next-queued.d.ts +13 -25
  192. package/dist/concurrency/queue-manager.d.ts +22 -54
  193. package/dist/concurrency/waiters.d.ts +15 -31
  194. package/dist/config/encryption.d.ts +0 -2
  195. package/dist/config/loader.d.ts +0 -1
  196. package/dist/config/reload.d.ts +0 -6
  197. package/dist/config/resolver.d.ts +0 -4
  198. package/dist/config/shared-store.d.ts +0 -2
  199. package/dist/config/types.d.ts +0 -3
  200. package/dist/content-requirements-cache.d.ts +0 -1
  201. package/dist/contexts/context-store.d.ts +0 -2
  202. package/dist/contexts/held-runs.d.ts +0 -2
  203. package/dist/contexts/variable-store.d.ts +0 -2
  204. package/dist/dashboard/attestation-filters.d.ts +13 -18
  205. package/dist/dashboard/handler.d.ts +143 -307
  206. package/dist/db/client.d.ts +2 -6
  207. package/dist/db/deferred-index-build.d.ts +8 -10
  208. package/dist/db/deferred-indexes.d.ts +15 -22
  209. package/dist/db/migration-provider.d.ts +2 -4
  210. package/dist/db/migration-test-harness.d.ts +26 -44
  211. package/dist/db/migrations/001_initial.d.ts +1 -4
  212. package/dist/db/migrations/002_config_versions_key_version.d.ts +5 -8
  213. package/dist/db/migrations/003_access_log.d.ts +10 -15
  214. package/dist/db/migrations/004_rename_bundle_to_source.d.ts +7 -9
  215. package/dist/db/migrations/005_cold_store_chunk_counter.d.ts +4 -10
  216. package/dist/db/migrations/006_runs_jobs_steps_archived_at.d.ts +9 -29
  217. package/dist/db/migrations/007_audit_logs_archived_at.d.ts +11 -30
  218. package/dist/db/migrations/008_event_log_archived_at.d.ts +14 -29
  219. package/dist/db/migrations/009_access_log_trigram.d.ts +4 -8
  220. package/dist/db/migrations/010_cold_store_chunks.d.ts +16 -26
  221. package/dist/db/migrations/011_drop_source_secrets_notify.d.ts +6 -14
  222. package/dist/db/migrations/012_peer_credentials_active_uniq.d.ts +9 -14
  223. package/dist/db/migrations/013_execution_log_bytes.d.ts +5 -13
  224. package/dist/db/migrations/014_kici_events_lease_retry.d.ts +14 -27
  225. package/dist/db/migrations/015_org_settings_customer_scoped.d.ts +12 -17
  226. package/dist/db/migrations/016_org_settings_allow_http_npm.d.ts +4 -6
  227. package/dist/db/migrations/019_generic_sources_change_notify.d.ts +13 -17
  228. package/dist/db/migrations/020_org_settings_dashboard_write_policy.d.ts +5 -8
  229. package/dist/db/migrations/021_check_run_tracking.d.ts +5 -17
  230. package/dist/db/migrations/022_scaler_manager_state.d.ts +10 -20
  231. package/dist/db/migrations/023_dispatch_queue_recovery_deadline.d.ts +7 -18
  232. package/dist/db/migrations/024_dispatch_queue_provisioning_error.d.ts +5 -11
  233. package/dist/db/migrations/025_init_failure.d.ts +5 -9
  234. package/dist/db/migrations/027_workflow_timeout.d.ts +5 -13
  235. package/dist/db/migrations/029_dispatch_queue_attempts.d.ts +4 -10
  236. package/dist/db/migrations/030_held_runs_env_set_null.d.ts +5 -6
  237. package/dist/db/migrations/031_dispatch_queue_ack_deadline.d.ts +6 -12
  238. package/dist/db/migrations/033_org_settings_approval.d.ts +7 -10
  239. package/dist/db/migrations/034_held_runs_generalize.d.ts +10 -18
  240. package/dist/db/migrations/035_pending_workflow_contexts.d.ts +4 -5
  241. package/dist/db/migrations/037_generic_sources_provider_type_local.d.ts +5 -6
  242. package/dist/db/migrations/038_remote_sources.d.ts +5 -7
  243. package/dist/db/migrations/045_host_properties.d.ts +5 -9
  244. package/dist/db/migrations/046_join_token_consumed_by_instance.d.ts +5 -10
  245. package/dist/db/migrations/048_host_reboot_pending.d.ts +6 -12
  246. package/dist/db/migrations/049_held_runs_payload.d.ts +3 -8
  247. package/dist/db/migrations/050_sources_slug.d.ts +4 -5
  248. package/dist/db/migrations/051_binding_host_pattern.d.ts +7 -10
  249. package/dist/db/migrations/052_host_reach_metadata.d.ts +8 -11
  250. package/dist/db/migrations/053_agent_token_single_use.d.ts +6 -10
  251. package/dist/db/migrations/054_local_working_tree.d.ts +4 -6
  252. package/dist/db/migrations/055_agent_token_mandatory_labels.d.ts +6 -11
  253. package/dist/db/migrations/056_execution_jobs_environments.d.ts +4 -11
  254. package/dist/db/migrations/064_execution_jobs_skipped_environments.d.ts +6 -9
  255. package/dist/db/migrations/065_pending_attestations.d.ts +9 -12
  256. package/dist/db/migrations/066_pending_attestations_rejected.d.ts +4 -7
  257. package/dist/db/migrations/068_request_idempotency.d.ts +7 -9
  258. package/dist/db/migrations/069_reroute_tunables.d.ts +6 -14
  259. package/dist/db/migrations/071_batch_accumulation.d.ts +8 -16
  260. package/dist/db/migrations/072_dispatch_queue_run_id_index.d.ts +3 -9
  261. package/dist/db/migrations/074_normalize_zero_concurrency_limit.d.ts +4 -7
  262. package/dist/db/migrations/076_artifacts.d.ts +7 -8
  263. package/dist/db/migrations/078_org_settings_backup_staleness.d.ts +3 -6
  264. package/dist/db/migrations/079_org_settings_scaler_spawn_timeout.d.ts +4 -8
  265. package/dist/db/migrations/080_cluster_settings.d.ts +6 -10
  266. package/dist/db/migrations/081_org_settings_queue_timeout.d.ts +7 -11
  267. package/dist/db/migrations/082_host_s3_reachable.d.ts +5 -10
  268. package/dist/db/migrations/083_org_settings_artifact_caps.d.ts +6 -8
  269. package/dist/db/migrations/084_orchestrator_signing_keys.d.ts +8 -13
  270. package/dist/db/migrations/085_cluster_settings_reroute_flap_grace_ms.d.ts +4 -5
  271. package/dist/db/migrations/086_cluster_settings_max_fanout_hosts.d.ts +3 -3
  272. package/dist/db/migrations/087_cluster_settings_event_router_rate_limit.d.ts +5 -3
  273. package/dist/db/migrations/088_cluster_settings_cache_max_tarball_bytes.d.ts +3 -3
  274. package/dist/db/migrations/089_cluster_settings_cache_ttl_days.d.ts +3 -5
  275. package/dist/db/migrations/090_cluster_settings_concurrency_wait_timeout_ms.d.ts +3 -6
  276. package/dist/db/migrations/091_cluster_settings_agent_token_ttl_ms.d.ts +3 -6
  277. package/dist/db/migrations/092_cluster_settings_version.d.ts +5 -9
  278. package/dist/db/migrations/094_dashboard_encryption_keys.d.ts +9 -14
  279. package/dist/db/migrations/095_dashboard_write_policy_tristate.d.ts +2 -6
  280. package/dist/db/migrations/099_cluster_settings_dashboard_verified_issuer.d.ts +8 -11
  281. package/dist/db/migrations/101_contexts_hold_expiry_drop_default.d.ts +9 -15
  282. package/dist/db/migrations/102_dispatch_queue_agent_id.d.ts +6 -11
  283. package/dist/db/migrations/103_cluster_settings_ownership_db_check_timeout_ms.d.ts +6 -12
  284. package/dist/db/migrations/104_check_run_terminal_sent.d.ts +8 -14
  285. package/dist/db/migrations/105_org_trust_policy.d.ts +5 -7
  286. package/dist/db/migrations/106_cluster_settings_check_run_tracking_ttl_days.d.ts +4 -12
  287. package/dist/db/migrations/107_check_run_tracking_updated_at_index.d.ts +8 -14
  288. package/dist/db/migrations/113_execution_runs_workflow_repo_index.d.ts +10 -19
  289. package/dist/db/migrations/115_global_workflows_cluster_switch.d.ts +2 -3
  290. package/dist/db/migrations/121_org_settings_allow_untrusted_dockerfile_builds.d.ts +7 -8
  291. package/dist/db/migrations/122_host_roster_scaler_managed.d.ts +9 -16
  292. package/dist/db/migrations/123_workflow_registrations_default_branch.d.ts +8 -16
  293. package/dist/db/migrations/125_org_trust_directory.d.ts +13 -21
  294. package/dist/db/migrations/126_held_runs_posted_pending_check.d.ts +10 -16
  295. package/dist/db/migrations/127_org_trust_policy_approval_expiry_seconds.d.ts +8 -16
  296. package/dist/db/migrations/128_execution_runs_global_eval_round.d.ts +6 -11
  297. package/dist/db/migrations/129_execution_runs_dispatch_routing_key.d.ts +8 -11
  298. package/dist/db/migrations/130_scaler_provision_outcomes.d.ts +13 -20
  299. package/dist/db/migrations/131_execution_jobs_git_credentials.d.ts +12 -22
  300. package/dist/db/migrations/132_execution_runs_trigger_event.d.ts +6 -16
  301. package/dist/db/migrations/133_dispatch_queue_source_tar_digest.d.ts +6 -9
  302. package/dist/db/migrations/140_dispatch_queue_canonical_labels.d.ts +22 -40
  303. package/dist/db/migrations/143_execution_jobs_precursor_result.d.ts +9 -18
  304. package/dist/db/migrations/144_execution_runs_registration_window.d.ts +12 -20
  305. package/dist/db/migrations/147_sealed_job_secrets.d.ts +5 -9
  306. package/dist/db/migrations/148_cluster_settings_sealed_secrets_retry_backoff_ms.d.ts +7 -13
  307. package/dist/db/migrations/150_execution_runs_status_epoch.d.ts +6 -11
  308. package/dist/db/migrations/152_org_settings_cache_upload_settle_timeout.d.ts +5 -8
  309. package/dist/db/migrations/157_host_roster_identity_source.d.ts +9 -13
  310. package/dist/db/migrations/159_org_id_widen_to_source_width.d.ts +5 -0
  311. package/dist/db/migrator.d.ts +17 -25
  312. package/dist/db/notify-listener.d.ts +8 -11
  313. package/dist/db/repos/dashboard-encryption-keys-repo.d.ts +9 -10
  314. package/dist/db/synthetic-job-ids.d.ts +6 -8
  315. package/dist/db/types.d.ts +531 -1058
  316. package/dist/deployment/deployment-identity.d.ts +42 -60
  317. package/dist/diagnostics/bundle-reader.d.ts +3 -15
  318. package/dist/diagnostics/bundle-writer.d.ts +5 -12
  319. package/dist/diagnostics/checks/agents.d.ts +1 -7
  320. package/dist/diagnostics/checks/backup.d.ts +7 -10
  321. package/dist/diagnostics/checks/certs.d.ts +1 -6
  322. package/dist/diagnostics/checks/config.d.ts +2 -5
  323. package/dist/diagnostics/checks/db.d.ts +1 -1
  324. package/dist/diagnostics/checks/disk.d.ts +1 -6
  325. package/dist/diagnostics/checks/firecracker-network.d.ts +3 -3
  326. package/dist/diagnostics/checks/index.d.ts +0 -6
  327. package/dist/diagnostics/checks/retention.d.ts +3 -6
  328. package/dist/diagnostics/checks/scaler.d.ts +3 -7
  329. package/dist/diagnostics/checks/ws.d.ts +1 -6
  330. package/dist/diagnostics/fleet-collector.d.ts +11 -14
  331. package/dist/diagnostics/fleet-constants.d.ts +1 -3
  332. package/dist/diagnostics/fleet-selection.d.ts +3 -7
  333. package/dist/diagnostics/fleet-topology.d.ts +2 -6
  334. package/dist/diagnostics/fleet-wiring.d.ts +6 -12
  335. package/dist/diagnostics/index.d.ts +0 -5
  336. package/dist/diagnostics/runner.d.ts +4 -8
  337. package/dist/diagnostics/types.d.ts +10 -21
  338. package/dist/drain/drain-controller.d.ts +16 -22
  339. package/dist/entry-helpers.d.ts +0 -2
  340. package/dist/events/circuit-breaker.d.ts +0 -3
  341. package/dist/events/event-router.d.ts +1 -2
  342. package/dist/events/event-store.d.ts +0 -9
  343. package/dist/events/trust-store.d.ts +0 -3
  344. package/dist/events/types.d.ts +0 -9
  345. package/dist/firecracker/host-network.d.ts +24 -30
  346. package/dist/firecracker/net-interfaces.d.ts +4 -7
  347. package/dist/helpers/rate-limiter.d.ts +0 -1
  348. package/dist/index.js +251 -486
  349. package/dist/metrics/agent-metrics-aggregator.d.ts +24 -67
  350. package/dist/metrics/metrics-reporter.d.ts +4 -13
  351. package/dist/metrics/prometheus.d.ts +47 -81
  352. package/dist/orchestrator-core.d.ts +3 -7
  353. package/dist/pipeline/content-filter.d.ts +20 -30
  354. package/dist/pipeline/cross-source-tally.d.ts +7 -8
  355. package/dist/pipeline/decorating-secret-resolver.d.ts +4 -8
  356. package/dist/pipeline/degraded-reason.d.ts +3 -4
  357. package/dist/pipeline/direct-ingress-deps.d.ts +15 -24
  358. package/dist/pipeline/dispatch-matched-workflow.d.ts +289 -525
  359. package/dist/pipeline/flatten-lock-steps.d.ts +4 -6
  360. package/dist/pipeline/global-dispatch-identity.d.ts +5 -10
  361. package/dist/pipeline/global-eval-round.d.ts +65 -151
  362. package/dist/pipeline/global-round-hold.d.ts +5 -7
  363. package/dist/pipeline/held-context-data.d.ts +5 -8
  364. package/dist/pipeline/held-round-registrations.d.ts +8 -11
  365. package/dist/pipeline/install-gate.d.ts +15 -25
  366. package/dist/pipeline/install-secrets-resolver.d.ts +24 -34
  367. package/dist/pipeline/internal-event-pipeline.d.ts +45 -62
  368. package/dist/pipeline/invoke-gate.d.ts +27 -40
  369. package/dist/pipeline/job-contexts.d.ts +19 -36
  370. package/dist/pipeline/manual-schedule.d.ts +7 -13
  371. package/dist/pipeline/needs-scheduler.d.ts +29 -46
  372. package/dist/pipeline/pending-workflow-context.d.ts +14 -28
  373. package/dist/pipeline/process-webhook.d.ts +88 -123
  374. package/dist/pipeline/processor.d.ts +209 -355
  375. package/dist/pipeline/remote-source-store.d.ts +5 -8
  376. package/dist/pipeline/request-idempotency.d.ts +10 -13
  377. package/dist/pipeline/rerun-global.d.ts +6 -7
  378. package/dist/pipeline/rerun-inputs.d.ts +23 -37
  379. package/dist/pipeline/rerun.d.ts +34 -57
  380. package/dist/pipeline/resolve-sandbox-grant.d.ts +5 -10
  381. package/dist/pipeline/resume-workflow.d.ts +28 -34
  382. package/dist/pipeline/route-or-dispatch-jobs.d.ts +19 -30
  383. package/dist/pipeline/run-event-context.d.ts +16 -21
  384. package/dist/pipeline/sandbox-allowlist-reader.d.ts +7 -11
  385. package/dist/pipeline/security-hold-check.d.ts +56 -80
  386. package/dist/pipeline/test-pipeline.d.ts +21 -46
  387. package/dist/pipeline/unbound-job-context.d.ts +6 -9
  388. package/dist/pipeline/undispatched-hold-checks.d.ts +4 -6
  389. package/dist/pipeline/wave-scheduler.d.ts +15 -22
  390. package/dist/pipeline/webhook-payload-store.d.ts +3 -10
  391. package/dist/pipeline/workflow-lock-at-sha.d.ts +14 -20
  392. package/dist/policy/dashboard-write-policy.d.ts +1 -1
  393. package/dist/provider-registry.d.ts +0 -19
  394. package/dist/providers/generic/index.d.ts +0 -2
  395. package/dist/providers/generic/normalizer.d.ts +0 -1
  396. package/dist/providers/generic/verification.d.ts +0 -8
  397. package/dist/providers/github/auth.d.ts +0 -1
  398. package/dist/providers/github/changed-files.d.ts +2 -1
  399. package/dist/providers/github/file-contents.d.ts +0 -4
  400. package/dist/providers/github/lock-file.d.ts +0 -4
  401. package/dist/providers/github/normalizer.d.ts +1 -9
  402. package/dist/providers/local/index.d.ts +1 -2
  403. package/dist/providers/local/lock-file-fetcher.d.ts +0 -4
  404. package/dist/providers/local/normalizer.d.ts +0 -10
  405. package/dist/providers/local/repo-url-builder.d.ts +0 -2
  406. package/dist/providers/universal-git/clone-token.d.ts +0 -2
  407. package/dist/providers/universal-git/lock-file.d.ts +0 -2
  408. package/dist/providers/universal-git/normalizer.d.ts +0 -2
  409. package/dist/queue/bootstrap.d.ts +3 -15
  410. package/dist/queue/cleanup.d.ts +29 -50
  411. package/dist/queue/depth-refresher.d.ts +21 -39
  412. package/dist/queue/job-queue.d.ts +112 -163
  413. package/dist/queue/retention.d.ts +23 -36
  414. package/dist/queue/scheduled-job.d.ts +17 -39
  415. package/dist/queue/terminalize-unroutable.d.ts +21 -18
  416. package/dist/queue/unroutable-probe.d.ts +15 -22
  417. package/dist/registration/extractor.d.ts +0 -6
  418. package/dist/registration/registration-index.d.ts +0 -6
  419. package/dist/registration/registration-store.d.ts +0 -9
  420. package/dist/reporting/agent-failure-category.d.ts +3 -5
  421. package/dist/reporting/check-run-reporter.d.ts +142 -316
  422. package/dist/reporting/check-run-summary.d.ts +16 -35
  423. package/dist/reporting/check-run-tracking-store.d.ts +54 -107
  424. package/dist/reporting/execution-tracker.d.ts +356 -526
  425. package/dist/reporting/fs-log-storage.d.ts +31 -33
  426. package/dist/reporting/job-check-run-completion.d.ts +14 -22
  427. package/dist/reporting/log-chunk-sink.d.ts +11 -16
  428. package/dist/reporting/log-pull-handler.d.ts +4 -13
  429. package/dist/reporting/log-retention.d.ts +3 -4
  430. package/dist/reporting/log-storage.d.ts +24 -52
  431. package/dist/reporting/log-writer.d.ts +28 -56
  432. package/dist/reporting/peer-log-normalize.d.ts +4 -8
  433. package/dist/reporting/run-aggregator.d.ts +6 -11
  434. package/dist/reporting/run-summary-fields.d.ts +5 -8
  435. package/dist/reporting/s3-log-storage.d.ts +31 -50
  436. package/dist/reporting/step-display-order.d.ts +11 -21
  437. package/dist/reporting/step-log-buffer.d.ts +7 -28
  438. package/dist/reporting/step-log-reader.d.ts +11 -19
  439. package/dist/routes/admin-access-log.d.ts +4 -6
  440. package/dist/routes/admin-auth.d.ts +19 -38
  441. package/dist/routes/admin-backends.d.ts +3 -14
  442. package/dist/routes/admin-cluster-name.d.ts +13 -21
  443. package/dist/routes/admin-cluster-settings.d.ts +6 -9
  444. package/dist/routes/admin-config.d.ts +7 -19
  445. package/dist/routes/admin-contexts.d.ts +6 -13
  446. package/dist/routes/admin-db.d.ts +5 -10
  447. package/dist/routes/admin-event-dlq.d.ts +3 -5
  448. package/dist/routes/admin-event-inspect.d.ts +3 -4
  449. package/dist/routes/admin-event-log.d.ts +3 -7
  450. package/dist/routes/admin-events.d.ts +16 -30
  451. package/dist/routes/admin-held-runs.d.ts +52 -76
  452. package/dist/routes/admin-maintenance.d.ts +3 -6
  453. package/dist/routes/admin-orchestrator-drain.d.ts +0 -2
  454. package/dist/routes/admin-org-settings.d.ts +8 -14
  455. package/dist/routes/admin-peer-forget.d.ts +0 -1
  456. package/dist/routes/admin-queue-execution.d.ts +3 -3
  457. package/dist/routes/admin-registrations.d.ts +3 -12
  458. package/dist/routes/admin-runs.d.ts +16 -29
  459. package/dist/routes/admin-scaler-orphans.d.ts +4 -4
  460. package/dist/routes/admin-scaler-reload.d.ts +5 -8
  461. package/dist/routes/admin-scheduled-jobs.d.ts +3 -4
  462. package/dist/routes/admin-sources.d.ts +20 -29
  463. package/dist/routes/admin-trust-policy.d.ts +31 -48
  464. package/dist/routes/admin.d.ts +67 -94
  465. package/dist/routes/caller-identity.d.ts +45 -0
  466. package/dist/routes/capabilities.d.ts +8 -13
  467. package/dist/routes/diagnostics.d.ts +2 -10
  468. package/dist/routes/fleet.d.ts +1 -2
  469. package/dist/routes/github-webhook.d.ts +2 -5
  470. package/dist/routes/health.d.ts +6 -19
  471. package/dist/routes/provenance-oidc.d.ts +3 -3
  472. package/dist/routes/test-direct.d.ts +46 -0
  473. package/dist/routes/uploads.d.ts +14 -19
  474. package/dist/routes/webhooks.d.ts +2 -24
  475. package/dist/scaler/agent-binary-location.d.ts +77 -0
  476. package/dist/scaler/agent-fit.d.ts +2 -2
  477. package/dist/scaler/backend-factory.d.ts +10 -18
  478. package/dist/scaler/bare-metal-backend.d.ts +28 -45
  479. package/dist/scaler/claim-store.d.ts +20 -43
  480. package/dist/scaler/config.d.ts +10 -37
  481. package/dist/scaler/container-backend.d.ts +22 -35
  482. package/dist/scaler/container-hostconfig.d.ts +10 -16
  483. package/dist/scaler/container-routing.d.ts +10 -13
  484. package/dist/scaler/disk-guard.d.ts +2 -3
  485. package/dist/scaler/event-backend.d.ts +33 -58
  486. package/dist/scaler/event-provision-reaper.d.ts +61 -84
  487. package/dist/scaler/failure-tracker.d.ts +6 -13
  488. package/dist/scaler/file-tail.d.ts +6 -19
  489. package/dist/scaler/firecracker-api.d.ts +19 -63
  490. package/dist/scaler/firecracker-backend.d.ts +142 -258
  491. package/dist/scaler/host-access.d.ts +7 -10
  492. package/dist/scaler/index.d.ts +3 -6
  493. package/dist/scaler/ip-allocator.d.ts +30 -82
  494. package/dist/scaler/label-matcher.d.ts +14 -35
  495. package/dist/scaler/log-forwarder.d.ts +6 -25
  496. package/dist/scaler/machine-ledger.d.ts +39 -86
  497. package/dist/scaler/manager.d.ts +517 -787
  498. package/dist/scaler/pending-scale-sweeper.d.ts +8 -11
  499. package/dist/scaler/reap-orphans.d.ts +8 -12
  500. package/dist/scaler/resolve-container-auth.d.ts +12 -21
  501. package/dist/scaler/scaler-events.d.ts +2 -5
  502. package/dist/scaler/scaler-state-store.d.ts +127 -189
  503. package/dist/scaler/spawn-errors.d.ts +1 -4
  504. package/dist/scaler/startup-orphan-sweep.d.ts +3 -4
  505. package/dist/scaler/startup-tools-check.d.ts +21 -0
  506. package/dist/scaler/types.d.ts +142 -255
  507. package/dist/scaler/vm-job-bindings.d.ts +5 -9
  508. package/dist/scaler/warm-pool.d.ts +46 -93
  509. package/dist/secrets/audit-logger.d.ts +0 -3
  510. package/dist/secrets/backend-health.d.ts +0 -9
  511. package/dist/secrets/backend-registry.d.ts +0 -9
  512. package/dist/secrets/cleanup.d.ts +0 -1
  513. package/dist/secrets/config.d.ts +0 -3
  514. package/dist/secrets/ephemeral-keys.d.ts +0 -3
  515. package/dist/secrets/pg-secret-store.d.ts +0 -3
  516. package/dist/secrets/rbac.d.ts +5 -7
  517. package/dist/secrets/routing-key-scope.d.ts +1 -0
  518. package/dist/secrets/secret-output-store.d.ts +0 -6
  519. package/dist/secrets/secret-resolver.d.ts +2 -6
  520. package/dist/secrets/token-manager.d.ts +1 -4
  521. package/dist/secrets/vault-secret-store.d.ts +0 -13
  522. package/dist/security/comment-handler.d.ts +0 -2
  523. package/dist/security/workflow-diff.d.ts +1 -1
  524. package/dist/server.js +49004 -53782
  525. package/dist/sources/source-manager.d.ts +0 -3
  526. package/dist/sources/source-store.d.ts +0 -9
  527. package/dist/sources/source-validator.d.ts +0 -3
  528. package/dist/stale-detector/all-terminal-runs.d.ts +8 -11
  529. package/dist/stale-detector/gate-deadline-detector.d.ts +14 -19
  530. package/dist/stale-detector/reboot-deadline-sweep.d.ts +7 -11
  531. package/dist/stale-detector/stale-dispatch-candidates.d.ts +13 -20
  532. package/dist/stale-detector/stale-run-detector.d.ts +74 -121
  533. package/dist/stale-detector/workflow-deadline-detector.d.ts +15 -43
  534. package/dist/standalone.js +9342 -13077
  535. package/dist/storage/blob-routes.d.ts +10 -19
  536. package/dist/storage/filesystem.d.ts +19 -29
  537. package/dist/storage/index.d.ts +2 -10
  538. package/dist/storage/key-discriminator.d.ts +8 -10
  539. package/dist/storage/loopback-guard.d.ts +27 -45
  540. package/dist/storage/s3.d.ts +26 -53
  541. package/dist/storage/sign-url.d.ts +7 -22
  542. package/dist/storage/types.d.ts +46 -72
  543. package/dist/webhook/event-log.d.ts +0 -2
  544. package/dist/webhook/generic-sources.d.ts +0 -21
  545. package/dist/webhook/ingest-admission.d.ts +0 -1
  546. package/dist/webhook/relay-buffer.d.ts +2 -2
  547. package/dist/webhook/verify-inbound.d.ts +0 -1
  548. package/dist/worker/in-memory-execution-tracker.d.ts +9 -31
  549. package/dist/worker/in-memory-job-queue.d.ts +32 -72
  550. package/dist/worker/peer-outbox.d.ts +5 -8
  551. package/dist/worker/reroute-dispatch.d.ts +4 -6
  552. package/dist/worker/static-agent-token-store.d.ts +6 -27
  553. package/dist/worker/worker-outbox-relay.d.ts +2 -2
  554. package/dist/worker/worker-status.d.ts +3 -20
  555. package/dist/ws/agent-api-registry.d.ts +8 -27
  556. package/dist/ws/agent-handler.d.ts +63 -110
  557. package/dist/ws/agent-heartbeat.d.ts +5 -12
  558. package/dist/ws/bringup-api.d.ts +31 -38
  559. package/dist/ws/dashboard-backends-handler.d.ts +12 -22
  560. package/dist/ws/dashboard-context-handler.d.ts +37 -80
  561. package/dist/ws/dashboard-diagnostics-handler.d.ts +8 -25
  562. package/dist/ws/dashboard-dispatch-guard.d.ts +3 -4
  563. package/dist/ws/dashboard-fleet-handler.d.ts +9 -13
  564. package/dist/ws/dashboard-fleet-write-handler.d.ts +7 -21
  565. package/dist/ws/dashboard-global-workflows-handler.d.ts +22 -44
  566. package/dist/ws/dashboard-registrations-handler.d.ts +10 -27
  567. package/dist/ws/dashboard-run-state-handler.d.ts +10 -13
  568. package/dist/ws/event-buffer.d.ts +3 -6
  569. package/dist/ws/execution-status-frame.d.ts +7 -10
  570. package/dist/ws/failure-messages.d.ts +9 -14
  571. package/dist/ws/fleet-agent-collector.d.ts +6 -12
  572. package/dist/ws/fleet-runs-on-all.d.ts +1 -6
  573. package/dist/ws/git-credential-relay.d.ts +12 -20
  574. package/dist/ws/inventory-api.d.ts +4 -5
  575. package/dist/ws/observer-registry.d.ts +14 -59
  576. package/dist/ws/ownership-gate.d.ts +1 -1
  577. package/dist/ws/platform-client.d.ts +21 -30
  578. package/dist/ws/replay-chunker.d.ts +7 -10
  579. package/dist/ws/server-options.d.ts +5 -9
  580. package/dist/ws/test-relay-handlers.d.ts +46 -63
  581. package/installer-image-digests.json +3 -3
  582. package/package.json +6 -5
  583. package/sbom.spdx.json +53 -53
  584. package/dist/__test-helpers__/fake-scaler-state-store.d.ts +0 -24
  585. package/dist/__test-helpers__/mock-db-query.d.ts +0 -129
  586. package/dist/__test-helpers__/mock-db.d.ts +0 -134
  587. package/dist/__test-helpers__/mock-ws.d.ts +0 -13
  588. package/dist/__test-helpers__/peer-mutual-auth.d.ts +0 -42
  589. package/dist/__test-helpers__/scratch-db.d.ts +0 -16
  590. package/dist/__test-helpers__/test-db.d.ts +0 -34
  591. package/dist/pipeline/dispatch-matched-workflow.test-helpers.d.ts +0 -121
  592. package/dist/scaler/container-spawn.d.ts +0 -28
@@ -1,10 +1,8 @@
1
1
  /**
2
- * ScalerManager: central coordinator for all scaler backends.
3
- *
4
- * Sits between the orchestrator's Dispatcher and individual scaler backends.
5
- * Receives "no agent available" signals, routes to the correct backend,
6
- * enforces global limits, tracks spawning agents to prevent over-provisioning,
7
- * and manages the agent lifecycle from spawn to destroy.
2
+ * ScalerManager: coordinator between the Dispatcher and the scaler backends.
3
+ * Routes "no agent available" signals to a backend, enforces global limits,
4
+ * tracks spawning agents against over-provisioning, and owns the agent
5
+ * lifecycle from spawn to destroy.
8
6
  */
9
7
  import { ScalerVmTracker } from '@kici-dev/engine';
10
8
  import type { ResourceRequest, ScalerCapacitySummary, ScalerLiveVm, ScalerReloadPlan, ScalerVmStopResult } from '@kici-dev/engine';
@@ -14,17 +12,15 @@ import type { ScalerEventEmitterLike } from './event-backend.js';
14
12
  import type { ClaimStore, ClaimedCredentials } from './claim-store.js';
15
13
  /**
16
14
  * How long a graceful shutdown waits for the scaler backends to stop their
17
- * agents. It is the first window inside the coordinator's 30 s shutdown limit,
18
- * and every later step (the HTTP stop grace, the database drain) still has to
19
- * fit after it.
15
+ * agents: the first window inside the coordinator's 30 s shutdown limit, so the
16
+ * later steps (HTTP stop grace, database drain) still fit after it.
20
17
  *
21
- * A container stop normally takes at most its own 10 s stop timeout, but the
22
- * container runtime can take far longer: a Docker daemon that misses a
23
- * container's exit waits out the stop timeout and then another 10 s for its
24
- * kill, and one such container held the whole step for 29.7 s. Past this
25
- * window the orchestrator logs each backend still stopping and continues. The
26
- * runtime finishes the stop on its own, and the next start's orphan cleanup
27
- * removes any agent container that is still there.
18
+ * A container stop normally takes at most its own 10 s stop timeout, but a
19
+ * Docker daemon that misses a container's exit waits out that timeout plus
20
+ * another 10 s for its kill; one such container held the step for 29.7 s. Past
21
+ * this window the orchestrator logs each backend still stopping and continues:
22
+ * the runtime finishes the stop, and the next start's orphan cleanup removes
23
+ * any agent container left behind.
28
24
  */
29
25
  export declare const SCALER_SHUTDOWN_GRACE_MS = 15000;
30
26
  import type { BackendFailureSummary } from './failure-tracker.js';
@@ -33,11 +29,10 @@ import type { ScalerStateStore, ScalerStateRecovery, ReapCandidate } from './sca
33
29
  /** The `skipped` reason for a job whose agent is already provisioning or registered. */
34
30
  export declare const PROVISION_IN_FLIGHT = "provision-in-flight";
35
31
  /**
36
- * Raised when a registering agent presents a scaler-minted id that the
37
- * orchestrator cannot back with a spawn record — either because the lookup
38
- * could not be completed (`cause` set) or because no record exists at all
39
- * (`cause` omitted). The WS handler answers it by refusing the registration, so
40
- * the agent reconnects rather than registering with no `mandatoryLabels` gate.
32
+ * Raised when a registering agent presents a scaler-minted id with no spawn
33
+ * record behind it: the lookup failed (`cause` set) or no record exists
34
+ * (`cause` omitted). The WS handler refuses the registration, so the agent
35
+ * reconnects rather than registering with no `mandatoryLabels` gate.
41
36
  */
42
37
  export declare class ScalerAdoptionLookupError extends Error {
43
38
  readonly agentId: string;
@@ -45,17 +40,14 @@ export declare class ScalerAdoptionLookupError extends Error {
45
40
  constructor(agentId: string, cause?: unknown);
46
41
  }
47
42
  /**
48
- * Counting semaphore that throttles the number of concurrent async operations.
49
- *
50
- * Used to bound how many `backend.spawn` provisioning operations (image pull +
43
+ * Counting semaphore bounding how many `backend.spawn` operations (image pull +
51
44
  * container create + start) run at once per backend, so a burst of in-cap jobs
52
45
  * cannot storm the container socket / registry.
53
46
  *
54
- * Slots are handed directly to the next waiter on release (rather than bumping a
47
+ * Release hands the slot directly to the next waiter (rather than bumping a
55
48
  * counter the woken task re-decrements), so a task arriving in the microtask gap
56
- * between a release and a woken waiter cannot jump the queue and transiently
57
- * exceed the cap. The slot is always released in `run`'s `finally`, so a failed
58
- * or throwing operation never leaks a slot and wedges the queue.
49
+ * cannot jump the queue and transiently exceed the cap. `run` releases in
50
+ * `finally`, so a throwing operation never leaks a slot and wedges the queue.
59
51
  */
60
52
  export declare class SpawnSemaphore {
61
53
  private available;
@@ -66,13 +58,7 @@ export declare class SpawnSemaphore {
66
58
  run<T>(fn: () => Promise<T>): Promise<T>;
67
59
  }
68
60
  /**
69
- * Resolved per-job resource amounts (cpus + bytes) for both `requests` and
70
- * `limits`. The scaler manager produces this from the job's declared resources
71
- * combined with the scaler's `defaults.resources`, applying the request<->limit
72
- * mirroring rule. Caps aggregate `requests`; backends use `limits`.
73
- */
74
- /**
75
- * Resolve the orchestrator WebSocket URL a scaler-spawned agent should dial.
61
+ * Resolve the orchestrator WebSocket URL a scaler-spawned agent dials.
76
62
  *
77
63
  * 1. Per-scaler `orchestratorUrl` (scalers.yaml) wins — required for container
78
64
  * agents (host.docker.internal / LAN IP) and Firecracker VMs (bridge gateway
@@ -84,6 +70,11 @@ export declare class SpawnSemaphore {
84
70
  * agents at 8080 leaves them unable to reach the orchestrator.
85
71
  */
86
72
  export declare function resolveScalerOrchestratorUrl(configUrl: string | undefined, envUrl: string | undefined, port: string | number | undefined): string;
73
+ /**
74
+ * Resolved per-job `requests` and `limits` (cpus + bytes): the job's declared
75
+ * resources over the scaler's `defaults.resources`, with request<->limit
76
+ * mirroring applied. Caps aggregate `requests`; backends use `limits`.
77
+ */
87
78
  export interface ResolvedResources {
88
79
  requests: {
89
80
  cpus: number;
@@ -111,9 +102,7 @@ export declare function buildScalerUsageRows(perScalerUsage: ReadonlyMap<string,
111
102
  cpus: number;
112
103
  memBytes: number;
113
104
  }>;
114
- /**
115
- * Status summary for metrics and health endpoints.
116
- */
105
+ /** Status summary for metrics and health endpoints. */
117
106
  export interface ScalerStatus {
118
107
  globalMaxAgents: number;
119
108
  globalActiveCount: number;
@@ -206,16 +195,13 @@ export interface ScalerManagerDeps {
206
195
  backend: ScalerBackend;
207
196
  }>;
208
197
  /**
209
- * Read-only view of the agent registry, for the warm pool's deficit
210
- * calculation. Structural on purpose — the scaler must not depend on the
211
- * agent layer's concrete class.
198
+ * Read-only agent-registry view for the warm pool's deficit. Structural on
199
+ * purpose: the scaler must not depend on the agent layer's concrete class.
212
200
  *
213
- * Both hosts pass one, and a host that skips it does not get an inert warm
214
- * pool: the readiness count falls back to 0, so a configured pool never sees
215
- * the agents it started and keeps filling until `warmCapacityRemaining`
216
- * clamps it at `maxAgents` — starving the job-bound spawns that share the
217
- * cap. Optional only because the unit tests drive the spawn path without a
218
- * registry.
201
+ * Both hosts pass one. Without it the readiness count falls back to 0, so a
202
+ * configured pool never sees its own agents and fills until
203
+ * `warmCapacityRemaining` clamps it at `maxAgents`, starving the job-bound
204
+ * spawns that share the cap. Optional only for unit tests.
219
205
  */
220
206
  agentRegistry?: {
221
207
  findAvailable(requiredLabels: string[]): Array<{
@@ -238,10 +224,7 @@ export interface ScalerManagerDeps {
238
224
  instanceId: string;
239
225
  /** Callback for relaying scaler events with runId/jobId context. */
240
226
  onScalerEvent?: (runId: string, jobId: string, event: ScalerEvent) => void;
241
- /**
242
- * Optional DB-backed state store. Tests omit it; production wires it
243
- * up via the orchestrator-core bootstrap.
244
- */
227
+ /** DB-backed state store, wired by the orchestrator-core bootstrap; tests omit it. */
245
228
  stateStore?: ScalerStateStore;
246
229
  /**
247
230
  * Late-bound reserved-event emitter, resolved per call because the event
@@ -256,10 +239,9 @@ export interface ScalerManagerDeps {
256
239
  */
257
240
  claimStore?: ClaimStore;
258
241
  /**
259
- * Optional machine-ledger options. When `machinePools` are configured,
260
- * the manager initializes a `MachineLedger` keyed off this directory and
261
- * the orchestrator's instance id; reservations sum across orchestrators
262
- * on the same host.
242
+ * Machine-ledger options. With `machinePools` configured, the manager builds a
243
+ * `MachineLedger` keyed off this directory and the instance id; reservations
244
+ * sum across orchestrators on the same host.
263
245
  */
264
246
  machineLedger?: {
265
247
  /** Override the on-disk ledger directory. Falls back to `KICI_MACHINE_LEDGER_DIR`. */
@@ -277,11 +259,10 @@ export interface ScalerManagerDeps {
277
259
  */
278
260
  spawnTimeoutMs: number;
279
261
  /**
280
- * Optional per-org resolver for the spawn deadline: given the job's org
281
- * (jobConfig.cacheOrgId, undefined for warm-pool spawns), returns the
282
- * effective timeout, falling back to the cluster default internally. Wired
283
- * in orchestrator-core (has DB); omitted by the worker + unit tests → the
284
- * cluster default spawnTimeoutMs is used.
262
+ * Per-org spawn-deadline resolver, keyed by the job's org
263
+ * (jobConfig.cacheOrgId, undefined for warm-pool spawns); falls back to the
264
+ * cluster default internally. Wired in orchestrator-core (has DB); the worker
265
+ * and unit tests omit it and use spawnTimeoutMs.
285
266
  */
286
267
  resolveSpawnTimeoutMs?: (orgId: string | undefined) => Promise<number>;
287
268
  /**
@@ -301,15 +282,15 @@ export interface ScalerManagerDeps {
301
282
  */
302
283
  readJobStatuses?: (jobs: readonly BoundJobRef[]) => Promise<ReadonlyMap<string, string>>;
303
284
  /**
304
- * Constructs a backend for a scaler added by a config reload. Supplied by
305
- * both hosts from the shared backend factory, so a reload can only build
306
- * what startup already exercises. When omitted (unit tests), a reload that
307
- * adds a scaler is rejected rather than silently ignored.
285
+ * Builds a backend for a scaler a config reload adds. Both hosts supply it
286
+ * from the shared backend factory, so a reload builds only what startup
287
+ * exercises. When omitted (unit tests), a reload that adds a scaler is
288
+ * rejected rather than silently ignored.
308
289
  *
309
- * `config` is the config being reloaded, not the boot-time one. The factory
310
- * reads `defaults.resources` and the whole `firecracker` network block off
311
- * it, so an added scaler built from a stale config would allocate IPs from
312
- * the old CIDR and attach TAPs to the old bridge — silently unreachable.
290
+ * `config` is the reloaded config, not the boot-time one: the factory reads
291
+ * `defaults.resources` and the `firecracker` network block off it, and a
292
+ * stale one would allocate IPs from the old CIDR and attach TAPs to the old
293
+ * bridge, silently unreachable.
313
294
  */
314
295
  createBackend?: (entry: ScalerEntry, config: ScalerConfig) => Promise<ScalerBackend | null>;
315
296
  }
@@ -331,17 +312,16 @@ export declare class ScalerManager {
331
312
  private readonly failureTracker;
332
313
  /**
333
314
  * Consecutive provisioning failures per scaler NAME, and the deferral they
334
- * earned: an event scaler's external provision that never registered an
335
- * agent, or a local launch that produced no agent (refused outright, or the
336
- * agent stopped before it registered).
315
+ * earned: an event provision that never registered an agent, or a local
316
+ * launch that produced none (refused, or the agent stopped before
317
+ * registering).
337
318
  *
338
- * Keyed by name rather than by backend type so one failing scaler never
339
- * defers spawns for an unrelated one — two event scalers routinely drive two
340
- * different providers, and an outage at one says nothing about the other.
319
+ * Keyed by name, not backend type, so one failing scaler never defers an
320
+ * unrelated one: two event scalers often drive two different providers.
341
321
  *
342
- * In-memory and leader-local, like `ScalerFailureTracker`: the state resets on
343
- * restart, which is the conservative direction (a fresh coordinator retries
344
- * once immediately rather than inheriting a deferral it never observed).
322
+ * In-memory and leader-local, like `ScalerFailureTracker`: a restart resets
323
+ * it, the conservative direction (a fresh coordinator retries once at once
324
+ * rather than inheriting a deferral it never observed).
345
325
  */
346
326
  private readonly provisionFailures;
347
327
  private readonly resolveProvisionBackoff;
@@ -405,81 +385,54 @@ export declare class ScalerManager {
405
385
  * teardown workflow when the agent it adopted disconnects.
406
386
  */
407
387
  private readonly scalerProvisioningTargets;
408
- /**
409
- * Tracks agents being spawned but not yet registered via WS.
410
- * Keyed by pre-generated agentId.
411
- */
388
+ /** Agents spawned but not yet registered via WS, keyed by pre-generated agentId. */
412
389
  private readonly spawningAgents;
413
- /**
414
- * Maps registered agentId to backendName for lifecycle events.
415
- */
390
+ /** Registered agentId → backendName, for lifecycle events. */
416
391
  private readonly managedAgentIndex;
417
392
  /**
418
- * Agents this instance pre-spawned to wait for work, rather than for a
419
- * specific queued job. The agent registry tracks readiness (`activeJobs`,
420
- * `registeredAt`) and has never tracked provenance — it cannot derive this,
421
- * and the spawning entry that carries it is deleted at registration.
422
- *
423
- * This is not a second copy of the pool's readiness. It is provenance, which
424
- * nothing else knows, and only the reaper reads it: it may destroy an agent
425
- * the pool itself created, never one that registered for a bound job whose
426
- * dispatch has not yet arrived.
393
+ * Agents this instance pre-spawned to wait for work rather than for a queued
394
+ * job. This is provenance, not readiness: the agent registry cannot derive it,
395
+ * and the spawning entry that carries it is deleted at registration. Only the
396
+ * reaper reads it, so it destroys agents the pool created, never one that
397
+ * registered for a bound job whose dispatch has not arrived.
427
398
  */
428
399
  private readonly warmAgents;
429
400
  /**
430
- * Registered agents this instance started for a specific queued job, keyed
431
- * by agent id. The spawning entry that carries the binding is deleted at
432
- * registration, so this is where {@link agentView} reads it: the agent always
433
- * runs its own job, and an agent started inside a job's image runs nothing
434
- * else.
401
+ * Registered agents this instance started for a queued job, keyed by agent
402
+ * id. The spawning entry is deleted at registration, so {@link agentView}
403
+ * reads the binding here: the agent always runs its own job, and one started
404
+ * inside a job's image runs nothing else.
435
405
  */
436
406
  private readonly boundAgents;
437
407
  /** Jobs a `requestScale` call is provisioning right now, before its spawning entry exists. */
438
408
  private readonly jobsBeingProvisioned;
439
409
  /**
440
- * Warm agents whose destroy is issued but has not settled yet.
441
- *
442
- * An agent leaves `warmAgents` only once its destroy resolves, so until then
443
- * `listIdle` keeps returning it and the next reap pass — 30 seconds later,
444
- * or a reload that lowers the target — selects the very same agent again. A
445
- * backend whose teardown outlives a tick would be handed a second destroy for
446
- * an agent it is already destroying, and every extra pass would count another
447
- * `kici_orch_scaler_warm_pool_reaped_total`. One entry per agent, cleared
448
- * when the destroy settles either way.
410
+ * Warm agents whose destroy is issued but not settled. Until it resolves the
411
+ * agent stays in `warmAgents` and `listIdle`, so the next reap pass (30 s
412
+ * later, or a reload lowering the target) would hand the backend a second
413
+ * destroy and count another `kici_orch_scaler_warm_pool_reaped_total`.
414
+ * Cleared when the destroy settles either way.
449
415
  */
450
416
  private readonly warmDestroying;
451
417
  /** Agents this instance adopted from another instance's spawn record. */
452
418
  private readonly adoptedAgents;
453
- /**
454
- * Active log forwarders for scaler-managed agents (container/bare-metal).
455
- * Keyed by agentId. Each forwarder consumes a LogCapture stream.
456
- */
419
+ /** Log forwarders for scaler-managed agents (container/bare-metal), keyed by agentId. */
457
420
  private readonly logForwarders;
458
- /**
459
- * Correlation map: agentId -> { runId, jobId }.
460
- * Populated by correlateAgentToJob() after job dispatch to a scaler-managed agent.
461
- */
421
+ /** agentId → { runId, jobId }, set by correlateAgentToJob() after dispatch. */
462
422
  private readonly agentJobCorrelation;
463
- /**
464
- * Buffer for scaler events emitted before job correlation is established.
465
- * Keyed by agentId. Flushed when correlateAgentToJob() is called.
466
- */
423
+ /** Events emitted before job correlation, by agentId; flushed by correlateAgentToJob(). */
467
424
  private readonly eventBuffer;
468
- /**
469
- * External callback for relaying correlated scaler events (e.g. to execution tracker).
470
- */
425
+ /** Relays correlated scaler events (e.g. to the execution tracker). */
471
426
  private readonly onScalerEvent?;
472
427
  /** Coordinator drain predicate; when true, requestScale() no-ops (no fresh
473
428
  * agents spawned for the held Pending backlog). Defaults to never-draining. */
474
429
  private readonly isDraining;
475
430
  /**
476
- * Optional callback fired (debounced) whenever scaler capacity frees — an
477
- * ephemeral agent released its reservation on disconnect, a spawn failed, or a
478
- * managed agent completed its job. Wired by the composition roots to the
479
- * dispatcher's capacity-freed re-drive so jobs that got an `at-capacity`
480
- * verdict get re-offered to the scaler the moment a slot opens, instead of
481
- * sitting pending until they time out. Assigned after construction because it
482
- * references the dispatcher, which is built after the manager.
431
+ * Fired (debounced) when scaler capacity frees: a reservation released on
432
+ * disconnect, a failed spawn, or a managed agent finishing its job. Wired to
433
+ * the dispatcher's capacity-freed re-drive, so `at-capacity` jobs are
434
+ * re-offered the moment a slot opens instead of pending until timeout.
435
+ * Assigned after construction: the dispatcher is built after the manager.
483
436
  */
484
437
  onCapacityFreed?: () => void;
485
438
  /** Trailing-debounce timer coalescing a burst of releases into one re-drive. */
@@ -493,11 +446,10 @@ export declare class ScalerManager {
493
446
  */
494
447
  private readonly warmPoolCallbacks;
495
448
  /**
496
- * Optional DB-backed state store. When wired (production path), every
497
- * mutation to `spawningAgents` / `agentJobCorrelation` / `reservations`
498
- * is write-through-cached to Postgres so a coord crash mid-spawn no
499
- * longer orphans agents, strands reservations, or loses correlation.
500
- * Unit tests can omit the store and operate from in-memory Maps only.
449
+ * DB-backed state store. When wired (production), every mutation to
450
+ * `spawningAgents` / `agentJobCorrelation` / `reservations` writes through
451
+ * to Postgres, so a coordinator crash mid-spawn does not orphan agents,
452
+ * strand reservations or lose correlation. Unit tests omit it.
501
453
  */
502
454
  private readonly stateStore?;
503
455
  /** Id of the orchestrator instance this manager runs on. */
@@ -518,46 +470,35 @@ export declare class ScalerManager {
518
470
  private readonly resolveSpawnTimeoutMs?;
519
471
  constructor(deps: ScalerManagerDeps);
520
472
  /**
521
- * Pre-spawn one warm agent for a pool's label set.
522
- *
523
- * The pool's counterpart of `requestScale`, with the job term of every
524
- * resolution absent: no bound job, no run, and resources resolved
525
- * from the label set (then the scaler defaults) alone. It takes the same
526
- * reservations through the same {@link reserveForSpawn} helper — a warm
527
- * agent occupies the same cpu and memory as a job-bound one, and on an event
528
- * backend it claims a cluster slot too, so a pool can no longer carry a
529
- * scaler past `maxAgents` fleet-wide.
473
+ * Pre-spawn one warm agent for a pool's label set: `requestScale` with no
474
+ * job term (no bound job, no run, resources from the label set then the
475
+ * scaler defaults). It reserves through the same {@link reserveForSpawn}: a
476
+ * warm agent occupies the same cpu and memory, and on an event backend claims
477
+ * a cluster slot, so a pool cannot carry a scaler past `maxAgents` fleet-wide.
530
478
  *
531
- * A refused reservation is reported back to the pool as a failed spawn, so
532
- * the in-flight slot it holds is released and the deficit is retried on the
533
- * next tick rather than the pool stalling below target.
479
+ * A refused reservation reports a failed spawn to the pool, releasing its
480
+ * in-flight slot so the deficit retries next tick instead of stalling.
534
481
  */
535
482
  private spawnWarmAgent;
536
483
  /**
537
484
  * Whether a warm pool for this label set can count its own agents.
538
485
  *
539
- * The pool measures readiness with `AgentRegistry.findAvailable(labels)`, and
540
- * that query applies the agent's `mandatoryLabels` gate: an agent is dropped
541
- * unless every label in its gate also appears in the query. A scaler-spawned
542
- * agent's gate is `mandatoryGate()`, which unions the configured gate with
543
- * the pool's platform taints. `warmPoolQueryLabels` puts the structured
544
- * taints into the query, so that part of the gate is satisfied by
545
- * construction.
486
+ * Readiness is `AgentRegistry.findAvailable(labels)`, which drops an agent
487
+ * unless its whole `mandatoryLabels` gate appears in the query. The gate is
488
+ * `mandatoryGate()` (configured gate ∪ platform taints), and
489
+ * `warmPoolQueryLabels` puts the taints into the query by construction.
546
490
  *
547
- * That leaves one residual case: a configured `mandatoryLabels` entry absent
548
- * from the label set, which `scaler/config.ts` validation already rejects.
549
- * The check stays as the fail-safe — without it a pool in that state would
550
- * spawn to `maxAgents` with `ready` stuck at 0, starving the on-demand spawns
551
- * that share the cap. A refused pool is inert, exactly as it is before it
552
- * fills at all — never partially filled.
491
+ * The residual case is a configured `mandatoryLabels` entry absent from the
492
+ * label set, which `scaler/config.ts` validation already rejects. This check
493
+ * is the fail-safe: such a pool would spawn to `maxAgents` with `ready` stuck
494
+ * at 0, starving the on-demand spawns sharing the cap. A refused pool is
495
+ * inert, never partially filled.
553
496
  *
554
- * The gate is compared against `queryLabels`: comparing it against
555
- * `declaredLabels` alone would refuse every structurally-tainted pool.
497
+ * The gate is compared against `queryLabels`: against `declaredLabels` alone
498
+ * it would refuse every structurally-tainted pool.
556
499
  */
557
500
  private warmPoolLabelSetFillable;
558
- /**
559
- * Whether this scaler pre-spawned (warm-filled) the agent.
560
- */
501
+ /** Whether this scaler pre-spawned (warm-filled) the agent. */
561
502
  isPrespawnedAgent(agentId: string): boolean;
562
503
  /** Every local Firecracker backend, under its scaler name. */
563
504
  private firecrackerBackends;
@@ -588,99 +529,69 @@ export declare class ScalerManager {
588
529
  /**
589
530
  * Whether a pre-spawned (warm) agent's shape fits the shape a job declares.
590
531
  *
591
- * A warm agent is generic by construction: it was started before the job
592
- * existed, at the pool's declared shape. The shape is applied when the agent
593
- * starts and cannot be changed afterwards — nothing in the agent reads a
594
- * job's `resources`. So a job that needs another shape must get its own
595
- * agent instead of silently running with the pool's.
596
- *
597
- * Returns true for any agent this scaler did not pre-spawn: a job-bound agent
598
- * was started at its job's shape.
599
- *
600
- * Only the fields the job actually declares are compared. `resolveEffective`
601
- * layers a job's declaration over the label set and the scaler defaults field
602
- * by field, and the warm agent's own reservation is what those two layers
603
- * resolve to — so a field the job leaves out already matches whatever the
604
- * agent was started at, and comparing it would refuse a job that fits.
605
- *
606
- * A field the job DOES declare must match exactly, in both directions. A
607
- * bigger ready agent is not a superset: limits are what the kernel enforces
608
- * on the running agent, so a job asking to be capped at 2 cpus, served by an
609
- * agent started at 4, runs uncapped. That is the same wrong execution
610
- * environment as the under-provisioned case, in the other direction, so `>=`
611
- * is not a safe relaxation.
612
- *
613
- * Both sides of the shape are compared, not just `requests`. The two answer
614
- * different questions — `requests` is what the agent bills against the caps,
615
- * `limits` is what the kernel enforces — and a declaration may legally set
616
- * them apart, so a job matched on `requests` alone can still land on an agent
617
- * capped somewhere else entirely.
532
+ * A warm agent starts before the job exists, at the pool's declared shape,
533
+ * and nothing in the agent reads a job's `resources` afterwards. So a job
534
+ * needing another shape gets its own agent instead of silently running with
535
+ * the pool's. True for any agent this scaler did not pre-spawn: a job-bound
536
+ * agent started at its job's shape.
537
+ *
538
+ * Only fields the job declares are compared: `resolveEffective` layers the
539
+ * job over the label set and scaler defaults field by field, so an omitted
540
+ * field already matches the warm agent's reservation.
541
+ *
542
+ * A declared field must match exactly. A bigger agent is not a superset:
543
+ * limits are kernel-enforced, so a job capped at 2 cpus on an agent started
544
+ * at 4 runs uncapped, so `>=` is not a safe relaxation.
545
+ *
546
+ * Both `requests` (billed against the caps) and `limits` (kernel-enforced)
547
+ * are compared: a declaration may set them apart, so matching `requests`
548
+ * alone can land a job on an agent capped elsewhere.
618
549
  */
619
550
  private prespawnedShapeFits;
620
551
  /**
621
552
  * Whether one side (`requests` or `limits`) of a job's declared shape leaves
622
- * the agent's resolved side unchanged.
623
- *
624
- * True when the job declares nothing on this side, and true when `resolved`
625
- * is absent — both mean "this cannot be told", and the predicate's rule is to
626
- * fail open rather than strand a job.
553
+ * the agent's resolved side unchanged. Fails open (true) when the job declares
554
+ * nothing on this side or `resolved` is absent, rather than strand a job.
627
555
  */
628
556
  private shapeSideMatches;
629
557
  /**
630
558
  * Whether a warm pool for this label set has a declared agent shape.
631
559
  *
632
- * A warm agent starts before any job exists, so its cpu and memory come from
633
- * the label set (or the scaler defaults) and from nowhere else. Both are
634
- * applied when the agent starts and cannot be changed afterwards, so an
635
- * undeclared shape means the pool reserves a guess and no job can be tested
636
- * for fit against it. Refusing is the fail-safe reading, and it is what makes
637
- * a pool predictable — on a cloud backend it is what lets an operator reason
638
- * about instance size and cost at all.
639
- *
640
- * Limits alone count as a declaration: `resolveEffective` mirrors limits into
641
- * requests, so a label set naming only `limits` still resolves to a concrete
642
- * shape.
643
- *
644
- * A refused pool is inert, exactly as it was before it filled at all — never
645
- * partially filled.
560
+ * A warm agent's cpu and memory come only from the label set (or the scaler
561
+ * defaults) and are fixed at start, so an undeclared shape reserves a guess
562
+ * no job can be tested against. Refusing is the fail-safe, and on a cloud
563
+ * backend it is what lets an operator reason about instance size and cost.
564
+ * Limits alone count: `resolveEffective` mirrors limits into requests. A
565
+ * refused pool is inert, never partially filled.
646
566
  *
647
- * A declaration this cannot resolve is refused the same way, and must never
648
- * throw: the config schema types `resources.*.memory` as a plain string, so
649
- * an unparseable value such as `4Gi` reaches `resolveEffective` and
650
- * `parseMemoryString` rejects it. This runs inside the constructor and inside
651
- * the commit block of {@link reload}, where an exception would abort
652
- * orchestrator startup or leave a half-applied config behind.
567
+ * An unresolvable declaration is refused the same way and must never throw:
568
+ * the schema types `resources.*.memory` as a plain string, so a value such as
569
+ * `4Gi` reaches `parseMemoryString` and is rejected. This runs in the
570
+ * constructor and in {@link reload}'s commit block, where an exception would
571
+ * abort startup or leave a half-applied config.
653
572
  */
654
573
  private warmPoolShapeDeclared;
655
574
  /**
656
575
  * Agents this backend may still start before it reaches the scaler's
657
576
  * `maxAgents` or the orchestrator's `globalMaxAgents`, whichever binds first.
658
577
  *
659
- * The warm pool clamps its deficit to this, so a pool that has reached a
660
- * count cap asks for nothing at all rather than `size` spawns on every tick.
661
- * Only the count caps are visible here: a pool with count headroom but no
662
- * cpu/memory headroom still asks, and {@link reserveForSpawn} refuses each
663
- * request under the reservation lock. Those refusals cost a log line and a
664
- * `spawn_refusals` increment per tick, never a spawned agent.
665
- *
666
- * It is a best-effort read, not a reservation: the authority is
667
- * {@link reserveForSpawn}, which every warm pre-spawn goes through and which
668
- * re-checks both count caps under the reservation lock. The two backend
669
- * families read differently here. A local backend (container, bare-metal,
670
- * Firecracker) pins its compute to this host, so `getActiveCount()` is the
671
- * whole population and the room it reports is exact. An **event** backend is
672
- * capped cluster-wide — the reserve sequence claims a spawn row under
673
- * `withScalerCapLock` — while `getActiveCount()` sees only the provisions
674
- * this coordinator tracks, so the room reported here counts slots a peer may
675
- * already hold. The cluster-wide cap is still enforced, by that claim rather
676
- * than by this clamp.
578
+ * The warm pool clamps its deficit to this, so a pool at a count cap asks for
579
+ * nothing rather than `size` spawns every tick. Cpu/memory caps are not seen
580
+ * here: {@link reserveForSpawn} refuses those under the reservation lock, at
581
+ * the cost of a log line and a `spawn_refusals` increment per tick.
582
+ *
583
+ * A best-effort read, not a reservation: {@link reserveForSpawn} re-checks both
584
+ * count caps under the lock. For a local backend (container, bare-metal,
585
+ * Firecracker) `getActiveCount()` is the whole population, so the room is
586
+ * exact. An **event** backend is capped cluster-wide (a spawn row claimed
587
+ * under `withScalerCapLock`) while `getActiveCount()` sees only this
588
+ * coordinator's provisions, so the room here may count slots a peer holds;
589
+ * the claim, not this clamp, enforces the cluster-wide cap.
677
590
  */
678
591
  private warmCapacityRemaining;
679
592
  /**
680
- * Publish the warm-pool gauges from the pool's current fill state.
681
- *
682
- * Called after each pool tick and after a config reload, so an operator sees
683
- * a newly configured target immediately rather than at the next tick.
593
+ * Publish the warm-pool gauges. Called after each pool tick and after a
594
+ * reload, so a newly configured target shows immediately.
684
595
  */
685
596
  private publishWarmPoolGauges;
686
597
  /** Per-scaler mandatoryLabels (taint-style opt-in gate). */
@@ -690,26 +601,24 @@ export declare class ScalerManager {
690
601
  /**
691
602
  * Resolve the pool's platform for label + taint derivation. Precedence:
692
603
  * 1. A declared `platform` field always wins (even on a linux host).
693
- * 2. Otherwise a bare-metal pool derives its platform from the host OS/arch.
694
- * 3. Otherwise (container / firecracker, undeclared) returns null — those
695
- * pools run linux agents and are not auto-tainted; only an explicit
696
- * declared platform taints them.
697
- * Returns null when a bare-metal host's OS/arch is not a supported enum value
698
- * (the caller falls back to the raw host-label derivation for labels).
604
+ * 2. Otherwise a bare-metal pool derives it from the host OS/arch.
605
+ * 3. Otherwise (container / firecracker) null: linux agents, not
606
+ * auto-tainted unless a platform is declared.
607
+ * Also null for a bare-metal host whose OS/arch is not a supported enum value
608
+ * (the caller falls back to raw host labels).
699
609
  */
700
610
  private resolveScalerPlatform;
701
611
  /**
702
612
  * The plain platform-taint tokens (`windows`, `macos`, `arm64`) for a scaler,
703
613
  * derived from the same resolved platform `mandatoryGate` gates on.
704
614
  *
705
- * This is the single source for the taint: it feeds the matcher's view of the
706
- * label sets, the taint gate, the labels the spawned agent registers with, and
707
- * the warm-pool readiness query. Deriving it anywhere else — in a backend, say
708
- * — is what let the gate and the agent's labels disagree, so a tainted pool
709
- * spawned agents `AgentRegistry.findAvailable` could never return.
615
+ * The single source for the taint: the matcher's label sets, the taint gate,
616
+ * the agent's registered labels and the warm-pool readiness query. Deriving it
617
+ * elsewhere (in a backend, say) lets the gate and the agent's labels disagree,
618
+ * so a tainted pool spawns agents `AgentRegistry.findAvailable` never returns.
710
619
  *
711
- * `backendType` is passed rather than looked up because the constructor
712
- * configures warm pools from config entries whose backend may not be indexed.
620
+ * `backendType` is passed, not looked up: the constructor configures warm
621
+ * pools from config entries whose backend may not be indexed.
713
622
  */
714
623
  private platformTaintsFor;
715
624
  /**
@@ -717,41 +626,34 @@ export declare class ScalerManager {
717
626
  * plus the pool's platform taints.
718
627
  *
719
628
  * `AgentRegistry.findAvailable` drops an agent unless its whole
720
- * `mandatoryLabels` gate appears in the query, and a tainted pool's gate
721
- * carries the taints — so a query built from the declared labels alone counts
722
- * none of the pool's own agents and the pool fills to `maxAgents` with `ready`
723
- * stuck at 0. Both construction sites (the constructor and the reload path)
724
- * call this so the two can never diverge.
725
- *
726
- * These labels are for the QUERY only. The pool is keyed by, and spawns with,
727
- * the DECLARED label set: a backend matches the requested set against its own
728
- * `labelSets` to resolve the image / binary / VM config and throws on a set it
729
- * does not have, and every in-flight release path normalizes the spawn labels
730
- * back to the map key.
629
+ * `mandatoryLabels` gate (taints included) appears in the query, so declared
630
+ * labels alone count none of the pool's agents and it fills to `maxAgents`
631
+ * with `ready` stuck at 0. The constructor and the reload path both call this
632
+ * so they never diverge.
633
+ *
634
+ * QUERY only. The pool is keyed by, and spawns with, the DECLARED set: a
635
+ * backend resolves its image / binary / VM config by matching its own
636
+ * `labelSets` and throws on any other set, and every in-flight release path
637
+ * normalizes the spawn labels back to the map key.
731
638
  */
732
639
  private warmPoolQueryLabels;
733
640
  /**
734
641
  * Taint gate for a scaler's label sets: the configured `mandatoryLabels` plus
735
642
  * the platform taints derived from the pool's resolved structured platform.
736
643
  *
737
- * This is the gate every routing and stamping decision uses. `backend`
738
- * carries only `type`, so a caller that holds one label set never has to
739
- * reach for the whole backend.
644
+ * Every routing and stamping decision uses it. `backend` needs only `type`,
645
+ * so a caller holding one label set never reaches for the whole backend.
740
646
  */
741
647
  private mandatoryGate;
742
648
  /**
743
- * Build enriched scaler entries with auto-labels injected into each label set.
744
- * This ensures label matching accounts for auto-injected labels (kici:role:*,
745
- * kici:os:*, kici:arch:*, kici:agent:*, kici:scaler:*) that agents receive
746
- * at spawn time.
747
- *
748
- * The os/arch labels derive from the pool's resolved structured platform
749
- * (declared field wins, else host-derive for bare-metal, else linux), and the
750
- * mandatory labels combine the per-scaler config gate with the platform taint,
751
- * so the label matcher applies the gate alongside subset matching.
649
+ * Scaler entries with the auto-labels agents receive at spawn (kici:role:*,
650
+ * kici:os:*, kici:arch:*, kici:agent:*, kici:scaler:*) injected into each
651
+ * label set, so label matching accounts for them.
752
652
  *
753
- * Retiring scalers are skipped: they were removed from the config and must
754
- * take no new work while their remaining agents finish.
653
+ * os/arch derive from the resolved structured platform, and the mandatory
654
+ * labels combine the config gate with the platform taint, so the matcher
655
+ * applies the gate alongside subset matching. Retiring scalers are skipped:
656
+ * they take no new work while their remaining agents finish.
755
657
  */
756
658
  private getEnrichedScalerEntries;
757
659
  /**
@@ -768,35 +670,30 @@ export declare class ScalerManager {
768
670
  private sweepRetiredBackends;
769
671
  /**
770
672
  * True while any spawn this manager started for `name` has neither registered
771
- * nor failed — including one still queued behind the spawn semaphore, which
772
- * has not reached `backend.spawn` at all. Such a spawn counts toward the
773
- * scaler's drain even though the backend reports no active agent.
673
+ * nor failed, including one still queued behind the spawn semaphore. Such a
674
+ * spawn holds the drain open though the backend reports no active agent.
774
675
  */
775
676
  private hasSpawnInFlight;
776
677
  /** Drop every trace of a scaler: the backend and all its per-scaler metadata. */
777
678
  private forgetScaler;
778
679
  /**
779
- * Main entry point: called by the Dispatcher when no agent is available.
780
- *
781
- * `resources` is the per-job request/limit declaration (or `undefined` to fall
782
- * back to scaler defaults). Used by per-scaler / per-orchestrator / per-machine
783
- * resource caps; the cap math is wired up by ScalerManager itself.
680
+ * Run one backend spawn under the resolved spawn deadline. On deadline it
681
+ * aborts the signal (so the backend's container-runtime/HTTP calls cancel and
682
+ * its own catch cleans up) AND rejects, so the caller's semaphore `run()`
683
+ * releases the slot even if the backend's unwind stalls somewhere
684
+ * non-abortable.
784
685
  */
686
+ private runSpawnWithTimeout;
785
687
  /**
786
- * Run a single backend spawn under the resolved spawn deadline. Returns the
787
- * ManagedAgent on success. On deadline: aborts the signal (so the backend's
788
- * threaded container-runtime/HTTP calls cancel and its own catch cleans up)
789
- * AND rejects, so the caller's semaphore `run()` finally releases the slot
790
- * even if the backend's unwind stalls in a non-abortable spot. The backing
791
- * spawn promise gets a no-op catch attached so a late abort-rejection doesn't
792
- * surface as an unhandled rejection.
688
+ * Main entry point: called by the Dispatcher when no agent is available.
689
+ * `resources` is the per-job request/limit declaration (`undefined` falls
690
+ * back to scaler defaults), billed against the per-scaler, per-orchestrator
691
+ * and per-machine resource caps.
793
692
  */
794
- private runSpawnWithTimeout;
795
693
  requestScale(labels: string[], jobId: string, runId: string, excludeLabels?: string[], resources?: ResourceRequest, orgId?: string,
796
694
  /**
797
- * The job's resolved container image + registry credentials, when it
798
- * declared one. Threaded to the backend so it can spawn the job's own
799
- * image instead of the pool's fixed agent image.
695
+ * The job's resolved container image + registry credentials, when declared,
696
+ * so the backend can spawn the job's own image instead of the pool's.
800
697
  */
801
698
  containerSpawn?: ResolvedContainerSpawn): Promise<ScaleResult>;
802
699
  /** Whether this coordinator already provisions, or holds an agent bound to, `jobId`. */
@@ -804,210 +701,160 @@ export declare class ScalerManager {
804
701
  /** `requestScale` past its drain and per-job checks: match, reserve, spawn. */
805
702
  private provisionForJob;
806
703
  /**
807
- * Take every reservation one spawn needs, or take none at all.
808
- *
809
- * The single reserve sequence both spawn paths share — the job-bound
810
- * `requestScale` and the warm pool's pre-spawn. The only difference between
811
- * them is `boundJobId`, which is `undefined` for a pre-spawn; a warm agent
812
- * costs the same host resources as a job-bound one, so it takes the same
813
- * reservations.
704
+ * Take every reservation one spawn needs, or none. Shared by `requestScale`
705
+ * and the warm pre-spawn (`boundJobId` `undefined`): a warm agent costs the
706
+ * same host resources, so it takes the same reservations.
814
707
  *
815
708
  * Ordering, and why:
816
709
  * - Count caps and the in-memory resource reservation run under the
817
- * in-process reservation lock, so the check and the reserve are atomic
818
- * against a concurrent caller. A local backend counts its own host's
819
- * agents in process, which is what its cap means: the compute is pinned to
820
- * this machine. An event backend provisions cloud instances any
821
- * coordinator can start, so its cap is counted — and its slot claimed —
822
- * cluster-wide, inside one advisory-locked transaction. The in-process
823
- * lock stays the outer guard so the two never contend.
824
- * - The cross-process machine-pool ledger is taken outside that lock,
825
- * because the ledger has its own cross-process mutex; holding both at once
826
- * buys nothing.
827
- *
828
- * Every refusal rolls back exactly what it already took, so a `false` return
829
- * leaves no reservation behind. On success the caller owns the release,
830
- * which is always `releaseAll(agentId)`.
710
+ * in-process reservation lock, so check and reserve are atomic. A local
711
+ * backend counts its own host's agents in process (its compute is pinned
712
+ * here). An event backend's cloud instances can be started by any
713
+ * coordinator, so its cap is counted, and its slot claimed, cluster-wide in
714
+ * one advisory-locked transaction; the in-process lock stays the outer
715
+ * guard so the two never contend.
716
+ * - The machine-pool ledger is taken outside that lock: it has its own
717
+ * cross-process mutex, and holding both buys nothing.
718
+ *
719
+ * Every refusal rolls back what it took, so `false` leaves no reservation.
720
+ * On success the caller owns the release: `releaseAll(agentId)`.
831
721
  */
832
722
  private reserveForSpawn;
833
723
  /**
834
- * How many reservations one scaler currently holds.
835
- *
836
- * Derived from `this.reservations` rather than kept as a counter beside it:
837
- * the map IS the admitted set, so a derived count cannot drift out of step
838
- * with it — including on the boot/leader-switch rehydrate path, which
839
- * repopulates the map directly.
840
- */
841
- /**
842
- * Record a refusal taken on one of the two agent-count caps.
724
+ * Record a refusal on one of the two agent-count caps.
843
725
  *
844
- * The refusal itself is returned to the dispatcher as `at-capacity`, which
845
- * leaves the job queued and logs nothing — correct while agents are genuinely
846
- * busy, and indistinguishable from a scaler that has stopped spawning
847
- * altogether. So the counter the metric's own help text promises
848
- * (`maxAgents`, `globalMaxAgents`) is incremented here, beside the resource
849
- * caps that already increment it.
726
+ * The dispatcher gets `at-capacity`, which queues the job and logs nothing,
727
+ * indistinguishable from a scaler that stopped spawning. So the counter the
728
+ * metric's help text promises (`maxAgents`, `globalMaxAgents`) is
729
+ * incremented here, beside the resource caps.
850
730
  *
851
- * The warning is narrower than the counter on purpose. Ordinary saturation
852
- * refuses with live agents holding the cap, and warning on that would emit a
853
- * line per queued job for as long as the cluster is busy. A refusal where the
854
- * reservation ledger outruns every backend's active count is the other shape:
855
- * capacity billed to agents that are not running, which does not resolve on
856
- * its own.
731
+ * The warning is narrower on purpose: ordinary saturation would warn per
732
+ * queued job while busy. It fires only when the reservation ledger outruns
733
+ * every backend's active count: capacity billed to agents not running, which
734
+ * does not resolve on its own.
857
735
  */
858
736
  private reportCountCapRefusal;
859
- private reservedCountForScaler;
860
737
  /**
861
- * Run a critical section under the reservation lock. Serializes concurrent
862
- * `requestScale()` calls so the check+reserve sequence is atomic.
738
+ * Reservations one scaler holds, derived from `this.reservations` rather than
739
+ * a separate counter: the map IS the admitted set, so the count cannot drift,
740
+ * even on the boot/leader-switch rehydrate path that repopulates the map.
863
741
  */
742
+ private reservedCountForScaler;
743
+ /** Serializes concurrent `requestScale()` calls so check+reserve is atomic. */
864
744
  private runWithReservationLock;
865
745
  /**
866
- * Lazily resolve the per-backend spawn throttle, sized by the scaler's
867
- * `maxConcurrentSpawns` (falling back to the cluster default). One semaphore
868
- * per backend name, since each backend has its own container socket.
746
+ * Lazily resolve the per-backend spawn throttle, sized by `maxConcurrentSpawns`
747
+ * (else the cluster default). One per backend: each has its own container socket.
869
748
  */
870
749
  private spawnSemaphoreFor;
871
750
  /**
872
751
  * Resolve the effective `{ requests, limits }` for a job at this scaler.
873
752
  *
874
- * Layered defaults (highest priority first):
753
+ * Layered per field, highest priority first:
875
754
  * 1. Job's own `resources` (after request<->limit mirroring).
876
755
  * 2. Label-set's `resources`.
877
756
  * 3. Scaler's `defaults.resources` (already merged from global defaults at construction).
878
- * 4. Zero (`{ cpus: 0, memBytes: 0 }`).
879
- *
880
- * Returns both sides fully resolved as numeric `{ cpus, memBytes }` pairs.
757
+ * 4. Zero.
881
758
  */
882
759
  private resolveEffective;
883
760
  /**
884
- * Attempt to reserve `requests` against the per-scaler and per-orchestrator
885
- * caps. Returns false (no state change) if any cap would be exceeded.
886
- *
887
- * Caller must hold the reservation lock.
761
+ * Reserve `requests` against the per-scaler and per-orchestrator caps, or
762
+ * return false with no state change. Caller must hold the reservation lock.
888
763
  */
889
764
  private tryReserveAll;
890
765
  /**
891
- * Push the current `perScalerUsage` and `globalUsage` snapshot into the
892
- * Prometheus observable gauges. Called on every successful reservation /
893
- * release so the gauge callback always reflects the latest state without
894
- * doing any work itself (gauge callbacks are synchronous and MUST NOT
895
- * touch this map directly — they just read whatever was last published).
766
+ * Push the `perScalerUsage` / `globalUsage` snapshot into the observable
767
+ * gauges on every reservation / release. Gauge callbacks are synchronous and
768
+ * MUST NOT touch this map; they read whatever was last published.
896
769
  *
897
- * Note: pool-level usage is intentionally not emitted here. The on-disk
898
- * machine-pool ledger requires async I/O + a file lock to read accurately,
899
- * and OTel observable-gauge callbacks must be synchronous. Operators
900
- * dashboarding pool utilization can derive a lower bound from the sum of
901
- * `kici_orch_scaler_cpus_used` rows whose scaler belongs to the pool.
770
+ * Pool-level usage is deliberately not emitted: reading the on-disk ledger
771
+ * needs async I/O + a file lock, and OTel gauge callbacks are synchronous. A
772
+ * lower bound is the sum of `kici_orch_scaler_cpus_used` over the pool's
773
+ * scalers.
902
774
  */
903
775
  private publishUsageMetrics;
904
776
  /**
905
- * Release the reservation tracked for an agent. Idempotent. Also releases
906
- * the cross-process pool reservation (best-effort) when the scaler
907
- * references a pool.
777
+ * Release an agent's reservation, and (best-effort) its cross-process pool
778
+ * reservation. Idempotent.
908
779
  */
909
780
  private releaseAll;
910
781
  /**
911
782
  * Signal that scaler capacity freed. Trailing-debounced by
912
- * {@link CAPACITY_FREED_DEBOUNCE_MS} so a burst of simultaneous releases (and
913
- * the onJobComplete → onAgentDisconnect pair for a single-use agent) coalesces
914
- * into one re-drive pass that runs after the last release settles. No-op when
915
- * no callback is wired. The timer is `unref`'d so it never keeps the process
916
- * alive, and cleared on shutdown.
783
+ * {@link CAPACITY_FREED_DEBOUNCE_MS} so a burst of releases (and a single-use
784
+ * agent's onJobComplete → onAgentDisconnect pair) coalesces into one re-drive
785
+ * after the last release. The timer is `unref`'d and cleared on shutdown.
917
786
  */
918
787
  private notifyCapacityFreed;
919
788
  /**
920
- * Subtract the given `requests` from the per-scaler and global in-memory
921
- * counters. Used both by `releaseAll()` and by the rollback path when a
922
- * machine-pool reservation refuses after the in-memory reservation
923
- * already succeeded.
789
+ * Subtract `requests` from the per-scaler and global in-memory counters; used
790
+ * by `releaseAll()` and by rollbacks after the in-memory reservation succeeded.
924
791
  */
925
792
  private releaseInMemory;
926
793
  /**
927
- * Resolve the scaler-level default resources, merging the global
928
- * `defaults.resources` underneath any scaler-specific override.
929
- *
930
- * Currently scaler entries themselves do not carry their own `defaults` (the
931
- * label-set level does). The merged result is stored per-scaler so that
932
- * future scaler-level defaults can be added without touching the resolver.
794
+ * Resolve a scaler's default resources. Scaler entries carry no `defaults` of
795
+ * their own (label sets do), so this is the global `defaults.resources`;
796
+ * storing it per scaler leaves room for scaler-level defaults.
933
797
  */
934
798
  private resolveScalerDefaults;
935
799
  /**
936
- * Called from agent-handler.ts when an agent registers via WS.
937
- * Correlates the registered agent to a spawned tracking entry.
800
+ * Called from agent-handler.ts when an agent registers via WS; correlates it to
801
+ * its spawning entry.
938
802
  *
939
803
  * Returns:
940
- * - `boundJobId` (optional): the queued jobId this agent was spawned for.
941
- * Used by the caller to eagerly dispatch the bound job before the
942
- * agent's idle timer fires, skipping the generic queue drain race.
943
- * - `mandatoryLabels` (always populated for scaler-managed agents): the
944
- * spawning scaler's effective `mandatoryLabels` gate — its configured
945
- * labels plus the platform taint derived from the pool's declared OS/arch.
946
- * Threaded into the
947
- * AgentRegistry so the queue-drain path (`onAgentAvailable` →
948
- * `dequeueForLabels`) and the eager-dispatch path
949
- * (`dispatchBoundJob` → `dequeueById`) both reject queued jobs whose
950
- * `runsOn` does not include every gate label.
951
- *
952
- * Returns `null` for a static agent — no spawning entry exists, and no other
953
- * instance's spawn record claims the agent either. A warm-pool spawn is not
954
- * that case: its spawning entry (and durable row) survives until it
804
+ * - `boundJobId` (optional): the queued job this agent was spawned for, which
805
+ * the caller dispatches eagerly before the idle timer fires, skipping the
806
+ * generic queue drain race.
807
+ * - `mandatoryLabels` (always set for scaler-managed agents): the scaler's
808
+ * effective gate (configured labels plus the platform taint). Threaded into
809
+ * the AgentRegistry so both the queue-drain path (`onAgentAvailable` →
810
+ * `dequeueForLabels`) and the eager-dispatch path (`dispatchBoundJob` →
811
+ * `dequeueById`) reject jobs whose `runsOn` lacks a gate label.
812
+ *
813
+ * Returns `null` for a static agent: no spawning entry and no peer spawn
814
+ * record. A warm-pool spawn keeps its entry (and durable row) until it
955
815
  * registers, so it resolves with `mandatoryLabels` and no `boundJobId`, which
956
- * is exactly what marks the agent warm on `register.ack`.
816
+ * marks the agent warm on `register.ack`.
957
817
  *
958
- * A local hit (this instance spawned the agent) is answered from memory. A
959
- * miss falls back to the shared spawn record: on an HA cluster behind one
960
- * endpoint the agent routinely reaches an instance that did not spawn it,
961
- * and adopting is the normal path, not an exception. Adoption is a
962
- * conditional UPDATE, so exactly one instance can win.
818
+ * A local hit is answered from memory; a miss falls back to the shared spawn
819
+ * record, since on an HA cluster behind one endpoint the agent routinely
820
+ * reaches an instance that did not spawn it. Adoption is a conditional
821
+ * UPDATE, so exactly one instance wins.
963
822
  */
964
823
  onAgentRegistered(agentId: string, labels: string[]): Promise<{
965
824
  boundJobId?: string;
966
825
  mandatoryLabels: string[];
967
826
  } | null>;
968
827
  /**
969
- * Reclaim the host-local compute of a scaler-spawned agent this coordinator
970
- * refused for want of a spawn record.
828
+ * Reclaim the host-local compute of a scaler-spawned agent refused for want of
829
+ * a spawn record.
971
830
  *
972
- * Refusing closes the WS before registration completes, so `onAgentDisconnected`
973
- * never runs for that agent, and `managedAgentIndex` never held it in the first
974
- * place — the two writes that populate it (`completeLocalRegistration` and the
975
- * adoption branch) both come after this point. So even the disconnect hook is a
976
- * no-op here: it reads that index, finds nothing, and takes the static-agent
977
- * exit. The agent keeps its VM or host process, reconnects, is refused again,
978
- * and loops until an operator stops it by hand.
831
+ * Refusing closes the WS before registration completes, and
832
+ * `managedAgentIndex` is only populated after this point, so the disconnect
833
+ * hook takes the static-agent exit. Without this the agent keeps its VM or
834
+ * host process and loops reconnect-refuse until an operator stops it.
979
835
  *
980
836
  * **This force-destroys a live instance, so read what makes it safe.**
981
837
  *
982
- * - **Local backends only.** An event agent's compute is a customer cloud
983
- * instance, reachable from any coordinator, and its missing adoption is
984
- * ambiguous: `adoptSpawningAgent` resolves `null` both when no row exists AND
985
- * when a peer coordinator already holds the row, so an event agent that is
986
- * alive and legitimately owned elsewhere reaches this exact branch. Tearing
987
- * it down there would destroy a peer's healthy provision. A local backend's
988
- * compute cannot be reached that way: it lives on the host that spawned it.
989
- * - **Host-local evidence decides, not this verdict.** The verdict alone is NOT
990
- * evidence the agent is this host's: `adoptSpawningAgent` adopts `event` rows
991
- * only, so a `container` / `bare-metal` / `firecracker` agent that reconnects
992
- * to a peer behind a shared endpoint is refused here even though its spawn
993
- * row is intact and its compute is alive on the peer's host. What keeps that
994
- * harmless is that each backend reclaims only what its own tracking or its
995
- * own on-host artifacts name — never a shared database row, which on an HA
996
- * pair names an identically-configured scaler on both hosts. A wrongly-routed
997
- * agent id finds nothing here and nothing is touched.
838
+ * - **Local backends only.** `adoptSpawningAgent` resolves `null` both when no
839
+ * row exists AND when a peer already holds it, so a live event agent owned
840
+ * elsewhere reaches this branch; tearing down its cloud instance would
841
+ * destroy a peer's healthy provision. Local compute lives on the host that
842
+ * spawned it.
843
+ * - **Host-local evidence decides, not this verdict.** `adoptSpawningAgent`
844
+ * adopts `event` rows only, so a `container` / `bare-metal` / `firecracker`
845
+ * agent reconnecting to a peer behind a shared endpoint is refused here with
846
+ * its row intact and compute alive on the peer's host. Each backend reclaims
847
+ * only what its own tracking or on-host artifacts name, never a shared
848
+ * database row (which on an HA pair names an identically-configured scaler
849
+ * on both hosts), so a wrongly-routed id touches nothing.
998
850
  * - **Never on a transient failure.** The caller must have positive evidence
999
- * the agent is unowned. A store read that merely *failed* is not evidence —
1000
- * the record may exist and be perfectly healthy.
851
+ * the agent is unowned; a store read that merely *failed* is not evidence.
1001
852
  *
1002
- * Fire-and-forget, matching `onAgentDisconnected`'s own teardown: the caller
1003
- * refuses the registration immediately rather than holding the agent's socket
1004
- * open for a VM shutdown.
853
+ * Fire-and-forget, like `onAgentDisconnected`'s teardown: the registration is
854
+ * refused immediately rather than holding the socket open for a VM shutdown.
1005
855
  */
1006
856
  private reclaimUnownedScalerAgent;
1007
- /**
1008
- * Finish registration for an agent this instance spawned itself, from the
1009
- * in-memory spawning entry.
1010
- */
857
+ /** Finish registration for an agent this instance spawned, from its spawning entry. */
1011
858
  private completeLocalRegistration;
1012
859
  /**
1013
860
  * Called from agent-handler.ts when an agent disconnects. `reason` is what an
@@ -1020,105 +867,86 @@ export declare class ScalerManager {
1020
867
  * Tear down a stranded provision the `EventProvisionReaper` found.
1021
868
  *
1022
869
  * Public because the reaper is leader-gated and lives outside the manager,
1023
- * while the emit and the row delete must stay in one place. The reason comes
1024
- * from the reaper's verdict — a provision whose agent never registered is a
1025
- * `spawn-timeout`, one whose agent went away is a `heartbeat-timeout` — so
1026
- * unlike the disconnect path this does not default to `shutdown`.
870
+ * while the emit and the row delete stay in one place. The reason is the
871
+ * reaper's verdict (never registered → `spawn-timeout`, went away →
872
+ * `heartbeat-timeout`), never a `shutdown` default.
1027
873
  *
1028
874
  * `kici_orch_scaler_external_provision_timeout_total` counts the spawn-timeout
1029
- * arm. Nothing else increments it in production: `runSpawnWithTimeout` bounds
1030
- * only the `spawn()` call, and an event backend's `spawn()` returns the moment
1031
- * the scale-up event is emitted.
875
+ * arm, and nothing else increments it in production: an event backend's
876
+ * `spawn()` returns once the scale-up is emitted, inside
877
+ * `runSpawnWithTimeout`'s bound.
1032
878
  *
1033
879
  * The spawn-timeout arm is also where an external provisioning failure enters
1034
- * the ordinary failure machinery. The local backends report a dead spawn
1035
- * through the `scaler.failed` event their `spawn()` emits; an event scaler's
1036
- * `spawn()` succeeds by definition (it only emits a scale-up), so the failure
1037
- * is not observable until the provision it asked for never registers — which
1038
- * is exactly this verdict. Synthesizing the event here gives the fleet
1039
- * counter, the operator's `diagnose scaler` view, and `last_provisioning_error`
1040
- * the same input every other backend already supplies, so a job whose external
1041
- * provisioning failed settles with the provisioning cause instead of a
1042
- * label-mismatch complaint that sends its operator to fix a correct `runsOn`.
880
+ * the failure machinery. Local backends emit `scaler.failed` from `spawn()`;
881
+ * an event scaler's `spawn()` always succeeds, so the failure shows only when
882
+ * the provision never registers. Synthesizing the event here feeds the fleet
883
+ * counter, `diagnose scaler` and `last_provisioning_error`, so the job
884
+ * settles with the provisioning cause instead of a label-mismatch complaint
885
+ * that sends its operator to fix a correct `runsOn`.
1043
886
  */
1044
887
  emitOrphanScaleDown(candidate: ReapCandidate, reason: ScaleDownReason): Promise<void>;
1045
888
  /**
1046
889
  * Refuse a spawn request while the scaler is inside its backoff window.
1047
890
  *
1048
- * Returns the refusal to hand back, or null to proceed. The message names how
1049
- * many consecutive failures produced the deferral and when it lifts. It
1050
- * reaches the operator through this method's own log line: the dispatcher
1051
- * branches on `action` alone and discards `reason`, so a job deferred here
1052
- * carries no record of the deferral on its queue row. The job whose provision
1053
- * actually failed is the one that gets `last_provisioning_error`, written by
1054
- * the `scaler.failed` this backoff's own trigger emits.
891
+ * Returns the refusal, or null to proceed. The operator sees it only through
892
+ * this method's log line: the dispatcher branches on `action` and discards
893
+ * `reason`, so a deferred job's queue row records nothing. The job whose
894
+ * provision failed gets `last_provisioning_error` from the triggering
895
+ * `scaler.failed`.
1055
896
  *
1056
- * The knobs are read here per request rather than per process, so an operator
1057
- * lowering the failure limit mid-outage sees the repeated-failure wording on
1058
- * the next refusal. The window itself was armed by `recordProvisionFailure`
1059
- * from the base and ceiling live at that moment, so a retuned window applies
1060
- * from the next recorded failure.
897
+ * The knobs are read per request, so a lowered failure limit changes the
898
+ * wording on the next refusal; the window itself was armed from the base and
899
+ * ceiling live at the last `recordProvisionFailure`, so a retuned window
900
+ * applies from the next failure.
1061
901
  */
1062
902
  private provisionBackoffRefusal;
1063
903
  /**
1064
- * Record a consecutive provisioning failure for a scaler and arm its next
1065
- * deferral.
1066
- *
1067
- * The delay doubles per consecutive failure and is capped, so a provider
1068
- * outage settles into a steady retry cadence rather than either hammering the
1069
- * provider or growing without bound. `2 ** (n - 1)` is bounded by the cap
1070
- * before it is used, so a long outage cannot overflow the shift.
904
+ * Record a consecutive provisioning failure and arm the next deferral. The
905
+ * delay doubles per failure up to the cap, so an outage settles into a steady
906
+ * retry cadence. `2 ** (n - 1)` is capped before use, so it cannot overflow.
1071
907
  */
1072
908
  private recordProvisionFailure;
1073
909
  /**
1074
- * Claim a dead provision as counted, or report that someone already did.
1075
- *
1076
- * The same provision is seen twice on the leader — once by its own
1077
- * stale-spawn prune, once by the leader-gated reaper — and every consequence
1078
- * of a failure is counted per provision, not per observer. The claim is
1079
- * synchronous and happens before any await, so two observers arriving in the
1080
- * same tick cannot both pass it.
910
+ * Claim a dead provision as counted, or report that someone already did. The
911
+ * leader sees one provision twice (its stale-spawn prune and the reaper), and
912
+ * a failure counts per provision, not per observer. Synchronous, before any
913
+ * await, so two observers in the same tick cannot both pass.
1081
914
  */
1082
915
  private claimFailedProvision;
1083
916
  /**
1084
917
  * Report one dead external provision, wherever this coordinator noticed it.
1085
918
  *
1086
- * Everything a failed provision owes an operator happens here: the fleet
1087
- * spawn-failure counter, the structured warn, the `ScalerFailureTracker`
1088
- * record `kici-admin diagnose scaler` reads, the `last_provisioning_error`
1089
- * that tells the waiting job why it never ran, and the backoff.
919
+ * Everything a failed provision owes an operator: the fleet spawn-failure
920
+ * counter, the structured warn, the `ScalerFailureTracker` record
921
+ * `kici-admin diagnose scaler` reads, the job's `last_provisioning_error`,
922
+ * and the backoff.
1090
923
  *
1091
- * It is deliberately reachable from BOTH observers rather than from the
1092
- * reaper alone. `ScalerFailureTracker` and `provisionFailures` are
1093
- * per-process, and the reaper is leader-gated — so a reaper-only report left
1094
- * every non-leader with an empty `diagnose scaler` for a scaler failing
1095
- * fleet-wide, and dispatching at the un-deferred cadence while only the
1096
- * leader backed off. The local prune is the observation every coordinator has
1097
- * of its own spawns.
924
+ * Reachable from BOTH observers, not the reaper alone: the tracker and
925
+ * `provisionFailures` are per-process and the reaper is leader-gated, so a
926
+ * reaper-only report leaves every non-leader with an empty `diagnose scaler`
927
+ * and an un-deferred cadence. The local prune is every coordinator's view of
928
+ * its own spawns.
1098
929
  *
1099
- * Synchronous up to the backoff arm, so a caller that must clean up after the
1100
- * event (the prune drops the agent's event buffer) sees the effect
1101
- * immediately.
930
+ * Synchronous up to the backoff arm, so a caller cleaning up after the event
931
+ * (the prune drops the agent's event buffer) sees the effect at once.
1102
932
  */
1103
933
  private reportProvisionFailure;
1104
934
  /**
1105
935
  * Defer a scaler whose host refused to start an agent, then free the spawn's
1106
936
  * reservation.
1107
937
  *
1108
- * Arming comes first: the release notifies freed capacity, and the re-drive
1109
- * that follows 250 ms later must meet the deferral rather than spawn straight
1110
- * into the same refusal. The release still notifies, because a job waiting on
1111
- * another scaler, or on the global cap, is legitimately unblocked by it.
1112
- * Two failures come here: a `spawn()` that rejected with a
938
+ * Arm first: the release notifies freed capacity, and the re-drive 250 ms
939
+ * later must meet the deferral rather than spawn into the same refusal. The
940
+ * release still notifies, since it legitimately unblocks a job waiting on
941
+ * another scaler or the global cap. Callers: a `spawn()` rejected with a
1113
942
  * `DeterministicSpawnError`, and a settled spawn whose compute stopped before
1114
- * its agent registered (`dropFailedSettledSpawn`). Every other spawn
1115
- * rejection is released at once and retried.
943
+ * its agent registered (`dropFailedSettledSpawn`). Any other rejection is
944
+ * released at once and retried.
1116
945
  */
1117
946
  private deferRefusedLaunch;
1118
947
  /**
1119
- * Whether a bound job has ended. A read error, a missing row, an unknown
1120
- * run, or no reader wired all read as "not ended", so the caller keeps
1121
- * reporting the provision as it would without the check.
948
+ * Whether a bound job has ended. A read error, missing row, unknown run or
949
+ * missing reader all read as "not ended", so the caller reports as usual.
1122
950
  */
1123
951
  private boundJobEnded;
1124
952
  /**
@@ -1127,62 +955,51 @@ export declare class ScalerManager {
1127
955
  *
1128
956
  * The job's row is the shared truth: a cancel reaches whichever coordinator
1129
957
  * the Platform picks, while the spawning entry lives on the one that asked.
1130
- * A withdrawal is not a provisioning failure, so nothing is reported and no
1131
- * backoff is armed. Returns how many provisions were withdrawn.
958
+ * A withdrawal is not a failure: nothing is reported, no backoff armed.
959
+ * Returns the withdrawn count.
1132
960
  */
1133
961
  withdrawEndedBoundProvisions(): Promise<number>;
1134
962
  /**
1135
963
  * Forget one pending provision and tear it down as `job-complete` — unless a
1136
964
  * peer adopted its agent, in which case the adopter owns the teardown.
1137
965
  *
1138
- * The teardown goes through `emitScaleDownForSpec`, which drops the spawn
1139
- * row only once the scale-down is out: a failed emit keeps the row for the
1140
- * reaper, whose spawn-timeout verdict re-checks the job and sends
1141
- * `job-complete` too. `backend.destroy` is not used — a rehydrated entry is
1142
- * not tracked by the backend, so it would emit nothing.
966
+ * Via `emitScaleDownForSpec`, which drops the spawn row only once the
967
+ * scale-down is out; a failed emit leaves it for the reaper, whose
968
+ * spawn-timeout verdict re-checks the job and also sends `job-complete`. Not
969
+ * `backend.destroy`: the backend does not track a rehydrated entry, so it
970
+ * would emit nothing.
1143
971
  */
1144
972
  private withdrawProvision;
1145
973
  /**
1146
974
  * Report a pruned event spawn as a dead provision — unless a peer adopted it.
1147
975
  *
1148
- * The prune is NOT proof of failure on an HA cluster. Nothing tells the
1149
- * spawning coordinator that its agent registered on a peer, so the entry
1150
- * survives adoption and lands in the prune the stale window later next to a
1151
- * perfectly live agent — the same asymmetry `deleteUnadoptedSpawningAgent`
1152
- * exists for. Reporting that as a dead provision writes a `scaler.failed` and
1153
- * a `last_provisioning_error` onto a job that is running, and backs a healthy
1154
- * scaler off, which is the misattribution this whole report exists to remove.
1155
- * The reaper already applies the same test: it returns `spawn-timeout` only
1156
- * for a candidate with no `adoptedBy`, so without this the two observers of
1157
- * one provision disagree.
1158
- *
1159
- * The question is "was this provision ever adopted?", and the spawn row alone
1160
- * cannot answer it: the row is deleted on teardown, so its absence is BOTH
1161
- * "never adopted" and "adopted, then torn down". `provisionAdopter` reads the
1162
- * live row first and the durable provision-outcome record second, which
1163
- * survives the delete — so an adopted provision the reaper has since
1164
- * condemned no longer reads as a dead one.
1165
- *
1166
- * The lookup is fire-and-forget so the prune stays off the request path, and
1167
- * it fails open. Both "the store errored" and "there is no verdict" leave the
1168
- * failure reported, deliberately: a provision with no positive evidence of
1169
- * adoption is what a single coordinator sees on a real provisioning outage,
1170
- * and suppressing there would disarm the per-coordinator backoff on every
1171
- * follower — an invisible missing report in place of a visible false one. The
1172
- * only rows with no verdict are those adopted before the outcome table
1173
- * existed, and they age out within one stale-prune window of the deploy.
976
+ * The prune is NOT proof of failure on an HA cluster: the spawning coordinator
977
+ * never learns its agent registered on a peer, so the entry reaches the prune
978
+ * next to a live agent (the asymmetry `deleteUnadoptedSpawningAgent` exists
979
+ * for). Reporting it would write `scaler.failed` and
980
+ * `last_provisioning_error` onto a running job and back off a healthy scaler.
981
+ * The reaper applies the same test (`spawn-timeout` only without
982
+ * `adoptedBy`), so the two observers agree.
983
+ *
984
+ * The spawn row alone cannot say "ever adopted?": it is deleted on teardown.
985
+ * `provisionAdopter` reads the live row, then the durable provision-outcome
986
+ * record that survives the delete, so an adopted provision the reaper has
987
+ * since condemned does not read as dead.
988
+ *
989
+ * The lookup is fire-and-forget (off the request path) and fails open: a
990
+ * store error or no verdict leaves the failure reported, because that is what
991
+ * a real outage looks like to one coordinator, and suppressing it would
992
+ * disarm every follower's backoff. The only rows with no verdict were adopted
993
+ * before the outcome table existed and age out within one stale-prune window
994
+ * of the deploy.
1174
995
  */
1175
996
  private reportPrunedProvisionFailure;
1176
997
  /** The adoption half of {@link reportPrunedProvisionFailure}. */
1177
998
  private reportPrunedUnlessAdopted;
1178
999
  /**
1179
- * Clear a scaler's consecutive-failure state after a successful provision.
1180
- *
1181
- * Called from `onAgentRegistered`, which is the single entry point for BOTH a
1182
- * locally-spawned agent and one this coordinator adopted from a peer. An
1183
- * adopted agent is a successful provision — the instance that spawned it
1184
- * is not the one it reached — so counting only local registrations
1185
- * would leave a healthy scaler deferred on an HA cluster.
1000
+ * Clear a scaler's consecutive-failure state after a successful provision,
1001
+ * from both the local and the adopted registration paths (see
1002
+ * `onAgentRegistered`).
1186
1003
  */
1187
1004
  private clearProvisionFailures;
1188
1005
  /**
@@ -1190,68 +1007,52 @@ export declare class ScalerManager {
1190
1007
  * drop the durable row.
1191
1008
  *
1192
1009
  * Used where the instance holding the agent has no local backend for its
1193
- * scaler — the ordinary shape on an HA cluster behind one shared endpoint,
1194
- * where the coordinator the agent reached may not configure the scaler that
1195
- * spawned it. The row's own targets are therefore primary, with local config
1196
- * as the fallback for a row that recorded none.
1197
- *
1198
- * A failed emit leaves the row in place: it is the only durable pointer at
1199
- * the customer's running instance, so the reaper needs it to retry from.
1010
+ * scaler, the ordinary shape on an HA cluster behind one shared endpoint.
1200
1011
  *
1201
- * Returns whether the teardown was actually emitted, so a caller that counts
1202
- * teardowns counts deliveries rather than attempts.
1012
+ * A failed emit leaves the row: it is the only durable pointer at the
1013
+ * customer's running instance, and the reaper retries from it. Returns
1014
+ * whether the teardown was emitted, so callers count deliveries, not attempts.
1203
1015
  */
1204
1016
  private emitScaleDownForSpec;
1205
1017
  /**
1206
- * Look up the scaler backend TYPE (a `ScalerBackendType` — `container`,
1207
- * `firecracker`, `bare-metal`, `event`, …) for a registered agent.
1208
- * Returns null if the agent is not scaler-managed (a static / stateful
1209
- * agent).
1018
+ * The backend TYPE (`container`, `firecracker`, `bare-metal`, `event`, …) of
1019
+ * a registered agent, or null for a static / stateful agent.
1210
1020
  *
1211
- * Used by AgentMetricsAggregator to stamp the `scaler` label on each
1212
- * kici_agent_* series. The label MUST be the backend type, not the
1213
- * operator-chosen scaler name: the Platform catalog filter constrains
1214
- * it to the ScalerBackendType set (plus `stateful`), so a free-form name is
1215
- * dropped as bad_label_value. `managedAgentIndex` stores the scaler
1216
- * name, so resolve it through `backends` to the type.
1021
+ * AgentMetricsAggregator stamps it as the `scaler` label on each
1022
+ * kici_agent_* series. It MUST be the type, not the operator-chosen name: the
1023
+ * Platform catalog filter allows only the ScalerBackendType set (plus
1024
+ * `stateful`) and drops anything else as bad_label_value.
1217
1025
  */
1218
1026
  getBackendType(agentId: string): string | null;
1219
1027
  /**
1220
1028
  * Exchange a provisioning claim code for freshly minted ephemeral credentials.
1221
1029
  *
1222
- * Reads the shared claim table directly rather than routing through the
1223
- * backend that minted the code: the mint needs only the agent-token store,
1224
- * which is already DB-backed, so redemption succeeds on an instance that has
1225
- * never heard of the emitting scaler. That is what lets an HA cluster front
1226
- * its coordinators with one shared endpoint — the provisioned agent reaches
1227
- * whichever coordinator the load balancer picks.
1030
+ * Reads the shared claim table directly, not through the minting backend: the
1031
+ * mint needs only the DB-backed agent-token store, so redemption works on an
1032
+ * instance that never heard of the emitting scaler. That lets an HA cluster
1033
+ * front its coordinators with one shared endpoint.
1228
1034
  *
1229
- * Returns `{ error }` when this instance has no claim store (no database), or
1230
- * when the redeem is rejected (unknown / consumed / expired / minting error).
1231
- * Called from the `scaler.claim-credentials` WS handler.
1035
+ * Returns `{ error }` with no claim store (no database) or a rejected redeem
1036
+ * (unknown / consumed / expired / minting error). Called from the
1037
+ * `scaler.claim-credentials` WS handler.
1232
1038
  */
1233
1039
  claimScalerCredentials(code: string): Promise<{
1234
1040
  credentials?: ClaimedCredentials;
1235
1041
  error?: string;
1236
1042
  }>;
1237
1043
  /**
1238
- * Called from agent-handler.ts when an agent sends config.ack.
1239
- * For Firecracker agents, clears MMDS data (belt-and-suspenders with in-VM iptables).
1240
- * For non-Firecracker agents, this is a no-op.
1044
+ * Called from agent-handler.ts on config.ack. Clears a Firecracker agent's MMDS
1045
+ * data (belt-and-suspenders with in-VM iptables); a no-op for other agents.
1241
1046
  */
1242
1047
  onConfigAck(agentId: string): void;
1243
- /**
1244
- * Called when a scaler-managed agent completes a job.
1245
- */
1048
+ /** Called when a scaler-managed agent completes a job. */
1246
1049
  onJobComplete(agentId: string): void;
1247
1050
  /**
1248
- * Correlate an agentId to a runId/jobId after the dispatcher assigns a job
1249
- * to a scaler-managed agent. Flushes any buffered pre-dispatch events.
1051
+ * Correlate an agent to a runId/jobId after dispatch, flushing buffered
1052
+ * pre-dispatch events.
1250
1053
  */
1251
1054
  correlateAgentToJob(agentId: string, runId: string, jobId: string): void;
1252
- /**
1253
- * Sum of all backends' active count + spawning agents.
1254
- */
1055
+ /** Sum of all backends' active counts, which include spawning agents. */
1255
1056
  getGlobalActiveCount(): number;
1256
1057
  /**
1257
1058
  * Shut every backend down in parallel, waiting at most
@@ -1259,9 +1060,7 @@ export declare class ScalerManager {
1259
1060
  * when the window closes is logged by name and left to finish on its own.
1260
1061
  */
1261
1062
  private shutdownBackendsWithinGrace;
1262
- /**
1263
- * Stop warm pool, shutdown all backends, clear tracking maps.
1264
- */
1063
+ /** Stop the warm pool, shut down all backends, clear tracking maps. */
1265
1064
  shutdownAll(): Promise<void>;
1266
1065
  /**
1267
1066
  * Classify every configured scaler against the currently loaded backends.
@@ -1272,22 +1071,18 @@ export declare class ScalerManager {
1272
1071
  * Validate the new config: label-set overlaps, backend type changes, and
1273
1072
  * each kept backend's own view of its label sets.
1274
1073
  *
1275
- * `backend.reload` validates and applies in one call, so a backend rejected
1276
- * late in the loop would otherwise leave the ones before it holding the new
1277
- * label sets and `maxAgents` while the commit never runs. Each successful
1278
- * call's previous state is therefore snapshotted and restored when any
1279
- * backend rejects — a rejected reload leaves every backend as it was.
1280
- *
1281
- * The snapshots are returned as well as used here: a later stage can still
1282
- * fail (an added scaler whose backend does not construct), and the same
1283
- * rollback is what keeps that rejection from half-applying too.
1074
+ * `backend.reload` validates and applies in one call, so each successful
1075
+ * call's previous state is snapshotted and restored if any backend rejects:
1076
+ * a rejected reload leaves every backend as it was. The snapshots are also
1077
+ * returned, so a later failing stage (an added backend that does not
1078
+ * construct) can roll back the same way.
1284
1079
  */
1285
1080
  private validateReload;
1286
1081
  /** Put every backend a rejected reload had already mutated back as it was. */
1287
1082
  private restoreReloadedBackends;
1288
1083
  /**
1289
- * Construct every added backend into a staging list. On any failure the
1290
- * partially built set is torn down and the caller keeps the old config.
1084
+ * Construct every added backend into a staging list; on any failure the
1085
+ * partial set is torn down and the old config stays.
1291
1086
  */
1292
1087
  private buildAddedBackends;
1293
1088
  /**
@@ -1298,15 +1093,14 @@ export declare class ScalerManager {
1298
1093
  */
1299
1094
  private prepareAddedBackends;
1300
1095
  /**
1301
- * Refresh per-scaler metadata from the new config. Keys are deleted only for
1302
- * scalers that are neither configured nor retiring — a retiring scaler keeps
1303
- * its cap, URL, roles and defaults until its last agent is gone.
1096
+ * Refresh per-scaler metadata. A retiring scaler keeps its cap, URL, roles and
1097
+ * defaults until its last agent is gone.
1304
1098
  */
1305
1099
  private applyScalerMetadata;
1306
1100
  /**
1307
- * Register every configured machine pool with the ledger, building the
1308
- * ledger when this reload is the first config to declare a pool. `cap` is
1309
- * re-registered on every reload, so a changed cap applies without a restart.
1101
+ * Register every configured machine pool, building the ledger when this reload
1102
+ * first declares a pool. `cap` is re-registered every reload, so a changed cap
1103
+ * applies without a restart.
1310
1104
  */
1311
1105
  private applyMachinePools;
1312
1106
  /** Rebuild the warm-pool config map from the new config (see WarmPoolManager.reload). */
@@ -1314,10 +1108,10 @@ export declare class ScalerManager {
1314
1108
  /**
1315
1109
  * Validate and reload configuration: plan → validate → build → commit.
1316
1110
  *
1317
- * Either the whole new config applies, or nothing does and the previous one
1318
- * keeps running. A scaler the new config no longer names is retired, not
1319
- * dropped: it stops taking work at once, its idle warm agents are destroyed,
1320
- * its job-bound agents finish, and its backend is torn down when it drains.
1111
+ * All or nothing: a rejected config leaves the previous one running. A scaler
1112
+ * the new config drops is retired: it stops taking work at once, its idle
1113
+ * warm agents are destroyed, its job-bound agents finish, and its backend is
1114
+ * torn down when it drains.
1321
1115
  */
1322
1116
  reload(newConfig: ScalerConfig): Promise<ScalerReloadResult>;
1323
1117
  /**
@@ -1325,199 +1119,149 @@ export declare class ScalerManager {
1325
1119
  * the commit, while the previous entries and global limits are still in place.
1326
1120
  */
1327
1121
  private describeReload;
1328
- /**
1329
- * Return status summary for metrics and health endpoints.
1330
- */
1331
- /**
1332
- * Recent scaler spawn failures grouped per backend instance, for the
1333
- * diagnose scaler check. `nowMs` is injected by the caller.
1334
- */
1122
+ /** Recent spawn failures per backend instance, for the diagnose scaler check. */
1335
1123
  recentSpawnFailures(windowMs: number, nowMs: number): Map<string, BackendFailureSummary>;
1124
+ /** Status summary for metrics and health endpoints. */
1336
1125
  getStatus(): ScalerStatus;
1337
1126
  /**
1338
1127
  * Scaler capacity advertised to peers in the cluster heartbeat.
1339
1128
  *
1340
- * Retiring backends are excluded: they accept no new work locally, so a peer
1341
- * that saw them would keep selecting this host for jobs it can no longer
1342
- * serve. Agents still draining on a retiring backend stay visible through the
1343
- * separate per-agent inventory, so in-flight work is unaffected.
1129
+ * Retiring backends are excluded, or peers would keep selecting this host for
1130
+ * jobs it can no longer serve. Their draining agents stay visible through the
1131
+ * per-agent inventory, so in-flight work is unaffected.
1344
1132
  */
1345
1133
  getRoutableCapacity(): ScalerCapacitySummary[];
1346
- /**
1347
- * Get the backend name managing a specific agent.
1348
- * Returns null if the agent is not scaler-managed (standalone).
1349
- */
1134
+ /** The scaler name managing an agent, or null for a standalone agent. */
1350
1135
  getBackendForAgent(agentId: string): string | null;
1351
1136
  /**
1352
- * Get a backend instance by scaler name. Returns undefined for an unknown
1353
- * name. Used by diagnostics to reach backend-specific accessors (e.g. the
1354
- * Firecracker backend's `getBridgeConfig()`).
1137
+ * A backend by scaler name, for diagnostics reaching backend-specific
1138
+ * accessors (e.g. the Firecracker backend's `getBridgeConfig()`).
1355
1139
  */
1356
1140
  getBackend(name: string): ScalerBackend | undefined;
1357
1141
  /**
1358
- * Whether some configured backend could spawn capacity for this label set —
1359
- * the same `findBackendForLabels` match `requestScale` makes, asked without
1360
- * spawning anything.
1361
- *
1362
- * Read by the queue-expiry sweep to tell a fleet/label problem apart from a
1363
- * capacity one: a job whose labels no agent AND no backend can serve could
1364
- * never have run (`unroutable`), while a job whose backend matched but whose
1365
- * spawn kept failing is a provisioning/capacity problem (`timed_out_stale`),
1366
- * and mislabelling the latter would tell an operator to fix a `runsOn` that
1367
- * is perfectly correct.
1368
- *
1369
- * Exact labels only — the scaler's own routing (`requestScale`) matches on
1370
- * exact labels, so a regex `runsOn` matcher is not expressible here. On a
1371
- * scaler-configured orchestrator that makes the answer conservative for a
1372
- * pattern-only `runsOn`: a backend whose labelSets carry no mandatory labels
1373
- * matches the empty exact set, so such a job is reported routable and settles
1374
- * `timed_out_stale` rather than `unroutable`. That loses precision, never
1375
- * safety — the job is still terminal and the run still fails, which is the
1376
- * invariant that matters. Tightening it means teaching scaler routing about
1377
- * patterns, which would change where jobs actually spawn.
1142
+ * Whether some configured backend could spawn for this label set: the
1143
+ * `findBackendForLabels` match `requestScale` makes, without spawning.
1144
+ *
1145
+ * The queue-expiry sweep uses it to separate a label problem (no agent AND no
1146
+ * backend: `unroutable`) from a provisioning/capacity one (`timed_out_stale`),
1147
+ * so an operator is not told to fix a correct `runsOn`.
1148
+ *
1149
+ * Exact labels only, as in `requestScale`, so a regex `runsOn` is not
1150
+ * expressible. For a pattern-only `runsOn` a backend with no mandatory labels
1151
+ * matches the empty set, so the job settles `timed_out_stale` instead of
1152
+ * `unroutable`: less precise, never unsafe (the job is terminal and the run
1153
+ * fails either way). Tightening it means teaching scaler routing about
1154
+ * patterns, which would change where jobs spawn.
1378
1155
  */
1379
1156
  hasBackendForLabels(labels: string[], excludeLabels?: string[]): boolean;
1380
1157
  /**
1381
- * Get scaler-specific configuration metadata for a managed agent.
1382
- * Returns undefined for non-scaler-managed (static) agents.
1383
- * Used to enrich job.context before forwarding to Platform.
1158
+ * Scaler-specific metadata for a managed agent (undefined for a static one),
1159
+ * used to enrich job.context before forwarding to the Platform.
1384
1160
  */
1385
1161
  getScalerContextForAgent(agentId: string): Record<string, unknown> | undefined;
1386
1162
  /**
1387
- * Start the machine-pool ledger reaper and the retirement sweep that tears
1388
- * down drained retiring backends, and publish the warm-pool gauges so each
1389
- * pool's target is visible before its first fill.
1390
- *
1391
- * The warm pools themselves are NOT started here — see
1392
- * {@link startWarmPools}, which the host calls once every dependency a fill
1393
- * can reach is wired.
1163
+ * Start the machine-pool ledger reaper and the retirement sweep, and publish
1164
+ * the warm-pool gauges so each target is visible before its first fill. The
1165
+ * warm pools themselves start in {@link startWarmPools}.
1394
1166
  */
1395
1167
  start(): void;
1396
1168
  /**
1397
1169
  * Run the warm pools' first deficit pass and arm their periodic tick.
1398
- *
1399
- * Separate from {@link start} because the first pass spawns immediately,
1400
- * and on an event backend a spawn emits through the event emitter — which
1401
- * the coordinator assigns after the scaler is built. The coordinator calls
1402
- * this once the emitter exists; the worker, which hosts no event backend,
1403
- * calls it right after `start()`.
1170
+ * Separate from {@link start}: the first pass spawns at once, and an event
1171
+ * backend's spawn needs the event emitter, which the coordinator assigns
1172
+ * after building the scaler. The worker (no event backend) calls it right
1173
+ * after `start()`.
1404
1174
  */
1405
1175
  startWarmPools(): void;
1406
1176
  /**
1407
- * Provision/heal every backend's host prerequisites before spawning starts.
1408
- * Awaits each backend's optional ensureHostReady, catching per-backend so one
1409
- * scaler's host-prep failure degrades only that scaler (its spawns will fail
1410
- * with clear errors + the firecracker-network diagnostic reports it) rather
1411
- * than aborting orchestrator startup.
1177
+ * Provision/heal every backend's host prerequisites before spawning starts. A
1178
+ * failure degrades only that scaler (its spawns fail with clear errors and
1179
+ * the firecracker-network diagnostic reports it), never aborts startup.
1412
1180
  */
1413
1181
  ensureHostsReady(): Promise<void>;
1414
1182
  /**
1415
1183
  * The window a spawning entry may sit unregistered before it is pruned.
1416
1184
  *
1417
- * An entry cannot be stale before its own spawn deadline has passed, so the
1418
- * window is floored at the configured deadline: an operator raising
1419
- * `KICI_SCALER_SPAWN_TIMEOUT_MS` for a slow cloud must not have this prune
1420
- * fire first and drop the correlation a provision booting inside the raised
1421
- * deadline still needs. The floor only ever extends the window — a lowered
1422
- * deadline keeps the default five minutes, because a local backend's agent
1423
- * boots and registers well after `spawn()` itself returns.
1185
+ * Floored at the configured spawn deadline, so raising
1186
+ * `KICI_SCALER_SPAWN_TIMEOUT_MS` for a slow cloud never has the prune drop
1187
+ * the correlation of a provision still booting inside it. A lowered deadline
1188
+ * keeps the default five minutes: a local backend's agent registers well
1189
+ * after `spawn()` returns.
1424
1190
  */
1425
1191
  private stalePruneWindowMs;
1426
1192
  /**
1427
1193
  * Remove spawning entries whose `backend.spawn` started longer ago than the
1428
- * stale window but never registered via WS (e.g., process/container crashed
1429
- * on startup). Without cleanup, these entries would leak in spawningAgents
1430
- * forever.
1431
- *
1432
- * The window is measured from `spawnStartedAt` (when the throttled spawn
1433
- * actually began), NOT from `spawnedAt` (enqueue time): a spawn still waiting
1434
- * behind the per-backend spawn semaphore has not started and must not be
1435
- * reaped mid-queue during a large burst. Recovered orphan entries carry
1436
- * `spawnStartedAt` set to their persisted enqueue time, so they are subject to
1437
- * the normal window.
1438
- *
1439
- * What happens to the durable row depends on the backend type the spawn
1440
- * recorded, which the entry carries so a scaler since removed from this
1441
- * coordinator's config still resolves. A local backend's
1442
- * compute is pinned to a host this process manages, so a pruned entry means
1443
- * the spawn is dead and the row goes. An event backend's row is the only
1444
- * durable pointer at a customer cloud instance, and it is precisely the
1445
- * reaper's `spawn-timeout` candidate — so the row STAYS, and the reaper
1446
- * decides. Deleting it here would beat the reaper by the flap grace on any
1447
- * coordinator serving jobs, so the teardown the row exists to trigger would
1448
- * never be emitted.
1194
+ * stale window and never registered via WS (e.g. a crash on startup), which
1195
+ * would otherwise leak forever.
1196
+ *
1197
+ * Measured from `spawnStartedAt`, NOT `spawnedAt` (enqueue): a spawn still
1198
+ * queued behind the semaphore must not be reaped mid-burst. Recovered entries
1199
+ * carry `spawnStartedAt` = their persisted enqueue time.
1200
+ *
1201
+ * The durable row's fate follows the backend type the spawn recorded. A local
1202
+ * backend's pruned spawn is dead, so the row goes. An event backend's row is
1203
+ * the only pointer at a customer cloud instance and is the reaper's
1204
+ * `spawn-timeout` candidate, so it STAYS: deleting it would beat the reaper by
1205
+ * the flap grace and the teardown would never be emitted.
1449
1206
  */
1450
1207
  private pruneStaleSpawningEntries;
1451
1208
  /**
1452
1209
  * The spawn row for one agent, as every writer of it must build it.
1453
1210
  *
1454
- * Ownership and self-describing columns are assembled only here.
1455
- * `backend_type` is what `adoptSpawningAgent`'s conditional UPDATE matches
1456
- * on, so a row without it can never be adopted by any instance — including
1457
- * the one that spawned it. `owner_instance_id` scopes recovery to that
1458
- * instance, and `mandatory_labels` carries the platform taint to a
1459
- * coordinator that has no config entry for this scaler and can therefore
1460
- * derive it from nothing else. `provisioning_targets` is the same story for
1461
- * teardown: it is the only address such a coordinator has for the customer's
1462
- * teardown workflow, and an emit to an empty target list reaches nobody, so
1463
- * the provision would bill forever. `roles` travels with it so the row stays
1464
- * a complete description of the spawn.
1211
+ * Ownership and self-describing columns are assembled only here:
1212
+ * `backend_type` is what `adoptSpawningAgent`'s conditional UPDATE matches, so
1213
+ * a row without it is never adoptable; `owner_instance_id` scopes recovery;
1214
+ * `mandatory_labels` and `provisioning_targets` give a coordinator with no
1215
+ * config entry for the scaler its platform taint and its only teardown
1216
+ * address (an empty target list reaches nobody, so the provision would bill
1217
+ * forever); `roles` keeps the row a complete description of the spawn.
1465
1218
  */
1466
1219
  private spawningAgentSnapshot;
1467
1220
  /**
1468
1221
  * Claim one slot against an event scaler's cluster-wide `maxAgents`.
1469
1222
  *
1470
- * The count and the claim share a single advisory-locked transaction, so the
1471
- * row this writes is already visible to the next coordinator that takes the
1472
- * lock. A store failure is never an admission: returning false costs a job a
1473
- * queue wait, while admitting on a count nobody recorded provisions a cloud
1474
- * instance the cap cannot see. Failing closed is not silent — a refusal
1475
- * increments the spawn-refusal counter and a store failure its own counter,
1476
- * labelled so a database outage, a contended lock, and a full cluster are
1477
- * three distinguishable things rather than one.
1478
- *
1479
- * The failure arm bounds the admission, not the row: a transaction that
1480
- * commits and then loses its acknowledgement leaves `slotClaimed` false, so
1481
- * neither rollback path fires and the row counts against the cap for an
1482
- * agent that never spawned. Closing that window needs two-phase commit; the
1483
- * event-provision reaper is the backstop, clearing the row once the spawn
1484
- * deadline passes with nothing adopted.
1223
+ * Count and claim share one advisory-locked transaction, so the row is visible
1224
+ * to the next coordinator taking the lock. A store failure never admits:
1225
+ * refusing costs a queue wait, admitting provisions an instance the cap
1226
+ * cannot see. Refusals and store failures are counted, labelled so a database
1227
+ * outage, a contended lock and a full cluster are distinguishable.
1228
+ *
1229
+ * A transaction that commits but loses its acknowledgement leaves
1230
+ * `slotClaimed` false, so the row counts against the cap for an agent that
1231
+ * never spawned. Closing that needs two-phase commit; the event-provision
1232
+ * reaper is the backstop, clearing the row once the spawn deadline passes
1233
+ * with nothing adopted.
1485
1234
  */
1486
1235
  private claimClusterSlot;
1487
1236
  private persistSpawningAgent;
1488
1237
  private deleteSpawningAgentFromStore;
1489
1238
  /**
1490
- * Stamp `adopted_by = this instance` on the spawning row of an agent that just
1491
- * registered here. Reuses the cross-instance conditional UPDATE, so a row some
1492
- * other instance already claimed is left exactly as it is.
1239
+ * Stamp `adopted_by = this instance` on a just-registered agent's row, via the
1240
+ * conditional UPDATE, so a row another instance claimed is left as it is.
1493
1241
  */
1494
1242
  private markSpawningAgentAdopted;
1495
1243
  /**
1496
1244
  * Record that `backend.spawn()` resolved for a spawn this instance started.
1497
1245
  *
1498
- * Returns false when the stale-spawn prune dropped the entry while
1499
- * `spawn()` was still running (a per-org spawn deadline can outlast the
1500
- * prune window). Nothing else would ever tear that compute down, so a local
1501
- * backend's is destroyed here. An event backend's provision is left to the
1502
- * reaper, as the prune leaves it.
1246
+ * False when the prune dropped the entry while `spawn()` ran (a per-org
1247
+ * deadline can outlast the prune window). Nothing else would tear that
1248
+ * compute down, so a local backend's is destroyed here; an event provision is
1249
+ * left to the reaper.
1503
1250
  */
1504
1251
  private onSpawnSettled;
1505
1252
  /**
1506
1253
  * Drop a settled local spawn whose backend reported it failed.
1507
1254
  *
1508
- * After `spawn()` resolved, a backend that finds its compute gone before the
1509
- * agent registered (a Firecracker guest that exited, a bare-metal agent
1510
- * process that exited) tears the compute down itself and emits
1511
- * `scaler.failed`. The agent will never register, so the spawning entry, its
1512
- * durable row and its reservation go now — waiting for the stale-spawn prune
1513
- * would hold the scaler's slot for minutes.
1255
+ * A backend whose compute dies before registration (a Firecracker guest or a
1256
+ * bare-metal agent process that exited) tears it down and emits
1257
+ * `scaler.failed`. The entry, durable row and reservation go now; waiting
1258
+ * for the prune would hold the slot for minutes.
1514
1259
  *
1515
- * A spawn that produced no agent is a refused launch, the same rule an event
1516
- * scaler's dead provision follows: the scaler is deferred before the
1517
- * reservation is freed, so an agent that dies at startup every time is not
1518
- * respawned at its own startup cadence. Only a spawn that has not registered
1519
- * reaches here — registration deletes the spawning entry — so an agent that
1520
- * registered and exits later never arms the backoff.
1260
+ * Like an event scaler's dead provision, it is a refused launch: the scaler is
1261
+ * deferred before the reservation is freed, so an agent dying at every
1262
+ * startup is not respawned at its startup cadence. Only an unregistered spawn
1263
+ * reaches here, so an agent that exits after registering never arms the
1264
+ * backoff.
1521
1265
  */
1522
1266
  private dropFailedSettledSpawn;
1523
1267
  /** Store-side half of the stale prune: drops the row only if nobody adopted it. */
@@ -1527,75 +1271,61 @@ export declare class ScalerManager {
1527
1271
  private persistAgentJob;
1528
1272
  private deleteAgentJobFromStore;
1529
1273
  /**
1530
- * Hydrate the in-memory Maps from the DB-backed state store after a
1531
- * coord boot or Raft leader switch. Reconstructs:
1274
+ * Hydrate the in-memory maps from the state store after a coordinator boot
1275
+ * or Raft leader switch:
1532
1276
  *
1533
- * - `spawningAgents` (with `boundJobId` preserved for eager-dispatch on register)
1277
+ * - `spawningAgents` (with `boundJobId` for eager-dispatch on register)
1534
1278
  * — event-backend rows only; a local-backend row is dropped, see below
1535
1279
  * - `agentJobCorrelation` (so scaler-lifecycle events route correctly)
1536
- * - `reservations` + `perScalerUsage` (so the cap-check critical
1537
- * section reflects the cluster-wide truth, not the local empty
1538
- * starting state)
1280
+ * - `reservations` + `perScalerUsage` (so the cap check sees the real
1281
+ * state, not an empty start)
1539
1282
  *
1540
- * The `globalUsage` counter is recomputed from `perScalerUsage` to
1541
- * keep the cap math consistent. `eventBuffer` is NOT restored — events
1542
- * emitted by the previous coord before correlation are lost (they are
1543
- * observability, not correctness: nothing downstream waits on them).
1283
+ * `globalUsage` is recomputed from `perScalerUsage`. `eventBuffer` is NOT
1284
+ * restored: its events are observability, and nothing waits on them.
1544
1285
  *
1545
- * Both reads are scoped to this instance's own rows. An unscoped read
1546
- * hydrates a peer's in-flight spawns and reservations as our own, so our
1547
- * spawn-timeout reaper destroys agents that peer is still legitimately
1548
- * waiting on and our caps double-count its reservations. Rows owned by a
1549
- * dead instance are the leader-gated reaper's business, not ours.
1550
- *
1551
- * No-op when no store is wired (unit-test path).
1286
+ * Both reads are scoped to this instance's own rows. An unscoped read would
1287
+ * hydrate a peer's in-flight spawns as ours, so our spawn-timeout reaper
1288
+ * destroys agents the peer still waits on and our caps double-count. A dead
1289
+ * instance's rows belong to the leader-gated reaper. No-op with no store.
1552
1290
  */
1553
1291
  recoverState(): Promise<ScalerStateRecovery>;
1554
1292
  /**
1555
- * Mint a scaler-managed agent id.
1556
- *
1557
- * Built from {@link SCALER_AGENT_ID_PREFIX} rather than a second copy of the
1558
- * literal: the registration guard in `onAgentRegistered` recognises a
1559
- * scaler-managed agent by that prefix alone when the spawn-record lookup
1560
- * fails, so a minting prefix that drifts from the constant leaves the guard
1561
- * matching nothing and failing open.
1293
+ * Mint a scaler-managed agent id from {@link SCALER_AGENT_ID_PREFIX}, never a
1294
+ * copy of the literal: `onAgentRegistered`'s guard recognises scaler agents by
1295
+ * that prefix alone when the spawn-record lookup fails, so a drifted prefix
1296
+ * would fail open.
1562
1297
  */
1563
1298
  private generateAgentId;
1564
1299
  /**
1565
- * Start log forwarding for a scaler-managed agent if its backend supports LogCapture.
1566
- * Fire-and-forget: the forward() promise runs for the agent's lifetime.
1300
+ * Start log forwarding if the backend supports LogCapture. Fire-and-forget:
1301
+ * forward() runs for the agent's lifetime.
1567
1302
  */
1568
1303
  private startLogForwarding;
1569
1304
  private getOrchestratorUrl;
1570
1305
  /**
1571
- * Create a per-agent event emitter closure to pass to backend.spawn().
1572
- * The closure captures agentId and routes events through handleScalerEvent().
1306
+ * Per-agent event emitter for backend.spawn(), routing through
1307
+ * handleScalerEvent().
1573
1308
  */
1574
1309
  private createEventEmitter;
1575
1310
  /**
1576
1311
  * Handle a scaler event from a backend.
1577
1312
  *
1578
- * For spawn FAILURES the bound spawning entry is the usual resolver, since
1579
- * correlation isn't set until a job is dispatched post-registration. The
1580
- * correlation map wins only in the rarer post-registration failure window
1581
- * (e.g. a bare-metal child 'error' firing after the agent already
1582
- * registered, by which point the spawning entry is gone). Warm-pool /
1583
- * unbound spawns resolve to neither. When attributed, the event is relayed
1584
- * immediately; otherwise it is buffered until correlateAgentToJob() flushes
1585
- * it.
1586
- *
1587
- * Every `scaler.failed` also increments the fleet-wide spawn-failure counter
1588
- * and emits a structured warn, regardless of whether it could be attributed.
1589
- * The backend label is resolved from the spawning entry first, then the
1590
- * managed-agent index — so a late failure after the spawning entry is gone
1591
- * still buckets under the real backend instead of "unknown".
1592
- *
1593
- * `attribution` is the last resort for a failure this coordinator did not
1594
- * spawn and never registered: the reaper is leader-gated, so the instance that
1595
- * condemns a stranded external provision routinely has neither a spawning
1596
- * entry nor a managed-agent index row for it, and every in-memory resolver
1597
- * above comes back empty. It is read only where the maps are silent, so a live
1598
- * local entry always wins over a durable row that may be a sweep behind.
1313
+ * A spawn FAILURE usually resolves through the bound spawning entry, since
1314
+ * correlation is set only at dispatch. The correlation map wins in the rarer
1315
+ * post-registration window (e.g. a bare-metal child 'error' after
1316
+ * registration, when the spawning entry is gone). Unbound spawns resolve to
1317
+ * neither. An attributed event is relayed at once; otherwise it is buffered
1318
+ * until correlateAgentToJob() flushes it.
1319
+ *
1320
+ * Every `scaler.failed` increments the fleet-wide spawn-failure counter and
1321
+ * warns, attributed or not. The backend label comes from the spawning entry,
1322
+ * then the managed-agent index, so a late failure still buckets under the
1323
+ * real backend, not "unknown".
1324
+ *
1325
+ * `attribution` is the last resort for a failure this coordinator neither
1326
+ * spawned nor registered (typical for the leader-gated reaper). It is read
1327
+ * only where the maps are silent, so a live local entry beats a durable row
1328
+ * that may be a sweep behind.
1599
1329
  */
1600
1330
  private handleScalerEvent;
1601
1331
  }