@kici-dev/orchestrator 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent-version.d.ts +12 -21
- package/dist/agent/dispatcher.d.ts +283 -437
- package/dist/agent/host-roster-reaper.d.ts +8 -10
- package/dist/agent/host-roster.d.ts +77 -100
- package/dist/agent/ownership-refusal.d.ts +3 -5
- package/dist/agent/ownership-tracker.d.ts +42 -76
- package/dist/agent/registry.d.ts +121 -233
- package/dist/agent/token-store.d.ts +34 -77
- package/dist/agent-packaging/availability.d.ts +6 -9
- package/dist/agent-packaging/download.d.ts +6 -9
- package/dist/agent-packaging/install-closure.d.ts +9 -12
- package/dist/agent-packaging/node-binary.d.ts +6 -10
- package/dist/agent-packaging/node-runtime-members.d.ts +3 -5
- package/dist/agent-packaging/refresh.d.ts +12 -16
- package/dist/agent-packaging/store.d.ts +5 -6
- package/dist/agent-packaging/upload.d.ts +3 -4
- package/dist/app.d.ts +0 -3
- package/dist/approvals/apply-decision.d.ts +39 -45
- package/dist/approvals/approval-resolver.d.ts +11 -23
- package/dist/approvals/held-run-release-wiring.d.ts +14 -21
- package/dist/approvals/independent-wiring.d.ts +13 -20
- package/dist/approvals/resume-router.d.ts +1 -1
- package/dist/approvals/step-approval-bridge.d.ts +11 -21
- package/dist/approvals/triggerer-subject.d.ts +8 -10
- package/dist/artifacts/artifact-store.d.ts +55 -95
- package/dist/artifacts/failure-messages.d.ts +15 -20
- package/dist/audit/access-log.d.ts +27 -45
- package/dist/audit/sampling-rate-limiter.d.ts +11 -23
- package/dist/cache/agent-job-failed-error.d.ts +2 -3
- package/dist/cache/build-coordinator.d.ts +5 -28
- package/dist/cache/cache-upload-settle.d.ts +11 -11
- package/dist/cache/dep-cache.d.ts +32 -60
- package/dist/cache/dispatch-cache-ref-tracker.d.ts +15 -23
- package/dist/cache/global-eval-round-cache.d.ts +33 -41
- package/dist/cache/pending-builds.d.ts +2 -6
- package/dist/cache/pending-dynamics.d.ts +2 -7
- package/dist/cache/pending-global-evals.d.ts +9 -12
- package/dist/cache/pending-inits.d.ts +11 -22
- package/dist/cache/pending-precursor-db-watcher.d.ts +24 -32
- package/dist/cache/pending-tracker.d.ts +17 -45
- package/dist/cache/precursor-result.d.ts +17 -30
- package/dist/cache/source-cache.d.ts +36 -65
- package/dist/cache/user-cache.d.ts +21 -25
- package/dist/cancel/cancel-held-run.d.ts +13 -16
- package/dist/cancel/cancel-route-answer.d.ts +1 -2
- package/dist/cancel/cancel-run.d.ts +26 -50
- package/dist/cancel/dashboard-cancel-handler.d.ts +2 -2
- package/dist/cancel/peer-job-cancel.d.ts +11 -14
- package/dist/cancel/sweep-stuck-cancelling.d.ts +9 -13
- package/dist/cli/api-client.d.ts +33 -76
- package/dist/cli/commands/access-log.d.ts +4 -24
- package/dist/cli/commands/agent-package.d.ts +7 -15
- package/dist/cli/commands/agent-service/agent-env.d.ts +3 -6
- package/dist/cli/commands/agent-service/index.d.ts +4 -10
- package/dist/cli/commands/agent-service/install.d.ts +4 -11
- package/dist/cli/commands/agent-service/status.d.ts +4 -9
- package/dist/cli/commands/agent-service/upgrade.d.ts +4 -16
- package/dist/cli/commands/agent.d.ts +12 -25
- package/dist/cli/commands/api-key.d.ts +7 -7
- package/dist/cli/commands/attestations-list.d.ts +7 -17
- package/dist/cli/commands/attestations-retry.d.ts +5 -11
- package/dist/cli/commands/attestations-reverify.d.ts +4 -5
- package/dist/cli/commands/attestations.d.ts +3 -8
- package/dist/cli/commands/audit.d.ts +1 -5
- package/dist/cli/commands/backend.d.ts +2 -11
- package/dist/cli/commands/check-run.d.ts +7 -18
- package/dist/cli/commands/cluster-name.d.ts +5 -12
- package/dist/cli/commands/cluster-settings.d.ts +28 -53
- package/dist/cli/commands/cold-store.d.ts +7 -15
- package/dist/cli/commands/config.d.ts +9 -20
- package/dist/cli/commands/context.d.ts +8 -20
- package/dist/cli/commands/dashboard-encryption-key.d.ts +9 -14
- package/dist/cli/commands/db-backup.d.ts +17 -32
- package/dist/cli/commands/db.d.ts +3 -4
- package/dist/cli/commands/debug-bundle.d.ts +3 -10
- package/dist/cli/commands/diagnose.d.ts +3 -14
- package/dist/cli/commands/event-dlq.d.ts +4 -12
- package/dist/cli/commands/event-log.d.ts +2 -20
- package/dist/cli/commands/execution.d.ts +3 -9
- package/dist/cli/commands/firecracker/index.d.ts +3 -6
- package/dist/cli/commands/held-run.d.ts +23 -37
- package/dist/cli/commands/host.d.ts +4 -11
- package/dist/cli/commands/inspect-bundle.d.ts +3 -11
- package/dist/cli/commands/local-hook.d.ts +2 -10
- package/dist/cli/commands/local-trigger.d.ts +2 -2
- package/dist/cli/commands/orchestrator-service/drain.d.ts +3 -6
- package/dist/cli/commands/orchestrator-service/index.d.ts +6 -14
- package/dist/cli/commands/orchestrator-service/install-env.d.ts +25 -49
- package/dist/cli/commands/orchestrator-service/install.d.ts +2 -5
- package/dist/cli/commands/orchestrator-service/status.d.ts +10 -22
- package/dist/cli/commands/orchestrator-service/upgrade.d.ts +6 -15
- package/dist/cli/commands/org-settings.d.ts +5 -11
- package/dist/cli/commands/peer.d.ts +5 -10
- package/dist/cli/commands/queue.d.ts +3 -9
- package/dist/cli/commands/registration.d.ts +4 -8
- package/dist/cli/commands/remote-source.d.ts +6 -12
- package/dist/cli/commands/rotate.d.ts +1 -5
- package/dist/cli/commands/runs.d.ts +1 -30
- package/dist/cli/commands/scaler.d.ts +5 -12
- package/dist/cli/commands/secret.d.ts +1 -9
- package/dist/cli/commands/service-health.d.ts +5 -10
- package/dist/cli/commands/shared/admin-cli-access-log.d.ts +10 -19
- package/dist/cli/commands/shared/bare-metal-binary-check.d.ts +86 -0
- package/dist/cli/commands/shared/cli-action.d.ts +1 -4
- package/dist/cli/commands/shared/confirm.d.ts +5 -10
- package/dist/cli/commands/shared/db.d.ts +1 -12
- package/dist/cli/commands/shared/env-file-mode.d.ts +6 -9
- package/dist/cli/commands/shared/secret-input.d.ts +7 -21
- package/dist/cli/commands/shared/service-verbs.d.ts +6 -12
- package/dist/cli/commands/shared/unbound-context-warning.d.ts +2 -4
- package/dist/cli/commands/shared/upgrade-hooks.d.ts +7 -16
- package/dist/cli/commands/shared/upgrade-safety.d.ts +20 -31
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +83 -161
- package/dist/cli/commands/signing-key.d.ts +4 -9
- package/dist/cli/commands/source-manifest.d.ts +11 -14
- package/dist/cli/commands/source.d.ts +7 -32
- package/dist/cli/commands/token.d.ts +4 -15
- package/dist/cli/commands/trust-policy.d.ts +35 -88
- package/dist/cli/commands/variable.d.ts +3 -5
- package/dist/cli/commands/workflow.d.ts +3 -24
- package/dist/cli/join.d.ts +3 -6
- package/dist/cli/kici-admin.d.ts +11 -29
- package/dist/cli/lazy-deps/cache.d.ts +1 -14
- package/dist/cli/lazy-deps/downloader.d.ts +4 -27
- package/dist/cli/lazy-deps/registry.d.ts +5 -25
- package/dist/cli/local-github-ingress-url.d.ts +5 -7
- package/dist/cli/loopback-callback.d.ts +6 -7
- package/dist/cli/open-browser.d.ts +4 -8
- package/dist/cli/root-option-routing.d.ts +17 -29
- package/dist/cli/service/backup-timer.d.ts +16 -39
- package/dist/cli/service/compose-path.d.ts +3 -7
- package/dist/cli/service/compose.d.ts +12 -30
- package/dist/cli/service/deploy-env.d.ts +6 -18
- package/dist/cli/service/entrypoint.d.ts +14 -25
- package/dist/cli/service/env-assignment.d.ts +6 -17
- package/dist/cli/service/image-digests.d.ts +6 -15
- package/dist/cli/service/index.d.ts +2 -18
- package/dist/cli/service/instance/index-file.d.ts +9 -16
- package/dist/cli/service/instance/manifest.d.ts +33 -59
- package/dist/cli/service/instance/resolve.d.ts +32 -68
- package/dist/cli/service/instance/types.d.ts +12 -29
- package/dist/cli/service/launchd.d.ts +29 -60
- package/dist/cli/service/platform-detect.d.ts +13 -68
- package/dist/cli/service/privilege.d.ts +12 -25
- package/dist/cli/service/shutdown-grace.d.ts +11 -20
- package/dist/cli/service/systemd.d.ts +7 -20
- package/dist/cli/service/types.d.ts +63 -114
- package/dist/cli/service/windows-acl.d.ts +14 -21
- package/dist/cli/service/windows-env-file.d.ts +10 -28
- package/dist/cli/service/windows.d.ts +15 -31
- package/dist/cli/webhook-url-notes.d.ts +4 -6
- package/dist/cli/wizard/agent-wizard.d.ts +3 -14
- package/dist/cli/wizard/orchestrator-wizard.d.ts +12 -37
- package/dist/cli/wizard/prompts.d.ts +6 -16
- package/dist/cli.js +4553 -6556
- package/dist/cluster/cluster-identity.d.ts +20 -32
- package/dist/cluster/cluster-settings-reader.d.ts +50 -77
- package/dist/cluster/coordinator-credential.d.ts +4 -5
- package/dist/cluster/coordinator.d.ts +76 -148
- package/dist/cluster/health-api.d.ts +2 -10
- package/dist/cluster/index.d.ts +1 -6
- package/dist/cluster/instance-heartbeat.d.ts +26 -42
- package/dist/cluster/join-client.d.ts +4 -7
- package/dist/cluster/join-handler.d.ts +3 -7
- package/dist/cluster/join-token.d.ts +13 -27
- package/dist/cluster/leader-gated-scheduler.d.ts +8 -17
- package/dist/cluster/orphan-recovery.d.ts +13 -35
- package/dist/cluster/peer-auth-coordinator.d.ts +2 -8
- package/dist/cluster/peer-client.d.ts +46 -76
- package/dist/cluster/peer-credentials.d.ts +24 -70
- package/dist/cluster/peer-crypto.d.ts +5 -26
- package/dist/cluster/peer-forget.d.ts +3 -3
- package/dist/cluster/peer-handler.d.ts +47 -71
- package/dist/cluster/peer-registry.d.ts +37 -106
- package/dist/cluster/plan-headroom-store.d.ts +4 -6
- package/dist/cluster/raft-state.d.ts +4 -13
- package/dist/cluster/raft.d.ts +18 -53
- package/dist/cluster/reconcile-identity.d.ts +8 -11
- package/dist/cluster/rerouted-job-guard.d.ts +7 -11
- package/dist/cluster/verified-issuer.d.ts +2 -3
- package/dist/cluster/worker-cluster-settings.d.ts +7 -10
- package/dist/cluster/worker-eviction.d.ts +16 -26
- package/dist/cold-store/load-access-log-range.d.ts +15 -31
- package/dist/cold-store/load-event-log-range.d.ts +13 -27
- package/dist/cold-store/load-secret-audit-log-range.d.ts +8 -12
- package/dist/cold-store/orchestrator-cold-store.d.ts +8 -28
- package/dist/cold-store/tables/access-log.d.ts +12 -14
- package/dist/cold-store/tables/event-log.d.ts +4 -8
- package/dist/cold-store/tables/execution-runs.d.ts +6 -11
- package/dist/cold-store/tables/secret-audit-log.d.ts +5 -12
- package/dist/concurrency/dispatch-next-queued.d.ts +13 -25
- package/dist/concurrency/queue-manager.d.ts +22 -54
- package/dist/concurrency/waiters.d.ts +15 -31
- package/dist/config/encryption.d.ts +0 -2
- package/dist/config/loader.d.ts +0 -1
- package/dist/config/reload.d.ts +0 -6
- package/dist/config/resolver.d.ts +0 -4
- package/dist/config/shared-store.d.ts +0 -2
- package/dist/config/types.d.ts +0 -3
- package/dist/content-requirements-cache.d.ts +0 -1
- package/dist/contexts/context-store.d.ts +0 -2
- package/dist/contexts/held-runs.d.ts +0 -2
- package/dist/contexts/variable-store.d.ts +0 -2
- package/dist/dashboard/attestation-filters.d.ts +13 -18
- package/dist/dashboard/handler.d.ts +143 -307
- package/dist/db/client.d.ts +2 -6
- package/dist/db/deferred-index-build.d.ts +8 -10
- package/dist/db/deferred-indexes.d.ts +15 -22
- package/dist/db/migration-provider.d.ts +2 -4
- package/dist/db/migration-test-harness.d.ts +26 -44
- package/dist/db/migrations/001_initial.d.ts +1 -4
- package/dist/db/migrations/002_config_versions_key_version.d.ts +5 -8
- package/dist/db/migrations/003_access_log.d.ts +10 -15
- package/dist/db/migrations/004_rename_bundle_to_source.d.ts +7 -9
- package/dist/db/migrations/005_cold_store_chunk_counter.d.ts +4 -10
- package/dist/db/migrations/006_runs_jobs_steps_archived_at.d.ts +9 -29
- package/dist/db/migrations/007_audit_logs_archived_at.d.ts +11 -30
- package/dist/db/migrations/008_event_log_archived_at.d.ts +14 -29
- package/dist/db/migrations/009_access_log_trigram.d.ts +4 -8
- package/dist/db/migrations/010_cold_store_chunks.d.ts +16 -26
- package/dist/db/migrations/011_drop_source_secrets_notify.d.ts +6 -14
- package/dist/db/migrations/012_peer_credentials_active_uniq.d.ts +9 -14
- package/dist/db/migrations/013_execution_log_bytes.d.ts +5 -13
- package/dist/db/migrations/014_kici_events_lease_retry.d.ts +14 -27
- package/dist/db/migrations/015_org_settings_customer_scoped.d.ts +12 -17
- package/dist/db/migrations/016_org_settings_allow_http_npm.d.ts +4 -6
- package/dist/db/migrations/019_generic_sources_change_notify.d.ts +13 -17
- package/dist/db/migrations/020_org_settings_dashboard_write_policy.d.ts +5 -8
- package/dist/db/migrations/021_check_run_tracking.d.ts +5 -17
- package/dist/db/migrations/022_scaler_manager_state.d.ts +10 -20
- package/dist/db/migrations/023_dispatch_queue_recovery_deadline.d.ts +7 -18
- package/dist/db/migrations/024_dispatch_queue_provisioning_error.d.ts +5 -11
- package/dist/db/migrations/025_init_failure.d.ts +5 -9
- package/dist/db/migrations/027_workflow_timeout.d.ts +5 -13
- package/dist/db/migrations/029_dispatch_queue_attempts.d.ts +4 -10
- package/dist/db/migrations/030_held_runs_env_set_null.d.ts +5 -6
- package/dist/db/migrations/031_dispatch_queue_ack_deadline.d.ts +6 -12
- package/dist/db/migrations/033_org_settings_approval.d.ts +7 -10
- package/dist/db/migrations/034_held_runs_generalize.d.ts +10 -18
- package/dist/db/migrations/035_pending_workflow_contexts.d.ts +4 -5
- package/dist/db/migrations/037_generic_sources_provider_type_local.d.ts +5 -6
- package/dist/db/migrations/038_remote_sources.d.ts +5 -7
- package/dist/db/migrations/045_host_properties.d.ts +5 -9
- package/dist/db/migrations/046_join_token_consumed_by_instance.d.ts +5 -10
- package/dist/db/migrations/048_host_reboot_pending.d.ts +6 -12
- package/dist/db/migrations/049_held_runs_payload.d.ts +3 -8
- package/dist/db/migrations/050_sources_slug.d.ts +4 -5
- package/dist/db/migrations/051_binding_host_pattern.d.ts +7 -10
- package/dist/db/migrations/052_host_reach_metadata.d.ts +8 -11
- package/dist/db/migrations/053_agent_token_single_use.d.ts +6 -10
- package/dist/db/migrations/054_local_working_tree.d.ts +4 -6
- package/dist/db/migrations/055_agent_token_mandatory_labels.d.ts +6 -11
- package/dist/db/migrations/056_execution_jobs_environments.d.ts +4 -11
- package/dist/db/migrations/064_execution_jobs_skipped_environments.d.ts +6 -9
- package/dist/db/migrations/065_pending_attestations.d.ts +9 -12
- package/dist/db/migrations/066_pending_attestations_rejected.d.ts +4 -7
- package/dist/db/migrations/068_request_idempotency.d.ts +7 -9
- package/dist/db/migrations/069_reroute_tunables.d.ts +6 -14
- package/dist/db/migrations/071_batch_accumulation.d.ts +8 -16
- package/dist/db/migrations/072_dispatch_queue_run_id_index.d.ts +3 -9
- package/dist/db/migrations/074_normalize_zero_concurrency_limit.d.ts +4 -7
- package/dist/db/migrations/076_artifacts.d.ts +7 -8
- package/dist/db/migrations/078_org_settings_backup_staleness.d.ts +3 -6
- package/dist/db/migrations/079_org_settings_scaler_spawn_timeout.d.ts +4 -8
- package/dist/db/migrations/080_cluster_settings.d.ts +6 -10
- package/dist/db/migrations/081_org_settings_queue_timeout.d.ts +7 -11
- package/dist/db/migrations/082_host_s3_reachable.d.ts +5 -10
- package/dist/db/migrations/083_org_settings_artifact_caps.d.ts +6 -8
- package/dist/db/migrations/084_orchestrator_signing_keys.d.ts +8 -13
- package/dist/db/migrations/085_cluster_settings_reroute_flap_grace_ms.d.ts +4 -5
- package/dist/db/migrations/086_cluster_settings_max_fanout_hosts.d.ts +3 -3
- package/dist/db/migrations/087_cluster_settings_event_router_rate_limit.d.ts +5 -3
- package/dist/db/migrations/088_cluster_settings_cache_max_tarball_bytes.d.ts +3 -3
- package/dist/db/migrations/089_cluster_settings_cache_ttl_days.d.ts +3 -5
- package/dist/db/migrations/090_cluster_settings_concurrency_wait_timeout_ms.d.ts +3 -6
- package/dist/db/migrations/091_cluster_settings_agent_token_ttl_ms.d.ts +3 -6
- package/dist/db/migrations/092_cluster_settings_version.d.ts +5 -9
- package/dist/db/migrations/094_dashboard_encryption_keys.d.ts +9 -14
- package/dist/db/migrations/095_dashboard_write_policy_tristate.d.ts +2 -6
- package/dist/db/migrations/099_cluster_settings_dashboard_verified_issuer.d.ts +8 -11
- package/dist/db/migrations/101_contexts_hold_expiry_drop_default.d.ts +9 -15
- package/dist/db/migrations/102_dispatch_queue_agent_id.d.ts +6 -11
- package/dist/db/migrations/103_cluster_settings_ownership_db_check_timeout_ms.d.ts +6 -12
- package/dist/db/migrations/104_check_run_terminal_sent.d.ts +8 -14
- package/dist/db/migrations/105_org_trust_policy.d.ts +5 -7
- package/dist/db/migrations/106_cluster_settings_check_run_tracking_ttl_days.d.ts +4 -12
- package/dist/db/migrations/107_check_run_tracking_updated_at_index.d.ts +8 -14
- package/dist/db/migrations/113_execution_runs_workflow_repo_index.d.ts +10 -19
- package/dist/db/migrations/115_global_workflows_cluster_switch.d.ts +2 -3
- package/dist/db/migrations/121_org_settings_allow_untrusted_dockerfile_builds.d.ts +7 -8
- package/dist/db/migrations/122_host_roster_scaler_managed.d.ts +9 -16
- package/dist/db/migrations/123_workflow_registrations_default_branch.d.ts +8 -16
- package/dist/db/migrations/125_org_trust_directory.d.ts +13 -21
- package/dist/db/migrations/126_held_runs_posted_pending_check.d.ts +10 -16
- package/dist/db/migrations/127_org_trust_policy_approval_expiry_seconds.d.ts +8 -16
- package/dist/db/migrations/128_execution_runs_global_eval_round.d.ts +6 -11
- package/dist/db/migrations/129_execution_runs_dispatch_routing_key.d.ts +8 -11
- package/dist/db/migrations/130_scaler_provision_outcomes.d.ts +13 -20
- package/dist/db/migrations/131_execution_jobs_git_credentials.d.ts +12 -22
- package/dist/db/migrations/132_execution_runs_trigger_event.d.ts +6 -16
- package/dist/db/migrations/133_dispatch_queue_source_tar_digest.d.ts +6 -9
- package/dist/db/migrations/140_dispatch_queue_canonical_labels.d.ts +22 -40
- package/dist/db/migrations/143_execution_jobs_precursor_result.d.ts +9 -18
- package/dist/db/migrations/144_execution_runs_registration_window.d.ts +12 -20
- package/dist/db/migrations/147_sealed_job_secrets.d.ts +5 -9
- package/dist/db/migrations/148_cluster_settings_sealed_secrets_retry_backoff_ms.d.ts +7 -13
- package/dist/db/migrations/150_execution_runs_status_epoch.d.ts +6 -11
- package/dist/db/migrations/152_org_settings_cache_upload_settle_timeout.d.ts +5 -8
- package/dist/db/migrations/157_host_roster_identity_source.d.ts +9 -13
- package/dist/db/migrations/159_org_id_widen_to_source_width.d.ts +5 -0
- package/dist/db/migrator.d.ts +17 -25
- package/dist/db/notify-listener.d.ts +8 -11
- package/dist/db/repos/dashboard-encryption-keys-repo.d.ts +9 -10
- package/dist/db/synthetic-job-ids.d.ts +6 -8
- package/dist/db/types.d.ts +531 -1058
- package/dist/deployment/deployment-identity.d.ts +42 -60
- package/dist/diagnostics/bundle-reader.d.ts +3 -15
- package/dist/diagnostics/bundle-writer.d.ts +5 -12
- package/dist/diagnostics/checks/agents.d.ts +1 -7
- package/dist/diagnostics/checks/backup.d.ts +7 -10
- package/dist/diagnostics/checks/certs.d.ts +1 -6
- package/dist/diagnostics/checks/config.d.ts +2 -5
- package/dist/diagnostics/checks/db.d.ts +1 -1
- package/dist/diagnostics/checks/disk.d.ts +1 -6
- package/dist/diagnostics/checks/firecracker-network.d.ts +3 -3
- package/dist/diagnostics/checks/index.d.ts +0 -6
- package/dist/diagnostics/checks/retention.d.ts +3 -6
- package/dist/diagnostics/checks/scaler.d.ts +3 -7
- package/dist/diagnostics/checks/ws.d.ts +1 -6
- package/dist/diagnostics/fleet-collector.d.ts +11 -14
- package/dist/diagnostics/fleet-constants.d.ts +1 -3
- package/dist/diagnostics/fleet-selection.d.ts +3 -7
- package/dist/diagnostics/fleet-topology.d.ts +2 -6
- package/dist/diagnostics/fleet-wiring.d.ts +6 -12
- package/dist/diagnostics/index.d.ts +0 -5
- package/dist/diagnostics/runner.d.ts +4 -8
- package/dist/diagnostics/types.d.ts +10 -21
- package/dist/drain/drain-controller.d.ts +16 -22
- package/dist/entry-helpers.d.ts +0 -2
- package/dist/events/circuit-breaker.d.ts +0 -3
- package/dist/events/event-router.d.ts +1 -2
- package/dist/events/event-store.d.ts +0 -9
- package/dist/events/trust-store.d.ts +0 -3
- package/dist/events/types.d.ts +0 -9
- package/dist/firecracker/host-network.d.ts +24 -30
- package/dist/firecracker/net-interfaces.d.ts +4 -7
- package/dist/helpers/rate-limiter.d.ts +0 -1
- package/dist/index.js +251 -486
- package/dist/metrics/agent-metrics-aggregator.d.ts +24 -67
- package/dist/metrics/metrics-reporter.d.ts +4 -13
- package/dist/metrics/prometheus.d.ts +47 -81
- package/dist/orchestrator-core.d.ts +3 -7
- package/dist/pipeline/content-filter.d.ts +20 -30
- package/dist/pipeline/cross-source-tally.d.ts +7 -8
- package/dist/pipeline/decorating-secret-resolver.d.ts +4 -8
- package/dist/pipeline/degraded-reason.d.ts +3 -4
- package/dist/pipeline/direct-ingress-deps.d.ts +15 -24
- package/dist/pipeline/dispatch-matched-workflow.d.ts +289 -525
- package/dist/pipeline/flatten-lock-steps.d.ts +4 -6
- package/dist/pipeline/global-dispatch-identity.d.ts +5 -10
- package/dist/pipeline/global-eval-round.d.ts +65 -151
- package/dist/pipeline/global-round-hold.d.ts +5 -7
- package/dist/pipeline/held-context-data.d.ts +5 -8
- package/dist/pipeline/held-round-registrations.d.ts +8 -11
- package/dist/pipeline/install-gate.d.ts +15 -25
- package/dist/pipeline/install-secrets-resolver.d.ts +24 -34
- package/dist/pipeline/internal-event-pipeline.d.ts +45 -62
- package/dist/pipeline/invoke-gate.d.ts +27 -40
- package/dist/pipeline/job-contexts.d.ts +19 -36
- package/dist/pipeline/manual-schedule.d.ts +7 -13
- package/dist/pipeline/needs-scheduler.d.ts +29 -46
- package/dist/pipeline/pending-workflow-context.d.ts +14 -28
- package/dist/pipeline/process-webhook.d.ts +88 -123
- package/dist/pipeline/processor.d.ts +209 -355
- package/dist/pipeline/remote-source-store.d.ts +5 -8
- package/dist/pipeline/request-idempotency.d.ts +10 -13
- package/dist/pipeline/rerun-global.d.ts +6 -7
- package/dist/pipeline/rerun-inputs.d.ts +23 -37
- package/dist/pipeline/rerun.d.ts +34 -57
- package/dist/pipeline/resolve-sandbox-grant.d.ts +5 -10
- package/dist/pipeline/resume-workflow.d.ts +28 -34
- package/dist/pipeline/route-or-dispatch-jobs.d.ts +19 -30
- package/dist/pipeline/run-event-context.d.ts +16 -21
- package/dist/pipeline/sandbox-allowlist-reader.d.ts +7 -11
- package/dist/pipeline/security-hold-check.d.ts +56 -80
- package/dist/pipeline/test-pipeline.d.ts +21 -46
- package/dist/pipeline/unbound-job-context.d.ts +6 -9
- package/dist/pipeline/undispatched-hold-checks.d.ts +4 -6
- package/dist/pipeline/wave-scheduler.d.ts +15 -22
- package/dist/pipeline/webhook-payload-store.d.ts +3 -10
- package/dist/pipeline/workflow-lock-at-sha.d.ts +14 -20
- package/dist/policy/dashboard-write-policy.d.ts +1 -1
- package/dist/provider-registry.d.ts +0 -19
- package/dist/providers/generic/index.d.ts +0 -2
- package/dist/providers/generic/normalizer.d.ts +0 -1
- package/dist/providers/generic/verification.d.ts +0 -8
- package/dist/providers/github/auth.d.ts +0 -1
- package/dist/providers/github/changed-files.d.ts +2 -1
- package/dist/providers/github/file-contents.d.ts +0 -4
- package/dist/providers/github/lock-file.d.ts +0 -4
- package/dist/providers/github/normalizer.d.ts +1 -9
- package/dist/providers/local/index.d.ts +1 -2
- package/dist/providers/local/lock-file-fetcher.d.ts +0 -4
- package/dist/providers/local/normalizer.d.ts +0 -10
- package/dist/providers/local/repo-url-builder.d.ts +0 -2
- package/dist/providers/universal-git/clone-token.d.ts +0 -2
- package/dist/providers/universal-git/lock-file.d.ts +0 -2
- package/dist/providers/universal-git/normalizer.d.ts +0 -2
- package/dist/queue/bootstrap.d.ts +3 -15
- package/dist/queue/cleanup.d.ts +29 -50
- package/dist/queue/depth-refresher.d.ts +21 -39
- package/dist/queue/job-queue.d.ts +112 -163
- package/dist/queue/retention.d.ts +23 -36
- package/dist/queue/scheduled-job.d.ts +17 -39
- package/dist/queue/terminalize-unroutable.d.ts +21 -18
- package/dist/queue/unroutable-probe.d.ts +15 -22
- package/dist/registration/extractor.d.ts +0 -6
- package/dist/registration/registration-index.d.ts +0 -6
- package/dist/registration/registration-store.d.ts +0 -9
- package/dist/reporting/agent-failure-category.d.ts +3 -5
- package/dist/reporting/check-run-reporter.d.ts +142 -316
- package/dist/reporting/check-run-summary.d.ts +16 -35
- package/dist/reporting/check-run-tracking-store.d.ts +54 -107
- package/dist/reporting/execution-tracker.d.ts +356 -526
- package/dist/reporting/fs-log-storage.d.ts +31 -33
- package/dist/reporting/job-check-run-completion.d.ts +14 -22
- package/dist/reporting/log-chunk-sink.d.ts +11 -16
- package/dist/reporting/log-pull-handler.d.ts +4 -13
- package/dist/reporting/log-retention.d.ts +3 -4
- package/dist/reporting/log-storage.d.ts +24 -52
- package/dist/reporting/log-writer.d.ts +28 -56
- package/dist/reporting/peer-log-normalize.d.ts +4 -8
- package/dist/reporting/run-aggregator.d.ts +6 -11
- package/dist/reporting/run-summary-fields.d.ts +5 -8
- package/dist/reporting/s3-log-storage.d.ts +31 -50
- package/dist/reporting/step-display-order.d.ts +11 -21
- package/dist/reporting/step-log-buffer.d.ts +7 -28
- package/dist/reporting/step-log-reader.d.ts +11 -19
- package/dist/routes/admin-access-log.d.ts +4 -6
- package/dist/routes/admin-auth.d.ts +19 -38
- package/dist/routes/admin-backends.d.ts +3 -14
- package/dist/routes/admin-cluster-name.d.ts +13 -21
- package/dist/routes/admin-cluster-settings.d.ts +6 -9
- package/dist/routes/admin-config.d.ts +7 -19
- package/dist/routes/admin-contexts.d.ts +6 -13
- package/dist/routes/admin-db.d.ts +5 -10
- package/dist/routes/admin-event-dlq.d.ts +3 -5
- package/dist/routes/admin-event-inspect.d.ts +3 -4
- package/dist/routes/admin-event-log.d.ts +3 -7
- package/dist/routes/admin-events.d.ts +16 -30
- package/dist/routes/admin-held-runs.d.ts +52 -76
- package/dist/routes/admin-maintenance.d.ts +3 -6
- package/dist/routes/admin-orchestrator-drain.d.ts +0 -2
- package/dist/routes/admin-org-settings.d.ts +8 -14
- package/dist/routes/admin-peer-forget.d.ts +0 -1
- package/dist/routes/admin-queue-execution.d.ts +3 -3
- package/dist/routes/admin-registrations.d.ts +3 -12
- package/dist/routes/admin-runs.d.ts +16 -29
- package/dist/routes/admin-scaler-orphans.d.ts +4 -4
- package/dist/routes/admin-scaler-reload.d.ts +5 -8
- package/dist/routes/admin-scheduled-jobs.d.ts +3 -4
- package/dist/routes/admin-sources.d.ts +20 -29
- package/dist/routes/admin-trust-policy.d.ts +31 -48
- package/dist/routes/admin.d.ts +67 -94
- package/dist/routes/caller-identity.d.ts +45 -0
- package/dist/routes/capabilities.d.ts +8 -13
- package/dist/routes/diagnostics.d.ts +2 -10
- package/dist/routes/fleet.d.ts +1 -2
- package/dist/routes/github-webhook.d.ts +2 -5
- package/dist/routes/health.d.ts +6 -19
- package/dist/routes/provenance-oidc.d.ts +3 -3
- package/dist/routes/test-direct.d.ts +46 -0
- package/dist/routes/uploads.d.ts +14 -19
- package/dist/routes/webhooks.d.ts +2 -24
- package/dist/scaler/agent-binary-location.d.ts +77 -0
- package/dist/scaler/agent-fit.d.ts +2 -2
- package/dist/scaler/backend-factory.d.ts +10 -18
- package/dist/scaler/bare-metal-backend.d.ts +28 -45
- package/dist/scaler/claim-store.d.ts +20 -43
- package/dist/scaler/config.d.ts +10 -37
- package/dist/scaler/container-backend.d.ts +22 -35
- package/dist/scaler/container-hostconfig.d.ts +10 -16
- package/dist/scaler/container-routing.d.ts +10 -13
- package/dist/scaler/disk-guard.d.ts +2 -3
- package/dist/scaler/event-backend.d.ts +33 -58
- package/dist/scaler/event-provision-reaper.d.ts +61 -84
- package/dist/scaler/failure-tracker.d.ts +6 -13
- package/dist/scaler/file-tail.d.ts +6 -19
- package/dist/scaler/firecracker-api.d.ts +19 -63
- package/dist/scaler/firecracker-backend.d.ts +142 -258
- package/dist/scaler/host-access.d.ts +7 -10
- package/dist/scaler/index.d.ts +3 -6
- package/dist/scaler/ip-allocator.d.ts +30 -82
- package/dist/scaler/label-matcher.d.ts +14 -35
- package/dist/scaler/log-forwarder.d.ts +6 -25
- package/dist/scaler/machine-ledger.d.ts +39 -86
- package/dist/scaler/manager.d.ts +517 -787
- package/dist/scaler/pending-scale-sweeper.d.ts +8 -11
- package/dist/scaler/reap-orphans.d.ts +8 -12
- package/dist/scaler/resolve-container-auth.d.ts +12 -21
- package/dist/scaler/scaler-events.d.ts +2 -5
- package/dist/scaler/scaler-state-store.d.ts +127 -189
- package/dist/scaler/spawn-errors.d.ts +1 -4
- package/dist/scaler/startup-orphan-sweep.d.ts +3 -4
- package/dist/scaler/startup-tools-check.d.ts +21 -0
- package/dist/scaler/types.d.ts +142 -255
- package/dist/scaler/vm-job-bindings.d.ts +5 -9
- package/dist/scaler/warm-pool.d.ts +46 -93
- package/dist/secrets/audit-logger.d.ts +0 -3
- package/dist/secrets/backend-health.d.ts +0 -9
- package/dist/secrets/backend-registry.d.ts +0 -9
- package/dist/secrets/cleanup.d.ts +0 -1
- package/dist/secrets/config.d.ts +0 -3
- package/dist/secrets/ephemeral-keys.d.ts +0 -3
- package/dist/secrets/pg-secret-store.d.ts +0 -3
- package/dist/secrets/rbac.d.ts +5 -7
- package/dist/secrets/routing-key-scope.d.ts +1 -0
- package/dist/secrets/secret-output-store.d.ts +0 -6
- package/dist/secrets/secret-resolver.d.ts +2 -6
- package/dist/secrets/token-manager.d.ts +1 -4
- package/dist/secrets/vault-secret-store.d.ts +0 -13
- package/dist/security/comment-handler.d.ts +0 -2
- package/dist/security/workflow-diff.d.ts +1 -1
- package/dist/server.js +49004 -53782
- package/dist/sources/source-manager.d.ts +0 -3
- package/dist/sources/source-store.d.ts +0 -9
- package/dist/sources/source-validator.d.ts +0 -3
- package/dist/stale-detector/all-terminal-runs.d.ts +8 -11
- package/dist/stale-detector/gate-deadline-detector.d.ts +14 -19
- package/dist/stale-detector/reboot-deadline-sweep.d.ts +7 -11
- package/dist/stale-detector/stale-dispatch-candidates.d.ts +13 -20
- package/dist/stale-detector/stale-run-detector.d.ts +74 -121
- package/dist/stale-detector/workflow-deadline-detector.d.ts +15 -43
- package/dist/standalone.js +9342 -13077
- package/dist/storage/blob-routes.d.ts +10 -19
- package/dist/storage/filesystem.d.ts +19 -29
- package/dist/storage/index.d.ts +2 -10
- package/dist/storage/key-discriminator.d.ts +8 -10
- package/dist/storage/loopback-guard.d.ts +27 -45
- package/dist/storage/s3.d.ts +26 -53
- package/dist/storage/sign-url.d.ts +7 -22
- package/dist/storage/types.d.ts +46 -72
- package/dist/webhook/event-log.d.ts +0 -2
- package/dist/webhook/generic-sources.d.ts +0 -21
- package/dist/webhook/ingest-admission.d.ts +0 -1
- package/dist/webhook/relay-buffer.d.ts +2 -2
- package/dist/webhook/verify-inbound.d.ts +0 -1
- package/dist/worker/in-memory-execution-tracker.d.ts +9 -31
- package/dist/worker/in-memory-job-queue.d.ts +32 -72
- package/dist/worker/peer-outbox.d.ts +5 -8
- package/dist/worker/reroute-dispatch.d.ts +4 -6
- package/dist/worker/static-agent-token-store.d.ts +6 -27
- package/dist/worker/worker-outbox-relay.d.ts +2 -2
- package/dist/worker/worker-status.d.ts +3 -20
- package/dist/ws/agent-api-registry.d.ts +8 -27
- package/dist/ws/agent-handler.d.ts +63 -110
- package/dist/ws/agent-heartbeat.d.ts +5 -12
- package/dist/ws/bringup-api.d.ts +31 -38
- package/dist/ws/dashboard-backends-handler.d.ts +12 -22
- package/dist/ws/dashboard-context-handler.d.ts +37 -80
- package/dist/ws/dashboard-diagnostics-handler.d.ts +8 -25
- package/dist/ws/dashboard-dispatch-guard.d.ts +3 -4
- package/dist/ws/dashboard-fleet-handler.d.ts +9 -13
- package/dist/ws/dashboard-fleet-write-handler.d.ts +7 -21
- package/dist/ws/dashboard-global-workflows-handler.d.ts +22 -44
- package/dist/ws/dashboard-registrations-handler.d.ts +10 -27
- package/dist/ws/dashboard-run-state-handler.d.ts +10 -13
- package/dist/ws/event-buffer.d.ts +3 -6
- package/dist/ws/execution-status-frame.d.ts +7 -10
- package/dist/ws/failure-messages.d.ts +9 -14
- package/dist/ws/fleet-agent-collector.d.ts +6 -12
- package/dist/ws/fleet-runs-on-all.d.ts +1 -6
- package/dist/ws/git-credential-relay.d.ts +12 -20
- package/dist/ws/inventory-api.d.ts +4 -5
- package/dist/ws/observer-registry.d.ts +14 -59
- package/dist/ws/ownership-gate.d.ts +1 -1
- package/dist/ws/platform-client.d.ts +21 -30
- package/dist/ws/replay-chunker.d.ts +7 -10
- package/dist/ws/server-options.d.ts +5 -9
- package/dist/ws/test-relay-handlers.d.ts +46 -63
- package/installer-image-digests.json +3 -3
- package/package.json +6 -5
- package/sbom.spdx.json +53 -53
- package/dist/__test-helpers__/fake-scaler-state-store.d.ts +0 -24
- package/dist/__test-helpers__/mock-db-query.d.ts +0 -129
- package/dist/__test-helpers__/mock-db.d.ts +0 -134
- package/dist/__test-helpers__/mock-ws.d.ts +0 -13
- package/dist/__test-helpers__/peer-mutual-auth.d.ts +0 -42
- package/dist/__test-helpers__/scratch-db.d.ts +0 -16
- package/dist/__test-helpers__/test-db.d.ts +0 -34
- package/dist/pipeline/dispatch-matched-workflow.test-helpers.d.ts +0 -121
- package/dist/scaler/container-spawn.d.ts +0 -28
package/dist/scaler/manager.d.ts
CHANGED
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* ScalerManager:
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* enforces global limits, tracks spawning agents to prevent over-provisioning,
|
|
7
|
-
* and manages the agent lifecycle from spawn to destroy.
|
|
2
|
+
* ScalerManager: coordinator between the Dispatcher and the scaler backends.
|
|
3
|
+
* Routes "no agent available" signals to a backend, enforces global limits,
|
|
4
|
+
* tracks spawning agents against over-provisioning, and owns the agent
|
|
5
|
+
* lifecycle from spawn to destroy.
|
|
8
6
|
*/
|
|
9
7
|
import { ScalerVmTracker } from '@kici-dev/engine';
|
|
10
8
|
import type { ResourceRequest, ScalerCapacitySummary, ScalerLiveVm, ScalerReloadPlan, ScalerVmStopResult } from '@kici-dev/engine';
|
|
@@ -14,17 +12,15 @@ import type { ScalerEventEmitterLike } from './event-backend.js';
|
|
|
14
12
|
import type { ClaimStore, ClaimedCredentials } from './claim-store.js';
|
|
15
13
|
/**
|
|
16
14
|
* How long a graceful shutdown waits for the scaler backends to stop their
|
|
17
|
-
* agents
|
|
18
|
-
*
|
|
19
|
-
* fit after it.
|
|
15
|
+
* agents: the first window inside the coordinator's 30 s shutdown limit, so the
|
|
16
|
+
* later steps (HTTP stop grace, database drain) still fit after it.
|
|
20
17
|
*
|
|
21
|
-
* A container stop normally takes at most its own 10 s stop timeout, but
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* removes any agent container that is still there.
|
|
18
|
+
* A container stop normally takes at most its own 10 s stop timeout, but a
|
|
19
|
+
* Docker daemon that misses a container's exit waits out that timeout plus
|
|
20
|
+
* another 10 s for its kill; one such container held the step for 29.7 s. Past
|
|
21
|
+
* this window the orchestrator logs each backend still stopping and continues:
|
|
22
|
+
* the runtime finishes the stop, and the next start's orphan cleanup removes
|
|
23
|
+
* any agent container left behind.
|
|
28
24
|
*/
|
|
29
25
|
export declare const SCALER_SHUTDOWN_GRACE_MS = 15000;
|
|
30
26
|
import type { BackendFailureSummary } from './failure-tracker.js';
|
|
@@ -33,11 +29,10 @@ import type { ScalerStateStore, ScalerStateRecovery, ReapCandidate } from './sca
|
|
|
33
29
|
/** The `skipped` reason for a job whose agent is already provisioning or registered. */
|
|
34
30
|
export declare const PROVISION_IN_FLIGHT = "provision-in-flight";
|
|
35
31
|
/**
|
|
36
|
-
* Raised when a registering agent presents a scaler-minted id
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
* the agent reconnects rather than registering with no `mandatoryLabels` gate.
|
|
32
|
+
* Raised when a registering agent presents a scaler-minted id with no spawn
|
|
33
|
+
* record behind it: the lookup failed (`cause` set) or no record exists
|
|
34
|
+
* (`cause` omitted). The WS handler refuses the registration, so the agent
|
|
35
|
+
* reconnects rather than registering with no `mandatoryLabels` gate.
|
|
41
36
|
*/
|
|
42
37
|
export declare class ScalerAdoptionLookupError extends Error {
|
|
43
38
|
readonly agentId: string;
|
|
@@ -45,17 +40,14 @@ export declare class ScalerAdoptionLookupError extends Error {
|
|
|
45
40
|
constructor(agentId: string, cause?: unknown);
|
|
46
41
|
}
|
|
47
42
|
/**
|
|
48
|
-
* Counting semaphore
|
|
49
|
-
*
|
|
50
|
-
* Used to bound how many `backend.spawn` provisioning operations (image pull +
|
|
43
|
+
* Counting semaphore bounding how many `backend.spawn` operations (image pull +
|
|
51
44
|
* container create + start) run at once per backend, so a burst of in-cap jobs
|
|
52
45
|
* cannot storm the container socket / registry.
|
|
53
46
|
*
|
|
54
|
-
*
|
|
47
|
+
* Release hands the slot directly to the next waiter (rather than bumping a
|
|
55
48
|
* counter the woken task re-decrements), so a task arriving in the microtask gap
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
* or throwing operation never leaks a slot and wedges the queue.
|
|
49
|
+
* cannot jump the queue and transiently exceed the cap. `run` releases in
|
|
50
|
+
* `finally`, so a throwing operation never leaks a slot and wedges the queue.
|
|
59
51
|
*/
|
|
60
52
|
export declare class SpawnSemaphore {
|
|
61
53
|
private available;
|
|
@@ -66,13 +58,7 @@ export declare class SpawnSemaphore {
|
|
|
66
58
|
run<T>(fn: () => Promise<T>): Promise<T>;
|
|
67
59
|
}
|
|
68
60
|
/**
|
|
69
|
-
*
|
|
70
|
-
* `limits`. The scaler manager produces this from the job's declared resources
|
|
71
|
-
* combined with the scaler's `defaults.resources`, applying the request<->limit
|
|
72
|
-
* mirroring rule. Caps aggregate `requests`; backends use `limits`.
|
|
73
|
-
*/
|
|
74
|
-
/**
|
|
75
|
-
* Resolve the orchestrator WebSocket URL a scaler-spawned agent should dial.
|
|
61
|
+
* Resolve the orchestrator WebSocket URL a scaler-spawned agent dials.
|
|
76
62
|
*
|
|
77
63
|
* 1. Per-scaler `orchestratorUrl` (scalers.yaml) wins — required for container
|
|
78
64
|
* agents (host.docker.internal / LAN IP) and Firecracker VMs (bridge gateway
|
|
@@ -84,6 +70,11 @@ export declare class SpawnSemaphore {
|
|
|
84
70
|
* agents at 8080 leaves them unable to reach the orchestrator.
|
|
85
71
|
*/
|
|
86
72
|
export declare function resolveScalerOrchestratorUrl(configUrl: string | undefined, envUrl: string | undefined, port: string | number | undefined): string;
|
|
73
|
+
/**
|
|
74
|
+
* Resolved per-job `requests` and `limits` (cpus + bytes): the job's declared
|
|
75
|
+
* resources over the scaler's `defaults.resources`, with request<->limit
|
|
76
|
+
* mirroring applied. Caps aggregate `requests`; backends use `limits`.
|
|
77
|
+
*/
|
|
87
78
|
export interface ResolvedResources {
|
|
88
79
|
requests: {
|
|
89
80
|
cpus: number;
|
|
@@ -111,9 +102,7 @@ export declare function buildScalerUsageRows(perScalerUsage: ReadonlyMap<string,
|
|
|
111
102
|
cpus: number;
|
|
112
103
|
memBytes: number;
|
|
113
104
|
}>;
|
|
114
|
-
/**
|
|
115
|
-
* Status summary for metrics and health endpoints.
|
|
116
|
-
*/
|
|
105
|
+
/** Status summary for metrics and health endpoints. */
|
|
117
106
|
export interface ScalerStatus {
|
|
118
107
|
globalMaxAgents: number;
|
|
119
108
|
globalActiveCount: number;
|
|
@@ -206,16 +195,13 @@ export interface ScalerManagerDeps {
|
|
|
206
195
|
backend: ScalerBackend;
|
|
207
196
|
}>;
|
|
208
197
|
/**
|
|
209
|
-
* Read-only
|
|
210
|
-
*
|
|
211
|
-
* agent layer's concrete class.
|
|
198
|
+
* Read-only agent-registry view for the warm pool's deficit. Structural on
|
|
199
|
+
* purpose: the scaler must not depend on the agent layer's concrete class.
|
|
212
200
|
*
|
|
213
|
-
* Both hosts pass one
|
|
214
|
-
* pool
|
|
215
|
-
*
|
|
216
|
-
*
|
|
217
|
-
* cap. Optional only because the unit tests drive the spawn path without a
|
|
218
|
-
* registry.
|
|
201
|
+
* Both hosts pass one. Without it the readiness count falls back to 0, so a
|
|
202
|
+
* configured pool never sees its own agents and fills until
|
|
203
|
+
* `warmCapacityRemaining` clamps it at `maxAgents`, starving the job-bound
|
|
204
|
+
* spawns that share the cap. Optional only for unit tests.
|
|
219
205
|
*/
|
|
220
206
|
agentRegistry?: {
|
|
221
207
|
findAvailable(requiredLabels: string[]): Array<{
|
|
@@ -238,10 +224,7 @@ export interface ScalerManagerDeps {
|
|
|
238
224
|
instanceId: string;
|
|
239
225
|
/** Callback for relaying scaler events with runId/jobId context. */
|
|
240
226
|
onScalerEvent?: (runId: string, jobId: string, event: ScalerEvent) => void;
|
|
241
|
-
/**
|
|
242
|
-
* Optional DB-backed state store. Tests omit it; production wires it
|
|
243
|
-
* up via the orchestrator-core bootstrap.
|
|
244
|
-
*/
|
|
227
|
+
/** DB-backed state store, wired by the orchestrator-core bootstrap; tests omit it. */
|
|
245
228
|
stateStore?: ScalerStateStore;
|
|
246
229
|
/**
|
|
247
230
|
* Late-bound reserved-event emitter, resolved per call because the event
|
|
@@ -256,10 +239,9 @@ export interface ScalerManagerDeps {
|
|
|
256
239
|
*/
|
|
257
240
|
claimStore?: ClaimStore;
|
|
258
241
|
/**
|
|
259
|
-
*
|
|
260
|
-
*
|
|
261
|
-
*
|
|
262
|
-
* on the same host.
|
|
242
|
+
* Machine-ledger options. With `machinePools` configured, the manager builds a
|
|
243
|
+
* `MachineLedger` keyed off this directory and the instance id; reservations
|
|
244
|
+
* sum across orchestrators on the same host.
|
|
263
245
|
*/
|
|
264
246
|
machineLedger?: {
|
|
265
247
|
/** Override the on-disk ledger directory. Falls back to `KICI_MACHINE_LEDGER_DIR`. */
|
|
@@ -277,11 +259,10 @@ export interface ScalerManagerDeps {
|
|
|
277
259
|
*/
|
|
278
260
|
spawnTimeoutMs: number;
|
|
279
261
|
/**
|
|
280
|
-
*
|
|
281
|
-
* (jobConfig.cacheOrgId, undefined for warm-pool spawns)
|
|
282
|
-
*
|
|
283
|
-
*
|
|
284
|
-
* cluster default spawnTimeoutMs is used.
|
|
262
|
+
* Per-org spawn-deadline resolver, keyed by the job's org
|
|
263
|
+
* (jobConfig.cacheOrgId, undefined for warm-pool spawns); falls back to the
|
|
264
|
+
* cluster default internally. Wired in orchestrator-core (has DB); the worker
|
|
265
|
+
* and unit tests omit it and use spawnTimeoutMs.
|
|
285
266
|
*/
|
|
286
267
|
resolveSpawnTimeoutMs?: (orgId: string | undefined) => Promise<number>;
|
|
287
268
|
/**
|
|
@@ -301,15 +282,15 @@ export interface ScalerManagerDeps {
|
|
|
301
282
|
*/
|
|
302
283
|
readJobStatuses?: (jobs: readonly BoundJobRef[]) => Promise<ReadonlyMap<string, string>>;
|
|
303
284
|
/**
|
|
304
|
-
*
|
|
305
|
-
*
|
|
306
|
-
*
|
|
307
|
-
*
|
|
285
|
+
* Builds a backend for a scaler a config reload adds. Both hosts supply it
|
|
286
|
+
* from the shared backend factory, so a reload builds only what startup
|
|
287
|
+
* exercises. When omitted (unit tests), a reload that adds a scaler is
|
|
288
|
+
* rejected rather than silently ignored.
|
|
308
289
|
*
|
|
309
|
-
* `config` is the config
|
|
310
|
-
*
|
|
311
|
-
*
|
|
312
|
-
*
|
|
290
|
+
* `config` is the reloaded config, not the boot-time one: the factory reads
|
|
291
|
+
* `defaults.resources` and the `firecracker` network block off it, and a
|
|
292
|
+
* stale one would allocate IPs from the old CIDR and attach TAPs to the old
|
|
293
|
+
* bridge, silently unreachable.
|
|
313
294
|
*/
|
|
314
295
|
createBackend?: (entry: ScalerEntry, config: ScalerConfig) => Promise<ScalerBackend | null>;
|
|
315
296
|
}
|
|
@@ -331,17 +312,16 @@ export declare class ScalerManager {
|
|
|
331
312
|
private readonly failureTracker;
|
|
332
313
|
/**
|
|
333
314
|
* Consecutive provisioning failures per scaler NAME, and the deferral they
|
|
334
|
-
* earned: an event
|
|
335
|
-
*
|
|
336
|
-
*
|
|
315
|
+
* earned: an event provision that never registered an agent, or a local
|
|
316
|
+
* launch that produced none (refused, or the agent stopped before
|
|
317
|
+
* registering).
|
|
337
318
|
*
|
|
338
|
-
* Keyed by name
|
|
339
|
-
*
|
|
340
|
-
* different providers, and an outage at one says nothing about the other.
|
|
319
|
+
* Keyed by name, not backend type, so one failing scaler never defers an
|
|
320
|
+
* unrelated one: two event scalers often drive two different providers.
|
|
341
321
|
*
|
|
342
|
-
* In-memory and leader-local, like `ScalerFailureTracker`:
|
|
343
|
-
*
|
|
344
|
-
*
|
|
322
|
+
* In-memory and leader-local, like `ScalerFailureTracker`: a restart resets
|
|
323
|
+
* it, the conservative direction (a fresh coordinator retries once at once
|
|
324
|
+
* rather than inheriting a deferral it never observed).
|
|
345
325
|
*/
|
|
346
326
|
private readonly provisionFailures;
|
|
347
327
|
private readonly resolveProvisionBackoff;
|
|
@@ -405,81 +385,54 @@ export declare class ScalerManager {
|
|
|
405
385
|
* teardown workflow when the agent it adopted disconnects.
|
|
406
386
|
*/
|
|
407
387
|
private readonly scalerProvisioningTargets;
|
|
408
|
-
/**
|
|
409
|
-
* Tracks agents being spawned but not yet registered via WS.
|
|
410
|
-
* Keyed by pre-generated agentId.
|
|
411
|
-
*/
|
|
388
|
+
/** Agents spawned but not yet registered via WS, keyed by pre-generated agentId. */
|
|
412
389
|
private readonly spawningAgents;
|
|
413
|
-
/**
|
|
414
|
-
* Maps registered agentId to backendName for lifecycle events.
|
|
415
|
-
*/
|
|
390
|
+
/** Registered agentId → backendName, for lifecycle events. */
|
|
416
391
|
private readonly managedAgentIndex;
|
|
417
392
|
/**
|
|
418
|
-
* Agents this instance pre-spawned to wait for work
|
|
419
|
-
*
|
|
420
|
-
*
|
|
421
|
-
*
|
|
422
|
-
*
|
|
423
|
-
* This is not a second copy of the pool's readiness. It is provenance, which
|
|
424
|
-
* nothing else knows, and only the reaper reads it: it may destroy an agent
|
|
425
|
-
* the pool itself created, never one that registered for a bound job whose
|
|
426
|
-
* dispatch has not yet arrived.
|
|
393
|
+
* Agents this instance pre-spawned to wait for work rather than for a queued
|
|
394
|
+
* job. This is provenance, not readiness: the agent registry cannot derive it,
|
|
395
|
+
* and the spawning entry that carries it is deleted at registration. Only the
|
|
396
|
+
* reaper reads it, so it destroys agents the pool created, never one that
|
|
397
|
+
* registered for a bound job whose dispatch has not arrived.
|
|
427
398
|
*/
|
|
428
399
|
private readonly warmAgents;
|
|
429
400
|
/**
|
|
430
|
-
* Registered agents this instance started for a
|
|
431
|
-
*
|
|
432
|
-
*
|
|
433
|
-
*
|
|
434
|
-
* else.
|
|
401
|
+
* Registered agents this instance started for a queued job, keyed by agent
|
|
402
|
+
* id. The spawning entry is deleted at registration, so {@link agentView}
|
|
403
|
+
* reads the binding here: the agent always runs its own job, and one started
|
|
404
|
+
* inside a job's image runs nothing else.
|
|
435
405
|
*/
|
|
436
406
|
private readonly boundAgents;
|
|
437
407
|
/** Jobs a `requestScale` call is provisioning right now, before its spawning entry exists. */
|
|
438
408
|
private readonly jobsBeingProvisioned;
|
|
439
409
|
/**
|
|
440
|
-
* Warm agents whose destroy is issued but
|
|
441
|
-
*
|
|
442
|
-
*
|
|
443
|
-
*
|
|
444
|
-
*
|
|
445
|
-
* backend whose teardown outlives a tick would be handed a second destroy for
|
|
446
|
-
* an agent it is already destroying, and every extra pass would count another
|
|
447
|
-
* `kici_orch_scaler_warm_pool_reaped_total`. One entry per agent, cleared
|
|
448
|
-
* when the destroy settles either way.
|
|
410
|
+
* Warm agents whose destroy is issued but not settled. Until it resolves the
|
|
411
|
+
* agent stays in `warmAgents` and `listIdle`, so the next reap pass (30 s
|
|
412
|
+
* later, or a reload lowering the target) would hand the backend a second
|
|
413
|
+
* destroy and count another `kici_orch_scaler_warm_pool_reaped_total`.
|
|
414
|
+
* Cleared when the destroy settles either way.
|
|
449
415
|
*/
|
|
450
416
|
private readonly warmDestroying;
|
|
451
417
|
/** Agents this instance adopted from another instance's spawn record. */
|
|
452
418
|
private readonly adoptedAgents;
|
|
453
|
-
/**
|
|
454
|
-
* Active log forwarders for scaler-managed agents (container/bare-metal).
|
|
455
|
-
* Keyed by agentId. Each forwarder consumes a LogCapture stream.
|
|
456
|
-
*/
|
|
419
|
+
/** Log forwarders for scaler-managed agents (container/bare-metal), keyed by agentId. */
|
|
457
420
|
private readonly logForwarders;
|
|
458
|
-
/**
|
|
459
|
-
* Correlation map: agentId -> { runId, jobId }.
|
|
460
|
-
* Populated by correlateAgentToJob() after job dispatch to a scaler-managed agent.
|
|
461
|
-
*/
|
|
421
|
+
/** agentId → { runId, jobId }, set by correlateAgentToJob() after dispatch. */
|
|
462
422
|
private readonly agentJobCorrelation;
|
|
463
|
-
/**
|
|
464
|
-
* Buffer for scaler events emitted before job correlation is established.
|
|
465
|
-
* Keyed by agentId. Flushed when correlateAgentToJob() is called.
|
|
466
|
-
*/
|
|
423
|
+
/** Events emitted before job correlation, by agentId; flushed by correlateAgentToJob(). */
|
|
467
424
|
private readonly eventBuffer;
|
|
468
|
-
/**
|
|
469
|
-
* External callback for relaying correlated scaler events (e.g. to execution tracker).
|
|
470
|
-
*/
|
|
425
|
+
/** Relays correlated scaler events (e.g. to the execution tracker). */
|
|
471
426
|
private readonly onScalerEvent?;
|
|
472
427
|
/** Coordinator drain predicate; when true, requestScale() no-ops (no fresh
|
|
473
428
|
* agents spawned for the held Pending backlog). Defaults to never-draining. */
|
|
474
429
|
private readonly isDraining;
|
|
475
430
|
/**
|
|
476
|
-
*
|
|
477
|
-
*
|
|
478
|
-
*
|
|
479
|
-
*
|
|
480
|
-
*
|
|
481
|
-
* sitting pending until they time out. Assigned after construction because it
|
|
482
|
-
* references the dispatcher, which is built after the manager.
|
|
431
|
+
* Fired (debounced) when scaler capacity frees: a reservation released on
|
|
432
|
+
* disconnect, a failed spawn, or a managed agent finishing its job. Wired to
|
|
433
|
+
* the dispatcher's capacity-freed re-drive, so `at-capacity` jobs are
|
|
434
|
+
* re-offered the moment a slot opens instead of pending until timeout.
|
|
435
|
+
* Assigned after construction: the dispatcher is built after the manager.
|
|
483
436
|
*/
|
|
484
437
|
onCapacityFreed?: () => void;
|
|
485
438
|
/** Trailing-debounce timer coalescing a burst of releases into one re-drive. */
|
|
@@ -493,11 +446,10 @@ export declare class ScalerManager {
|
|
|
493
446
|
*/
|
|
494
447
|
private readonly warmPoolCallbacks;
|
|
495
448
|
/**
|
|
496
|
-
*
|
|
497
|
-
*
|
|
498
|
-
*
|
|
499
|
-
*
|
|
500
|
-
* Unit tests can omit the store and operate from in-memory Maps only.
|
|
449
|
+
* DB-backed state store. When wired (production), every mutation to
|
|
450
|
+
* `spawningAgents` / `agentJobCorrelation` / `reservations` writes through
|
|
451
|
+
* to Postgres, so a coordinator crash mid-spawn does not orphan agents,
|
|
452
|
+
* strand reservations or lose correlation. Unit tests omit it.
|
|
501
453
|
*/
|
|
502
454
|
private readonly stateStore?;
|
|
503
455
|
/** Id of the orchestrator instance this manager runs on. */
|
|
@@ -518,46 +470,35 @@ export declare class ScalerManager {
|
|
|
518
470
|
private readonly resolveSpawnTimeoutMs?;
|
|
519
471
|
constructor(deps: ScalerManagerDeps);
|
|
520
472
|
/**
|
|
521
|
-
* Pre-spawn one warm agent for a pool's label set
|
|
522
|
-
*
|
|
523
|
-
*
|
|
524
|
-
*
|
|
525
|
-
*
|
|
526
|
-
* reservations through the same {@link reserveForSpawn} helper — a warm
|
|
527
|
-
* agent occupies the same cpu and memory as a job-bound one, and on an event
|
|
528
|
-
* backend it claims a cluster slot too, so a pool can no longer carry a
|
|
529
|
-
* scaler past `maxAgents` fleet-wide.
|
|
473
|
+
* Pre-spawn one warm agent for a pool's label set: `requestScale` with no
|
|
474
|
+
* job term (no bound job, no run, resources from the label set then the
|
|
475
|
+
* scaler defaults). It reserves through the same {@link reserveForSpawn}: a
|
|
476
|
+
* warm agent occupies the same cpu and memory, and on an event backend claims
|
|
477
|
+
* a cluster slot, so a pool cannot carry a scaler past `maxAgents` fleet-wide.
|
|
530
478
|
*
|
|
531
|
-
* A refused reservation
|
|
532
|
-
*
|
|
533
|
-
* next tick rather than the pool stalling below target.
|
|
479
|
+
* A refused reservation reports a failed spawn to the pool, releasing its
|
|
480
|
+
* in-flight slot so the deficit retries next tick instead of stalling.
|
|
534
481
|
*/
|
|
535
482
|
private spawnWarmAgent;
|
|
536
483
|
/**
|
|
537
484
|
* Whether a warm pool for this label set can count its own agents.
|
|
538
485
|
*
|
|
539
|
-
*
|
|
540
|
-
*
|
|
541
|
-
*
|
|
542
|
-
*
|
|
543
|
-
* the pool's platform taints. `warmPoolQueryLabels` puts the structured
|
|
544
|
-
* taints into the query, so that part of the gate is satisfied by
|
|
545
|
-
* construction.
|
|
486
|
+
* Readiness is `AgentRegistry.findAvailable(labels)`, which drops an agent
|
|
487
|
+
* unless its whole `mandatoryLabels` gate appears in the query. The gate is
|
|
488
|
+
* `mandatoryGate()` (configured gate ∪ platform taints), and
|
|
489
|
+
* `warmPoolQueryLabels` puts the taints into the query by construction.
|
|
546
490
|
*
|
|
547
|
-
*
|
|
548
|
-
*
|
|
549
|
-
*
|
|
550
|
-
*
|
|
551
|
-
*
|
|
552
|
-
* fills at all — never partially filled.
|
|
491
|
+
* The residual case is a configured `mandatoryLabels` entry absent from the
|
|
492
|
+
* label set, which `scaler/config.ts` validation already rejects. This check
|
|
493
|
+
* is the fail-safe: such a pool would spawn to `maxAgents` with `ready` stuck
|
|
494
|
+
* at 0, starving the on-demand spawns sharing the cap. A refused pool is
|
|
495
|
+
* inert, never partially filled.
|
|
553
496
|
*
|
|
554
|
-
* The gate is compared against `queryLabels`:
|
|
555
|
-
*
|
|
497
|
+
* The gate is compared against `queryLabels`: against `declaredLabels` alone
|
|
498
|
+
* it would refuse every structurally-tainted pool.
|
|
556
499
|
*/
|
|
557
500
|
private warmPoolLabelSetFillable;
|
|
558
|
-
/**
|
|
559
|
-
* Whether this scaler pre-spawned (warm-filled) the agent.
|
|
560
|
-
*/
|
|
501
|
+
/** Whether this scaler pre-spawned (warm-filled) the agent. */
|
|
561
502
|
isPrespawnedAgent(agentId: string): boolean;
|
|
562
503
|
/** Every local Firecracker backend, under its scaler name. */
|
|
563
504
|
private firecrackerBackends;
|
|
@@ -588,99 +529,69 @@ export declare class ScalerManager {
|
|
|
588
529
|
/**
|
|
589
530
|
* Whether a pre-spawned (warm) agent's shape fits the shape a job declares.
|
|
590
531
|
*
|
|
591
|
-
* A warm agent
|
|
592
|
-
*
|
|
593
|
-
*
|
|
594
|
-
*
|
|
595
|
-
* agent
|
|
596
|
-
*
|
|
597
|
-
*
|
|
598
|
-
*
|
|
599
|
-
*
|
|
600
|
-
*
|
|
601
|
-
*
|
|
602
|
-
*
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
*
|
|
607
|
-
*
|
|
608
|
-
* on the running agent, so a job asking to be capped at 2 cpus, served by an
|
|
609
|
-
* agent started at 4, runs uncapped. That is the same wrong execution
|
|
610
|
-
* environment as the under-provisioned case, in the other direction, so `>=`
|
|
611
|
-
* is not a safe relaxation.
|
|
612
|
-
*
|
|
613
|
-
* Both sides of the shape are compared, not just `requests`. The two answer
|
|
614
|
-
* different questions — `requests` is what the agent bills against the caps,
|
|
615
|
-
* `limits` is what the kernel enforces — and a declaration may legally set
|
|
616
|
-
* them apart, so a job matched on `requests` alone can still land on an agent
|
|
617
|
-
* capped somewhere else entirely.
|
|
532
|
+
* A warm agent starts before the job exists, at the pool's declared shape,
|
|
533
|
+
* and nothing in the agent reads a job's `resources` afterwards. So a job
|
|
534
|
+
* needing another shape gets its own agent instead of silently running with
|
|
535
|
+
* the pool's. True for any agent this scaler did not pre-spawn: a job-bound
|
|
536
|
+
* agent started at its job's shape.
|
|
537
|
+
*
|
|
538
|
+
* Only fields the job declares are compared: `resolveEffective` layers the
|
|
539
|
+
* job over the label set and scaler defaults field by field, so an omitted
|
|
540
|
+
* field already matches the warm agent's reservation.
|
|
541
|
+
*
|
|
542
|
+
* A declared field must match exactly. A bigger agent is not a superset:
|
|
543
|
+
* limits are kernel-enforced, so a job capped at 2 cpus on an agent started
|
|
544
|
+
* at 4 runs uncapped, so `>=` is not a safe relaxation.
|
|
545
|
+
*
|
|
546
|
+
* Both `requests` (billed against the caps) and `limits` (kernel-enforced)
|
|
547
|
+
* are compared: a declaration may set them apart, so matching `requests`
|
|
548
|
+
* alone can land a job on an agent capped elsewhere.
|
|
618
549
|
*/
|
|
619
550
|
private prespawnedShapeFits;
|
|
620
551
|
/**
|
|
621
552
|
* Whether one side (`requests` or `limits`) of a job's declared shape leaves
|
|
622
|
-
* the agent's resolved side unchanged.
|
|
623
|
-
*
|
|
624
|
-
* True when the job declares nothing on this side, and true when `resolved`
|
|
625
|
-
* is absent — both mean "this cannot be told", and the predicate's rule is to
|
|
626
|
-
* fail open rather than strand a job.
|
|
553
|
+
* the agent's resolved side unchanged. Fails open (true) when the job declares
|
|
554
|
+
* nothing on this side or `resolved` is absent, rather than strand a job.
|
|
627
555
|
*/
|
|
628
556
|
private shapeSideMatches;
|
|
629
557
|
/**
|
|
630
558
|
* Whether a warm pool for this label set has a declared agent shape.
|
|
631
559
|
*
|
|
632
|
-
* A warm agent
|
|
633
|
-
*
|
|
634
|
-
*
|
|
635
|
-
*
|
|
636
|
-
*
|
|
637
|
-
*
|
|
638
|
-
* about instance size and cost at all.
|
|
639
|
-
*
|
|
640
|
-
* Limits alone count as a declaration: `resolveEffective` mirrors limits into
|
|
641
|
-
* requests, so a label set naming only `limits` still resolves to a concrete
|
|
642
|
-
* shape.
|
|
643
|
-
*
|
|
644
|
-
* A refused pool is inert, exactly as it was before it filled at all — never
|
|
645
|
-
* partially filled.
|
|
560
|
+
* A warm agent's cpu and memory come only from the label set (or the scaler
|
|
561
|
+
* defaults) and are fixed at start, so an undeclared shape reserves a guess
|
|
562
|
+
* no job can be tested against. Refusing is the fail-safe, and on a cloud
|
|
563
|
+
* backend it is what lets an operator reason about instance size and cost.
|
|
564
|
+
* Limits alone count: `resolveEffective` mirrors limits into requests. A
|
|
565
|
+
* refused pool is inert, never partially filled.
|
|
646
566
|
*
|
|
647
|
-
*
|
|
648
|
-
*
|
|
649
|
-
*
|
|
650
|
-
*
|
|
651
|
-
*
|
|
652
|
-
* orchestrator startup or leave a half-applied config behind.
|
|
567
|
+
* An unresolvable declaration is refused the same way and must never throw:
|
|
568
|
+
* the schema types `resources.*.memory` as a plain string, so a value such as
|
|
569
|
+
* `4Gi` reaches `parseMemoryString` and is rejected. This runs in the
|
|
570
|
+
* constructor and in {@link reload}'s commit block, where an exception would
|
|
571
|
+
* abort startup or leave a half-applied config.
|
|
653
572
|
*/
|
|
654
573
|
private warmPoolShapeDeclared;
|
|
655
574
|
/**
|
|
656
575
|
* Agents this backend may still start before it reaches the scaler's
|
|
657
576
|
* `maxAgents` or the orchestrator's `globalMaxAgents`, whichever binds first.
|
|
658
577
|
*
|
|
659
|
-
* The warm pool clamps its deficit to this, so a pool
|
|
660
|
-
*
|
|
661
|
-
*
|
|
662
|
-
*
|
|
663
|
-
*
|
|
664
|
-
*
|
|
665
|
-
*
|
|
666
|
-
*
|
|
667
|
-
*
|
|
668
|
-
*
|
|
669
|
-
*
|
|
670
|
-
*
|
|
671
|
-
* whole population and the room it reports is exact. An **event** backend is
|
|
672
|
-
* capped cluster-wide — the reserve sequence claims a spawn row under
|
|
673
|
-
* `withScalerCapLock` — while `getActiveCount()` sees only the provisions
|
|
674
|
-
* this coordinator tracks, so the room reported here counts slots a peer may
|
|
675
|
-
* already hold. The cluster-wide cap is still enforced, by that claim rather
|
|
676
|
-
* than by this clamp.
|
|
578
|
+
* The warm pool clamps its deficit to this, so a pool at a count cap asks for
|
|
579
|
+
* nothing rather than `size` spawns every tick. Cpu/memory caps are not seen
|
|
580
|
+
* here: {@link reserveForSpawn} refuses those under the reservation lock, at
|
|
581
|
+
* the cost of a log line and a `spawn_refusals` increment per tick.
|
|
582
|
+
*
|
|
583
|
+
* A best-effort read, not a reservation: {@link reserveForSpawn} re-checks both
|
|
584
|
+
* count caps under the lock. For a local backend (container, bare-metal,
|
|
585
|
+
* Firecracker) `getActiveCount()` is the whole population, so the room is
|
|
586
|
+
* exact. An **event** backend is capped cluster-wide (a spawn row claimed
|
|
587
|
+
* under `withScalerCapLock`) while `getActiveCount()` sees only this
|
|
588
|
+
* coordinator's provisions, so the room here may count slots a peer holds;
|
|
589
|
+
* the claim, not this clamp, enforces the cluster-wide cap.
|
|
677
590
|
*/
|
|
678
591
|
private warmCapacityRemaining;
|
|
679
592
|
/**
|
|
680
|
-
* Publish the warm-pool gauges
|
|
681
|
-
*
|
|
682
|
-
* Called after each pool tick and after a config reload, so an operator sees
|
|
683
|
-
* a newly configured target immediately rather than at the next tick.
|
|
593
|
+
* Publish the warm-pool gauges. Called after each pool tick and after a
|
|
594
|
+
* reload, so a newly configured target shows immediately.
|
|
684
595
|
*/
|
|
685
596
|
private publishWarmPoolGauges;
|
|
686
597
|
/** Per-scaler mandatoryLabels (taint-style opt-in gate). */
|
|
@@ -690,26 +601,24 @@ export declare class ScalerManager {
|
|
|
690
601
|
/**
|
|
691
602
|
* Resolve the pool's platform for label + taint derivation. Precedence:
|
|
692
603
|
* 1. A declared `platform` field always wins (even on a linux host).
|
|
693
|
-
* 2. Otherwise a bare-metal pool derives
|
|
694
|
-
* 3. Otherwise (container / firecracker
|
|
695
|
-
*
|
|
696
|
-
*
|
|
697
|
-
*
|
|
698
|
-
* (the caller falls back to the raw host-label derivation for labels).
|
|
604
|
+
* 2. Otherwise a bare-metal pool derives it from the host OS/arch.
|
|
605
|
+
* 3. Otherwise (container / firecracker) null: linux agents, not
|
|
606
|
+
* auto-tainted unless a platform is declared.
|
|
607
|
+
* Also null for a bare-metal host whose OS/arch is not a supported enum value
|
|
608
|
+
* (the caller falls back to raw host labels).
|
|
699
609
|
*/
|
|
700
610
|
private resolveScalerPlatform;
|
|
701
611
|
/**
|
|
702
612
|
* The plain platform-taint tokens (`windows`, `macos`, `arm64`) for a scaler,
|
|
703
613
|
* derived from the same resolved platform `mandatoryGate` gates on.
|
|
704
614
|
*
|
|
705
|
-
*
|
|
706
|
-
*
|
|
707
|
-
*
|
|
708
|
-
*
|
|
709
|
-
* spawned agents `AgentRegistry.findAvailable` could never return.
|
|
615
|
+
* The single source for the taint: the matcher's label sets, the taint gate,
|
|
616
|
+
* the agent's registered labels and the warm-pool readiness query. Deriving it
|
|
617
|
+
* elsewhere (in a backend, say) lets the gate and the agent's labels disagree,
|
|
618
|
+
* so a tainted pool spawns agents `AgentRegistry.findAvailable` never returns.
|
|
710
619
|
*
|
|
711
|
-
* `backendType` is passed
|
|
712
|
-
*
|
|
620
|
+
* `backendType` is passed, not looked up: the constructor configures warm
|
|
621
|
+
* pools from config entries whose backend may not be indexed.
|
|
713
622
|
*/
|
|
714
623
|
private platformTaintsFor;
|
|
715
624
|
/**
|
|
@@ -717,41 +626,34 @@ export declare class ScalerManager {
|
|
|
717
626
|
* plus the pool's platform taints.
|
|
718
627
|
*
|
|
719
628
|
* `AgentRegistry.findAvailable` drops an agent unless its whole
|
|
720
|
-
* `mandatoryLabels` gate appears in the query,
|
|
721
|
-
*
|
|
722
|
-
*
|
|
723
|
-
*
|
|
724
|
-
*
|
|
725
|
-
*
|
|
726
|
-
*
|
|
727
|
-
*
|
|
728
|
-
*
|
|
729
|
-
* does not have, and every in-flight release path normalizes the spawn labels
|
|
730
|
-
* back to the map key.
|
|
629
|
+
* `mandatoryLabels` gate (taints included) appears in the query, so declared
|
|
630
|
+
* labels alone count none of the pool's agents and it fills to `maxAgents`
|
|
631
|
+
* with `ready` stuck at 0. The constructor and the reload path both call this
|
|
632
|
+
* so they never diverge.
|
|
633
|
+
*
|
|
634
|
+
* QUERY only. The pool is keyed by, and spawns with, the DECLARED set: a
|
|
635
|
+
* backend resolves its image / binary / VM config by matching its own
|
|
636
|
+
* `labelSets` and throws on any other set, and every in-flight release path
|
|
637
|
+
* normalizes the spawn labels back to the map key.
|
|
731
638
|
*/
|
|
732
639
|
private warmPoolQueryLabels;
|
|
733
640
|
/**
|
|
734
641
|
* Taint gate for a scaler's label sets: the configured `mandatoryLabels` plus
|
|
735
642
|
* the platform taints derived from the pool's resolved structured platform.
|
|
736
643
|
*
|
|
737
|
-
*
|
|
738
|
-
*
|
|
739
|
-
* reach for the whole backend.
|
|
644
|
+
* Every routing and stamping decision uses it. `backend` needs only `type`,
|
|
645
|
+
* so a caller holding one label set never reaches for the whole backend.
|
|
740
646
|
*/
|
|
741
647
|
private mandatoryGate;
|
|
742
648
|
/**
|
|
743
|
-
*
|
|
744
|
-
*
|
|
745
|
-
*
|
|
746
|
-
* at spawn time.
|
|
747
|
-
*
|
|
748
|
-
* The os/arch labels derive from the pool's resolved structured platform
|
|
749
|
-
* (declared field wins, else host-derive for bare-metal, else linux), and the
|
|
750
|
-
* mandatory labels combine the per-scaler config gate with the platform taint,
|
|
751
|
-
* so the label matcher applies the gate alongside subset matching.
|
|
649
|
+
* Scaler entries with the auto-labels agents receive at spawn (kici:role:*,
|
|
650
|
+
* kici:os:*, kici:arch:*, kici:agent:*, kici:scaler:*) injected into each
|
|
651
|
+
* label set, so label matching accounts for them.
|
|
752
652
|
*
|
|
753
|
-
*
|
|
754
|
-
*
|
|
653
|
+
* os/arch derive from the resolved structured platform, and the mandatory
|
|
654
|
+
* labels combine the config gate with the platform taint, so the matcher
|
|
655
|
+
* applies the gate alongside subset matching. Retiring scalers are skipped:
|
|
656
|
+
* they take no new work while their remaining agents finish.
|
|
755
657
|
*/
|
|
756
658
|
private getEnrichedScalerEntries;
|
|
757
659
|
/**
|
|
@@ -768,35 +670,30 @@ export declare class ScalerManager {
|
|
|
768
670
|
private sweepRetiredBackends;
|
|
769
671
|
/**
|
|
770
672
|
* True while any spawn this manager started for `name` has neither registered
|
|
771
|
-
* nor failed
|
|
772
|
-
*
|
|
773
|
-
* scaler's drain even though the backend reports no active agent.
|
|
673
|
+
* nor failed, including one still queued behind the spawn semaphore. Such a
|
|
674
|
+
* spawn holds the drain open though the backend reports no active agent.
|
|
774
675
|
*/
|
|
775
676
|
private hasSpawnInFlight;
|
|
776
677
|
/** Drop every trace of a scaler: the backend and all its per-scaler metadata. */
|
|
777
678
|
private forgetScaler;
|
|
778
679
|
/**
|
|
779
|
-
*
|
|
780
|
-
*
|
|
781
|
-
*
|
|
782
|
-
*
|
|
783
|
-
*
|
|
680
|
+
* Run one backend spawn under the resolved spawn deadline. On deadline it
|
|
681
|
+
* aborts the signal (so the backend's container-runtime/HTTP calls cancel and
|
|
682
|
+
* its own catch cleans up) AND rejects, so the caller's semaphore `run()`
|
|
683
|
+
* releases the slot even if the backend's unwind stalls somewhere
|
|
684
|
+
* non-abortable.
|
|
784
685
|
*/
|
|
686
|
+
private runSpawnWithTimeout;
|
|
785
687
|
/**
|
|
786
|
-
*
|
|
787
|
-
*
|
|
788
|
-
*
|
|
789
|
-
*
|
|
790
|
-
* even if the backend's unwind stalls in a non-abortable spot. The backing
|
|
791
|
-
* spawn promise gets a no-op catch attached so a late abort-rejection doesn't
|
|
792
|
-
* surface as an unhandled rejection.
|
|
688
|
+
* Main entry point: called by the Dispatcher when no agent is available.
|
|
689
|
+
* `resources` is the per-job request/limit declaration (`undefined` falls
|
|
690
|
+
* back to scaler defaults), billed against the per-scaler, per-orchestrator
|
|
691
|
+
* and per-machine resource caps.
|
|
793
692
|
*/
|
|
794
|
-
private runSpawnWithTimeout;
|
|
795
693
|
requestScale(labels: string[], jobId: string, runId: string, excludeLabels?: string[], resources?: ResourceRequest, orgId?: string,
|
|
796
694
|
/**
|
|
797
|
-
* The job's resolved container image + registry credentials, when
|
|
798
|
-
*
|
|
799
|
-
* image instead of the pool's fixed agent image.
|
|
695
|
+
* The job's resolved container image + registry credentials, when declared,
|
|
696
|
+
* so the backend can spawn the job's own image instead of the pool's.
|
|
800
697
|
*/
|
|
801
698
|
containerSpawn?: ResolvedContainerSpawn): Promise<ScaleResult>;
|
|
802
699
|
/** Whether this coordinator already provisions, or holds an agent bound to, `jobId`. */
|
|
@@ -804,210 +701,160 @@ export declare class ScalerManager {
|
|
|
804
701
|
/** `requestScale` past its drain and per-job checks: match, reserve, spawn. */
|
|
805
702
|
private provisionForJob;
|
|
806
703
|
/**
|
|
807
|
-
* Take every reservation one spawn needs, or
|
|
808
|
-
*
|
|
809
|
-
*
|
|
810
|
-
* `requestScale` and the warm pool's pre-spawn. The only difference between
|
|
811
|
-
* them is `boundJobId`, which is `undefined` for a pre-spawn; a warm agent
|
|
812
|
-
* costs the same host resources as a job-bound one, so it takes the same
|
|
813
|
-
* reservations.
|
|
704
|
+
* Take every reservation one spawn needs, or none. Shared by `requestScale`
|
|
705
|
+
* and the warm pre-spawn (`boundJobId` `undefined`): a warm agent costs the
|
|
706
|
+
* same host resources, so it takes the same reservations.
|
|
814
707
|
*
|
|
815
708
|
* Ordering, and why:
|
|
816
709
|
* - Count caps and the in-memory resource reservation run under the
|
|
817
|
-
* in-process reservation lock, so
|
|
818
|
-
*
|
|
819
|
-
*
|
|
820
|
-
*
|
|
821
|
-
*
|
|
822
|
-
*
|
|
823
|
-
*
|
|
824
|
-
*
|
|
825
|
-
*
|
|
826
|
-
*
|
|
827
|
-
*
|
|
828
|
-
* Every refusal rolls back exactly what it already took, so a `false` return
|
|
829
|
-
* leaves no reservation behind. On success the caller owns the release,
|
|
830
|
-
* which is always `releaseAll(agentId)`.
|
|
710
|
+
* in-process reservation lock, so check and reserve are atomic. A local
|
|
711
|
+
* backend counts its own host's agents in process (its compute is pinned
|
|
712
|
+
* here). An event backend's cloud instances can be started by any
|
|
713
|
+
* coordinator, so its cap is counted, and its slot claimed, cluster-wide in
|
|
714
|
+
* one advisory-locked transaction; the in-process lock stays the outer
|
|
715
|
+
* guard so the two never contend.
|
|
716
|
+
* - The machine-pool ledger is taken outside that lock: it has its own
|
|
717
|
+
* cross-process mutex, and holding both buys nothing.
|
|
718
|
+
*
|
|
719
|
+
* Every refusal rolls back what it took, so `false` leaves no reservation.
|
|
720
|
+
* On success the caller owns the release: `releaseAll(agentId)`.
|
|
831
721
|
*/
|
|
832
722
|
private reserveForSpawn;
|
|
833
723
|
/**
|
|
834
|
-
*
|
|
835
|
-
*
|
|
836
|
-
* Derived from `this.reservations` rather than kept as a counter beside it:
|
|
837
|
-
* the map IS the admitted set, so a derived count cannot drift out of step
|
|
838
|
-
* with it — including on the boot/leader-switch rehydrate path, which
|
|
839
|
-
* repopulates the map directly.
|
|
840
|
-
*/
|
|
841
|
-
/**
|
|
842
|
-
* Record a refusal taken on one of the two agent-count caps.
|
|
724
|
+
* Record a refusal on one of the two agent-count caps.
|
|
843
725
|
*
|
|
844
|
-
* The
|
|
845
|
-
*
|
|
846
|
-
*
|
|
847
|
-
*
|
|
848
|
-
* (`maxAgents`, `globalMaxAgents`) is incremented here, beside the resource
|
|
849
|
-
* caps that already increment it.
|
|
726
|
+
* The dispatcher gets `at-capacity`, which queues the job and logs nothing,
|
|
727
|
+
* indistinguishable from a scaler that stopped spawning. So the counter the
|
|
728
|
+
* metric's help text promises (`maxAgents`, `globalMaxAgents`) is
|
|
729
|
+
* incremented here, beside the resource caps.
|
|
850
730
|
*
|
|
851
|
-
* The warning is narrower
|
|
852
|
-
*
|
|
853
|
-
*
|
|
854
|
-
*
|
|
855
|
-
* capacity billed to agents that are not running, which does not resolve on
|
|
856
|
-
* its own.
|
|
731
|
+
* The warning is narrower on purpose: ordinary saturation would warn per
|
|
732
|
+
* queued job while busy. It fires only when the reservation ledger outruns
|
|
733
|
+
* every backend's active count: capacity billed to agents not running, which
|
|
734
|
+
* does not resolve on its own.
|
|
857
735
|
*/
|
|
858
736
|
private reportCountCapRefusal;
|
|
859
|
-
private reservedCountForScaler;
|
|
860
737
|
/**
|
|
861
|
-
*
|
|
862
|
-
*
|
|
738
|
+
* Reservations one scaler holds, derived from `this.reservations` rather than
|
|
739
|
+
* a separate counter: the map IS the admitted set, so the count cannot drift,
|
|
740
|
+
* even on the boot/leader-switch rehydrate path that repopulates the map.
|
|
863
741
|
*/
|
|
742
|
+
private reservedCountForScaler;
|
|
743
|
+
/** Serializes concurrent `requestScale()` calls so check+reserve is atomic. */
|
|
864
744
|
private runWithReservationLock;
|
|
865
745
|
/**
|
|
866
|
-
* Lazily resolve the per-backend spawn throttle, sized by
|
|
867
|
-
*
|
|
868
|
-
* per backend name, since each backend has its own container socket.
|
|
746
|
+
* Lazily resolve the per-backend spawn throttle, sized by `maxConcurrentSpawns`
|
|
747
|
+
* (else the cluster default). One per backend: each has its own container socket.
|
|
869
748
|
*/
|
|
870
749
|
private spawnSemaphoreFor;
|
|
871
750
|
/**
|
|
872
751
|
* Resolve the effective `{ requests, limits }` for a job at this scaler.
|
|
873
752
|
*
|
|
874
|
-
* Layered
|
|
753
|
+
* Layered per field, highest priority first:
|
|
875
754
|
* 1. Job's own `resources` (after request<->limit mirroring).
|
|
876
755
|
* 2. Label-set's `resources`.
|
|
877
756
|
* 3. Scaler's `defaults.resources` (already merged from global defaults at construction).
|
|
878
|
-
* 4. Zero
|
|
879
|
-
*
|
|
880
|
-
* Returns both sides fully resolved as numeric `{ cpus, memBytes }` pairs.
|
|
757
|
+
* 4. Zero.
|
|
881
758
|
*/
|
|
882
759
|
private resolveEffective;
|
|
883
760
|
/**
|
|
884
|
-
*
|
|
885
|
-
*
|
|
886
|
-
*
|
|
887
|
-
* Caller must hold the reservation lock.
|
|
761
|
+
* Reserve `requests` against the per-scaler and per-orchestrator caps, or
|
|
762
|
+
* return false with no state change. Caller must hold the reservation lock.
|
|
888
763
|
*/
|
|
889
764
|
private tryReserveAll;
|
|
890
765
|
/**
|
|
891
|
-
* Push the
|
|
892
|
-
*
|
|
893
|
-
*
|
|
894
|
-
* doing any work itself (gauge callbacks are synchronous and MUST NOT
|
|
895
|
-
* touch this map directly — they just read whatever was last published).
|
|
766
|
+
* Push the `perScalerUsage` / `globalUsage` snapshot into the observable
|
|
767
|
+
* gauges on every reservation / release. Gauge callbacks are synchronous and
|
|
768
|
+
* MUST NOT touch this map; they read whatever was last published.
|
|
896
769
|
*
|
|
897
|
-
*
|
|
898
|
-
*
|
|
899
|
-
*
|
|
900
|
-
*
|
|
901
|
-
* `kici_orch_scaler_cpus_used` rows whose scaler belongs to the pool.
|
|
770
|
+
* Pool-level usage is deliberately not emitted: reading the on-disk ledger
|
|
771
|
+
* needs async I/O + a file lock, and OTel gauge callbacks are synchronous. A
|
|
772
|
+
* lower bound is the sum of `kici_orch_scaler_cpus_used` over the pool's
|
|
773
|
+
* scalers.
|
|
902
774
|
*/
|
|
903
775
|
private publishUsageMetrics;
|
|
904
776
|
/**
|
|
905
|
-
* Release
|
|
906
|
-
*
|
|
907
|
-
* references a pool.
|
|
777
|
+
* Release an agent's reservation, and (best-effort) its cross-process pool
|
|
778
|
+
* reservation. Idempotent.
|
|
908
779
|
*/
|
|
909
780
|
private releaseAll;
|
|
910
781
|
/**
|
|
911
782
|
* Signal that scaler capacity freed. Trailing-debounced by
|
|
912
|
-
* {@link CAPACITY_FREED_DEBOUNCE_MS} so a burst of
|
|
913
|
-
*
|
|
914
|
-
*
|
|
915
|
-
* no callback is wired. The timer is `unref`'d so it never keeps the process
|
|
916
|
-
* alive, and cleared on shutdown.
|
|
783
|
+
* {@link CAPACITY_FREED_DEBOUNCE_MS} so a burst of releases (and a single-use
|
|
784
|
+
* agent's onJobComplete → onAgentDisconnect pair) coalesces into one re-drive
|
|
785
|
+
* after the last release. The timer is `unref`'d and cleared on shutdown.
|
|
917
786
|
*/
|
|
918
787
|
private notifyCapacityFreed;
|
|
919
788
|
/**
|
|
920
|
-
* Subtract
|
|
921
|
-
*
|
|
922
|
-
* machine-pool reservation refuses after the in-memory reservation
|
|
923
|
-
* already succeeded.
|
|
789
|
+
* Subtract `requests` from the per-scaler and global in-memory counters; used
|
|
790
|
+
* by `releaseAll()` and by rollbacks after the in-memory reservation succeeded.
|
|
924
791
|
*/
|
|
925
792
|
private releaseInMemory;
|
|
926
793
|
/**
|
|
927
|
-
* Resolve
|
|
928
|
-
*
|
|
929
|
-
*
|
|
930
|
-
* Currently scaler entries themselves do not carry their own `defaults` (the
|
|
931
|
-
* label-set level does). The merged result is stored per-scaler so that
|
|
932
|
-
* future scaler-level defaults can be added without touching the resolver.
|
|
794
|
+
* Resolve a scaler's default resources. Scaler entries carry no `defaults` of
|
|
795
|
+
* their own (label sets do), so this is the global `defaults.resources`;
|
|
796
|
+
* storing it per scaler leaves room for scaler-level defaults.
|
|
933
797
|
*/
|
|
934
798
|
private resolveScalerDefaults;
|
|
935
799
|
/**
|
|
936
|
-
* Called from agent-handler.ts when an agent registers via WS
|
|
937
|
-
*
|
|
800
|
+
* Called from agent-handler.ts when an agent registers via WS; correlates it to
|
|
801
|
+
* its spawning entry.
|
|
938
802
|
*
|
|
939
803
|
* Returns:
|
|
940
|
-
* - `boundJobId` (optional): the queued
|
|
941
|
-
*
|
|
942
|
-
*
|
|
943
|
-
* - `mandatoryLabels` (always
|
|
944
|
-
*
|
|
945
|
-
*
|
|
946
|
-
*
|
|
947
|
-
*
|
|
948
|
-
*
|
|
949
|
-
*
|
|
950
|
-
*
|
|
951
|
-
*
|
|
952
|
-
* Returns `null` for a static agent — no spawning entry exists, and no other
|
|
953
|
-
* instance's spawn record claims the agent either. A warm-pool spawn is not
|
|
954
|
-
* that case: its spawning entry (and durable row) survives until it
|
|
804
|
+
* - `boundJobId` (optional): the queued job this agent was spawned for, which
|
|
805
|
+
* the caller dispatches eagerly before the idle timer fires, skipping the
|
|
806
|
+
* generic queue drain race.
|
|
807
|
+
* - `mandatoryLabels` (always set for scaler-managed agents): the scaler's
|
|
808
|
+
* effective gate (configured labels plus the platform taint). Threaded into
|
|
809
|
+
* the AgentRegistry so both the queue-drain path (`onAgentAvailable` →
|
|
810
|
+
* `dequeueForLabels`) and the eager-dispatch path (`dispatchBoundJob` →
|
|
811
|
+
* `dequeueById`) reject jobs whose `runsOn` lacks a gate label.
|
|
812
|
+
*
|
|
813
|
+
* Returns `null` for a static agent: no spawning entry and no peer spawn
|
|
814
|
+
* record. A warm-pool spawn keeps its entry (and durable row) until it
|
|
955
815
|
* registers, so it resolves with `mandatoryLabels` and no `boundJobId`, which
|
|
956
|
-
*
|
|
816
|
+
* marks the agent warm on `register.ack`.
|
|
957
817
|
*
|
|
958
|
-
* A local hit
|
|
959
|
-
*
|
|
960
|
-
*
|
|
961
|
-
*
|
|
962
|
-
* conditional UPDATE, so exactly one instance can win.
|
|
818
|
+
* A local hit is answered from memory; a miss falls back to the shared spawn
|
|
819
|
+
* record, since on an HA cluster behind one endpoint the agent routinely
|
|
820
|
+
* reaches an instance that did not spawn it. Adoption is a conditional
|
|
821
|
+
* UPDATE, so exactly one instance wins.
|
|
963
822
|
*/
|
|
964
823
|
onAgentRegistered(agentId: string, labels: string[]): Promise<{
|
|
965
824
|
boundJobId?: string;
|
|
966
825
|
mandatoryLabels: string[];
|
|
967
826
|
} | null>;
|
|
968
827
|
/**
|
|
969
|
-
* Reclaim the host-local compute of a scaler-spawned agent
|
|
970
|
-
*
|
|
828
|
+
* Reclaim the host-local compute of a scaler-spawned agent refused for want of
|
|
829
|
+
* a spawn record.
|
|
971
830
|
*
|
|
972
|
-
* Refusing closes the WS before registration completes,
|
|
973
|
-
*
|
|
974
|
-
*
|
|
975
|
-
*
|
|
976
|
-
* no-op here: it reads that index, finds nothing, and takes the static-agent
|
|
977
|
-
* exit. The agent keeps its VM or host process, reconnects, is refused again,
|
|
978
|
-
* and loops until an operator stops it by hand.
|
|
831
|
+
* Refusing closes the WS before registration completes, and
|
|
832
|
+
* `managedAgentIndex` is only populated after this point, so the disconnect
|
|
833
|
+
* hook takes the static-agent exit. Without this the agent keeps its VM or
|
|
834
|
+
* host process and loops reconnect-refuse until an operator stops it.
|
|
979
835
|
*
|
|
980
836
|
* **This force-destroys a live instance, so read what makes it safe.**
|
|
981
837
|
*
|
|
982
|
-
* - **Local backends only.**
|
|
983
|
-
*
|
|
984
|
-
*
|
|
985
|
-
*
|
|
986
|
-
*
|
|
987
|
-
*
|
|
988
|
-
*
|
|
989
|
-
*
|
|
990
|
-
*
|
|
991
|
-
* only
|
|
992
|
-
*
|
|
993
|
-
*
|
|
994
|
-
* harmless is that each backend reclaims only what its own tracking or its
|
|
995
|
-
* own on-host artifacts name — never a shared database row, which on an HA
|
|
996
|
-
* pair names an identically-configured scaler on both hosts. A wrongly-routed
|
|
997
|
-
* agent id finds nothing here and nothing is touched.
|
|
838
|
+
* - **Local backends only.** `adoptSpawningAgent` resolves `null` both when no
|
|
839
|
+
* row exists AND when a peer already holds it, so a live event agent owned
|
|
840
|
+
* elsewhere reaches this branch; tearing down its cloud instance would
|
|
841
|
+
* destroy a peer's healthy provision. Local compute lives on the host that
|
|
842
|
+
* spawned it.
|
|
843
|
+
* - **Host-local evidence decides, not this verdict.** `adoptSpawningAgent`
|
|
844
|
+
* adopts `event` rows only, so a `container` / `bare-metal` / `firecracker`
|
|
845
|
+
* agent reconnecting to a peer behind a shared endpoint is refused here with
|
|
846
|
+
* its row intact and compute alive on the peer's host. Each backend reclaims
|
|
847
|
+
* only what its own tracking or on-host artifacts name, never a shared
|
|
848
|
+
* database row (which on an HA pair names an identically-configured scaler
|
|
849
|
+
* on both hosts), so a wrongly-routed id touches nothing.
|
|
998
850
|
* - **Never on a transient failure.** The caller must have positive evidence
|
|
999
|
-
* the agent is unowned
|
|
1000
|
-
* the record may exist and be perfectly healthy.
|
|
851
|
+
* the agent is unowned; a store read that merely *failed* is not evidence.
|
|
1001
852
|
*
|
|
1002
|
-
* Fire-and-forget,
|
|
1003
|
-
*
|
|
1004
|
-
* open for a VM shutdown.
|
|
853
|
+
* Fire-and-forget, like `onAgentDisconnected`'s teardown: the registration is
|
|
854
|
+
* refused immediately rather than holding the socket open for a VM shutdown.
|
|
1005
855
|
*/
|
|
1006
856
|
private reclaimUnownedScalerAgent;
|
|
1007
|
-
/**
|
|
1008
|
-
* Finish registration for an agent this instance spawned itself, from the
|
|
1009
|
-
* in-memory spawning entry.
|
|
1010
|
-
*/
|
|
857
|
+
/** Finish registration for an agent this instance spawned, from its spawning entry. */
|
|
1011
858
|
private completeLocalRegistration;
|
|
1012
859
|
/**
|
|
1013
860
|
* Called from agent-handler.ts when an agent disconnects. `reason` is what an
|
|
@@ -1020,105 +867,86 @@ export declare class ScalerManager {
|
|
|
1020
867
|
* Tear down a stranded provision the `EventProvisionReaper` found.
|
|
1021
868
|
*
|
|
1022
869
|
* Public because the reaper is leader-gated and lives outside the manager,
|
|
1023
|
-
* while the emit and the row delete
|
|
1024
|
-
*
|
|
1025
|
-
* `
|
|
1026
|
-
* unlike the disconnect path this does not default to `shutdown`.
|
|
870
|
+
* while the emit and the row delete stay in one place. The reason is the
|
|
871
|
+
* reaper's verdict (never registered → `spawn-timeout`, went away →
|
|
872
|
+
* `heartbeat-timeout`), never a `shutdown` default.
|
|
1027
873
|
*
|
|
1028
874
|
* `kici_orch_scaler_external_provision_timeout_total` counts the spawn-timeout
|
|
1029
|
-
* arm
|
|
1030
|
-
*
|
|
1031
|
-
*
|
|
875
|
+
* arm, and nothing else increments it in production: an event backend's
|
|
876
|
+
* `spawn()` returns once the scale-up is emitted, inside
|
|
877
|
+
* `runSpawnWithTimeout`'s bound.
|
|
1032
878
|
*
|
|
1033
879
|
* The spawn-timeout arm is also where an external provisioning failure enters
|
|
1034
|
-
* the
|
|
1035
|
-
*
|
|
1036
|
-
*
|
|
1037
|
-
*
|
|
1038
|
-
*
|
|
1039
|
-
*
|
|
1040
|
-
* the same input every other backend already supplies, so a job whose external
|
|
1041
|
-
* provisioning failed settles with the provisioning cause instead of a
|
|
1042
|
-
* label-mismatch complaint that sends its operator to fix a correct `runsOn`.
|
|
880
|
+
* the failure machinery. Local backends emit `scaler.failed` from `spawn()`;
|
|
881
|
+
* an event scaler's `spawn()` always succeeds, so the failure shows only when
|
|
882
|
+
* the provision never registers. Synthesizing the event here feeds the fleet
|
|
883
|
+
* counter, `diagnose scaler` and `last_provisioning_error`, so the job
|
|
884
|
+
* settles with the provisioning cause instead of a label-mismatch complaint
|
|
885
|
+
* that sends its operator to fix a correct `runsOn`.
|
|
1043
886
|
*/
|
|
1044
887
|
emitOrphanScaleDown(candidate: ReapCandidate, reason: ScaleDownReason): Promise<void>;
|
|
1045
888
|
/**
|
|
1046
889
|
* Refuse a spawn request while the scaler is inside its backoff window.
|
|
1047
890
|
*
|
|
1048
|
-
* Returns the refusal
|
|
1049
|
-
*
|
|
1050
|
-
*
|
|
1051
|
-
*
|
|
1052
|
-
*
|
|
1053
|
-
* actually failed is the one that gets `last_provisioning_error`, written by
|
|
1054
|
-
* the `scaler.failed` this backoff's own trigger emits.
|
|
891
|
+
* Returns the refusal, or null to proceed. The operator sees it only through
|
|
892
|
+
* this method's log line: the dispatcher branches on `action` and discards
|
|
893
|
+
* `reason`, so a deferred job's queue row records nothing. The job whose
|
|
894
|
+
* provision failed gets `last_provisioning_error` from the triggering
|
|
895
|
+
* `scaler.failed`.
|
|
1055
896
|
*
|
|
1056
|
-
* The knobs are read
|
|
1057
|
-
*
|
|
1058
|
-
*
|
|
1059
|
-
* from the
|
|
1060
|
-
* from the next recorded failure.
|
|
897
|
+
* The knobs are read per request, so a lowered failure limit changes the
|
|
898
|
+
* wording on the next refusal; the window itself was armed from the base and
|
|
899
|
+
* ceiling live at the last `recordProvisionFailure`, so a retuned window
|
|
900
|
+
* applies from the next failure.
|
|
1061
901
|
*/
|
|
1062
902
|
private provisionBackoffRefusal;
|
|
1063
903
|
/**
|
|
1064
|
-
* Record a consecutive provisioning failure
|
|
1065
|
-
*
|
|
1066
|
-
*
|
|
1067
|
-
* The delay doubles per consecutive failure and is capped, so a provider
|
|
1068
|
-
* outage settles into a steady retry cadence rather than either hammering the
|
|
1069
|
-
* provider or growing without bound. `2 ** (n - 1)` is bounded by the cap
|
|
1070
|
-
* before it is used, so a long outage cannot overflow the shift.
|
|
904
|
+
* Record a consecutive provisioning failure and arm the next deferral. The
|
|
905
|
+
* delay doubles per failure up to the cap, so an outage settles into a steady
|
|
906
|
+
* retry cadence. `2 ** (n - 1)` is capped before use, so it cannot overflow.
|
|
1071
907
|
*/
|
|
1072
908
|
private recordProvisionFailure;
|
|
1073
909
|
/**
|
|
1074
|
-
* Claim a dead provision as counted, or report that someone already did.
|
|
1075
|
-
*
|
|
1076
|
-
*
|
|
1077
|
-
*
|
|
1078
|
-
* of a failure is counted per provision, not per observer. The claim is
|
|
1079
|
-
* synchronous and happens before any await, so two observers arriving in the
|
|
1080
|
-
* same tick cannot both pass it.
|
|
910
|
+
* Claim a dead provision as counted, or report that someone already did. The
|
|
911
|
+
* leader sees one provision twice (its stale-spawn prune and the reaper), and
|
|
912
|
+
* a failure counts per provision, not per observer. Synchronous, before any
|
|
913
|
+
* await, so two observers in the same tick cannot both pass.
|
|
1081
914
|
*/
|
|
1082
915
|
private claimFailedProvision;
|
|
1083
916
|
/**
|
|
1084
917
|
* Report one dead external provision, wherever this coordinator noticed it.
|
|
1085
918
|
*
|
|
1086
|
-
* Everything a failed provision owes an operator
|
|
1087
|
-
*
|
|
1088
|
-
*
|
|
1089
|
-
*
|
|
919
|
+
* Everything a failed provision owes an operator: the fleet spawn-failure
|
|
920
|
+
* counter, the structured warn, the `ScalerFailureTracker` record
|
|
921
|
+
* `kici-admin diagnose scaler` reads, the job's `last_provisioning_error`,
|
|
922
|
+
* and the backoff.
|
|
1090
923
|
*
|
|
1091
|
-
*
|
|
1092
|
-
*
|
|
1093
|
-
*
|
|
1094
|
-
*
|
|
1095
|
-
*
|
|
1096
|
-
* leader backed off. The local prune is the observation every coordinator has
|
|
1097
|
-
* of its own spawns.
|
|
924
|
+
* Reachable from BOTH observers, not the reaper alone: the tracker and
|
|
925
|
+
* `provisionFailures` are per-process and the reaper is leader-gated, so a
|
|
926
|
+
* reaper-only report leaves every non-leader with an empty `diagnose scaler`
|
|
927
|
+
* and an un-deferred cadence. The local prune is every coordinator's view of
|
|
928
|
+
* its own spawns.
|
|
1098
929
|
*
|
|
1099
|
-
* Synchronous up to the backoff arm, so a caller
|
|
1100
|
-
*
|
|
1101
|
-
* immediately.
|
|
930
|
+
* Synchronous up to the backoff arm, so a caller cleaning up after the event
|
|
931
|
+
* (the prune drops the agent's event buffer) sees the effect at once.
|
|
1102
932
|
*/
|
|
1103
933
|
private reportProvisionFailure;
|
|
1104
934
|
/**
|
|
1105
935
|
* Defer a scaler whose host refused to start an agent, then free the spawn's
|
|
1106
936
|
* reservation.
|
|
1107
937
|
*
|
|
1108
|
-
*
|
|
1109
|
-
*
|
|
1110
|
-
*
|
|
1111
|
-
* another scaler
|
|
1112
|
-
* Two failures come here: a `spawn()` that rejected with a
|
|
938
|
+
* Arm first: the release notifies freed capacity, and the re-drive 250 ms
|
|
939
|
+
* later must meet the deferral rather than spawn into the same refusal. The
|
|
940
|
+
* release still notifies, since it legitimately unblocks a job waiting on
|
|
941
|
+
* another scaler or the global cap. Callers: a `spawn()` rejected with a
|
|
1113
942
|
* `DeterministicSpawnError`, and a settled spawn whose compute stopped before
|
|
1114
|
-
* its agent registered (`dropFailedSettledSpawn`).
|
|
1115
|
-
*
|
|
943
|
+
* its agent registered (`dropFailedSettledSpawn`). Any other rejection is
|
|
944
|
+
* released at once and retried.
|
|
1116
945
|
*/
|
|
1117
946
|
private deferRefusedLaunch;
|
|
1118
947
|
/**
|
|
1119
|
-
* Whether a bound job has ended. A read error,
|
|
1120
|
-
*
|
|
1121
|
-
* reporting the provision as it would without the check.
|
|
948
|
+
* Whether a bound job has ended. A read error, missing row, unknown run or
|
|
949
|
+
* missing reader all read as "not ended", so the caller reports as usual.
|
|
1122
950
|
*/
|
|
1123
951
|
private boundJobEnded;
|
|
1124
952
|
/**
|
|
@@ -1127,62 +955,51 @@ export declare class ScalerManager {
|
|
|
1127
955
|
*
|
|
1128
956
|
* The job's row is the shared truth: a cancel reaches whichever coordinator
|
|
1129
957
|
* the Platform picks, while the spawning entry lives on the one that asked.
|
|
1130
|
-
* A withdrawal is not a
|
|
1131
|
-
*
|
|
958
|
+
* A withdrawal is not a failure: nothing is reported, no backoff armed.
|
|
959
|
+
* Returns the withdrawn count.
|
|
1132
960
|
*/
|
|
1133
961
|
withdrawEndedBoundProvisions(): Promise<number>;
|
|
1134
962
|
/**
|
|
1135
963
|
* Forget one pending provision and tear it down as `job-complete` — unless a
|
|
1136
964
|
* peer adopted its agent, in which case the adopter owns the teardown.
|
|
1137
965
|
*
|
|
1138
|
-
*
|
|
1139
|
-
*
|
|
1140
|
-
*
|
|
1141
|
-
* `
|
|
1142
|
-
*
|
|
966
|
+
* Via `emitScaleDownForSpec`, which drops the spawn row only once the
|
|
967
|
+
* scale-down is out; a failed emit leaves it for the reaper, whose
|
|
968
|
+
* spawn-timeout verdict re-checks the job and also sends `job-complete`. Not
|
|
969
|
+
* `backend.destroy`: the backend does not track a rehydrated entry, so it
|
|
970
|
+
* would emit nothing.
|
|
1143
971
|
*/
|
|
1144
972
|
private withdrawProvision;
|
|
1145
973
|
/**
|
|
1146
974
|
* Report a pruned event spawn as a dead provision — unless a peer adopted it.
|
|
1147
975
|
*
|
|
1148
|
-
* The prune is NOT proof of failure on an HA cluster
|
|
1149
|
-
*
|
|
1150
|
-
*
|
|
1151
|
-
*
|
|
1152
|
-
*
|
|
1153
|
-
*
|
|
1154
|
-
*
|
|
1155
|
-
*
|
|
1156
|
-
*
|
|
1157
|
-
*
|
|
1158
|
-
*
|
|
1159
|
-
*
|
|
1160
|
-
*
|
|
1161
|
-
*
|
|
1162
|
-
*
|
|
1163
|
-
*
|
|
1164
|
-
*
|
|
1165
|
-
*
|
|
1166
|
-
*
|
|
1167
|
-
* it fails open. Both "the store errored" and "there is no verdict" leave the
|
|
1168
|
-
* failure reported, deliberately: a provision with no positive evidence of
|
|
1169
|
-
* adoption is what a single coordinator sees on a real provisioning outage,
|
|
1170
|
-
* and suppressing there would disarm the per-coordinator backoff on every
|
|
1171
|
-
* follower — an invisible missing report in place of a visible false one. The
|
|
1172
|
-
* only rows with no verdict are those adopted before the outcome table
|
|
1173
|
-
* existed, and they age out within one stale-prune window of the deploy.
|
|
976
|
+
* The prune is NOT proof of failure on an HA cluster: the spawning coordinator
|
|
977
|
+
* never learns its agent registered on a peer, so the entry reaches the prune
|
|
978
|
+
* next to a live agent (the asymmetry `deleteUnadoptedSpawningAgent` exists
|
|
979
|
+
* for). Reporting it would write `scaler.failed` and
|
|
980
|
+
* `last_provisioning_error` onto a running job and back off a healthy scaler.
|
|
981
|
+
* The reaper applies the same test (`spawn-timeout` only without
|
|
982
|
+
* `adoptedBy`), so the two observers agree.
|
|
983
|
+
*
|
|
984
|
+
* The spawn row alone cannot say "ever adopted?": it is deleted on teardown.
|
|
985
|
+
* `provisionAdopter` reads the live row, then the durable provision-outcome
|
|
986
|
+
* record that survives the delete, so an adopted provision the reaper has
|
|
987
|
+
* since condemned does not read as dead.
|
|
988
|
+
*
|
|
989
|
+
* The lookup is fire-and-forget (off the request path) and fails open: a
|
|
990
|
+
* store error or no verdict leaves the failure reported, because that is what
|
|
991
|
+
* a real outage looks like to one coordinator, and suppressing it would
|
|
992
|
+
* disarm every follower's backoff. The only rows with no verdict were adopted
|
|
993
|
+
* before the outcome table existed and age out within one stale-prune window
|
|
994
|
+
* of the deploy.
|
|
1174
995
|
*/
|
|
1175
996
|
private reportPrunedProvisionFailure;
|
|
1176
997
|
/** The adoption half of {@link reportPrunedProvisionFailure}. */
|
|
1177
998
|
private reportPrunedUnlessAdopted;
|
|
1178
999
|
/**
|
|
1179
|
-
* Clear a scaler's consecutive-failure state after a successful provision
|
|
1180
|
-
*
|
|
1181
|
-
*
|
|
1182
|
-
* locally-spawned agent and one this coordinator adopted from a peer. An
|
|
1183
|
-
* adopted agent is a successful provision — the instance that spawned it
|
|
1184
|
-
* is not the one it reached — so counting only local registrations
|
|
1185
|
-
* would leave a healthy scaler deferred on an HA cluster.
|
|
1000
|
+
* Clear a scaler's consecutive-failure state after a successful provision,
|
|
1001
|
+
* from both the local and the adopted registration paths (see
|
|
1002
|
+
* `onAgentRegistered`).
|
|
1186
1003
|
*/
|
|
1187
1004
|
private clearProvisionFailures;
|
|
1188
1005
|
/**
|
|
@@ -1190,68 +1007,52 @@ export declare class ScalerManager {
|
|
|
1190
1007
|
* drop the durable row.
|
|
1191
1008
|
*
|
|
1192
1009
|
* Used where the instance holding the agent has no local backend for its
|
|
1193
|
-
* scaler
|
|
1194
|
-
* where the coordinator the agent reached may not configure the scaler that
|
|
1195
|
-
* spawned it. The row's own targets are therefore primary, with local config
|
|
1196
|
-
* as the fallback for a row that recorded none.
|
|
1197
|
-
*
|
|
1198
|
-
* A failed emit leaves the row in place: it is the only durable pointer at
|
|
1199
|
-
* the customer's running instance, so the reaper needs it to retry from.
|
|
1010
|
+
* scaler, the ordinary shape on an HA cluster behind one shared endpoint.
|
|
1200
1011
|
*
|
|
1201
|
-
*
|
|
1202
|
-
*
|
|
1012
|
+
* A failed emit leaves the row: it is the only durable pointer at the
|
|
1013
|
+
* customer's running instance, and the reaper retries from it. Returns
|
|
1014
|
+
* whether the teardown was emitted, so callers count deliveries, not attempts.
|
|
1203
1015
|
*/
|
|
1204
1016
|
private emitScaleDownForSpec;
|
|
1205
1017
|
/**
|
|
1206
|
-
*
|
|
1207
|
-
*
|
|
1208
|
-
* Returns null if the agent is not scaler-managed (a static / stateful
|
|
1209
|
-
* agent).
|
|
1018
|
+
* The backend TYPE (`container`, `firecracker`, `bare-metal`, `event`, …) of
|
|
1019
|
+
* a registered agent, or null for a static / stateful agent.
|
|
1210
1020
|
*
|
|
1211
|
-
*
|
|
1212
|
-
* kici_agent_* series.
|
|
1213
|
-
*
|
|
1214
|
-
*
|
|
1215
|
-
* dropped as bad_label_value. `managedAgentIndex` stores the scaler
|
|
1216
|
-
* name, so resolve it through `backends` to the type.
|
|
1021
|
+
* AgentMetricsAggregator stamps it as the `scaler` label on each
|
|
1022
|
+
* kici_agent_* series. It MUST be the type, not the operator-chosen name: the
|
|
1023
|
+
* Platform catalog filter allows only the ScalerBackendType set (plus
|
|
1024
|
+
* `stateful`) and drops anything else as bad_label_value.
|
|
1217
1025
|
*/
|
|
1218
1026
|
getBackendType(agentId: string): string | null;
|
|
1219
1027
|
/**
|
|
1220
1028
|
* Exchange a provisioning claim code for freshly minted ephemeral credentials.
|
|
1221
1029
|
*
|
|
1222
|
-
* Reads the shared claim table directly
|
|
1223
|
-
*
|
|
1224
|
-
*
|
|
1225
|
-
*
|
|
1226
|
-
* its coordinators with one shared endpoint — the provisioned agent reaches
|
|
1227
|
-
* whichever coordinator the load balancer picks.
|
|
1030
|
+
* Reads the shared claim table directly, not through the minting backend: the
|
|
1031
|
+
* mint needs only the DB-backed agent-token store, so redemption works on an
|
|
1032
|
+
* instance that never heard of the emitting scaler. That lets an HA cluster
|
|
1033
|
+
* front its coordinators with one shared endpoint.
|
|
1228
1034
|
*
|
|
1229
|
-
* Returns `{ error }`
|
|
1230
|
-
*
|
|
1231
|
-
*
|
|
1035
|
+
* Returns `{ error }` with no claim store (no database) or a rejected redeem
|
|
1036
|
+
* (unknown / consumed / expired / minting error). Called from the
|
|
1037
|
+
* `scaler.claim-credentials` WS handler.
|
|
1232
1038
|
*/
|
|
1233
1039
|
claimScalerCredentials(code: string): Promise<{
|
|
1234
1040
|
credentials?: ClaimedCredentials;
|
|
1235
1041
|
error?: string;
|
|
1236
1042
|
}>;
|
|
1237
1043
|
/**
|
|
1238
|
-
* Called from agent-handler.ts
|
|
1239
|
-
*
|
|
1240
|
-
* For non-Firecracker agents, this is a no-op.
|
|
1044
|
+
* Called from agent-handler.ts on config.ack. Clears a Firecracker agent's MMDS
|
|
1045
|
+
* data (belt-and-suspenders with in-VM iptables); a no-op for other agents.
|
|
1241
1046
|
*/
|
|
1242
1047
|
onConfigAck(agentId: string): void;
|
|
1243
|
-
/**
|
|
1244
|
-
* Called when a scaler-managed agent completes a job.
|
|
1245
|
-
*/
|
|
1048
|
+
/** Called when a scaler-managed agent completes a job. */
|
|
1246
1049
|
onJobComplete(agentId: string): void;
|
|
1247
1050
|
/**
|
|
1248
|
-
* Correlate an
|
|
1249
|
-
*
|
|
1051
|
+
* Correlate an agent to a runId/jobId after dispatch, flushing buffered
|
|
1052
|
+
* pre-dispatch events.
|
|
1250
1053
|
*/
|
|
1251
1054
|
correlateAgentToJob(agentId: string, runId: string, jobId: string): void;
|
|
1252
|
-
/**
|
|
1253
|
-
* Sum of all backends' active count + spawning agents.
|
|
1254
|
-
*/
|
|
1055
|
+
/** Sum of all backends' active counts, which include spawning agents. */
|
|
1255
1056
|
getGlobalActiveCount(): number;
|
|
1256
1057
|
/**
|
|
1257
1058
|
* Shut every backend down in parallel, waiting at most
|
|
@@ -1259,9 +1060,7 @@ export declare class ScalerManager {
|
|
|
1259
1060
|
* when the window closes is logged by name and left to finish on its own.
|
|
1260
1061
|
*/
|
|
1261
1062
|
private shutdownBackendsWithinGrace;
|
|
1262
|
-
/**
|
|
1263
|
-
* Stop warm pool, shutdown all backends, clear tracking maps.
|
|
1264
|
-
*/
|
|
1063
|
+
/** Stop the warm pool, shut down all backends, clear tracking maps. */
|
|
1265
1064
|
shutdownAll(): Promise<void>;
|
|
1266
1065
|
/**
|
|
1267
1066
|
* Classify every configured scaler against the currently loaded backends.
|
|
@@ -1272,22 +1071,18 @@ export declare class ScalerManager {
|
|
|
1272
1071
|
* Validate the new config: label-set overlaps, backend type changes, and
|
|
1273
1072
|
* each kept backend's own view of its label sets.
|
|
1274
1073
|
*
|
|
1275
|
-
* `backend.reload` validates and applies in one call, so
|
|
1276
|
-
*
|
|
1277
|
-
*
|
|
1278
|
-
*
|
|
1279
|
-
*
|
|
1280
|
-
*
|
|
1281
|
-
* The snapshots are returned as well as used here: a later stage can still
|
|
1282
|
-
* fail (an added scaler whose backend does not construct), and the same
|
|
1283
|
-
* rollback is what keeps that rejection from half-applying too.
|
|
1074
|
+
* `backend.reload` validates and applies in one call, so each successful
|
|
1075
|
+
* call's previous state is snapshotted and restored if any backend rejects:
|
|
1076
|
+
* a rejected reload leaves every backend as it was. The snapshots are also
|
|
1077
|
+
* returned, so a later failing stage (an added backend that does not
|
|
1078
|
+
* construct) can roll back the same way.
|
|
1284
1079
|
*/
|
|
1285
1080
|
private validateReload;
|
|
1286
1081
|
/** Put every backend a rejected reload had already mutated back as it was. */
|
|
1287
1082
|
private restoreReloadedBackends;
|
|
1288
1083
|
/**
|
|
1289
|
-
* Construct every added backend into a staging list
|
|
1290
|
-
*
|
|
1084
|
+
* Construct every added backend into a staging list; on any failure the
|
|
1085
|
+
* partial set is torn down and the old config stays.
|
|
1291
1086
|
*/
|
|
1292
1087
|
private buildAddedBackends;
|
|
1293
1088
|
/**
|
|
@@ -1298,15 +1093,14 @@ export declare class ScalerManager {
|
|
|
1298
1093
|
*/
|
|
1299
1094
|
private prepareAddedBackends;
|
|
1300
1095
|
/**
|
|
1301
|
-
* Refresh per-scaler metadata
|
|
1302
|
-
*
|
|
1303
|
-
* its cap, URL, roles and defaults until its last agent is gone.
|
|
1096
|
+
* Refresh per-scaler metadata. A retiring scaler keeps its cap, URL, roles and
|
|
1097
|
+
* defaults until its last agent is gone.
|
|
1304
1098
|
*/
|
|
1305
1099
|
private applyScalerMetadata;
|
|
1306
1100
|
/**
|
|
1307
|
-
* Register every configured machine pool
|
|
1308
|
-
*
|
|
1309
|
-
*
|
|
1101
|
+
* Register every configured machine pool, building the ledger when this reload
|
|
1102
|
+
* first declares a pool. `cap` is re-registered every reload, so a changed cap
|
|
1103
|
+
* applies without a restart.
|
|
1310
1104
|
*/
|
|
1311
1105
|
private applyMachinePools;
|
|
1312
1106
|
/** Rebuild the warm-pool config map from the new config (see WarmPoolManager.reload). */
|
|
@@ -1314,10 +1108,10 @@ export declare class ScalerManager {
|
|
|
1314
1108
|
/**
|
|
1315
1109
|
* Validate and reload configuration: plan → validate → build → commit.
|
|
1316
1110
|
*
|
|
1317
|
-
*
|
|
1318
|
-
*
|
|
1319
|
-
*
|
|
1320
|
-
*
|
|
1111
|
+
* All or nothing: a rejected config leaves the previous one running. A scaler
|
|
1112
|
+
* the new config drops is retired: it stops taking work at once, its idle
|
|
1113
|
+
* warm agents are destroyed, its job-bound agents finish, and its backend is
|
|
1114
|
+
* torn down when it drains.
|
|
1321
1115
|
*/
|
|
1322
1116
|
reload(newConfig: ScalerConfig): Promise<ScalerReloadResult>;
|
|
1323
1117
|
/**
|
|
@@ -1325,199 +1119,149 @@ export declare class ScalerManager {
|
|
|
1325
1119
|
* the commit, while the previous entries and global limits are still in place.
|
|
1326
1120
|
*/
|
|
1327
1121
|
private describeReload;
|
|
1328
|
-
/**
|
|
1329
|
-
* Return status summary for metrics and health endpoints.
|
|
1330
|
-
*/
|
|
1331
|
-
/**
|
|
1332
|
-
* Recent scaler spawn failures grouped per backend instance, for the
|
|
1333
|
-
* diagnose scaler check. `nowMs` is injected by the caller.
|
|
1334
|
-
*/
|
|
1122
|
+
/** Recent spawn failures per backend instance, for the diagnose scaler check. */
|
|
1335
1123
|
recentSpawnFailures(windowMs: number, nowMs: number): Map<string, BackendFailureSummary>;
|
|
1124
|
+
/** Status summary for metrics and health endpoints. */
|
|
1336
1125
|
getStatus(): ScalerStatus;
|
|
1337
1126
|
/**
|
|
1338
1127
|
* Scaler capacity advertised to peers in the cluster heartbeat.
|
|
1339
1128
|
*
|
|
1340
|
-
* Retiring backends are excluded
|
|
1341
|
-
*
|
|
1342
|
-
*
|
|
1343
|
-
* separate per-agent inventory, so in-flight work is unaffected.
|
|
1129
|
+
* Retiring backends are excluded, or peers would keep selecting this host for
|
|
1130
|
+
* jobs it can no longer serve. Their draining agents stay visible through the
|
|
1131
|
+
* per-agent inventory, so in-flight work is unaffected.
|
|
1344
1132
|
*/
|
|
1345
1133
|
getRoutableCapacity(): ScalerCapacitySummary[];
|
|
1346
|
-
/**
|
|
1347
|
-
* Get the backend name managing a specific agent.
|
|
1348
|
-
* Returns null if the agent is not scaler-managed (standalone).
|
|
1349
|
-
*/
|
|
1134
|
+
/** The scaler name managing an agent, or null for a standalone agent. */
|
|
1350
1135
|
getBackendForAgent(agentId: string): string | null;
|
|
1351
1136
|
/**
|
|
1352
|
-
*
|
|
1353
|
-
*
|
|
1354
|
-
* Firecracker backend's `getBridgeConfig()`).
|
|
1137
|
+
* A backend by scaler name, for diagnostics reaching backend-specific
|
|
1138
|
+
* accessors (e.g. the Firecracker backend's `getBridgeConfig()`).
|
|
1355
1139
|
*/
|
|
1356
1140
|
getBackend(name: string): ScalerBackend | undefined;
|
|
1357
1141
|
/**
|
|
1358
|
-
* Whether some configured backend could spawn
|
|
1359
|
-
*
|
|
1360
|
-
*
|
|
1361
|
-
*
|
|
1362
|
-
*
|
|
1363
|
-
*
|
|
1364
|
-
*
|
|
1365
|
-
*
|
|
1366
|
-
*
|
|
1367
|
-
*
|
|
1368
|
-
*
|
|
1369
|
-
*
|
|
1370
|
-
*
|
|
1371
|
-
* scaler-configured orchestrator that makes the answer conservative for a
|
|
1372
|
-
* pattern-only `runsOn`: a backend whose labelSets carry no mandatory labels
|
|
1373
|
-
* matches the empty exact set, so such a job is reported routable and settles
|
|
1374
|
-
* `timed_out_stale` rather than `unroutable`. That loses precision, never
|
|
1375
|
-
* safety — the job is still terminal and the run still fails, which is the
|
|
1376
|
-
* invariant that matters. Tightening it means teaching scaler routing about
|
|
1377
|
-
* patterns, which would change where jobs actually spawn.
|
|
1142
|
+
* Whether some configured backend could spawn for this label set: the
|
|
1143
|
+
* `findBackendForLabels` match `requestScale` makes, without spawning.
|
|
1144
|
+
*
|
|
1145
|
+
* The queue-expiry sweep uses it to separate a label problem (no agent AND no
|
|
1146
|
+
* backend: `unroutable`) from a provisioning/capacity one (`timed_out_stale`),
|
|
1147
|
+
* so an operator is not told to fix a correct `runsOn`.
|
|
1148
|
+
*
|
|
1149
|
+
* Exact labels only, as in `requestScale`, so a regex `runsOn` is not
|
|
1150
|
+
* expressible. For a pattern-only `runsOn` a backend with no mandatory labels
|
|
1151
|
+
* matches the empty set, so the job settles `timed_out_stale` instead of
|
|
1152
|
+
* `unroutable`: less precise, never unsafe (the job is terminal and the run
|
|
1153
|
+
* fails either way). Tightening it means teaching scaler routing about
|
|
1154
|
+
* patterns, which would change where jobs spawn.
|
|
1378
1155
|
*/
|
|
1379
1156
|
hasBackendForLabels(labels: string[], excludeLabels?: string[]): boolean;
|
|
1380
1157
|
/**
|
|
1381
|
-
*
|
|
1382
|
-
*
|
|
1383
|
-
* Used to enrich job.context before forwarding to Platform.
|
|
1158
|
+
* Scaler-specific metadata for a managed agent (undefined for a static one),
|
|
1159
|
+
* used to enrich job.context before forwarding to the Platform.
|
|
1384
1160
|
*/
|
|
1385
1161
|
getScalerContextForAgent(agentId: string): Record<string, unknown> | undefined;
|
|
1386
1162
|
/**
|
|
1387
|
-
* Start the machine-pool ledger reaper and the retirement sweep
|
|
1388
|
-
*
|
|
1389
|
-
*
|
|
1390
|
-
*
|
|
1391
|
-
* The warm pools themselves are NOT started here — see
|
|
1392
|
-
* {@link startWarmPools}, which the host calls once every dependency a fill
|
|
1393
|
-
* can reach is wired.
|
|
1163
|
+
* Start the machine-pool ledger reaper and the retirement sweep, and publish
|
|
1164
|
+
* the warm-pool gauges so each target is visible before its first fill. The
|
|
1165
|
+
* warm pools themselves start in {@link startWarmPools}.
|
|
1394
1166
|
*/
|
|
1395
1167
|
start(): void;
|
|
1396
1168
|
/**
|
|
1397
1169
|
* Run the warm pools' first deficit pass and arm their periodic tick.
|
|
1398
|
-
*
|
|
1399
|
-
*
|
|
1400
|
-
*
|
|
1401
|
-
*
|
|
1402
|
-
* this once the emitter exists; the worker, which hosts no event backend,
|
|
1403
|
-
* calls it right after `start()`.
|
|
1170
|
+
* Separate from {@link start}: the first pass spawns at once, and an event
|
|
1171
|
+
* backend's spawn needs the event emitter, which the coordinator assigns
|
|
1172
|
+
* after building the scaler. The worker (no event backend) calls it right
|
|
1173
|
+
* after `start()`.
|
|
1404
1174
|
*/
|
|
1405
1175
|
startWarmPools(): void;
|
|
1406
1176
|
/**
|
|
1407
|
-
* Provision/heal every backend's host prerequisites before spawning starts.
|
|
1408
|
-
*
|
|
1409
|
-
*
|
|
1410
|
-
* with clear errors + the firecracker-network diagnostic reports it) rather
|
|
1411
|
-
* than aborting orchestrator startup.
|
|
1177
|
+
* Provision/heal every backend's host prerequisites before spawning starts. A
|
|
1178
|
+
* failure degrades only that scaler (its spawns fail with clear errors and
|
|
1179
|
+
* the firecracker-network diagnostic reports it), never aborts startup.
|
|
1412
1180
|
*/
|
|
1413
1181
|
ensureHostsReady(): Promise<void>;
|
|
1414
1182
|
/**
|
|
1415
1183
|
* The window a spawning entry may sit unregistered before it is pruned.
|
|
1416
1184
|
*
|
|
1417
|
-
*
|
|
1418
|
-
*
|
|
1419
|
-
*
|
|
1420
|
-
*
|
|
1421
|
-
*
|
|
1422
|
-
* deadline keeps the default five minutes, because a local backend's agent
|
|
1423
|
-
* boots and registers well after `spawn()` itself returns.
|
|
1185
|
+
* Floored at the configured spawn deadline, so raising
|
|
1186
|
+
* `KICI_SCALER_SPAWN_TIMEOUT_MS` for a slow cloud never has the prune drop
|
|
1187
|
+
* the correlation of a provision still booting inside it. A lowered deadline
|
|
1188
|
+
* keeps the default five minutes: a local backend's agent registers well
|
|
1189
|
+
* after `spawn()` returns.
|
|
1424
1190
|
*/
|
|
1425
1191
|
private stalePruneWindowMs;
|
|
1426
1192
|
/**
|
|
1427
1193
|
* Remove spawning entries whose `backend.spawn` started longer ago than the
|
|
1428
|
-
* stale window
|
|
1429
|
-
*
|
|
1430
|
-
*
|
|
1431
|
-
*
|
|
1432
|
-
*
|
|
1433
|
-
*
|
|
1434
|
-
*
|
|
1435
|
-
*
|
|
1436
|
-
*
|
|
1437
|
-
* the
|
|
1438
|
-
*
|
|
1439
|
-
*
|
|
1440
|
-
* recorded, which the entry carries so a scaler since removed from this
|
|
1441
|
-
* coordinator's config still resolves. A local backend's
|
|
1442
|
-
* compute is pinned to a host this process manages, so a pruned entry means
|
|
1443
|
-
* the spawn is dead and the row goes. An event backend's row is the only
|
|
1444
|
-
* durable pointer at a customer cloud instance, and it is precisely the
|
|
1445
|
-
* reaper's `spawn-timeout` candidate — so the row STAYS, and the reaper
|
|
1446
|
-
* decides. Deleting it here would beat the reaper by the flap grace on any
|
|
1447
|
-
* coordinator serving jobs, so the teardown the row exists to trigger would
|
|
1448
|
-
* never be emitted.
|
|
1194
|
+
* stale window and never registered via WS (e.g. a crash on startup), which
|
|
1195
|
+
* would otherwise leak forever.
|
|
1196
|
+
*
|
|
1197
|
+
* Measured from `spawnStartedAt`, NOT `spawnedAt` (enqueue): a spawn still
|
|
1198
|
+
* queued behind the semaphore must not be reaped mid-burst. Recovered entries
|
|
1199
|
+
* carry `spawnStartedAt` = their persisted enqueue time.
|
|
1200
|
+
*
|
|
1201
|
+
* The durable row's fate follows the backend type the spawn recorded. A local
|
|
1202
|
+
* backend's pruned spawn is dead, so the row goes. An event backend's row is
|
|
1203
|
+
* the only pointer at a customer cloud instance and is the reaper's
|
|
1204
|
+
* `spawn-timeout` candidate, so it STAYS: deleting it would beat the reaper by
|
|
1205
|
+
* the flap grace and the teardown would never be emitted.
|
|
1449
1206
|
*/
|
|
1450
1207
|
private pruneStaleSpawningEntries;
|
|
1451
1208
|
/**
|
|
1452
1209
|
* The spawn row for one agent, as every writer of it must build it.
|
|
1453
1210
|
*
|
|
1454
|
-
* Ownership and self-describing columns are assembled only here
|
|
1455
|
-
* `backend_type` is what `adoptSpawningAgent`'s conditional UPDATE matches
|
|
1456
|
-
*
|
|
1457
|
-
*
|
|
1458
|
-
*
|
|
1459
|
-
*
|
|
1460
|
-
*
|
|
1461
|
-
* teardown: it is the only address such a coordinator has for the customer's
|
|
1462
|
-
* teardown workflow, and an emit to an empty target list reaches nobody, so
|
|
1463
|
-
* the provision would bill forever. `roles` travels with it so the row stays
|
|
1464
|
-
* a complete description of the spawn.
|
|
1211
|
+
* Ownership and self-describing columns are assembled only here:
|
|
1212
|
+
* `backend_type` is what `adoptSpawningAgent`'s conditional UPDATE matches, so
|
|
1213
|
+
* a row without it is never adoptable; `owner_instance_id` scopes recovery;
|
|
1214
|
+
* `mandatory_labels` and `provisioning_targets` give a coordinator with no
|
|
1215
|
+
* config entry for the scaler its platform taint and its only teardown
|
|
1216
|
+
* address (an empty target list reaches nobody, so the provision would bill
|
|
1217
|
+
* forever); `roles` keeps the row a complete description of the spawn.
|
|
1465
1218
|
*/
|
|
1466
1219
|
private spawningAgentSnapshot;
|
|
1467
1220
|
/**
|
|
1468
1221
|
* Claim one slot against an event scaler's cluster-wide `maxAgents`.
|
|
1469
1222
|
*
|
|
1470
|
-
*
|
|
1471
|
-
*
|
|
1472
|
-
*
|
|
1473
|
-
*
|
|
1474
|
-
*
|
|
1475
|
-
*
|
|
1476
|
-
*
|
|
1477
|
-
*
|
|
1478
|
-
*
|
|
1479
|
-
*
|
|
1480
|
-
*
|
|
1481
|
-
* neither rollback path fires and the row counts against the cap for an
|
|
1482
|
-
* agent that never spawned. Closing that window needs two-phase commit; the
|
|
1483
|
-
* event-provision reaper is the backstop, clearing the row once the spawn
|
|
1484
|
-
* deadline passes with nothing adopted.
|
|
1223
|
+
* Count and claim share one advisory-locked transaction, so the row is visible
|
|
1224
|
+
* to the next coordinator taking the lock. A store failure never admits:
|
|
1225
|
+
* refusing costs a queue wait, admitting provisions an instance the cap
|
|
1226
|
+
* cannot see. Refusals and store failures are counted, labelled so a database
|
|
1227
|
+
* outage, a contended lock and a full cluster are distinguishable.
|
|
1228
|
+
*
|
|
1229
|
+
* A transaction that commits but loses its acknowledgement leaves
|
|
1230
|
+
* `slotClaimed` false, so the row counts against the cap for an agent that
|
|
1231
|
+
* never spawned. Closing that needs two-phase commit; the event-provision
|
|
1232
|
+
* reaper is the backstop, clearing the row once the spawn deadline passes
|
|
1233
|
+
* with nothing adopted.
|
|
1485
1234
|
*/
|
|
1486
1235
|
private claimClusterSlot;
|
|
1487
1236
|
private persistSpawningAgent;
|
|
1488
1237
|
private deleteSpawningAgentFromStore;
|
|
1489
1238
|
/**
|
|
1490
|
-
* Stamp `adopted_by = this instance` on
|
|
1491
|
-
*
|
|
1492
|
-
* other instance already claimed is left exactly as it is.
|
|
1239
|
+
* Stamp `adopted_by = this instance` on a just-registered agent's row, via the
|
|
1240
|
+
* conditional UPDATE, so a row another instance claimed is left as it is.
|
|
1493
1241
|
*/
|
|
1494
1242
|
private markSpawningAgentAdopted;
|
|
1495
1243
|
/**
|
|
1496
1244
|
* Record that `backend.spawn()` resolved for a spawn this instance started.
|
|
1497
1245
|
*
|
|
1498
|
-
*
|
|
1499
|
-
*
|
|
1500
|
-
*
|
|
1501
|
-
*
|
|
1502
|
-
* reaper, as the prune leaves it.
|
|
1246
|
+
* False when the prune dropped the entry while `spawn()` ran (a per-org
|
|
1247
|
+
* deadline can outlast the prune window). Nothing else would tear that
|
|
1248
|
+
* compute down, so a local backend's is destroyed here; an event provision is
|
|
1249
|
+
* left to the reaper.
|
|
1503
1250
|
*/
|
|
1504
1251
|
private onSpawnSettled;
|
|
1505
1252
|
/**
|
|
1506
1253
|
* Drop a settled local spawn whose backend reported it failed.
|
|
1507
1254
|
*
|
|
1508
|
-
*
|
|
1509
|
-
* agent
|
|
1510
|
-
*
|
|
1511
|
-
*
|
|
1512
|
-
* durable row and its reservation go now — waiting for the stale-spawn prune
|
|
1513
|
-
* would hold the scaler's slot for minutes.
|
|
1255
|
+
* A backend whose compute dies before registration (a Firecracker guest or a
|
|
1256
|
+
* bare-metal agent process that exited) tears it down and emits
|
|
1257
|
+
* `scaler.failed`. The entry, durable row and reservation go now; waiting
|
|
1258
|
+
* for the prune would hold the slot for minutes.
|
|
1514
1259
|
*
|
|
1515
|
-
*
|
|
1516
|
-
*
|
|
1517
|
-
*
|
|
1518
|
-
*
|
|
1519
|
-
*
|
|
1520
|
-
* registered and exits later never arms the backoff.
|
|
1260
|
+
* Like an event scaler's dead provision, it is a refused launch: the scaler is
|
|
1261
|
+
* deferred before the reservation is freed, so an agent dying at every
|
|
1262
|
+
* startup is not respawned at its startup cadence. Only an unregistered spawn
|
|
1263
|
+
* reaches here, so an agent that exits after registering never arms the
|
|
1264
|
+
* backoff.
|
|
1521
1265
|
*/
|
|
1522
1266
|
private dropFailedSettledSpawn;
|
|
1523
1267
|
/** Store-side half of the stale prune: drops the row only if nobody adopted it. */
|
|
@@ -1527,75 +1271,61 @@ export declare class ScalerManager {
|
|
|
1527
1271
|
private persistAgentJob;
|
|
1528
1272
|
private deleteAgentJobFromStore;
|
|
1529
1273
|
/**
|
|
1530
|
-
* Hydrate the in-memory
|
|
1531
|
-
*
|
|
1274
|
+
* Hydrate the in-memory maps from the state store after a coordinator boot
|
|
1275
|
+
* or Raft leader switch:
|
|
1532
1276
|
*
|
|
1533
|
-
* - `spawningAgents` (with `boundJobId`
|
|
1277
|
+
* - `spawningAgents` (with `boundJobId` for eager-dispatch on register)
|
|
1534
1278
|
* — event-backend rows only; a local-backend row is dropped, see below
|
|
1535
1279
|
* - `agentJobCorrelation` (so scaler-lifecycle events route correctly)
|
|
1536
|
-
* - `reservations` + `perScalerUsage` (so the cap
|
|
1537
|
-
*
|
|
1538
|
-
* starting state)
|
|
1280
|
+
* - `reservations` + `perScalerUsage` (so the cap check sees the real
|
|
1281
|
+
* state, not an empty start)
|
|
1539
1282
|
*
|
|
1540
|
-
*
|
|
1541
|
-
*
|
|
1542
|
-
* emitted by the previous coord before correlation are lost (they are
|
|
1543
|
-
* observability, not correctness: nothing downstream waits on them).
|
|
1283
|
+
* `globalUsage` is recomputed from `perScalerUsage`. `eventBuffer` is NOT
|
|
1284
|
+
* restored: its events are observability, and nothing waits on them.
|
|
1544
1285
|
*
|
|
1545
|
-
* Both reads are scoped to this instance's own rows. An unscoped read
|
|
1546
|
-
*
|
|
1547
|
-
*
|
|
1548
|
-
*
|
|
1549
|
-
* dead instance are the leader-gated reaper's business, not ours.
|
|
1550
|
-
*
|
|
1551
|
-
* No-op when no store is wired (unit-test path).
|
|
1286
|
+
* Both reads are scoped to this instance's own rows. An unscoped read would
|
|
1287
|
+
* hydrate a peer's in-flight spawns as ours, so our spawn-timeout reaper
|
|
1288
|
+
* destroys agents the peer still waits on and our caps double-count. A dead
|
|
1289
|
+
* instance's rows belong to the leader-gated reaper. No-op with no store.
|
|
1552
1290
|
*/
|
|
1553
1291
|
recoverState(): Promise<ScalerStateRecovery>;
|
|
1554
1292
|
/**
|
|
1555
|
-
* Mint a scaler-managed agent id
|
|
1556
|
-
*
|
|
1557
|
-
*
|
|
1558
|
-
*
|
|
1559
|
-
* scaler-managed agent by that prefix alone when the spawn-record lookup
|
|
1560
|
-
* fails, so a minting prefix that drifts from the constant leaves the guard
|
|
1561
|
-
* matching nothing and failing open.
|
|
1293
|
+
* Mint a scaler-managed agent id from {@link SCALER_AGENT_ID_PREFIX}, never a
|
|
1294
|
+
* copy of the literal: `onAgentRegistered`'s guard recognises scaler agents by
|
|
1295
|
+
* that prefix alone when the spawn-record lookup fails, so a drifted prefix
|
|
1296
|
+
* would fail open.
|
|
1562
1297
|
*/
|
|
1563
1298
|
private generateAgentId;
|
|
1564
1299
|
/**
|
|
1565
|
-
* Start log forwarding
|
|
1566
|
-
*
|
|
1300
|
+
* Start log forwarding if the backend supports LogCapture. Fire-and-forget:
|
|
1301
|
+
* forward() runs for the agent's lifetime.
|
|
1567
1302
|
*/
|
|
1568
1303
|
private startLogForwarding;
|
|
1569
1304
|
private getOrchestratorUrl;
|
|
1570
1305
|
/**
|
|
1571
|
-
*
|
|
1572
|
-
*
|
|
1306
|
+
* Per-agent event emitter for backend.spawn(), routing through
|
|
1307
|
+
* handleScalerEvent().
|
|
1573
1308
|
*/
|
|
1574
1309
|
private createEventEmitter;
|
|
1575
1310
|
/**
|
|
1576
1311
|
* Handle a scaler event from a backend.
|
|
1577
1312
|
*
|
|
1578
|
-
*
|
|
1579
|
-
* correlation
|
|
1580
|
-
*
|
|
1581
|
-
*
|
|
1582
|
-
*
|
|
1583
|
-
*
|
|
1584
|
-
*
|
|
1585
|
-
*
|
|
1586
|
-
*
|
|
1587
|
-
*
|
|
1588
|
-
*
|
|
1589
|
-
*
|
|
1590
|
-
*
|
|
1591
|
-
*
|
|
1592
|
-
*
|
|
1593
|
-
*
|
|
1594
|
-
* spawn and never registered: the reaper is leader-gated, so the instance that
|
|
1595
|
-
* condemns a stranded external provision routinely has neither a spawning
|
|
1596
|
-
* entry nor a managed-agent index row for it, and every in-memory resolver
|
|
1597
|
-
* above comes back empty. It is read only where the maps are silent, so a live
|
|
1598
|
-
* local entry always wins over a durable row that may be a sweep behind.
|
|
1313
|
+
* A spawn FAILURE usually resolves through the bound spawning entry, since
|
|
1314
|
+
* correlation is set only at dispatch. The correlation map wins in the rarer
|
|
1315
|
+
* post-registration window (e.g. a bare-metal child 'error' after
|
|
1316
|
+
* registration, when the spawning entry is gone). Unbound spawns resolve to
|
|
1317
|
+
* neither. An attributed event is relayed at once; otherwise it is buffered
|
|
1318
|
+
* until correlateAgentToJob() flushes it.
|
|
1319
|
+
*
|
|
1320
|
+
* Every `scaler.failed` increments the fleet-wide spawn-failure counter and
|
|
1321
|
+
* warns, attributed or not. The backend label comes from the spawning entry,
|
|
1322
|
+
* then the managed-agent index, so a late failure still buckets under the
|
|
1323
|
+
* real backend, not "unknown".
|
|
1324
|
+
*
|
|
1325
|
+
* `attribution` is the last resort for a failure this coordinator neither
|
|
1326
|
+
* spawned nor registered (typical for the leader-gated reaper). It is read
|
|
1327
|
+
* only where the maps are silent, so a live local entry beats a durable row
|
|
1328
|
+
* that may be a sweep behind.
|
|
1599
1329
|
*/
|
|
1600
1330
|
private handleScalerEvent;
|
|
1601
1331
|
}
|