@kici-dev/orchestrator 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent-version.d.ts +12 -21
- package/dist/agent/dispatcher.d.ts +283 -437
- package/dist/agent/host-roster-reaper.d.ts +8 -10
- package/dist/agent/host-roster.d.ts +77 -100
- package/dist/agent/ownership-refusal.d.ts +3 -5
- package/dist/agent/ownership-tracker.d.ts +42 -76
- package/dist/agent/registry.d.ts +121 -233
- package/dist/agent/token-store.d.ts +34 -77
- package/dist/agent-packaging/availability.d.ts +6 -9
- package/dist/agent-packaging/download.d.ts +6 -9
- package/dist/agent-packaging/install-closure.d.ts +9 -12
- package/dist/agent-packaging/node-binary.d.ts +6 -10
- package/dist/agent-packaging/node-runtime-members.d.ts +3 -5
- package/dist/agent-packaging/refresh.d.ts +12 -16
- package/dist/agent-packaging/store.d.ts +5 -6
- package/dist/agent-packaging/upload.d.ts +3 -4
- package/dist/app.d.ts +0 -3
- package/dist/approvals/apply-decision.d.ts +35 -52
- package/dist/approvals/approval-resolver.d.ts +11 -23
- package/dist/approvals/held-run-release-wiring.d.ts +14 -21
- package/dist/approvals/independent-wiring.d.ts +13 -20
- package/dist/approvals/resume-router.d.ts +1 -1
- package/dist/approvals/step-approval-bridge.d.ts +11 -21
- package/dist/approvals/triggerer-subject.d.ts +8 -10
- package/dist/artifacts/artifact-store.d.ts +55 -95
- package/dist/artifacts/failure-messages.d.ts +15 -20
- package/dist/audit/access-log.d.ts +27 -45
- package/dist/audit/sampling-rate-limiter.d.ts +11 -23
- package/dist/cache/agent-job-failed-error.d.ts +2 -3
- package/dist/cache/build-coordinator.d.ts +5 -28
- package/dist/cache/cache-upload-settle.d.ts +11 -11
- package/dist/cache/dep-cache.d.ts +32 -60
- package/dist/cache/dispatch-cache-ref-tracker.d.ts +15 -23
- package/dist/cache/global-eval-round-cache.d.ts +33 -41
- package/dist/cache/pending-builds.d.ts +2 -6
- package/dist/cache/pending-dynamics.d.ts +2 -7
- package/dist/cache/pending-global-evals.d.ts +9 -12
- package/dist/cache/pending-inits.d.ts +11 -22
- package/dist/cache/pending-precursor-db-watcher.d.ts +24 -32
- package/dist/cache/pending-tracker.d.ts +17 -45
- package/dist/cache/precursor-result.d.ts +17 -30
- package/dist/cache/source-cache.d.ts +36 -65
- package/dist/cache/user-cache.d.ts +21 -25
- package/dist/cancel/cancel-held-run.d.ts +13 -16
- package/dist/cancel/cancel-route-answer.d.ts +1 -2
- package/dist/cancel/cancel-run.d.ts +26 -50
- package/dist/cancel/dashboard-cancel-handler.d.ts +2 -2
- package/dist/cancel/peer-job-cancel.d.ts +11 -14
- package/dist/cancel/sweep-stuck-cancelling.d.ts +9 -13
- package/dist/cli/api-client.d.ts +25 -76
- package/dist/cli/commands/access-log.d.ts +4 -24
- package/dist/cli/commands/agent-package.d.ts +7 -15
- package/dist/cli/commands/agent-service/agent-env.d.ts +3 -6
- package/dist/cli/commands/agent-service/index.d.ts +4 -10
- package/dist/cli/commands/agent-service/install.d.ts +4 -11
- package/dist/cli/commands/agent-service/status.d.ts +4 -9
- package/dist/cli/commands/agent-service/upgrade.d.ts +4 -16
- package/dist/cli/commands/agent.d.ts +12 -25
- package/dist/cli/commands/api-key.d.ts +5 -12
- package/dist/cli/commands/attestations-list.d.ts +7 -17
- package/dist/cli/commands/attestations-retry.d.ts +5 -11
- package/dist/cli/commands/attestations-reverify.d.ts +4 -5
- package/dist/cli/commands/attestations.d.ts +3 -8
- package/dist/cli/commands/audit.d.ts +1 -5
- package/dist/cli/commands/backend.d.ts +2 -11
- package/dist/cli/commands/check-run.d.ts +7 -18
- package/dist/cli/commands/cluster-name.d.ts +5 -12
- package/dist/cli/commands/cluster-settings.d.ts +28 -53
- package/dist/cli/commands/cold-store.d.ts +7 -15
- package/dist/cli/commands/config.d.ts +9 -20
- package/dist/cli/commands/context.d.ts +8 -20
- package/dist/cli/commands/dashboard-encryption-key.d.ts +9 -14
- package/dist/cli/commands/db-backup.d.ts +17 -32
- package/dist/cli/commands/db.d.ts +3 -4
- package/dist/cli/commands/debug-bundle.d.ts +3 -10
- package/dist/cli/commands/diagnose.d.ts +3 -14
- package/dist/cli/commands/event-dlq.d.ts +4 -12
- package/dist/cli/commands/event-log.d.ts +2 -20
- package/dist/cli/commands/execution.d.ts +3 -9
- package/dist/cli/commands/firecracker/index.d.ts +3 -6
- package/dist/cli/commands/held-run.d.ts +23 -37
- package/dist/cli/commands/host.d.ts +4 -11
- package/dist/cli/commands/inspect-bundle.d.ts +3 -11
- package/dist/cli/commands/local-hook.d.ts +2 -10
- package/dist/cli/commands/local-trigger.d.ts +2 -2
- package/dist/cli/commands/orchestrator-service/drain.d.ts +3 -6
- package/dist/cli/commands/orchestrator-service/index.d.ts +6 -14
- package/dist/cli/commands/orchestrator-service/install-env.d.ts +25 -49
- package/dist/cli/commands/orchestrator-service/install.d.ts +2 -5
- package/dist/cli/commands/orchestrator-service/status.d.ts +10 -22
- package/dist/cli/commands/orchestrator-service/upgrade.d.ts +6 -15
- package/dist/cli/commands/org-settings.d.ts +5 -11
- package/dist/cli/commands/peer.d.ts +5 -10
- package/dist/cli/commands/queue.d.ts +3 -9
- package/dist/cli/commands/registration.d.ts +4 -8
- package/dist/cli/commands/remote-source.d.ts +6 -12
- package/dist/cli/commands/rotate.d.ts +1 -5
- package/dist/cli/commands/runs.d.ts +1 -30
- package/dist/cli/commands/scaler.d.ts +5 -12
- package/dist/cli/commands/secret.d.ts +1 -9
- package/dist/cli/commands/service-health.d.ts +5 -10
- package/dist/cli/commands/shared/admin-cli-access-log.d.ts +10 -19
- package/dist/cli/commands/shared/bare-metal-binary-check.d.ts +86 -0
- package/dist/cli/commands/shared/cli-action.d.ts +1 -4
- package/dist/cli/commands/shared/confirm.d.ts +5 -10
- package/dist/cli/commands/shared/db.d.ts +1 -12
- package/dist/cli/commands/shared/env-file-mode.d.ts +6 -9
- package/dist/cli/commands/shared/secret-input.d.ts +7 -21
- package/dist/cli/commands/shared/service-verbs.d.ts +6 -12
- package/dist/cli/commands/shared/unbound-context-warning.d.ts +2 -4
- package/dist/cli/commands/shared/upgrade-hooks.d.ts +7 -16
- package/dist/cli/commands/shared/upgrade-safety.d.ts +20 -31
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +83 -161
- package/dist/cli/commands/signing-key.d.ts +4 -9
- package/dist/cli/commands/source-manifest.d.ts +11 -14
- package/dist/cli/commands/source.d.ts +7 -32
- package/dist/cli/commands/token.d.ts +4 -15
- package/dist/cli/commands/trust-policy.d.ts +35 -88
- package/dist/cli/commands/variable.d.ts +3 -5
- package/dist/cli/commands/workflow.d.ts +3 -24
- package/dist/cli/join.d.ts +3 -6
- package/dist/cli/kici-admin.d.ts +11 -29
- package/dist/cli/lazy-deps/cache.d.ts +1 -14
- package/dist/cli/lazy-deps/downloader.d.ts +4 -27
- package/dist/cli/lazy-deps/registry.d.ts +5 -25
- package/dist/cli/local-github-ingress-url.d.ts +5 -7
- package/dist/cli/loopback-callback.d.ts +6 -7
- package/dist/cli/open-browser.d.ts +4 -8
- package/dist/cli/root-option-routing.d.ts +17 -29
- package/dist/cli/service/backup-timer.d.ts +16 -39
- package/dist/cli/service/compose-path.d.ts +3 -7
- package/dist/cli/service/compose.d.ts +12 -30
- package/dist/cli/service/deploy-env.d.ts +6 -18
- package/dist/cli/service/entrypoint.d.ts +14 -25
- package/dist/cli/service/env-assignment.d.ts +6 -17
- package/dist/cli/service/image-digests.d.ts +6 -15
- package/dist/cli/service/index.d.ts +2 -18
- package/dist/cli/service/instance/index-file.d.ts +9 -16
- package/dist/cli/service/instance/manifest.d.ts +33 -59
- package/dist/cli/service/instance/resolve.d.ts +32 -68
- package/dist/cli/service/instance/types.d.ts +12 -29
- package/dist/cli/service/launchd.d.ts +29 -60
- package/dist/cli/service/platform-detect.d.ts +13 -68
- package/dist/cli/service/privilege.d.ts +12 -25
- package/dist/cli/service/shutdown-grace.d.ts +11 -20
- package/dist/cli/service/systemd.d.ts +7 -20
- package/dist/cli/service/types.d.ts +63 -114
- package/dist/cli/service/windows-acl.d.ts +14 -21
- package/dist/cli/service/windows-env-file.d.ts +10 -28
- package/dist/cli/service/windows.d.ts +15 -31
- package/dist/cli/webhook-url-notes.d.ts +4 -6
- package/dist/cli/wizard/agent-wizard.d.ts +3 -14
- package/dist/cli/wizard/orchestrator-wizard.d.ts +12 -37
- package/dist/cli/wizard/prompts.d.ts +6 -16
- package/dist/cli.js +4568 -6571
- package/dist/cluster/cluster-identity.d.ts +20 -32
- package/dist/cluster/cluster-settings-reader.d.ts +50 -77
- package/dist/cluster/coordinator-credential.d.ts +4 -5
- package/dist/cluster/coordinator.d.ts +76 -148
- package/dist/cluster/health-api.d.ts +2 -10
- package/dist/cluster/index.d.ts +1 -6
- package/dist/cluster/instance-heartbeat.d.ts +26 -42
- package/dist/cluster/join-client.d.ts +4 -7
- package/dist/cluster/join-handler.d.ts +3 -7
- package/dist/cluster/join-token.d.ts +13 -27
- package/dist/cluster/leader-gated-scheduler.d.ts +8 -17
- package/dist/cluster/orphan-recovery.d.ts +13 -35
- package/dist/cluster/peer-auth-coordinator.d.ts +2 -8
- package/dist/cluster/peer-client.d.ts +46 -76
- package/dist/cluster/peer-credentials.d.ts +24 -70
- package/dist/cluster/peer-crypto.d.ts +5 -26
- package/dist/cluster/peer-forget.d.ts +3 -3
- package/dist/cluster/peer-handler.d.ts +47 -71
- package/dist/cluster/peer-registry.d.ts +37 -106
- package/dist/cluster/plan-headroom-store.d.ts +4 -6
- package/dist/cluster/raft-state.d.ts +4 -13
- package/dist/cluster/raft.d.ts +18 -53
- package/dist/cluster/reconcile-identity.d.ts +8 -11
- package/dist/cluster/rerouted-job-guard.d.ts +7 -11
- package/dist/cluster/verified-issuer.d.ts +2 -3
- package/dist/cluster/worker-cluster-settings.d.ts +7 -10
- package/dist/cluster/worker-eviction.d.ts +16 -26
- package/dist/cold-store/load-access-log-range.d.ts +15 -31
- package/dist/cold-store/load-event-log-range.d.ts +13 -27
- package/dist/cold-store/load-secret-audit-log-range.d.ts +8 -12
- package/dist/cold-store/orchestrator-cold-store.d.ts +8 -28
- package/dist/cold-store/tables/access-log.d.ts +12 -14
- package/dist/cold-store/tables/event-log.d.ts +4 -8
- package/dist/cold-store/tables/execution-runs.d.ts +6 -11
- package/dist/cold-store/tables/secret-audit-log.d.ts +5 -12
- package/dist/concurrency/dispatch-next-queued.d.ts +13 -25
- package/dist/concurrency/queue-manager.d.ts +22 -54
- package/dist/concurrency/waiters.d.ts +15 -31
- package/dist/config/encryption.d.ts +0 -2
- package/dist/config/loader.d.ts +0 -1
- package/dist/config/reload.d.ts +0 -6
- package/dist/config/resolver.d.ts +0 -4
- package/dist/config/shared-store.d.ts +0 -2
- package/dist/config/types.d.ts +0 -3
- package/dist/content-requirements-cache.d.ts +0 -1
- package/dist/contexts/context-store.d.ts +0 -2
- package/dist/contexts/held-runs.d.ts +0 -2
- package/dist/contexts/variable-store.d.ts +0 -2
- package/dist/dashboard/attestation-filters.d.ts +13 -18
- package/dist/dashboard/handler.d.ts +143 -307
- package/dist/db/client.d.ts +2 -6
- package/dist/db/deferred-index-build.d.ts +8 -10
- package/dist/db/deferred-indexes.d.ts +15 -22
- package/dist/db/migration-provider.d.ts +2 -4
- package/dist/db/migration-test-harness.d.ts +26 -44
- package/dist/db/migrations/001_initial.d.ts +1 -4
- package/dist/db/migrations/002_config_versions_key_version.d.ts +5 -8
- package/dist/db/migrations/003_access_log.d.ts +10 -15
- package/dist/db/migrations/004_rename_bundle_to_source.d.ts +7 -9
- package/dist/db/migrations/005_cold_store_chunk_counter.d.ts +4 -10
- package/dist/db/migrations/006_runs_jobs_steps_archived_at.d.ts +9 -29
- package/dist/db/migrations/007_audit_logs_archived_at.d.ts +11 -30
- package/dist/db/migrations/008_event_log_archived_at.d.ts +14 -29
- package/dist/db/migrations/009_access_log_trigram.d.ts +4 -8
- package/dist/db/migrations/010_cold_store_chunks.d.ts +16 -26
- package/dist/db/migrations/011_drop_source_secrets_notify.d.ts +6 -14
- package/dist/db/migrations/012_peer_credentials_active_uniq.d.ts +9 -14
- package/dist/db/migrations/013_execution_log_bytes.d.ts +5 -13
- package/dist/db/migrations/014_kici_events_lease_retry.d.ts +14 -27
- package/dist/db/migrations/015_org_settings_customer_scoped.d.ts +12 -17
- package/dist/db/migrations/016_org_settings_allow_http_npm.d.ts +4 -6
- package/dist/db/migrations/019_generic_sources_change_notify.d.ts +13 -17
- package/dist/db/migrations/020_org_settings_dashboard_write_policy.d.ts +5 -8
- package/dist/db/migrations/021_check_run_tracking.d.ts +5 -17
- package/dist/db/migrations/022_scaler_manager_state.d.ts +10 -20
- package/dist/db/migrations/023_dispatch_queue_recovery_deadline.d.ts +7 -18
- package/dist/db/migrations/024_dispatch_queue_provisioning_error.d.ts +5 -11
- package/dist/db/migrations/025_init_failure.d.ts +5 -9
- package/dist/db/migrations/027_workflow_timeout.d.ts +5 -13
- package/dist/db/migrations/029_dispatch_queue_attempts.d.ts +4 -10
- package/dist/db/migrations/030_held_runs_env_set_null.d.ts +5 -6
- package/dist/db/migrations/031_dispatch_queue_ack_deadline.d.ts +6 -12
- package/dist/db/migrations/033_org_settings_approval.d.ts +7 -10
- package/dist/db/migrations/034_held_runs_generalize.d.ts +10 -18
- package/dist/db/migrations/035_pending_workflow_contexts.d.ts +4 -5
- package/dist/db/migrations/037_generic_sources_provider_type_local.d.ts +5 -6
- package/dist/db/migrations/038_remote_sources.d.ts +5 -7
- package/dist/db/migrations/045_host_properties.d.ts +5 -9
- package/dist/db/migrations/046_join_token_consumed_by_instance.d.ts +5 -10
- package/dist/db/migrations/048_host_reboot_pending.d.ts +6 -12
- package/dist/db/migrations/049_held_runs_payload.d.ts +3 -8
- package/dist/db/migrations/050_sources_slug.d.ts +4 -5
- package/dist/db/migrations/051_binding_host_pattern.d.ts +7 -10
- package/dist/db/migrations/052_host_reach_metadata.d.ts +8 -11
- package/dist/db/migrations/053_agent_token_single_use.d.ts +6 -10
- package/dist/db/migrations/054_local_working_tree.d.ts +4 -6
- package/dist/db/migrations/055_agent_token_mandatory_labels.d.ts +6 -11
- package/dist/db/migrations/056_execution_jobs_environments.d.ts +4 -11
- package/dist/db/migrations/064_execution_jobs_skipped_environments.d.ts +6 -9
- package/dist/db/migrations/065_pending_attestations.d.ts +9 -12
- package/dist/db/migrations/066_pending_attestations_rejected.d.ts +4 -7
- package/dist/db/migrations/068_request_idempotency.d.ts +7 -9
- package/dist/db/migrations/069_reroute_tunables.d.ts +6 -14
- package/dist/db/migrations/071_batch_accumulation.d.ts +8 -16
- package/dist/db/migrations/072_dispatch_queue_run_id_index.d.ts +3 -9
- package/dist/db/migrations/074_normalize_zero_concurrency_limit.d.ts +4 -7
- package/dist/db/migrations/076_artifacts.d.ts +7 -8
- package/dist/db/migrations/078_org_settings_backup_staleness.d.ts +3 -6
- package/dist/db/migrations/079_org_settings_scaler_spawn_timeout.d.ts +4 -8
- package/dist/db/migrations/080_cluster_settings.d.ts +6 -10
- package/dist/db/migrations/081_org_settings_queue_timeout.d.ts +7 -11
- package/dist/db/migrations/082_host_s3_reachable.d.ts +5 -10
- package/dist/db/migrations/083_org_settings_artifact_caps.d.ts +6 -8
- package/dist/db/migrations/084_orchestrator_signing_keys.d.ts +8 -13
- package/dist/db/migrations/085_cluster_settings_reroute_flap_grace_ms.d.ts +4 -5
- package/dist/db/migrations/086_cluster_settings_max_fanout_hosts.d.ts +3 -3
- package/dist/db/migrations/087_cluster_settings_event_router_rate_limit.d.ts +5 -3
- package/dist/db/migrations/088_cluster_settings_cache_max_tarball_bytes.d.ts +3 -3
- package/dist/db/migrations/089_cluster_settings_cache_ttl_days.d.ts +3 -5
- package/dist/db/migrations/090_cluster_settings_concurrency_wait_timeout_ms.d.ts +3 -6
- package/dist/db/migrations/091_cluster_settings_agent_token_ttl_ms.d.ts +3 -6
- package/dist/db/migrations/092_cluster_settings_version.d.ts +5 -9
- package/dist/db/migrations/094_dashboard_encryption_keys.d.ts +9 -14
- package/dist/db/migrations/095_dashboard_write_policy_tristate.d.ts +2 -6
- package/dist/db/migrations/099_cluster_settings_dashboard_verified_issuer.d.ts +8 -11
- package/dist/db/migrations/101_contexts_hold_expiry_drop_default.d.ts +9 -15
- package/dist/db/migrations/102_dispatch_queue_agent_id.d.ts +6 -11
- package/dist/db/migrations/103_cluster_settings_ownership_db_check_timeout_ms.d.ts +6 -12
- package/dist/db/migrations/104_check_run_terminal_sent.d.ts +8 -14
- package/dist/db/migrations/105_org_trust_policy.d.ts +5 -7
- package/dist/db/migrations/106_cluster_settings_check_run_tracking_ttl_days.d.ts +4 -12
- package/dist/db/migrations/107_check_run_tracking_updated_at_index.d.ts +8 -14
- package/dist/db/migrations/113_execution_runs_workflow_repo_index.d.ts +10 -19
- package/dist/db/migrations/115_global_workflows_cluster_switch.d.ts +2 -3
- package/dist/db/migrations/121_org_settings_allow_untrusted_dockerfile_builds.d.ts +7 -8
- package/dist/db/migrations/122_host_roster_scaler_managed.d.ts +9 -16
- package/dist/db/migrations/123_workflow_registrations_default_branch.d.ts +8 -16
- package/dist/db/migrations/125_org_trust_directory.d.ts +13 -21
- package/dist/db/migrations/126_held_runs_posted_pending_check.d.ts +10 -16
- package/dist/db/migrations/127_org_trust_policy_approval_expiry_seconds.d.ts +8 -16
- package/dist/db/migrations/128_execution_runs_global_eval_round.d.ts +6 -11
- package/dist/db/migrations/129_execution_runs_dispatch_routing_key.d.ts +8 -11
- package/dist/db/migrations/130_scaler_provision_outcomes.d.ts +13 -20
- package/dist/db/migrations/131_execution_jobs_git_credentials.d.ts +12 -22
- package/dist/db/migrations/132_execution_runs_trigger_event.d.ts +6 -16
- package/dist/db/migrations/133_dispatch_queue_source_tar_digest.d.ts +6 -9
- package/dist/db/migrations/140_dispatch_queue_canonical_labels.d.ts +22 -40
- package/dist/db/migrations/143_execution_jobs_precursor_result.d.ts +9 -18
- package/dist/db/migrations/144_execution_runs_registration_window.d.ts +12 -20
- package/dist/db/migrations/147_sealed_job_secrets.d.ts +5 -9
- package/dist/db/migrations/148_cluster_settings_sealed_secrets_retry_backoff_ms.d.ts +7 -13
- package/dist/db/migrations/150_execution_runs_status_epoch.d.ts +6 -11
- package/dist/db/migrations/152_org_settings_cache_upload_settle_timeout.d.ts +5 -8
- package/dist/db/migrations/157_host_roster_identity_source.d.ts +9 -13
- package/dist/db/migrations/159_org_id_widen_to_source_width.d.ts +5 -0
- package/dist/db/migrator.d.ts +17 -25
- package/dist/db/notify-listener.d.ts +8 -11
- package/dist/db/repos/dashboard-encryption-keys-repo.d.ts +9 -10
- package/dist/db/synthetic-job-ids.d.ts +6 -8
- package/dist/db/types.d.ts +531 -1058
- package/dist/deployment/deployment-identity.d.ts +42 -60
- package/dist/diagnostics/bundle-reader.d.ts +3 -15
- package/dist/diagnostics/bundle-writer.d.ts +5 -12
- package/dist/diagnostics/checks/agents.d.ts +1 -7
- package/dist/diagnostics/checks/backup.d.ts +7 -10
- package/dist/diagnostics/checks/certs.d.ts +1 -6
- package/dist/diagnostics/checks/config.d.ts +2 -5
- package/dist/diagnostics/checks/db.d.ts +1 -1
- package/dist/diagnostics/checks/disk.d.ts +1 -6
- package/dist/diagnostics/checks/firecracker-network.d.ts +3 -3
- package/dist/diagnostics/checks/index.d.ts +0 -6
- package/dist/diagnostics/checks/retention.d.ts +3 -6
- package/dist/diagnostics/checks/scaler.d.ts +3 -7
- package/dist/diagnostics/checks/ws.d.ts +1 -6
- package/dist/diagnostics/fleet-collector.d.ts +11 -14
- package/dist/diagnostics/fleet-constants.d.ts +1 -3
- package/dist/diagnostics/fleet-selection.d.ts +3 -7
- package/dist/diagnostics/fleet-topology.d.ts +2 -6
- package/dist/diagnostics/fleet-wiring.d.ts +6 -12
- package/dist/diagnostics/index.d.ts +0 -5
- package/dist/diagnostics/runner.d.ts +4 -8
- package/dist/diagnostics/types.d.ts +10 -21
- package/dist/drain/drain-controller.d.ts +16 -22
- package/dist/entry-helpers.d.ts +0 -2
- package/dist/events/circuit-breaker.d.ts +0 -3
- package/dist/events/event-router.d.ts +1 -2
- package/dist/events/event-store.d.ts +0 -9
- package/dist/events/trust-store.d.ts +0 -3
- package/dist/events/types.d.ts +0 -9
- package/dist/firecracker/host-network.d.ts +24 -30
- package/dist/firecracker/net-interfaces.d.ts +4 -7
- package/dist/helpers/rate-limiter.d.ts +0 -1
- package/dist/index.js +240 -482
- package/dist/metrics/agent-metrics-aggregator.d.ts +24 -67
- package/dist/metrics/metrics-reporter.d.ts +4 -13
- package/dist/metrics/prometheus.d.ts +47 -81
- package/dist/orchestrator-core.d.ts +3 -7
- package/dist/pipeline/content-filter.d.ts +20 -30
- package/dist/pipeline/cross-source-tally.d.ts +7 -8
- package/dist/pipeline/decorating-secret-resolver.d.ts +4 -8
- package/dist/pipeline/degraded-reason.d.ts +3 -4
- package/dist/pipeline/direct-ingress-deps.d.ts +15 -24
- package/dist/pipeline/dispatch-matched-workflow.d.ts +289 -525
- package/dist/pipeline/flatten-lock-steps.d.ts +4 -6
- package/dist/pipeline/global-dispatch-identity.d.ts +5 -10
- package/dist/pipeline/global-eval-round.d.ts +65 -151
- package/dist/pipeline/global-round-hold.d.ts +5 -7
- package/dist/pipeline/held-context-data.d.ts +5 -8
- package/dist/pipeline/held-round-registrations.d.ts +8 -11
- package/dist/pipeline/install-gate.d.ts +15 -25
- package/dist/pipeline/install-secrets-resolver.d.ts +24 -34
- package/dist/pipeline/internal-event-pipeline.d.ts +45 -62
- package/dist/pipeline/invoke-gate.d.ts +27 -40
- package/dist/pipeline/job-contexts.d.ts +19 -36
- package/dist/pipeline/manual-schedule.d.ts +7 -13
- package/dist/pipeline/needs-scheduler.d.ts +29 -46
- package/dist/pipeline/pending-workflow-context.d.ts +14 -28
- package/dist/pipeline/process-webhook.d.ts +88 -123
- package/dist/pipeline/processor.d.ts +209 -355
- package/dist/pipeline/remote-source-store.d.ts +5 -8
- package/dist/pipeline/request-idempotency.d.ts +10 -13
- package/dist/pipeline/rerun-global.d.ts +6 -7
- package/dist/pipeline/rerun-inputs.d.ts +23 -37
- package/dist/pipeline/rerun.d.ts +34 -57
- package/dist/pipeline/resolve-sandbox-grant.d.ts +5 -10
- package/dist/pipeline/resume-workflow.d.ts +28 -34
- package/dist/pipeline/route-or-dispatch-jobs.d.ts +19 -30
- package/dist/pipeline/run-event-context.d.ts +16 -21
- package/dist/pipeline/sandbox-allowlist-reader.d.ts +7 -11
- package/dist/pipeline/security-hold-check.d.ts +56 -80
- package/dist/pipeline/test-pipeline.d.ts +20 -46
- package/dist/pipeline/unbound-job-context.d.ts +6 -9
- package/dist/pipeline/undispatched-hold-checks.d.ts +4 -6
- package/dist/pipeline/wave-scheduler.d.ts +15 -22
- package/dist/pipeline/webhook-payload-store.d.ts +3 -10
- package/dist/pipeline/workflow-lock-at-sha.d.ts +14 -20
- package/dist/policy/dashboard-write-policy.d.ts +1 -1
- package/dist/provider-registry.d.ts +0 -19
- package/dist/providers/generic/index.d.ts +0 -2
- package/dist/providers/generic/normalizer.d.ts +0 -1
- package/dist/providers/generic/verification.d.ts +0 -8
- package/dist/providers/github/auth.d.ts +0 -1
- package/dist/providers/github/changed-files.d.ts +2 -1
- package/dist/providers/github/file-contents.d.ts +0 -4
- package/dist/providers/github/lock-file.d.ts +0 -4
- package/dist/providers/github/normalizer.d.ts +1 -9
- package/dist/providers/local/index.d.ts +1 -2
- package/dist/providers/local/lock-file-fetcher.d.ts +0 -4
- package/dist/providers/local/normalizer.d.ts +0 -10
- package/dist/providers/local/repo-url-builder.d.ts +0 -2
- package/dist/providers/universal-git/clone-token.d.ts +0 -2
- package/dist/providers/universal-git/lock-file.d.ts +0 -2
- package/dist/providers/universal-git/normalizer.d.ts +0 -2
- package/dist/queue/bootstrap.d.ts +3 -15
- package/dist/queue/cleanup.d.ts +29 -50
- package/dist/queue/depth-refresher.d.ts +21 -39
- package/dist/queue/job-queue.d.ts +112 -163
- package/dist/queue/retention.d.ts +23 -36
- package/dist/queue/scheduled-job.d.ts +17 -39
- package/dist/queue/terminalize-unroutable.d.ts +16 -18
- package/dist/queue/unroutable-probe.d.ts +15 -22
- package/dist/registration/extractor.d.ts +0 -6
- package/dist/registration/registration-index.d.ts +0 -6
- package/dist/registration/registration-store.d.ts +0 -9
- package/dist/reporting/agent-failure-category.d.ts +3 -5
- package/dist/reporting/check-run-reporter.d.ts +142 -316
- package/dist/reporting/check-run-summary.d.ts +16 -35
- package/dist/reporting/check-run-tracking-store.d.ts +54 -107
- package/dist/reporting/execution-tracker.d.ts +356 -526
- package/dist/reporting/fs-log-storage.d.ts +31 -33
- package/dist/reporting/job-check-run-completion.d.ts +14 -22
- package/dist/reporting/log-chunk-sink.d.ts +11 -16
- package/dist/reporting/log-pull-handler.d.ts +4 -13
- package/dist/reporting/log-retention.d.ts +3 -4
- package/dist/reporting/log-storage.d.ts +24 -52
- package/dist/reporting/log-writer.d.ts +28 -56
- package/dist/reporting/peer-log-normalize.d.ts +4 -8
- package/dist/reporting/run-aggregator.d.ts +6 -11
- package/dist/reporting/run-summary-fields.d.ts +5 -8
- package/dist/reporting/s3-log-storage.d.ts +31 -50
- package/dist/reporting/step-display-order.d.ts +11 -21
- package/dist/reporting/step-log-buffer.d.ts +7 -28
- package/dist/reporting/step-log-reader.d.ts +11 -19
- package/dist/routes/admin-access-log.d.ts +4 -6
- package/dist/routes/admin-auth.d.ts +17 -38
- package/dist/routes/admin-backends.d.ts +3 -14
- package/dist/routes/admin-cluster-name.d.ts +13 -21
- package/dist/routes/admin-cluster-settings.d.ts +6 -9
- package/dist/routes/admin-config.d.ts +7 -19
- package/dist/routes/admin-contexts.d.ts +6 -13
- package/dist/routes/admin-db.d.ts +5 -10
- package/dist/routes/admin-event-dlq.d.ts +3 -5
- package/dist/routes/admin-event-inspect.d.ts +3 -4
- package/dist/routes/admin-event-log.d.ts +3 -7
- package/dist/routes/admin-events.d.ts +16 -30
- package/dist/routes/admin-held-runs.d.ts +52 -76
- package/dist/routes/admin-maintenance.d.ts +3 -6
- package/dist/routes/admin-orchestrator-drain.d.ts +0 -2
- package/dist/routes/admin-org-settings.d.ts +8 -14
- package/dist/routes/admin-peer-forget.d.ts +0 -1
- package/dist/routes/admin-queue-execution.d.ts +3 -3
- package/dist/routes/admin-registrations.d.ts +3 -12
- package/dist/routes/admin-runs.d.ts +16 -29
- package/dist/routes/admin-scaler-orphans.d.ts +4 -4
- package/dist/routes/admin-scaler-reload.d.ts +5 -8
- package/dist/routes/admin-scheduled-jobs.d.ts +3 -4
- package/dist/routes/admin-sources.d.ts +20 -29
- package/dist/routes/admin-trust-policy.d.ts +31 -48
- package/dist/routes/admin.d.ts +67 -94
- package/dist/routes/caller-identity.d.ts +5 -12
- package/dist/routes/capabilities.d.ts +8 -13
- package/dist/routes/diagnostics.d.ts +2 -10
- package/dist/routes/fleet.d.ts +1 -2
- package/dist/routes/github-webhook.d.ts +2 -5
- package/dist/routes/health.d.ts +6 -19
- package/dist/routes/provenance-oidc.d.ts +3 -3
- package/dist/routes/uploads.d.ts +5 -11
- package/dist/routes/webhooks.d.ts +2 -24
- package/dist/scaler/agent-binary-location.d.ts +77 -0
- package/dist/scaler/agent-fit.d.ts +2 -2
- package/dist/scaler/backend-factory.d.ts +10 -18
- package/dist/scaler/bare-metal-backend.d.ts +28 -45
- package/dist/scaler/claim-store.d.ts +20 -43
- package/dist/scaler/config.d.ts +10 -37
- package/dist/scaler/container-backend.d.ts +22 -35
- package/dist/scaler/container-hostconfig.d.ts +10 -16
- package/dist/scaler/container-routing.d.ts +10 -13
- package/dist/scaler/disk-guard.d.ts +2 -3
- package/dist/scaler/event-backend.d.ts +33 -58
- package/dist/scaler/event-provision-reaper.d.ts +61 -84
- package/dist/scaler/failure-tracker.d.ts +6 -13
- package/dist/scaler/file-tail.d.ts +6 -19
- package/dist/scaler/firecracker-api.d.ts +19 -63
- package/dist/scaler/firecracker-backend.d.ts +142 -258
- package/dist/scaler/host-access.d.ts +7 -10
- package/dist/scaler/index.d.ts +3 -6
- package/dist/scaler/ip-allocator.d.ts +30 -82
- package/dist/scaler/label-matcher.d.ts +14 -35
- package/dist/scaler/log-forwarder.d.ts +6 -25
- package/dist/scaler/machine-ledger.d.ts +39 -86
- package/dist/scaler/manager.d.ts +517 -787
- package/dist/scaler/pending-scale-sweeper.d.ts +8 -11
- package/dist/scaler/reap-orphans.d.ts +8 -12
- package/dist/scaler/resolve-container-auth.d.ts +12 -21
- package/dist/scaler/scaler-events.d.ts +2 -5
- package/dist/scaler/scaler-state-store.d.ts +127 -189
- package/dist/scaler/spawn-errors.d.ts +1 -4
- package/dist/scaler/startup-orphan-sweep.d.ts +3 -4
- package/dist/scaler/startup-tools-check.d.ts +21 -0
- package/dist/scaler/types.d.ts +142 -255
- package/dist/scaler/vm-job-bindings.d.ts +5 -9
- package/dist/scaler/warm-pool.d.ts +46 -93
- package/dist/secrets/audit-logger.d.ts +0 -3
- package/dist/secrets/backend-health.d.ts +0 -9
- package/dist/secrets/backend-registry.d.ts +0 -9
- package/dist/secrets/cleanup.d.ts +0 -1
- package/dist/secrets/config.d.ts +0 -3
- package/dist/secrets/ephemeral-keys.d.ts +0 -3
- package/dist/secrets/pg-secret-store.d.ts +0 -3
- package/dist/secrets/rbac.d.ts +0 -6
- package/dist/secrets/secret-output-store.d.ts +0 -6
- package/dist/secrets/secret-resolver.d.ts +2 -6
- package/dist/secrets/token-manager.d.ts +0 -4
- package/dist/secrets/vault-secret-store.d.ts +0 -13
- package/dist/security/comment-handler.d.ts +0 -2
- package/dist/security/workflow-diff.d.ts +1 -1
- package/dist/server.js +7500 -12629
- package/dist/sources/source-manager.d.ts +0 -3
- package/dist/sources/source-store.d.ts +0 -9
- package/dist/sources/source-validator.d.ts +0 -3
- package/dist/stale-detector/all-terminal-runs.d.ts +8 -11
- package/dist/stale-detector/gate-deadline-detector.d.ts +14 -19
- package/dist/stale-detector/reboot-deadline-sweep.d.ts +7 -11
- package/dist/stale-detector/stale-dispatch-candidates.d.ts +13 -20
- package/dist/stale-detector/stale-run-detector.d.ts +74 -121
- package/dist/stale-detector/workflow-deadline-detector.d.ts +15 -43
- package/dist/standalone.js +7141 -11957
- package/dist/storage/blob-routes.d.ts +10 -19
- package/dist/storage/filesystem.d.ts +19 -29
- package/dist/storage/index.d.ts +2 -10
- package/dist/storage/key-discriminator.d.ts +8 -10
- package/dist/storage/loopback-guard.d.ts +27 -45
- package/dist/storage/s3.d.ts +25 -52
- package/dist/storage/sign-url.d.ts +7 -22
- package/dist/storage/types.d.ts +44 -70
- package/dist/webhook/event-log.d.ts +0 -2
- package/dist/webhook/generic-sources.d.ts +0 -21
- package/dist/webhook/ingest-admission.d.ts +0 -1
- package/dist/webhook/relay-buffer.d.ts +2 -2
- package/dist/webhook/verify-inbound.d.ts +0 -1
- package/dist/worker/in-memory-execution-tracker.d.ts +9 -31
- package/dist/worker/in-memory-job-queue.d.ts +32 -72
- package/dist/worker/peer-outbox.d.ts +5 -8
- package/dist/worker/reroute-dispatch.d.ts +4 -6
- package/dist/worker/static-agent-token-store.d.ts +6 -27
- package/dist/worker/worker-outbox-relay.d.ts +2 -2
- package/dist/worker/worker-status.d.ts +3 -20
- package/dist/ws/agent-api-registry.d.ts +8 -27
- package/dist/ws/agent-handler.d.ts +63 -110
- package/dist/ws/agent-heartbeat.d.ts +5 -12
- package/dist/ws/bringup-api.d.ts +31 -38
- package/dist/ws/dashboard-backends-handler.d.ts +12 -22
- package/dist/ws/dashboard-context-handler.d.ts +37 -80
- package/dist/ws/dashboard-diagnostics-handler.d.ts +8 -25
- package/dist/ws/dashboard-dispatch-guard.d.ts +3 -4
- package/dist/ws/dashboard-fleet-handler.d.ts +9 -13
- package/dist/ws/dashboard-fleet-write-handler.d.ts +7 -21
- package/dist/ws/dashboard-global-workflows-handler.d.ts +22 -44
- package/dist/ws/dashboard-registrations-handler.d.ts +10 -27
- package/dist/ws/dashboard-run-state-handler.d.ts +10 -13
- package/dist/ws/event-buffer.d.ts +3 -6
- package/dist/ws/execution-status-frame.d.ts +7 -10
- package/dist/ws/failure-messages.d.ts +9 -14
- package/dist/ws/fleet-agent-collector.d.ts +6 -12
- package/dist/ws/fleet-runs-on-all.d.ts +1 -6
- package/dist/ws/git-credential-relay.d.ts +12 -20
- package/dist/ws/inventory-api.d.ts +4 -5
- package/dist/ws/observer-registry.d.ts +14 -59
- package/dist/ws/ownership-gate.d.ts +1 -1
- package/dist/ws/platform-client.d.ts +21 -30
- package/dist/ws/replay-chunker.d.ts +7 -10
- package/dist/ws/server-options.d.ts +5 -9
- package/dist/ws/test-relay-handlers.d.ts +36 -60
- package/installer-image-digests.json +3 -3
- package/package.json +6 -5
- package/sbom.spdx.json +53 -53
- package/dist/__test-helpers__/fake-scaler-state-store.d.ts +0 -24
- package/dist/__test-helpers__/mock-db-query.d.ts +0 -129
- package/dist/__test-helpers__/mock-db.d.ts +0 -134
- package/dist/__test-helpers__/mock-ws.d.ts +0 -13
- package/dist/__test-helpers__/peer-mutual-auth.d.ts +0 -42
- package/dist/__test-helpers__/scratch-db.d.ts +0 -16
- package/dist/__test-helpers__/test-db.d.ts +0 -34
- package/dist/pipeline/dispatch-matched-workflow.test-helpers.d.ts +0 -121
|
@@ -1,20 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Firecracker
|
|
2
|
+
* Ephemeral agent microVMs under the Firecracker jailer. Lifecycle: prepare chroot, hardlink
|
|
3
|
+
* rootfs+kernel, create overlay drive, invoke jailer, inject MMDS metadata, destroy with cleanup.
|
|
4
|
+
* Mirrors the Docker backend's structure.
|
|
3
5
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* serial console log file and VMM logs captured via --log-path. Both files are tailed
|
|
11
|
-
* using fs.watchFile() and forwarded with distinct logsSource tags ('firecracker-serial',
|
|
12
|
-
* 'firecracker-vmm'). The serial-console path is best-effort only: Firecracker's Rust
|
|
13
|
-
* BufWriter around stdout + jailer's uid drop make sparse userspace writes unreliable,
|
|
14
|
-
* so it's treated as defense-in-depth. The canonical agent log path is WS agent.log
|
|
15
|
-
* (attached unconditionally by the agent; see packages/agent/src/server.ts step 6).
|
|
16
|
-
*
|
|
17
|
-
* Follows the Docker backend pattern closely for consistency.
|
|
6
|
+
* Log capture is internal: the jailer runs as a non-daemonized detached child with stdout
|
|
7
|
+
* redirected to a serial console log file and VMM logs captured via --log-path. Both files are
|
|
8
|
+
* tailed and forwarded with logsSource tags 'firecracker-serial' and 'firecracker-vmm'. The
|
|
9
|
+
* serial path is best-effort defense-in-depth: Firecracker's Rust BufWriter around stdout plus
|
|
10
|
+
* the jailer's uid drop make sparse userspace writes unreliable. The canonical agent log path
|
|
11
|
+
* is WS agent.log (attached unconditionally by the agent; see packages/agent/src/server.ts step 6).
|
|
18
12
|
*/
|
|
19
13
|
import { ScalerVmTracker, type ScalerVmStopResult } from '@kici-dev/engine';
|
|
20
14
|
import { type ToolRequirement } from '@kici-dev/shared';
|
|
@@ -36,17 +30,13 @@ export interface FirecrackerManagedAgent extends ManagedAgent {
|
|
|
36
30
|
/** TAP device name for cleanup */
|
|
37
31
|
tapDevice: string;
|
|
38
32
|
/**
|
|
39
|
-
* The `hostAccess` policy resolved
|
|
40
|
-
*
|
|
41
|
-
* Carried on the record because a bridge self-heal re-applies isolation for
|
|
42
|
-
* every live VM, and that path has no `orchestratorUrl` to re-derive the
|
|
43
|
-
* default from.
|
|
33
|
+
* The `hostAccess` policy resolved at provision time. Carried because a bridge self-heal
|
|
34
|
+
* re-applies isolation for every live VM and has no `orchestratorUrl` to re-derive it.
|
|
44
35
|
*/
|
|
45
36
|
hostAccess?: string[];
|
|
46
37
|
/**
|
|
47
|
-
* Set once the VM's agent registered
|
|
48
|
-
*
|
|
49
|
-
* down; afterwards the agent's disconnect does.
|
|
38
|
+
* Set once the VM's agent registered. Until then a VM that stops on its own is a failed
|
|
39
|
+
* spawn nothing else tears down; afterwards the agent's disconnect does.
|
|
50
40
|
*/
|
|
51
41
|
registered?: boolean;
|
|
52
42
|
}
|
|
@@ -55,19 +45,15 @@ export interface FirecrackerScalerBackendOptions {
|
|
|
55
45
|
name: string;
|
|
56
46
|
/** Label sets this backend can provision */
|
|
57
47
|
labelSets: LabelSetConfig[];
|
|
58
|
-
/** Maximum concurrent agents */
|
|
59
48
|
maxAgents: number;
|
|
60
49
|
/** Shared IP allocator instance */
|
|
61
50
|
ipAllocator: IpAllocator;
|
|
62
51
|
/**
|
|
63
|
-
* Extra `hostAccess` entries the orchestrator itself directed agents at
|
|
64
|
-
*
|
|
65
|
-
* narrowing does not cut off a host-local object store.
|
|
52
|
+
* Extra `hostAccess` entries the orchestrator itself directed agents at (its object-storage
|
|
53
|
+
* endpoint), folded into the agent default so the narrowing keeps a host-local object store.
|
|
66
54
|
*/
|
|
67
55
|
hostServices?: string[];
|
|
68
|
-
/** Path to firecracker binary */
|
|
69
56
|
firecrackerPath: string;
|
|
70
|
-
/** Path to jailer binary */
|
|
71
57
|
jailerPath: string;
|
|
72
58
|
/** Default kernel image path */
|
|
73
59
|
kernelPath: string;
|
|
@@ -96,41 +82,39 @@ export interface FirecrackerScalerBackendOptions {
|
|
|
96
82
|
netmask: string;
|
|
97
83
|
/** nft table name for host-network diagnostics. @default 'kici' */
|
|
98
84
|
table?: string;
|
|
99
|
-
/** Token store for
|
|
85
|
+
/** Token store for ephemeral agent auth tokens; when undefined, no token is injected. */
|
|
100
86
|
tokenStore?: AgentTokenStore;
|
|
101
87
|
/** TTL for ephemeral agent tokens in ms. Default: 1 hour. */
|
|
102
88
|
tokenTtlMs?: number;
|
|
103
89
|
/**
|
|
104
|
-
* Live resolver for the ephemeral agent-token TTL (ms)
|
|
105
|
-
*
|
|
106
|
-
* `cluster_settings.agent_token_ttl_ms` override; falls back to `tokenTtlMs`.
|
|
90
|
+
* Live per-spawn resolver for the ephemeral agent-token TTL (ms), so the leader serves the
|
|
91
|
+
* fleet-wide `cluster_settings.agent_token_ttl_ms` override; falls back to `tokenTtlMs`.
|
|
107
92
|
*/
|
|
108
93
|
tokenTtlProvider?: () => Promise<number>;
|
|
109
94
|
/**
|
|
110
|
-
* How long a spawn waits for the VM's API socket after the jailer starts
|
|
111
|
-
*
|
|
95
|
+
* How long (ms) a spawn waits for the VM's API socket after the jailer starts.
|
|
96
|
+
* Default {@link DEFAULT_API_SOCKET_WAIT_MS}.
|
|
112
97
|
*/
|
|
113
98
|
apiSocketWaitMs?: number;
|
|
114
99
|
/**
|
|
115
100
|
* Live per-spawn resolver for the API-socket wait, so the fleet-wide
|
|
116
|
-
* `cluster_settings.firecracker_api_socket_wait_ms` applies without a
|
|
117
|
-
*
|
|
101
|
+
* `cluster_settings.firecracker_api_socket_wait_ms` applies without a restart. Falls back
|
|
102
|
+
* to `apiSocketWaitMs`.
|
|
118
103
|
*/
|
|
119
104
|
apiSocketWaitMsProvider?: () => Promise<number>;
|
|
120
105
|
/** Agent roles for this scaler. undefined = all, [] = execution only. */
|
|
121
106
|
roles?: string[];
|
|
122
107
|
/**
|
|
123
|
-
* Wrap privileged commands (`ip`, `chown`) with `sudo -n`. Required when the
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
* On hosts where the orchestrator already runs as root, leave this false.
|
|
108
|
+
* Wrap privileged commands (`ip`, `chown`) with `sudo -n`. Required when the orchestrator
|
|
109
|
+
* runs as a non-root user (e.g. user-mode systemd on edge worker nodes) with a sudoers
|
|
110
|
+
* NOPASSWD rule for those binaries; leave false when it runs as root.
|
|
127
111
|
* @default false
|
|
128
112
|
*/
|
|
129
113
|
requireSudo?: boolean;
|
|
130
114
|
/**
|
|
131
|
-
* When true, `ensureHostReady()` verifies and provisions this host bridge on
|
|
132
|
-
*
|
|
133
|
-
*
|
|
115
|
+
* When true, `ensureHostReady()` verifies and provisions this host bridge on startup
|
|
116
|
+
* (self-heal); when false it is a no-op. Threaded from `firecracker.autoProvisionHost`.
|
|
117
|
+
* @default false (cores pass the real value)
|
|
134
118
|
*/
|
|
135
119
|
autoProvisionHost?: boolean;
|
|
136
120
|
/** Injectable bridge verifier (tests). @default host-network verifyBridge */
|
|
@@ -146,12 +130,9 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
146
130
|
/** AbortControllers for file tailing per managed VM (keyed by agent ID) */
|
|
147
131
|
private readonly tailAbortControllers;
|
|
148
132
|
/**
|
|
149
|
-
* Per-spawn `scaler.failed` emitters, keyed by agent id.
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
* `spawn`'s own scope, and the manager releases a spawn reservation only on a
|
|
153
|
-
* failure event. Holding the emitter here is what lets those two paths report
|
|
154
|
-
* the failure instead of leaving the reservation held until restart.
|
|
133
|
+
* Per-spawn `scaler.failed` emitters, keyed by agent id. The jailer's `exit` handler and the
|
|
134
|
+
* spawn-deadline abort fire outside `spawn`'s scope, and the manager releases a spawn
|
|
135
|
+
* reservation only on a failure event; without this the reservation stays held until restart.
|
|
155
136
|
*/
|
|
156
137
|
private readonly spawnFailureHandlers;
|
|
157
138
|
/** Liveness probes of spawned VMs whose agent has not registered yet, by agent id. */
|
|
@@ -231,44 +212,32 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
231
212
|
bridgeCidr: string;
|
|
232
213
|
table: string;
|
|
233
214
|
};
|
|
215
|
+
/**
|
|
216
|
+
* nft options for this backend's own rules. `table` is the operator-configured one, not the
|
|
217
|
+
* literal `kici`: with two coordinators on one host, B's per-VM rules in A's `kici` table
|
|
218
|
+
* would be swept by A's next re-provision and evaluated against A's subnet-scoped baseline.
|
|
219
|
+
*/
|
|
220
|
+
private nftOpts;
|
|
234
221
|
/**
|
|
235
222
|
* Verify and (if needed) provision this backend's host bridge on startup.
|
|
236
223
|
* No-op unless autoProvisionHost is set. Throws on a real provision failure;
|
|
237
224
|
* ScalerManager.ensureHostsReady catches per-backend and degrades this scaler.
|
|
238
225
|
*/
|
|
239
|
-
/**
|
|
240
|
-
* nft options for this backend's own rules.
|
|
241
|
-
*
|
|
242
|
-
* `table` is the operator-configured one, not the literal `kici`: with two
|
|
243
|
-
* coordinators on one host, coordinator B's baseline goes to `kici_b` while
|
|
244
|
-
* its per-VM rules used to go to `kici` — coordinator A's table. A's next
|
|
245
|
-
* re-provision then swept B's live VMs' isolation along with its own, and B's
|
|
246
|
-
* rules were evaluated against A's subnet-scoped baseline.
|
|
247
|
-
*/
|
|
248
|
-
private nftOpts;
|
|
249
226
|
ensureHostReady(): Promise<void>;
|
|
250
227
|
/**
|
|
251
|
-
* Fail loudly on a kernel that will not honour bridge port isolation.
|
|
252
|
-
*
|
|
253
|
-
*
|
|
254
|
-
* L2, and no nft rule sees that traffic — so the isolation the operator docs
|
|
255
|
-
* promise silently does not exist. The same fail-closed posture
|
|
256
|
-
* `validateNftablesAvailability` takes for a missing `nft`: better to refuse
|
|
257
|
-
* to run VMs than to run them unisolated.
|
|
228
|
+
* Fail loudly on a kernel that will not honour bridge port isolation. Without the flag every
|
|
229
|
+
* tenant VM reaches every other one at L2, unseen by nft, so the documented isolation silently
|
|
230
|
+
* does not exist. Fail-closed like `validateNftablesAvailability` for a missing `nft`.
|
|
258
231
|
*
|
|
259
|
-
* The probe is
|
|
260
|
-
*
|
|
261
|
-
* bridge itself, and only the real command exercises all three.
|
|
232
|
+
* The probe is end-to-end (create a TAP, enslave it isolated, delete it) because the kernel,
|
|
233
|
+
* iproute2 or the bridge can each reject the flag, and only the real command exercises all three.
|
|
262
234
|
*/
|
|
263
235
|
private assertBridgePortIsolationSupported;
|
|
264
236
|
/**
|
|
265
|
-
* Re-apply per-VM isolation rules for every VM
|
|
266
|
-
*
|
|
267
|
-
*
|
|
268
|
-
*
|
|
269
|
-
* baseline there too. Re-applying is what guarantees a self-heal cannot leave
|
|
270
|
-
* a running VM fail-open. Each identifier is removed before it is re-added,
|
|
271
|
-
* so a self-heal that fires twice does not duplicate the rules.
|
|
237
|
+
* Re-apply per-VM isolation rules for every tracked VM. Provisioning sweeps the `forward`
|
|
238
|
+
* chain of everything no live VM owns (an old-shaped host carries its baseline there too), so
|
|
239
|
+
* this keeps a self-heal from leaving a running VM fail-open. Each identifier is removed
|
|
240
|
+
* before it is re-added, so a self-heal that fires twice does not duplicate the rules.
|
|
272
241
|
*/
|
|
273
242
|
private reapplyIsolationForTrackedVms;
|
|
274
243
|
markRegistered(managedId: string): void;
|
|
@@ -280,13 +249,11 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
280
249
|
/**
|
|
281
250
|
* Keep loosening a rootless VM's chroot permissions while firecracker starts.
|
|
282
251
|
*
|
|
283
|
-
* The jailer chmods the chroot root and every directory it creates (`/run`)
|
|
284
|
-
*
|
|
285
|
-
*
|
|
286
|
-
*
|
|
287
|
-
*
|
|
288
|
-
* can outlast the tick, and overlapping ones would pile up for the whole
|
|
289
|
-
* wait.
|
|
252
|
+
* The jailer chmods the chroot root and every directory it creates (`/run`) to 0700 owned by
|
|
253
|
+
* its `--uid`, locking an orchestrator running as another user out of
|
|
254
|
+
* `/run/firecracker.socket`. A `chmod -R go+rX` every 100 ms reopens each one as it appears
|
|
255
|
+
* (host-side view only). One chmod runs at a time: on a slow disk a `sudo chmod -R` can
|
|
256
|
+
* outlast the tick, and overlapping ones would pile up for the whole wait.
|
|
290
257
|
*
|
|
291
258
|
* @returns a function that stops the loosening
|
|
292
259
|
*/
|
|
@@ -296,26 +263,19 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
296
263
|
/**
|
|
297
264
|
* Tear the VM down if its process stops before its agent registers.
|
|
298
265
|
*
|
|
299
|
-
* A guest whose init exits
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
*
|
|
303
|
-
*
|
|
304
|
-
* So the VM's process is probed through its PID file, and the probe stops
|
|
305
|
-
* once the agent registers (its disconnect owns the teardown from then on),
|
|
306
|
-
* the VM is being destroyed, or it is no longer tracked.
|
|
266
|
+
* A guest whose init exits (fatal agent startup error, kernel panic) reboots and firecracker
|
|
267
|
+
* exits; nothing else releases its `maxAgents` slot, TAP, nft rules, IP and chroot until the
|
|
268
|
+
* manager's stale-spawn prune. The jailer child's exit cannot tell: under `--new-pid-ns` it
|
|
269
|
+
* exits 0 right after the start. So the VM's PID file is probed until the agent registers (its
|
|
270
|
+
* disconnect then owns the teardown), the VM is being destroyed, or it is no longer tracked.
|
|
307
271
|
*/
|
|
308
272
|
private watchUntilRegistered;
|
|
309
273
|
/**
|
|
310
|
-
* Whether there is positive evidence
|
|
311
|
-
*
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
* Any other failure — an unreadable PID file, a `/proc` this user cannot
|
|
316
|
-
* see — is not evidence, so the answer is "not gone". The caller tears the
|
|
317
|
-
* VM down on a "gone", and tearing down a live VM leaves firecracker running
|
|
318
|
-
* without its TAP, IP and chroot.
|
|
274
|
+
* Whether there is positive evidence a VM's firecracker is gone: no PID file, no such PID, or
|
|
275
|
+
* the PID belongs to a process started after `sinceUptimeS` (seconds since boot, taken once
|
|
276
|
+
* firecracker was running). Any other failure (unreadable PID file, invisible `/proc`) is
|
|
277
|
+
* "not gone": the caller tears down on "gone", and tearing down a live VM leaves firecracker
|
|
278
|
+
* running without its TAP, IP and chroot.
|
|
319
279
|
*/
|
|
320
280
|
private isVmProcessGone;
|
|
321
281
|
/** Stop forwarding a VM's serial console and VMM logs. */
|
|
@@ -325,23 +285,17 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
325
285
|
/**
|
|
326
286
|
* Force-reclaim a VM this backend no longer tracks in memory.
|
|
327
287
|
*
|
|
328
|
-
* `destroy()`
|
|
329
|
-
*
|
|
330
|
-
*
|
|
331
|
-
* three passes skips an id whose jailer process is alive, deliberately, since
|
|
332
|
-
* a live process is the only thing that distinguishes a healthy VM from a
|
|
333
|
-
* leaked one when you are sweeping blind.
|
|
288
|
+
* After an orchestrator restart `destroy()` no-ops (`if (!managed) return`) while the VM keeps
|
|
289
|
+
* its RAM and IP, and `cleanupOrphans()` deliberately skips any id whose jailer process is
|
|
290
|
+
* alive: sweeping blind, a live process is all that tells a healthy VM from a leaked one.
|
|
334
291
|
*
|
|
335
|
-
* This is the same reclaim
|
|
336
|
-
*
|
|
337
|
-
*
|
|
338
|
-
*
|
|
339
|
-
*
|
|
340
|
-
*
|
|
341
|
-
*
|
|
342
|
-
* scalers, and routing key" (`docs/operator/orchestrator/clustering.md`), so a
|
|
343
|
-
* peer's live VM has a row naming this very scaler. The row is read only to
|
|
344
|
-
* finish a reclaim the chroot already authorized.
|
|
292
|
+
* This is the same reclaim for ONE id the caller already established as an orphan, so the
|
|
293
|
+
* liveness check must not apply. See `ScalerBackend.reapUnowned` for the two properties the
|
|
294
|
+
* caller relies on; the host-local half is enforced here by refusing to act unless the jailer
|
|
295
|
+
* chroot for `managedId` exists on THIS host. That directory is the only host-local artifact a
|
|
296
|
+
* VM leaves: an HA pair shares PostgreSQL, scalers and routing key
|
|
297
|
+
* (`docs/operator/orchestrator/clustering.md`), so a peer's live VM has an `ip_allocations`
|
|
298
|
+
* row naming this very scaler. The row is read only to finish a reclaim the chroot authorized.
|
|
345
299
|
*/
|
|
346
300
|
reapUnowned(managedId: string): Promise<boolean>;
|
|
347
301
|
private reapUnownedLocked;
|
|
@@ -355,15 +309,13 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
355
309
|
/** Whether this scaler's allocator holds the address of `vmId`. */
|
|
356
310
|
ownsAllocationFor(vmId: string): Promise<boolean>;
|
|
357
311
|
/**
|
|
358
|
-
* Reclaim the host resources of a VM
|
|
359
|
-
*
|
|
360
|
-
* and delete its chroot.
|
|
312
|
+
* Reclaim the host resources of a dead, untracked VM: forward its remaining logs, remove its
|
|
313
|
+
* isolation rules, TAP and IP, and delete its chroot.
|
|
361
314
|
*
|
|
362
|
-
*
|
|
363
|
-
*
|
|
364
|
-
* row
|
|
365
|
-
*
|
|
366
|
-
* alone, so it cannot be another VM's.
|
|
315
|
+
* Only a row naming THIS scaler is ours to release: a second firecracker scaler on this host
|
|
316
|
+
* has its own IP space and teardown. With no such row (a worker's in-memory allocator forgets
|
|
317
|
+
* every row on restart) the TAP named after the VM's id is deleted; that name cannot be
|
|
318
|
+
* another VM's.
|
|
367
319
|
*/
|
|
368
320
|
private teardownUntrackedVm;
|
|
369
321
|
/** Whether a live VM is one this backend spawned and still tracks. */
|
|
@@ -377,15 +329,13 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
377
329
|
/**
|
|
378
330
|
* Stop one live VM nothing on this node tracks, on an operator's request.
|
|
379
331
|
*
|
|
380
|
-
* The tracked check
|
|
381
|
-
*
|
|
382
|
-
*
|
|
383
|
-
*
|
|
384
|
-
* that window; `destroy` and `reapUnowned` carry the same window, and Node
|
|
385
|
-
* has no pidfd to close it.
|
|
332
|
+
* The tracked check and the first `kill(2)` run in one synchronous tick: tracking only changes
|
|
333
|
+
* between ticks (a registration, a rehydrated spawn row, a new spawn), so nothing can claim the
|
|
334
|
+
* VM in between. The kernel can still recycle the PID in that window, as in `destroy` and
|
|
335
|
+
* `reapUnowned`; Node has no pidfd to close it.
|
|
386
336
|
*
|
|
387
|
-
* Runs under the host reclaim lock, so a sweep cannot release this VM's
|
|
388
|
-
*
|
|
337
|
+
* Runs under the host reclaim lock, so a sweep cannot release this VM's address to a new VM
|
|
338
|
+
* while the stop is still removing its rules.
|
|
389
339
|
*
|
|
390
340
|
* @param trackersOf what else on this node tracks the VM, read in the
|
|
391
341
|
* kill's tick; it must not await.
|
|
@@ -395,22 +345,20 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
395
345
|
/**
|
|
396
346
|
* SIGKILL a VM's firecracker process.
|
|
397
347
|
*
|
|
398
|
-
* The jailer drops firecracker to `--uid`, so
|
|
399
|
-
*
|
|
400
|
-
*
|
|
401
|
-
*
|
|
402
|
-
* already gone, which is the outcome wanted.
|
|
348
|
+
* The jailer drops firecracker to `--uid`, so another unprivileged user gets EPERM from
|
|
349
|
+
* `kill(2)`. On a rootless host (`requireSudo`) the signal goes through
|
|
350
|
+
* `sudo -n -u #<uid> kill`; elsewhere EPERM means the process lacks CAP_KILL. ESRCH (already
|
|
351
|
+
* gone) is success.
|
|
403
352
|
*
|
|
404
|
-
* The first statement is the synchronous `kill(2)`: `stopUntrackedVm` relies
|
|
405
|
-
*
|
|
353
|
+
* The first statement is the synchronous `kill(2)`: `stopUntrackedVm` relies on the signal
|
|
354
|
+
* leaving in its caller's tick.
|
|
406
355
|
*/
|
|
407
356
|
private signalVmProcess;
|
|
408
357
|
/** Whether `/proc/<pid>` names a process right now. */
|
|
409
358
|
private procEntryExists;
|
|
410
359
|
/**
|
|
411
|
-
* {@link signalVmProcess} for the teardown paths: a failure is logged,
|
|
412
|
-
*
|
|
413
|
-
* still runs.
|
|
360
|
+
* {@link signalVmProcess} for the teardown paths: a failure is logged and reported, so the
|
|
361
|
+
* caller keeps the chroot of a process that still runs.
|
|
414
362
|
*
|
|
415
363
|
* @returns whether the process was signalled or was already gone
|
|
416
364
|
*/
|
|
@@ -424,15 +372,9 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
424
372
|
private withHostReclaimLock;
|
|
425
373
|
shutdownAll(): Promise<void>;
|
|
426
374
|
/**
|
|
427
|
-
* Clear MMDS data
|
|
428
|
-
*
|
|
429
|
-
*
|
|
430
|
-
*
|
|
431
|
-
* Called by `ScalerManager.onConfigAck`, which reaches it structurally
|
|
432
|
-
* (`'clearAgentMmds' in backend`) rather than through `ScalerBackend` — no
|
|
433
|
-
* other backend has an MMDS to clear.
|
|
434
|
-
*
|
|
435
|
-
* @param agentId - The agent ID whose VM MMDS should be cleared
|
|
375
|
+
* Clear an agent's MMDS data after its config.ack arrives via WS, so nothing is left to read
|
|
376
|
+
* post-startup (belt-and-suspenders). Called by `ScalerManager.onConfigAck` structurally
|
|
377
|
+
* (`'clearAgentMmds' in backend`), not through `ScalerBackend`: no other backend has an MMDS.
|
|
436
378
|
*/
|
|
437
379
|
clearAgentMmds(agentId: string): Promise<void>;
|
|
438
380
|
reload(labelSets: LabelSetConfig[], opts?: {
|
|
@@ -440,47 +382,38 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
440
382
|
entry?: ScalerEntry;
|
|
441
383
|
}): ValidationResult;
|
|
442
384
|
/**
|
|
443
|
-
* Clean up orphaned VMs.
|
|
385
|
+
* Clean up orphaned VMs; returns the count cleaned.
|
|
444
386
|
*
|
|
445
|
-
* Safe
|
|
446
|
-
* (
|
|
447
|
-
*
|
|
448
|
-
* re-reads DB allocations after listing host interfaces to close the spawn
|
|
449
|
-
* race (allocate → ip tuntap add is not atomic).
|
|
387
|
+
* Safe at startup AND periodically (startPeriodicOrphanSweep): each pass skips VMs tracked
|
|
388
|
+
* in-memory (spawning / running / destroying), and Pass 3 re-reads DB allocations after
|
|
389
|
+
* listing host interfaces to close the spawn race (allocate → ip tuntap add is not atomic).
|
|
450
390
|
*
|
|
451
|
-
*
|
|
391
|
+
* Passes:
|
|
452
392
|
* 1. DB allocations: check if VM process is still running, clean dead ones
|
|
453
393
|
* 2. Filesystem: scan chroot dir for directories not in DB allocations
|
|
454
394
|
* 3. Network: scan host interfaces for orphan TAP devices
|
|
455
|
-
*
|
|
456
|
-
* Returns the count of cleaned orphans.
|
|
395
|
+
* 4. nftables: reap isolation rules no live VM owns
|
|
457
396
|
*/
|
|
458
397
|
cleanupOrphans(): Promise<number>;
|
|
459
398
|
private cleanupOrphansLocked;
|
|
460
399
|
/**
|
|
461
400
|
* Delete isolation rules for every identifier no live VM owns.
|
|
462
401
|
*
|
|
463
|
-
* The live set is the VM ids
|
|
464
|
-
*
|
|
465
|
-
*
|
|
466
|
-
* writes to the same chain, and reaping its rules would cut its running jobs
|
|
467
|
-
* off the network — the mistake this whole sweep exists to stop making.
|
|
402
|
+
* The live set is the passes' live VM ids, mapped to this scaler's allocated IPs. An
|
|
403
|
+
* identifier the DB does not attribute to this scaler is left alone: a second scaler on the
|
|
404
|
+
* host writes to the same chain, and reaping its rules would cut its running jobs off.
|
|
468
405
|
*
|
|
469
406
|
* @returns the number of rules deleted
|
|
470
407
|
*/
|
|
471
408
|
private reapUnownedIsolationRules;
|
|
472
409
|
/**
|
|
473
|
-
* Default
|
|
474
|
-
*
|
|
475
|
-
*
|
|
476
|
-
* host-level cleanup job skips TAP cleanup while an orchestrator is active
|
|
477
|
-
* (to avoid racing spawns). Without this, leaked
|
|
478
|
-
* TAPs accumulate until the orchestrator restarts — which can be weeks.
|
|
410
|
+
* Default periodic orphan sweep interval (15 minutes). A host-level cleanup job skips TAP
|
|
411
|
+
* cleanup while an orchestrator is active (to avoid racing spawns), so without this sweep
|
|
412
|
+
* leaked TAPs accumulate until the orchestrator restarts, which can be weeks.
|
|
479
413
|
*/
|
|
480
414
|
private static readonly ORPHAN_SWEEP_INTERVAL_MS;
|
|
481
415
|
/**
|
|
482
|
-
* Start a periodic orphan sweep. Idempotent
|
|
483
|
-
* beyond the first call. Use stopPeriodicOrphanSweep() to stop.
|
|
416
|
+
* Start a periodic orphan sweep. Idempotent; stopPeriodicOrphanSweep() stops it.
|
|
484
417
|
*
|
|
485
418
|
* @param intervalMs override the default interval (primarily for tests)
|
|
486
419
|
*/
|
|
@@ -494,17 +427,14 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
494
427
|
* Build the merged forwarded-env map for an agent's MMDS payload.
|
|
495
428
|
*
|
|
496
429
|
* Mirrors bare-metal/container precedence: KICI_AGENT_ENV_*-prefixed vars from the
|
|
497
|
-
* orchestrator's process.env (prefix stripped)
|
|
498
|
-
* `env:` overlays them (yaml wins on conflict).
|
|
430
|
+
* orchestrator's process.env (prefix stripped) first, then scalers.yaml `env:` (yaml wins).
|
|
499
431
|
*
|
|
500
|
-
*
|
|
501
|
-
* - Keys must match POSIX_ENV_NAME_PATTERN
|
|
502
|
-
*
|
|
503
|
-
*
|
|
504
|
-
*
|
|
505
|
-
*
|
|
506
|
-
* skipped and warnings logged. This protects the ~51 KiB MMDS data store cap
|
|
507
|
-
* from being blown by an accidentally-huge env value.
|
|
432
|
+
* Safety filters:
|
|
433
|
+
* - Keys must match POSIX_ENV_NAME_PATTERN, keeping MMDS path construction safe and never
|
|
434
|
+
* asking the guest to `export` something the shell can't represent.
|
|
435
|
+
* - The cumulative cost (key + value + 2 bytes per entry) must stay under
|
|
436
|
+
* MMDS_FORWARDED_ENV_BUDGET_BYTES; entries past it are skipped with a warning, protecting
|
|
437
|
+
* the ~51 KiB MMDS store cap from a huge env value.
|
|
508
438
|
*/
|
|
509
439
|
buildForwardedEnv(matchedLabelSet: LabelSetConfig, agentId: string): Record<string, string>;
|
|
510
440
|
/**
|
|
@@ -515,10 +445,9 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
515
445
|
* 2. label-set's vcpuCount/memSizeMib.
|
|
516
446
|
* 3. backend-level default vcpuCount/memSizeMib.
|
|
517
447
|
*
|
|
518
|
-
* Firecracker requires integer vCPUs and
|
|
519
|
-
*
|
|
520
|
-
*
|
|
521
|
-
* changes the value, so operators can spot frequent fractional requests.
|
|
448
|
+
* Firecracker requires integer vCPUs and MiB: CPUs round UP (a 0.5-CPU request gets a whole
|
|
449
|
+
* vCPU, never under-provisioned) and bytes floor-divide to MiB. A rounding that changes the
|
|
450
|
+
* value logs at debug, so operators can spot frequent fractional requests.
|
|
522
451
|
*/
|
|
523
452
|
buildVmConfig(alloc: IpAllocationResult, labelSet: LabelSetConfig, effectiveLimits?: EffectiveLimits): Record<string, unknown>;
|
|
524
453
|
/**
|
|
@@ -545,26 +474,15 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
545
474
|
/**
|
|
546
475
|
* Remove a VM's jailer chroot directory.
|
|
547
476
|
*
|
|
548
|
-
* On rootless nodes (requireSudo)
|
|
549
|
-
*
|
|
550
|
-
*
|
|
551
|
-
*
|
|
552
|
-
*
|
|
553
|
-
*
|
|
554
|
-
* data disk fills and the orchestrator crash-loops on ENOSPC (it can no
|
|
555
|
-
* longer write its own files to start, so the in-process orphan sweep never
|
|
556
|
-
* runs to recover). Reclaim ownership via the same sudo-wrapped `chown` path
|
|
557
|
-
* used for `ip` before the `rm` runs as ourselves. On root nodes (requireSudo
|
|
558
|
-
* false) the chown is skipped and `rm` works directly.
|
|
477
|
+
* On rootless nodes (requireSudo) the jailer leaves inner directories owned by its uid with
|
|
478
|
+
* mode 0755, so a plain `rm` by the orchestrator's uid fails with EACCES. destroy() and both
|
|
479
|
+
* cleanupOrphans passes swallow that failure, so each spawn would leak a multi-GiB chroot
|
|
480
|
+
* until the data disk fills and the orchestrator crash-loops on ENOSPC (unable to start, so
|
|
481
|
+
* its orphan sweep never recovers). Ownership is reclaimed via the sudo-wrapped `chown` first;
|
|
482
|
+
* root nodes skip it.
|
|
559
483
|
*/
|
|
560
484
|
private removeChrootDir;
|
|
561
|
-
/**
|
|
562
|
-
* Get the API socket path for an agent's Firecracker VM.
|
|
563
|
-
*/
|
|
564
485
|
getSocketPath(agentId: string): string;
|
|
565
|
-
/**
|
|
566
|
-
* Get the chroot directory path for an agent.
|
|
567
|
-
*/
|
|
568
486
|
getChrootDir(agentId: string): string;
|
|
569
487
|
/**
|
|
570
488
|
* Probe the process a VM's PID file names.
|
|
@@ -585,80 +503,46 @@ export declare class FirecrackerScalerBackend implements ScalerBackend, LiveVmBa
|
|
|
585
503
|
*/
|
|
586
504
|
private probeVmPid;
|
|
587
505
|
/**
|
|
588
|
-
*
|
|
589
|
-
*
|
|
590
|
-
*
|
|
591
|
-
* This is the guard for acting ON the process — `destroy` and `reapUnowned`
|
|
592
|
-
* SIGKILL through it — so it fails closed: anything short of a confirmed
|
|
593
|
-
* identity yields `undefined` and nothing is signalled.
|
|
594
|
-
*
|
|
595
|
-
* @returns the PID when it is provably this VM's, otherwise `undefined`
|
|
506
|
+
* The PID from a VM's PID file, only when it is provably still *this* VM's firecracker.
|
|
507
|
+
* `destroy` and `reapUnowned` SIGKILL through it, so it fails closed: anything short of a
|
|
508
|
+
* confirmed identity yields `undefined` and nothing is signalled.
|
|
596
509
|
*/
|
|
597
510
|
private readVmPid;
|
|
598
511
|
/**
|
|
599
512
|
* Whether a chroot's PID file names a process that is running right now.
|
|
600
513
|
*
|
|
601
|
-
*
|
|
602
|
-
*
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
*
|
|
606
|
-
* files it is running out of corrupts whatever it is — while sparing it costs
|
|
607
|
-
* only the disk one chroot occupies.
|
|
608
|
-
*
|
|
609
|
-
* The leak the identity checks exist to prevent is unaffected: a recycled
|
|
610
|
-
* number is detected by start time, which this guard honours, so a chroot
|
|
611
|
-
* whose VM is genuinely over is still reclaimed.
|
|
514
|
+
* The orphan reaper's guard fails closed opposite to {@link readVmPid}: that refuses to signal
|
|
515
|
+
* a PID it cannot claim; the reaper must refuse to DELETE THE ROOTFS under one. A process whose
|
|
516
|
+
* `comm` or `/proc/<pid>/root` does not match is still a process, and deleting its files
|
|
517
|
+
* corrupts it, while sparing it costs one chroot's disk. A recycled PID is still detected by
|
|
518
|
+
* start time, so a chroot whose VM is genuinely over is still reclaimed.
|
|
612
519
|
*/
|
|
613
520
|
private isChrootPidRunning;
|
|
614
521
|
/**
|
|
615
|
-
* Whether *this* VM's own firecracker process is still up
|
|
616
|
-
*
|
|
617
|
-
*
|
|
618
|
-
*
|
|
619
|
-
* {@link waitForProcessExit} needs. Without the identity checks a recycled
|
|
620
|
-
* PID answers signal 0 successfully and the VM is spared forever.
|
|
621
|
-
*
|
|
622
|
-
* The orphan reaper asks a different question and uses
|
|
623
|
-
* {@link isChrootPidRunning} instead: a PID it cannot claim is still a live
|
|
624
|
-
* process, and its rootfs must not be deleted underneath it.
|
|
522
|
+
* Whether *this* VM's own firecracker process is still up: `readVmPid` plus signal 0, the
|
|
523
|
+
* identity-bearing question {@link waitForProcessExit} needs. Without the identity checks a
|
|
524
|
+
* recycled PID answers signal 0 and the VM is spared forever. The orphan reaper uses
|
|
525
|
+
* {@link isChrootPidRunning} instead.
|
|
625
526
|
*/
|
|
626
527
|
private isVmProcessAlive;
|
|
627
|
-
/**
|
|
628
|
-
* Wait for a VM's jailer process to exit within a timeout.
|
|
629
|
-
*/
|
|
630
528
|
private waitForProcessExit;
|
|
631
|
-
/**
|
|
632
|
-
* Clean up resources after a failed spawn attempt.
|
|
633
|
-
*/
|
|
634
529
|
private cleanupFailedSpawn;
|
|
635
530
|
/**
|
|
636
|
-
*
|
|
637
|
-
*
|
|
638
|
-
*
|
|
639
|
-
* Also monitors the jailer child process for early exit (boot failure detection).
|
|
640
|
-
* Forwarding is handled internally (not through ScalerManager/AgentLogForwarder)
|
|
641
|
-
* because Firecracker has two distinct log sources per VM.
|
|
531
|
+
* Tail the serial console and VMM log files, forwarding lines via forwardLine() with distinct
|
|
532
|
+
* logsSource tags, and watch the jailer child for early exit (boot failure). Forwarding is
|
|
533
|
+
* internal, not through ScalerManager/AgentLogForwarder, because each VM has two log sources.
|
|
642
534
|
*/
|
|
643
535
|
private startLogTailing;
|
|
644
536
|
/**
|
|
645
|
-
* Tail
|
|
646
|
-
* For serial console lines: non-JSON lines (kernel boot output) are wrapped
|
|
537
|
+
* Tail one log file into forwardLine(). Non-JSON serial lines (kernel boot output) are wrapped
|
|
647
538
|
* as debug-level JSON; failure patterns (kernel panic, init failure) are elevated to error.
|
|
648
539
|
*/
|
|
649
540
|
private tailAndForward;
|
|
650
|
-
/**
|
|
651
|
-
* Check if a serial console line matches known boot failure patterns.
|
|
652
|
-
*/
|
|
541
|
+
/** Whether a serial console line matches a known boot failure pattern. */
|
|
653
542
|
private isFailurePattern;
|
|
654
|
-
/**
|
|
655
|
-
* Read the serial console log file and detect the type of boot failure.
|
|
656
|
-
*/
|
|
543
|
+
/** Classify a boot failure from the serial console log. */
|
|
657
544
|
private detectFailureType;
|
|
658
|
-
/**
|
|
659
|
-
* Read and forward any remaining log file content for an orphaned/dead VM.
|
|
660
|
-
* Called during cleanupOrphans() before deleting the chroot directory.
|
|
661
|
-
*/
|
|
545
|
+
/** Forward a dead VM's remaining log content before its chroot is deleted. */
|
|
662
546
|
private forwardRemainingLogs;
|
|
663
547
|
}
|
|
664
548
|
//# sourceMappingURL=firecracker-backend.d.ts.map
|