@kici-dev/orchestrator 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/agent-version.d.ts +12 -21
- package/dist/agent/dispatcher.d.ts +283 -437
- package/dist/agent/host-roster-reaper.d.ts +8 -10
- package/dist/agent/host-roster.d.ts +77 -100
- package/dist/agent/ownership-refusal.d.ts +3 -5
- package/dist/agent/ownership-tracker.d.ts +42 -76
- package/dist/agent/registry.d.ts +121 -233
- package/dist/agent/token-store.d.ts +34 -77
- package/dist/agent-packaging/availability.d.ts +6 -9
- package/dist/agent-packaging/download.d.ts +6 -9
- package/dist/agent-packaging/install-closure.d.ts +9 -12
- package/dist/agent-packaging/node-binary.d.ts +6 -10
- package/dist/agent-packaging/node-runtime-members.d.ts +3 -5
- package/dist/agent-packaging/refresh.d.ts +12 -16
- package/dist/agent-packaging/store.d.ts +5 -6
- package/dist/agent-packaging/upload.d.ts +3 -4
- package/dist/app.d.ts +0 -3
- package/dist/approvals/apply-decision.d.ts +39 -45
- package/dist/approvals/approval-resolver.d.ts +11 -23
- package/dist/approvals/held-run-release-wiring.d.ts +14 -21
- package/dist/approvals/independent-wiring.d.ts +13 -20
- package/dist/approvals/resume-router.d.ts +1 -1
- package/dist/approvals/step-approval-bridge.d.ts +11 -21
- package/dist/approvals/triggerer-subject.d.ts +8 -10
- package/dist/artifacts/artifact-store.d.ts +55 -95
- package/dist/artifacts/failure-messages.d.ts +15 -20
- package/dist/audit/access-log.d.ts +27 -45
- package/dist/audit/sampling-rate-limiter.d.ts +11 -23
- package/dist/cache/agent-job-failed-error.d.ts +2 -3
- package/dist/cache/build-coordinator.d.ts +5 -28
- package/dist/cache/cache-upload-settle.d.ts +11 -11
- package/dist/cache/dep-cache.d.ts +32 -60
- package/dist/cache/dispatch-cache-ref-tracker.d.ts +15 -23
- package/dist/cache/global-eval-round-cache.d.ts +33 -41
- package/dist/cache/pending-builds.d.ts +2 -6
- package/dist/cache/pending-dynamics.d.ts +2 -7
- package/dist/cache/pending-global-evals.d.ts +9 -12
- package/dist/cache/pending-inits.d.ts +11 -22
- package/dist/cache/pending-precursor-db-watcher.d.ts +24 -32
- package/dist/cache/pending-tracker.d.ts +17 -45
- package/dist/cache/precursor-result.d.ts +17 -30
- package/dist/cache/source-cache.d.ts +36 -65
- package/dist/cache/user-cache.d.ts +21 -25
- package/dist/cancel/cancel-held-run.d.ts +13 -16
- package/dist/cancel/cancel-route-answer.d.ts +1 -2
- package/dist/cancel/cancel-run.d.ts +26 -50
- package/dist/cancel/dashboard-cancel-handler.d.ts +2 -2
- package/dist/cancel/peer-job-cancel.d.ts +11 -14
- package/dist/cancel/sweep-stuck-cancelling.d.ts +9 -13
- package/dist/cli/api-client.d.ts +33 -76
- package/dist/cli/commands/access-log.d.ts +4 -24
- package/dist/cli/commands/agent-package.d.ts +7 -15
- package/dist/cli/commands/agent-service/agent-env.d.ts +3 -6
- package/dist/cli/commands/agent-service/index.d.ts +4 -10
- package/dist/cli/commands/agent-service/install.d.ts +4 -11
- package/dist/cli/commands/agent-service/status.d.ts +4 -9
- package/dist/cli/commands/agent-service/upgrade.d.ts +4 -16
- package/dist/cli/commands/agent.d.ts +12 -25
- package/dist/cli/commands/api-key.d.ts +7 -7
- package/dist/cli/commands/attestations-list.d.ts +7 -17
- package/dist/cli/commands/attestations-retry.d.ts +5 -11
- package/dist/cli/commands/attestations-reverify.d.ts +4 -5
- package/dist/cli/commands/attestations.d.ts +3 -8
- package/dist/cli/commands/audit.d.ts +1 -5
- package/dist/cli/commands/backend.d.ts +2 -11
- package/dist/cli/commands/check-run.d.ts +7 -18
- package/dist/cli/commands/cluster-name.d.ts +5 -12
- package/dist/cli/commands/cluster-settings.d.ts +28 -53
- package/dist/cli/commands/cold-store.d.ts +7 -15
- package/dist/cli/commands/config.d.ts +9 -20
- package/dist/cli/commands/context.d.ts +8 -20
- package/dist/cli/commands/dashboard-encryption-key.d.ts +9 -14
- package/dist/cli/commands/db-backup.d.ts +17 -32
- package/dist/cli/commands/db.d.ts +3 -4
- package/dist/cli/commands/debug-bundle.d.ts +3 -10
- package/dist/cli/commands/diagnose.d.ts +3 -14
- package/dist/cli/commands/event-dlq.d.ts +4 -12
- package/dist/cli/commands/event-log.d.ts +2 -20
- package/dist/cli/commands/execution.d.ts +3 -9
- package/dist/cli/commands/firecracker/index.d.ts +3 -6
- package/dist/cli/commands/held-run.d.ts +23 -37
- package/dist/cli/commands/host.d.ts +4 -11
- package/dist/cli/commands/inspect-bundle.d.ts +3 -11
- package/dist/cli/commands/local-hook.d.ts +2 -10
- package/dist/cli/commands/local-trigger.d.ts +2 -2
- package/dist/cli/commands/orchestrator-service/drain.d.ts +3 -6
- package/dist/cli/commands/orchestrator-service/index.d.ts +6 -14
- package/dist/cli/commands/orchestrator-service/install-env.d.ts +25 -49
- package/dist/cli/commands/orchestrator-service/install.d.ts +2 -5
- package/dist/cli/commands/orchestrator-service/status.d.ts +10 -22
- package/dist/cli/commands/orchestrator-service/upgrade.d.ts +6 -15
- package/dist/cli/commands/org-settings.d.ts +5 -11
- package/dist/cli/commands/peer.d.ts +5 -10
- package/dist/cli/commands/queue.d.ts +3 -9
- package/dist/cli/commands/registration.d.ts +4 -8
- package/dist/cli/commands/remote-source.d.ts +6 -12
- package/dist/cli/commands/rotate.d.ts +1 -5
- package/dist/cli/commands/runs.d.ts +1 -30
- package/dist/cli/commands/scaler.d.ts +5 -12
- package/dist/cli/commands/secret.d.ts +1 -9
- package/dist/cli/commands/service-health.d.ts +5 -10
- package/dist/cli/commands/shared/admin-cli-access-log.d.ts +10 -19
- package/dist/cli/commands/shared/bare-metal-binary-check.d.ts +86 -0
- package/dist/cli/commands/shared/cli-action.d.ts +1 -4
- package/dist/cli/commands/shared/confirm.d.ts +5 -10
- package/dist/cli/commands/shared/db.d.ts +1 -12
- package/dist/cli/commands/shared/env-file-mode.d.ts +6 -9
- package/dist/cli/commands/shared/secret-input.d.ts +7 -21
- package/dist/cli/commands/shared/service-verbs.d.ts +6 -12
- package/dist/cli/commands/shared/unbound-context-warning.d.ts +2 -4
- package/dist/cli/commands/shared/upgrade-hooks.d.ts +7 -16
- package/dist/cli/commands/shared/upgrade-safety.d.ts +20 -31
- package/dist/cli/commands/shared/versioned-upgrade.d.ts +83 -161
- package/dist/cli/commands/signing-key.d.ts +4 -9
- package/dist/cli/commands/source-manifest.d.ts +11 -14
- package/dist/cli/commands/source.d.ts +7 -32
- package/dist/cli/commands/token.d.ts +4 -15
- package/dist/cli/commands/trust-policy.d.ts +35 -88
- package/dist/cli/commands/variable.d.ts +3 -5
- package/dist/cli/commands/workflow.d.ts +3 -24
- package/dist/cli/join.d.ts +3 -6
- package/dist/cli/kici-admin.d.ts +11 -29
- package/dist/cli/lazy-deps/cache.d.ts +1 -14
- package/dist/cli/lazy-deps/downloader.d.ts +4 -27
- package/dist/cli/lazy-deps/registry.d.ts +5 -25
- package/dist/cli/local-github-ingress-url.d.ts +5 -7
- package/dist/cli/loopback-callback.d.ts +6 -7
- package/dist/cli/open-browser.d.ts +4 -8
- package/dist/cli/root-option-routing.d.ts +17 -29
- package/dist/cli/service/backup-timer.d.ts +16 -39
- package/dist/cli/service/compose-path.d.ts +3 -7
- package/dist/cli/service/compose.d.ts +12 -30
- package/dist/cli/service/deploy-env.d.ts +6 -18
- package/dist/cli/service/entrypoint.d.ts +14 -25
- package/dist/cli/service/env-assignment.d.ts +6 -17
- package/dist/cli/service/image-digests.d.ts +6 -15
- package/dist/cli/service/index.d.ts +2 -18
- package/dist/cli/service/instance/index-file.d.ts +9 -16
- package/dist/cli/service/instance/manifest.d.ts +33 -59
- package/dist/cli/service/instance/resolve.d.ts +32 -68
- package/dist/cli/service/instance/types.d.ts +12 -29
- package/dist/cli/service/launchd.d.ts +29 -60
- package/dist/cli/service/platform-detect.d.ts +13 -68
- package/dist/cli/service/privilege.d.ts +12 -25
- package/dist/cli/service/shutdown-grace.d.ts +11 -20
- package/dist/cli/service/systemd.d.ts +7 -20
- package/dist/cli/service/types.d.ts +63 -114
- package/dist/cli/service/windows-acl.d.ts +14 -21
- package/dist/cli/service/windows-env-file.d.ts +10 -28
- package/dist/cli/service/windows.d.ts +15 -31
- package/dist/cli/webhook-url-notes.d.ts +4 -6
- package/dist/cli/wizard/agent-wizard.d.ts +3 -14
- package/dist/cli/wizard/orchestrator-wizard.d.ts +12 -37
- package/dist/cli/wizard/prompts.d.ts +6 -16
- package/dist/cli.js +4553 -6556
- package/dist/cluster/cluster-identity.d.ts +20 -32
- package/dist/cluster/cluster-settings-reader.d.ts +50 -77
- package/dist/cluster/coordinator-credential.d.ts +4 -5
- package/dist/cluster/coordinator.d.ts +76 -148
- package/dist/cluster/health-api.d.ts +2 -10
- package/dist/cluster/index.d.ts +1 -6
- package/dist/cluster/instance-heartbeat.d.ts +26 -42
- package/dist/cluster/join-client.d.ts +4 -7
- package/dist/cluster/join-handler.d.ts +3 -7
- package/dist/cluster/join-token.d.ts +13 -27
- package/dist/cluster/leader-gated-scheduler.d.ts +8 -17
- package/dist/cluster/orphan-recovery.d.ts +13 -35
- package/dist/cluster/peer-auth-coordinator.d.ts +2 -8
- package/dist/cluster/peer-client.d.ts +46 -76
- package/dist/cluster/peer-credentials.d.ts +24 -70
- package/dist/cluster/peer-crypto.d.ts +5 -26
- package/dist/cluster/peer-forget.d.ts +3 -3
- package/dist/cluster/peer-handler.d.ts +47 -71
- package/dist/cluster/peer-registry.d.ts +37 -106
- package/dist/cluster/plan-headroom-store.d.ts +4 -6
- package/dist/cluster/raft-state.d.ts +4 -13
- package/dist/cluster/raft.d.ts +18 -53
- package/dist/cluster/reconcile-identity.d.ts +8 -11
- package/dist/cluster/rerouted-job-guard.d.ts +7 -11
- package/dist/cluster/verified-issuer.d.ts +2 -3
- package/dist/cluster/worker-cluster-settings.d.ts +7 -10
- package/dist/cluster/worker-eviction.d.ts +16 -26
- package/dist/cold-store/load-access-log-range.d.ts +15 -31
- package/dist/cold-store/load-event-log-range.d.ts +13 -27
- package/dist/cold-store/load-secret-audit-log-range.d.ts +8 -12
- package/dist/cold-store/orchestrator-cold-store.d.ts +8 -28
- package/dist/cold-store/tables/access-log.d.ts +12 -14
- package/dist/cold-store/tables/event-log.d.ts +4 -8
- package/dist/cold-store/tables/execution-runs.d.ts +6 -11
- package/dist/cold-store/tables/secret-audit-log.d.ts +5 -12
- package/dist/concurrency/dispatch-next-queued.d.ts +13 -25
- package/dist/concurrency/queue-manager.d.ts +22 -54
- package/dist/concurrency/waiters.d.ts +15 -31
- package/dist/config/encryption.d.ts +0 -2
- package/dist/config/loader.d.ts +0 -1
- package/dist/config/reload.d.ts +0 -6
- package/dist/config/resolver.d.ts +0 -4
- package/dist/config/shared-store.d.ts +0 -2
- package/dist/config/types.d.ts +0 -3
- package/dist/content-requirements-cache.d.ts +0 -1
- package/dist/contexts/context-store.d.ts +0 -2
- package/dist/contexts/held-runs.d.ts +0 -2
- package/dist/contexts/variable-store.d.ts +0 -2
- package/dist/dashboard/attestation-filters.d.ts +13 -18
- package/dist/dashboard/handler.d.ts +143 -307
- package/dist/db/client.d.ts +2 -6
- package/dist/db/deferred-index-build.d.ts +8 -10
- package/dist/db/deferred-indexes.d.ts +15 -22
- package/dist/db/migration-provider.d.ts +2 -4
- package/dist/db/migration-test-harness.d.ts +26 -44
- package/dist/db/migrations/001_initial.d.ts +1 -4
- package/dist/db/migrations/002_config_versions_key_version.d.ts +5 -8
- package/dist/db/migrations/003_access_log.d.ts +10 -15
- package/dist/db/migrations/004_rename_bundle_to_source.d.ts +7 -9
- package/dist/db/migrations/005_cold_store_chunk_counter.d.ts +4 -10
- package/dist/db/migrations/006_runs_jobs_steps_archived_at.d.ts +9 -29
- package/dist/db/migrations/007_audit_logs_archived_at.d.ts +11 -30
- package/dist/db/migrations/008_event_log_archived_at.d.ts +14 -29
- package/dist/db/migrations/009_access_log_trigram.d.ts +4 -8
- package/dist/db/migrations/010_cold_store_chunks.d.ts +16 -26
- package/dist/db/migrations/011_drop_source_secrets_notify.d.ts +6 -14
- package/dist/db/migrations/012_peer_credentials_active_uniq.d.ts +9 -14
- package/dist/db/migrations/013_execution_log_bytes.d.ts +5 -13
- package/dist/db/migrations/014_kici_events_lease_retry.d.ts +14 -27
- package/dist/db/migrations/015_org_settings_customer_scoped.d.ts +12 -17
- package/dist/db/migrations/016_org_settings_allow_http_npm.d.ts +4 -6
- package/dist/db/migrations/019_generic_sources_change_notify.d.ts +13 -17
- package/dist/db/migrations/020_org_settings_dashboard_write_policy.d.ts +5 -8
- package/dist/db/migrations/021_check_run_tracking.d.ts +5 -17
- package/dist/db/migrations/022_scaler_manager_state.d.ts +10 -20
- package/dist/db/migrations/023_dispatch_queue_recovery_deadline.d.ts +7 -18
- package/dist/db/migrations/024_dispatch_queue_provisioning_error.d.ts +5 -11
- package/dist/db/migrations/025_init_failure.d.ts +5 -9
- package/dist/db/migrations/027_workflow_timeout.d.ts +5 -13
- package/dist/db/migrations/029_dispatch_queue_attempts.d.ts +4 -10
- package/dist/db/migrations/030_held_runs_env_set_null.d.ts +5 -6
- package/dist/db/migrations/031_dispatch_queue_ack_deadline.d.ts +6 -12
- package/dist/db/migrations/033_org_settings_approval.d.ts +7 -10
- package/dist/db/migrations/034_held_runs_generalize.d.ts +10 -18
- package/dist/db/migrations/035_pending_workflow_contexts.d.ts +4 -5
- package/dist/db/migrations/037_generic_sources_provider_type_local.d.ts +5 -6
- package/dist/db/migrations/038_remote_sources.d.ts +5 -7
- package/dist/db/migrations/045_host_properties.d.ts +5 -9
- package/dist/db/migrations/046_join_token_consumed_by_instance.d.ts +5 -10
- package/dist/db/migrations/048_host_reboot_pending.d.ts +6 -12
- package/dist/db/migrations/049_held_runs_payload.d.ts +3 -8
- package/dist/db/migrations/050_sources_slug.d.ts +4 -5
- package/dist/db/migrations/051_binding_host_pattern.d.ts +7 -10
- package/dist/db/migrations/052_host_reach_metadata.d.ts +8 -11
- package/dist/db/migrations/053_agent_token_single_use.d.ts +6 -10
- package/dist/db/migrations/054_local_working_tree.d.ts +4 -6
- package/dist/db/migrations/055_agent_token_mandatory_labels.d.ts +6 -11
- package/dist/db/migrations/056_execution_jobs_environments.d.ts +4 -11
- package/dist/db/migrations/064_execution_jobs_skipped_environments.d.ts +6 -9
- package/dist/db/migrations/065_pending_attestations.d.ts +9 -12
- package/dist/db/migrations/066_pending_attestations_rejected.d.ts +4 -7
- package/dist/db/migrations/068_request_idempotency.d.ts +7 -9
- package/dist/db/migrations/069_reroute_tunables.d.ts +6 -14
- package/dist/db/migrations/071_batch_accumulation.d.ts +8 -16
- package/dist/db/migrations/072_dispatch_queue_run_id_index.d.ts +3 -9
- package/dist/db/migrations/074_normalize_zero_concurrency_limit.d.ts +4 -7
- package/dist/db/migrations/076_artifacts.d.ts +7 -8
- package/dist/db/migrations/078_org_settings_backup_staleness.d.ts +3 -6
- package/dist/db/migrations/079_org_settings_scaler_spawn_timeout.d.ts +4 -8
- package/dist/db/migrations/080_cluster_settings.d.ts +6 -10
- package/dist/db/migrations/081_org_settings_queue_timeout.d.ts +7 -11
- package/dist/db/migrations/082_host_s3_reachable.d.ts +5 -10
- package/dist/db/migrations/083_org_settings_artifact_caps.d.ts +6 -8
- package/dist/db/migrations/084_orchestrator_signing_keys.d.ts +8 -13
- package/dist/db/migrations/085_cluster_settings_reroute_flap_grace_ms.d.ts +4 -5
- package/dist/db/migrations/086_cluster_settings_max_fanout_hosts.d.ts +3 -3
- package/dist/db/migrations/087_cluster_settings_event_router_rate_limit.d.ts +5 -3
- package/dist/db/migrations/088_cluster_settings_cache_max_tarball_bytes.d.ts +3 -3
- package/dist/db/migrations/089_cluster_settings_cache_ttl_days.d.ts +3 -5
- package/dist/db/migrations/090_cluster_settings_concurrency_wait_timeout_ms.d.ts +3 -6
- package/dist/db/migrations/091_cluster_settings_agent_token_ttl_ms.d.ts +3 -6
- package/dist/db/migrations/092_cluster_settings_version.d.ts +5 -9
- package/dist/db/migrations/094_dashboard_encryption_keys.d.ts +9 -14
- package/dist/db/migrations/095_dashboard_write_policy_tristate.d.ts +2 -6
- package/dist/db/migrations/099_cluster_settings_dashboard_verified_issuer.d.ts +8 -11
- package/dist/db/migrations/101_contexts_hold_expiry_drop_default.d.ts +9 -15
- package/dist/db/migrations/102_dispatch_queue_agent_id.d.ts +6 -11
- package/dist/db/migrations/103_cluster_settings_ownership_db_check_timeout_ms.d.ts +6 -12
- package/dist/db/migrations/104_check_run_terminal_sent.d.ts +8 -14
- package/dist/db/migrations/105_org_trust_policy.d.ts +5 -7
- package/dist/db/migrations/106_cluster_settings_check_run_tracking_ttl_days.d.ts +4 -12
- package/dist/db/migrations/107_check_run_tracking_updated_at_index.d.ts +8 -14
- package/dist/db/migrations/113_execution_runs_workflow_repo_index.d.ts +10 -19
- package/dist/db/migrations/115_global_workflows_cluster_switch.d.ts +2 -3
- package/dist/db/migrations/121_org_settings_allow_untrusted_dockerfile_builds.d.ts +7 -8
- package/dist/db/migrations/122_host_roster_scaler_managed.d.ts +9 -16
- package/dist/db/migrations/123_workflow_registrations_default_branch.d.ts +8 -16
- package/dist/db/migrations/125_org_trust_directory.d.ts +13 -21
- package/dist/db/migrations/126_held_runs_posted_pending_check.d.ts +10 -16
- package/dist/db/migrations/127_org_trust_policy_approval_expiry_seconds.d.ts +8 -16
- package/dist/db/migrations/128_execution_runs_global_eval_round.d.ts +6 -11
- package/dist/db/migrations/129_execution_runs_dispatch_routing_key.d.ts +8 -11
- package/dist/db/migrations/130_scaler_provision_outcomes.d.ts +13 -20
- package/dist/db/migrations/131_execution_jobs_git_credentials.d.ts +12 -22
- package/dist/db/migrations/132_execution_runs_trigger_event.d.ts +6 -16
- package/dist/db/migrations/133_dispatch_queue_source_tar_digest.d.ts +6 -9
- package/dist/db/migrations/140_dispatch_queue_canonical_labels.d.ts +22 -40
- package/dist/db/migrations/143_execution_jobs_precursor_result.d.ts +9 -18
- package/dist/db/migrations/144_execution_runs_registration_window.d.ts +12 -20
- package/dist/db/migrations/147_sealed_job_secrets.d.ts +5 -9
- package/dist/db/migrations/148_cluster_settings_sealed_secrets_retry_backoff_ms.d.ts +7 -13
- package/dist/db/migrations/150_execution_runs_status_epoch.d.ts +6 -11
- package/dist/db/migrations/152_org_settings_cache_upload_settle_timeout.d.ts +5 -8
- package/dist/db/migrations/157_host_roster_identity_source.d.ts +9 -13
- package/dist/db/migrations/159_org_id_widen_to_source_width.d.ts +5 -0
- package/dist/db/migrator.d.ts +17 -25
- package/dist/db/notify-listener.d.ts +8 -11
- package/dist/db/repos/dashboard-encryption-keys-repo.d.ts +9 -10
- package/dist/db/synthetic-job-ids.d.ts +6 -8
- package/dist/db/types.d.ts +531 -1058
- package/dist/deployment/deployment-identity.d.ts +42 -60
- package/dist/diagnostics/bundle-reader.d.ts +3 -15
- package/dist/diagnostics/bundle-writer.d.ts +5 -12
- package/dist/diagnostics/checks/agents.d.ts +1 -7
- package/dist/diagnostics/checks/backup.d.ts +7 -10
- package/dist/diagnostics/checks/certs.d.ts +1 -6
- package/dist/diagnostics/checks/config.d.ts +2 -5
- package/dist/diagnostics/checks/db.d.ts +1 -1
- package/dist/diagnostics/checks/disk.d.ts +1 -6
- package/dist/diagnostics/checks/firecracker-network.d.ts +3 -3
- package/dist/diagnostics/checks/index.d.ts +0 -6
- package/dist/diagnostics/checks/retention.d.ts +3 -6
- package/dist/diagnostics/checks/scaler.d.ts +3 -7
- package/dist/diagnostics/checks/ws.d.ts +1 -6
- package/dist/diagnostics/fleet-collector.d.ts +11 -14
- package/dist/diagnostics/fleet-constants.d.ts +1 -3
- package/dist/diagnostics/fleet-selection.d.ts +3 -7
- package/dist/diagnostics/fleet-topology.d.ts +2 -6
- package/dist/diagnostics/fleet-wiring.d.ts +6 -12
- package/dist/diagnostics/index.d.ts +0 -5
- package/dist/diagnostics/runner.d.ts +4 -8
- package/dist/diagnostics/types.d.ts +10 -21
- package/dist/drain/drain-controller.d.ts +16 -22
- package/dist/entry-helpers.d.ts +0 -2
- package/dist/events/circuit-breaker.d.ts +0 -3
- package/dist/events/event-router.d.ts +1 -2
- package/dist/events/event-store.d.ts +0 -9
- package/dist/events/trust-store.d.ts +0 -3
- package/dist/events/types.d.ts +0 -9
- package/dist/firecracker/host-network.d.ts +24 -30
- package/dist/firecracker/net-interfaces.d.ts +4 -7
- package/dist/helpers/rate-limiter.d.ts +0 -1
- package/dist/index.js +251 -486
- package/dist/metrics/agent-metrics-aggregator.d.ts +24 -67
- package/dist/metrics/metrics-reporter.d.ts +4 -13
- package/dist/metrics/prometheus.d.ts +47 -81
- package/dist/orchestrator-core.d.ts +3 -7
- package/dist/pipeline/content-filter.d.ts +20 -30
- package/dist/pipeline/cross-source-tally.d.ts +7 -8
- package/dist/pipeline/decorating-secret-resolver.d.ts +4 -8
- package/dist/pipeline/degraded-reason.d.ts +3 -4
- package/dist/pipeline/direct-ingress-deps.d.ts +15 -24
- package/dist/pipeline/dispatch-matched-workflow.d.ts +289 -525
- package/dist/pipeline/flatten-lock-steps.d.ts +4 -6
- package/dist/pipeline/global-dispatch-identity.d.ts +5 -10
- package/dist/pipeline/global-eval-round.d.ts +65 -151
- package/dist/pipeline/global-round-hold.d.ts +5 -7
- package/dist/pipeline/held-context-data.d.ts +5 -8
- package/dist/pipeline/held-round-registrations.d.ts +8 -11
- package/dist/pipeline/install-gate.d.ts +15 -25
- package/dist/pipeline/install-secrets-resolver.d.ts +24 -34
- package/dist/pipeline/internal-event-pipeline.d.ts +45 -62
- package/dist/pipeline/invoke-gate.d.ts +27 -40
- package/dist/pipeline/job-contexts.d.ts +19 -36
- package/dist/pipeline/manual-schedule.d.ts +7 -13
- package/dist/pipeline/needs-scheduler.d.ts +29 -46
- package/dist/pipeline/pending-workflow-context.d.ts +14 -28
- package/dist/pipeline/process-webhook.d.ts +88 -123
- package/dist/pipeline/processor.d.ts +209 -355
- package/dist/pipeline/remote-source-store.d.ts +5 -8
- package/dist/pipeline/request-idempotency.d.ts +10 -13
- package/dist/pipeline/rerun-global.d.ts +6 -7
- package/dist/pipeline/rerun-inputs.d.ts +23 -37
- package/dist/pipeline/rerun.d.ts +34 -57
- package/dist/pipeline/resolve-sandbox-grant.d.ts +5 -10
- package/dist/pipeline/resume-workflow.d.ts +28 -34
- package/dist/pipeline/route-or-dispatch-jobs.d.ts +19 -30
- package/dist/pipeline/run-event-context.d.ts +16 -21
- package/dist/pipeline/sandbox-allowlist-reader.d.ts +7 -11
- package/dist/pipeline/security-hold-check.d.ts +56 -80
- package/dist/pipeline/test-pipeline.d.ts +21 -46
- package/dist/pipeline/unbound-job-context.d.ts +6 -9
- package/dist/pipeline/undispatched-hold-checks.d.ts +4 -6
- package/dist/pipeline/wave-scheduler.d.ts +15 -22
- package/dist/pipeline/webhook-payload-store.d.ts +3 -10
- package/dist/pipeline/workflow-lock-at-sha.d.ts +14 -20
- package/dist/policy/dashboard-write-policy.d.ts +1 -1
- package/dist/provider-registry.d.ts +0 -19
- package/dist/providers/generic/index.d.ts +0 -2
- package/dist/providers/generic/normalizer.d.ts +0 -1
- package/dist/providers/generic/verification.d.ts +0 -8
- package/dist/providers/github/auth.d.ts +0 -1
- package/dist/providers/github/changed-files.d.ts +2 -1
- package/dist/providers/github/file-contents.d.ts +0 -4
- package/dist/providers/github/lock-file.d.ts +0 -4
- package/dist/providers/github/normalizer.d.ts +1 -9
- package/dist/providers/local/index.d.ts +1 -2
- package/dist/providers/local/lock-file-fetcher.d.ts +0 -4
- package/dist/providers/local/normalizer.d.ts +0 -10
- package/dist/providers/local/repo-url-builder.d.ts +0 -2
- package/dist/providers/universal-git/clone-token.d.ts +0 -2
- package/dist/providers/universal-git/lock-file.d.ts +0 -2
- package/dist/providers/universal-git/normalizer.d.ts +0 -2
- package/dist/queue/bootstrap.d.ts +3 -15
- package/dist/queue/cleanup.d.ts +29 -50
- package/dist/queue/depth-refresher.d.ts +21 -39
- package/dist/queue/job-queue.d.ts +112 -163
- package/dist/queue/retention.d.ts +23 -36
- package/dist/queue/scheduled-job.d.ts +17 -39
- package/dist/queue/terminalize-unroutable.d.ts +21 -18
- package/dist/queue/unroutable-probe.d.ts +15 -22
- package/dist/registration/extractor.d.ts +0 -6
- package/dist/registration/registration-index.d.ts +0 -6
- package/dist/registration/registration-store.d.ts +0 -9
- package/dist/reporting/agent-failure-category.d.ts +3 -5
- package/dist/reporting/check-run-reporter.d.ts +142 -316
- package/dist/reporting/check-run-summary.d.ts +16 -35
- package/dist/reporting/check-run-tracking-store.d.ts +54 -107
- package/dist/reporting/execution-tracker.d.ts +356 -526
- package/dist/reporting/fs-log-storage.d.ts +31 -33
- package/dist/reporting/job-check-run-completion.d.ts +14 -22
- package/dist/reporting/log-chunk-sink.d.ts +11 -16
- package/dist/reporting/log-pull-handler.d.ts +4 -13
- package/dist/reporting/log-retention.d.ts +3 -4
- package/dist/reporting/log-storage.d.ts +24 -52
- package/dist/reporting/log-writer.d.ts +28 -56
- package/dist/reporting/peer-log-normalize.d.ts +4 -8
- package/dist/reporting/run-aggregator.d.ts +6 -11
- package/dist/reporting/run-summary-fields.d.ts +5 -8
- package/dist/reporting/s3-log-storage.d.ts +31 -50
- package/dist/reporting/step-display-order.d.ts +11 -21
- package/dist/reporting/step-log-buffer.d.ts +7 -28
- package/dist/reporting/step-log-reader.d.ts +11 -19
- package/dist/routes/admin-access-log.d.ts +4 -6
- package/dist/routes/admin-auth.d.ts +19 -38
- package/dist/routes/admin-backends.d.ts +3 -14
- package/dist/routes/admin-cluster-name.d.ts +13 -21
- package/dist/routes/admin-cluster-settings.d.ts +6 -9
- package/dist/routes/admin-config.d.ts +7 -19
- package/dist/routes/admin-contexts.d.ts +6 -13
- package/dist/routes/admin-db.d.ts +5 -10
- package/dist/routes/admin-event-dlq.d.ts +3 -5
- package/dist/routes/admin-event-inspect.d.ts +3 -4
- package/dist/routes/admin-event-log.d.ts +3 -7
- package/dist/routes/admin-events.d.ts +16 -30
- package/dist/routes/admin-held-runs.d.ts +52 -76
- package/dist/routes/admin-maintenance.d.ts +3 -6
- package/dist/routes/admin-orchestrator-drain.d.ts +0 -2
- package/dist/routes/admin-org-settings.d.ts +8 -14
- package/dist/routes/admin-peer-forget.d.ts +0 -1
- package/dist/routes/admin-queue-execution.d.ts +3 -3
- package/dist/routes/admin-registrations.d.ts +3 -12
- package/dist/routes/admin-runs.d.ts +16 -29
- package/dist/routes/admin-scaler-orphans.d.ts +4 -4
- package/dist/routes/admin-scaler-reload.d.ts +5 -8
- package/dist/routes/admin-scheduled-jobs.d.ts +3 -4
- package/dist/routes/admin-sources.d.ts +20 -29
- package/dist/routes/admin-trust-policy.d.ts +31 -48
- package/dist/routes/admin.d.ts +67 -94
- package/dist/routes/caller-identity.d.ts +45 -0
- package/dist/routes/capabilities.d.ts +8 -13
- package/dist/routes/diagnostics.d.ts +2 -10
- package/dist/routes/fleet.d.ts +1 -2
- package/dist/routes/github-webhook.d.ts +2 -5
- package/dist/routes/health.d.ts +6 -19
- package/dist/routes/provenance-oidc.d.ts +3 -3
- package/dist/routes/test-direct.d.ts +46 -0
- package/dist/routes/uploads.d.ts +14 -19
- package/dist/routes/webhooks.d.ts +2 -24
- package/dist/scaler/agent-binary-location.d.ts +77 -0
- package/dist/scaler/agent-fit.d.ts +2 -2
- package/dist/scaler/backend-factory.d.ts +10 -18
- package/dist/scaler/bare-metal-backend.d.ts +28 -45
- package/dist/scaler/claim-store.d.ts +20 -43
- package/dist/scaler/config.d.ts +10 -37
- package/dist/scaler/container-backend.d.ts +22 -35
- package/dist/scaler/container-hostconfig.d.ts +10 -16
- package/dist/scaler/container-routing.d.ts +10 -13
- package/dist/scaler/disk-guard.d.ts +2 -3
- package/dist/scaler/event-backend.d.ts +33 -58
- package/dist/scaler/event-provision-reaper.d.ts +61 -84
- package/dist/scaler/failure-tracker.d.ts +6 -13
- package/dist/scaler/file-tail.d.ts +6 -19
- package/dist/scaler/firecracker-api.d.ts +19 -63
- package/dist/scaler/firecracker-backend.d.ts +142 -258
- package/dist/scaler/host-access.d.ts +7 -10
- package/dist/scaler/index.d.ts +3 -6
- package/dist/scaler/ip-allocator.d.ts +30 -82
- package/dist/scaler/label-matcher.d.ts +14 -35
- package/dist/scaler/log-forwarder.d.ts +6 -25
- package/dist/scaler/machine-ledger.d.ts +39 -86
- package/dist/scaler/manager.d.ts +517 -787
- package/dist/scaler/pending-scale-sweeper.d.ts +8 -11
- package/dist/scaler/reap-orphans.d.ts +8 -12
- package/dist/scaler/resolve-container-auth.d.ts +12 -21
- package/dist/scaler/scaler-events.d.ts +2 -5
- package/dist/scaler/scaler-state-store.d.ts +127 -189
- package/dist/scaler/spawn-errors.d.ts +1 -4
- package/dist/scaler/startup-orphan-sweep.d.ts +3 -4
- package/dist/scaler/startup-tools-check.d.ts +21 -0
- package/dist/scaler/types.d.ts +142 -255
- package/dist/scaler/vm-job-bindings.d.ts +5 -9
- package/dist/scaler/warm-pool.d.ts +46 -93
- package/dist/secrets/audit-logger.d.ts +0 -3
- package/dist/secrets/backend-health.d.ts +0 -9
- package/dist/secrets/backend-registry.d.ts +0 -9
- package/dist/secrets/cleanup.d.ts +0 -1
- package/dist/secrets/config.d.ts +0 -3
- package/dist/secrets/ephemeral-keys.d.ts +0 -3
- package/dist/secrets/pg-secret-store.d.ts +0 -3
- package/dist/secrets/rbac.d.ts +5 -7
- package/dist/secrets/routing-key-scope.d.ts +1 -0
- package/dist/secrets/secret-output-store.d.ts +0 -6
- package/dist/secrets/secret-resolver.d.ts +2 -6
- package/dist/secrets/token-manager.d.ts +1 -4
- package/dist/secrets/vault-secret-store.d.ts +0 -13
- package/dist/security/comment-handler.d.ts +0 -2
- package/dist/security/workflow-diff.d.ts +1 -1
- package/dist/server.js +49004 -53782
- package/dist/sources/source-manager.d.ts +0 -3
- package/dist/sources/source-store.d.ts +0 -9
- package/dist/sources/source-validator.d.ts +0 -3
- package/dist/stale-detector/all-terminal-runs.d.ts +8 -11
- package/dist/stale-detector/gate-deadline-detector.d.ts +14 -19
- package/dist/stale-detector/reboot-deadline-sweep.d.ts +7 -11
- package/dist/stale-detector/stale-dispatch-candidates.d.ts +13 -20
- package/dist/stale-detector/stale-run-detector.d.ts +74 -121
- package/dist/stale-detector/workflow-deadline-detector.d.ts +15 -43
- package/dist/standalone.js +9342 -13077
- package/dist/storage/blob-routes.d.ts +10 -19
- package/dist/storage/filesystem.d.ts +19 -29
- package/dist/storage/index.d.ts +2 -10
- package/dist/storage/key-discriminator.d.ts +8 -10
- package/dist/storage/loopback-guard.d.ts +27 -45
- package/dist/storage/s3.d.ts +26 -53
- package/dist/storage/sign-url.d.ts +7 -22
- package/dist/storage/types.d.ts +46 -72
- package/dist/webhook/event-log.d.ts +0 -2
- package/dist/webhook/generic-sources.d.ts +0 -21
- package/dist/webhook/ingest-admission.d.ts +0 -1
- package/dist/webhook/relay-buffer.d.ts +2 -2
- package/dist/webhook/verify-inbound.d.ts +0 -1
- package/dist/worker/in-memory-execution-tracker.d.ts +9 -31
- package/dist/worker/in-memory-job-queue.d.ts +32 -72
- package/dist/worker/peer-outbox.d.ts +5 -8
- package/dist/worker/reroute-dispatch.d.ts +4 -6
- package/dist/worker/static-agent-token-store.d.ts +6 -27
- package/dist/worker/worker-outbox-relay.d.ts +2 -2
- package/dist/worker/worker-status.d.ts +3 -20
- package/dist/ws/agent-api-registry.d.ts +8 -27
- package/dist/ws/agent-handler.d.ts +63 -110
- package/dist/ws/agent-heartbeat.d.ts +5 -12
- package/dist/ws/bringup-api.d.ts +31 -38
- package/dist/ws/dashboard-backends-handler.d.ts +12 -22
- package/dist/ws/dashboard-context-handler.d.ts +37 -80
- package/dist/ws/dashboard-diagnostics-handler.d.ts +8 -25
- package/dist/ws/dashboard-dispatch-guard.d.ts +3 -4
- package/dist/ws/dashboard-fleet-handler.d.ts +9 -13
- package/dist/ws/dashboard-fleet-write-handler.d.ts +7 -21
- package/dist/ws/dashboard-global-workflows-handler.d.ts +22 -44
- package/dist/ws/dashboard-registrations-handler.d.ts +10 -27
- package/dist/ws/dashboard-run-state-handler.d.ts +10 -13
- package/dist/ws/event-buffer.d.ts +3 -6
- package/dist/ws/execution-status-frame.d.ts +7 -10
- package/dist/ws/failure-messages.d.ts +9 -14
- package/dist/ws/fleet-agent-collector.d.ts +6 -12
- package/dist/ws/fleet-runs-on-all.d.ts +1 -6
- package/dist/ws/git-credential-relay.d.ts +12 -20
- package/dist/ws/inventory-api.d.ts +4 -5
- package/dist/ws/observer-registry.d.ts +14 -59
- package/dist/ws/ownership-gate.d.ts +1 -1
- package/dist/ws/platform-client.d.ts +21 -30
- package/dist/ws/replay-chunker.d.ts +7 -10
- package/dist/ws/server-options.d.ts +5 -9
- package/dist/ws/test-relay-handlers.d.ts +46 -63
- package/installer-image-digests.json +3 -3
- package/package.json +6 -5
- package/sbom.spdx.json +53 -53
- package/dist/__test-helpers__/fake-scaler-state-store.d.ts +0 -24
- package/dist/__test-helpers__/mock-db-query.d.ts +0 -129
- package/dist/__test-helpers__/mock-db.d.ts +0 -134
- package/dist/__test-helpers__/mock-ws.d.ts +0 -13
- package/dist/__test-helpers__/peer-mutual-auth.d.ts +0 -42
- package/dist/__test-helpers__/scratch-db.d.ts +0 -16
- package/dist/__test-helpers__/test-db.d.ts +0 -34
- package/dist/pipeline/dispatch-matched-workflow.test-helpers.d.ts +0 -121
- package/dist/scaler/container-spawn.d.ts +0 -28
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Execution state tracker with write-through DB persistence.
|
|
3
3
|
*
|
|
4
|
-
* Aggregates job/step status updates
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* to the execution_runs/execution_jobs/execution_steps tables.
|
|
4
|
+
* Aggregates agent job/step status updates in memory (job name resolution, run
|
|
5
|
+
* completion detection) and writes through to the
|
|
6
|
+
* execution_runs/execution_jobs/execution_steps tables.
|
|
8
7
|
*
|
|
9
|
-
* Overall run status
|
|
8
|
+
* Overall run status:
|
|
10
9
|
* - success ONLY if ALL jobs succeed
|
|
11
10
|
* - failed if ANY job fails
|
|
12
11
|
* - cancelled if ANY job is cancelled (and none failed)
|
|
@@ -153,18 +152,17 @@ export interface ExecutionTrackerDeps {
|
|
|
153
152
|
* Drop the in-process state a run leaves behind — its pending job contexts
|
|
154
153
|
* and its still-open eval gates.
|
|
155
154
|
*
|
|
156
|
-
* Separate from {@link ExecutionTrackerDeps.onExecutionComplete}
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
* a context is only stored for a job that was gated rather than dispatched.
|
|
155
|
+
* Separate from {@link ExecutionTrackerDeps.onExecutionComplete}, which is NOT
|
|
156
|
+
* a terminalization chokepoint: it fires on the paths that finish a run whose
|
|
157
|
+
* jobs ran (`completeRun`, and both stale-detector arms) and on none that
|
|
158
|
+
* terminalize a run whose jobs never did — `failRun` (the expired-approval
|
|
159
|
+
* sweep and every pre-dispatch abort) and `cancelHeldRun` (an install gate
|
|
160
|
+
* rejected). Those runs are the likeliest to hold a pending job context, since
|
|
161
|
+
* a context is stored only for a job that was gated rather than dispatched.
|
|
164
162
|
*
|
|
165
|
-
* Synchronous by contract: the in-memory half
|
|
166
|
-
*
|
|
167
|
-
*
|
|
163
|
+
* Synchronous by contract: the in-memory half must not be deferred behind an
|
|
164
|
+
* await the caller does not hold; the DB half is fire-and-forget inside the
|
|
165
|
+
* callback.
|
|
168
166
|
*/
|
|
169
167
|
onRunTerminalCleanup?: (runId: string) => void;
|
|
170
168
|
/** Optional callback to forward step status to Platform. */
|
|
@@ -250,12 +248,11 @@ export interface ExecutionTrackerDeps {
|
|
|
250
248
|
/** Optional job queue for cascading run failures to dispatch_queue entries. */
|
|
251
249
|
jobQueue?: JobQueue;
|
|
252
250
|
/**
|
|
253
|
-
* Resolves a routing key to its owning org
|
|
254
|
-
* `execution_runs.customer_id` at insert
|
|
255
|
-
*
|
|
256
|
-
*
|
|
257
|
-
*
|
|
258
|
-
* `'__default__'` column default (the no-source fallback org).
|
|
251
|
+
* Resolves a routing key to its owning org, stamped on
|
|
252
|
+
* `execution_runs.customer_id` at insert so the concurrency-gate running count
|
|
253
|
+
* is scoped per tenant. Injected (bound to `(rk) => resolveOrgId(db, rk)`) to
|
|
254
|
+
* avoid a pipeline import cycle. Absent, or for a run with no routing key, the
|
|
255
|
+
* row falls back to the `'__default__'` column default (the no-source org).
|
|
259
256
|
*/
|
|
260
257
|
resolveOrgId?: (routingKey: string) => Promise<string>;
|
|
261
258
|
/**
|
|
@@ -278,11 +275,9 @@ export interface ExecutionTrackerDeps {
|
|
|
278
275
|
/**
|
|
279
276
|
* The per-job fields the tracker persists to an execution_jobs row.
|
|
280
277
|
*
|
|
281
|
-
*
|
|
282
|
-
* `addJobsToRun`) rather than restated inline
|
|
283
|
-
*
|
|
284
|
-
* new column had to be added in three places or silently go unwritten from one
|
|
285
|
-
* of them.
|
|
278
|
+
* Shared by every registration entry point (`onExecutionStarted`,
|
|
279
|
+
* `addJobsToRun`) rather than restated inline: inline copies drift on which
|
|
280
|
+
* optional fields they carry, so a new column silently goes unwritten from one.
|
|
286
281
|
*/
|
|
287
282
|
export interface TrackedJobRow {
|
|
288
283
|
jobId: string;
|
|
@@ -358,15 +353,11 @@ export declare class ExecutionTracker {
|
|
|
358
353
|
/** Tracks which runs are test runs for observer broadcasting. */
|
|
359
354
|
private readonly testRunIds;
|
|
360
355
|
/**
|
|
361
|
-
* Per-(run, job) raw log
|
|
356
|
+
* Per-(run, job) raw log bytes: runId -> jobId -> bytes.
|
|
362
357
|
*
|
|
363
|
-
*
|
|
364
|
-
* `
|
|
365
|
-
*
|
|
366
|
-
* summing the per-job entries and written into `execution_runs.log_bytes`
|
|
367
|
-
* at run completion. Deleted on run prune to avoid Map growth.
|
|
368
|
-
*
|
|
369
|
-
* Map shape: runId -> jobId -> bytes.
|
|
358
|
+
* Fed by `logBytesStreamed` on terminal `step.status` messages, written to
|
|
359
|
+
* `execution_jobs.log_bytes` at job terminal state and summed into
|
|
360
|
+
* `execution_runs.log_bytes` at run completion. Deleted on run prune.
|
|
370
361
|
*/
|
|
371
362
|
private readonly jobLogBytes;
|
|
372
363
|
/** Callback fired when a job's needs are satisfied and it's ready for dispatch. */
|
|
@@ -384,10 +375,8 @@ export declare class ExecutionTracker {
|
|
|
384
375
|
*/
|
|
385
376
|
setOnJobReadyCallback(cb: (runId: string, jobName: string) => Promise<void>): void;
|
|
386
377
|
/**
|
|
387
|
-
* Record a new execution run with its initial jobs
|
|
388
|
-
*
|
|
389
|
-
* Inserts execution_runs and execution_jobs rows in the DB and
|
|
390
|
-
* sets up in-memory tracking state.
|
|
378
|
+
* Record a new execution run with its initial jobs: inserts the
|
|
379
|
+
* execution_runs and execution_jobs rows and sets up in-memory state.
|
|
391
380
|
*/
|
|
392
381
|
onExecutionStarted(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, triggerDecision: Record<string, unknown> | null, jobs: TrackedJobRow[], routingKey?: string,
|
|
393
382
|
/** Secret context names dispatched with jobs (for context-disable job lookup). */
|
|
@@ -467,21 +456,18 @@ export declare class ExecutionTracker {
|
|
|
467
456
|
* Cancel job rows just registered under a run that has already stopped.
|
|
468
457
|
*
|
|
469
458
|
* A job registers after its dispatch, so a cancel landing in between leaves
|
|
470
|
-
* the run terminal
|
|
471
|
-
*
|
|
472
|
-
*
|
|
473
|
-
* the queued jobs it can see.
|
|
459
|
+
* the run terminal and the job row `pending`. The queue refuses that dispatch
|
|
460
|
+
* (`JobQueue.insertDispatched`), so no agent will report on it; the row is
|
|
461
|
+
* cancelled here, as the cancel cancels the queued jobs it can see.
|
|
474
462
|
*/
|
|
475
463
|
private cancelJobsOfStoppedRun;
|
|
476
464
|
/**
|
|
477
|
-
* Find the synthetic needs-pending job ID
|
|
478
|
-
* Used by dispatchReadyJob to locate the placeholder entry before replacing it.
|
|
465
|
+
* Find the synthetic needs-pending job ID that dispatchReadyJob replaces.
|
|
479
466
|
*
|
|
480
|
-
* Cluster correctness: the
|
|
481
|
-
*
|
|
482
|
-
*
|
|
483
|
-
*
|
|
484
|
-
* which peer owns the downstream dispatch.
|
|
467
|
+
* Cluster correctness: the peer that ingested the webhook inserts the
|
|
468
|
+
* synthetic row, but `dispatchReadyJob` can fire on a peer whose in-memory map
|
|
469
|
+
* has no entry for this run. The DB fallback cleans up the synthetic row
|
|
470
|
+
* whichever peer owns the downstream dispatch.
|
|
485
471
|
*/
|
|
486
472
|
findSyntheticJobId(runId: string, jobName: string): Promise<string | undefined>;
|
|
487
473
|
/**
|
|
@@ -502,28 +488,23 @@ export declare class ExecutionTracker {
|
|
|
502
488
|
* Whether a rerouted job has visibly started, read from the shared
|
|
503
489
|
* `execution_jobs` row rather than from a relayed progress frame.
|
|
504
490
|
*
|
|
505
|
-
* The
|
|
506
|
-
*
|
|
507
|
-
*
|
|
508
|
-
*
|
|
509
|
-
* whether the job is running, and the backstop would cancel a healthy job.
|
|
491
|
+
* The spawn-window backstop learns progress from `job.progress`, which only a
|
|
492
|
+
* WORKER peer (no database) sends. A peer COORDINATOR writes the status
|
|
493
|
+
* straight into this table, so wire silence says nothing and the backstop
|
|
494
|
+
* would cancel a healthy job.
|
|
510
495
|
*
|
|
511
|
-
* False when no row exists yet
|
|
512
|
-
*
|
|
513
|
-
* already moved on to a terminal value.
|
|
496
|
+
* False when no row exists yet or the row is still `pending`; the
|
|
497
|
+
* `started_at` half covers a status already moved on to a terminal value.
|
|
514
498
|
*/
|
|
515
499
|
hasJobStarted(runId: string, jobId: string): Promise<boolean>;
|
|
516
500
|
/**
|
|
517
|
-
* Whether
|
|
518
|
-
* row.
|
|
501
|
+
* Whether the shared `execution_jobs` row records a terminal state.
|
|
519
502
|
*
|
|
520
|
-
*
|
|
521
|
-
*
|
|
522
|
-
*
|
|
523
|
-
* over and its reroute tracking can be released.
|
|
503
|
+
* Companion to {@link hasJobStarted}: a peer COORDINATOR relays no terminal
|
|
504
|
+
* `job.progress`, so the row is the routing coordinator's only signal that the
|
|
505
|
+
* job is over and its reroute tracking can be released.
|
|
524
506
|
*
|
|
525
|
-
* False when no row exists
|
|
526
|
-
* never a finished one.
|
|
507
|
+
* False when no row exists — an absent row is a job not yet started.
|
|
527
508
|
*/
|
|
528
509
|
isJobTerminal(runId: string, jobId: string): Promise<boolean>;
|
|
529
510
|
/**
|
|
@@ -543,14 +524,13 @@ export declare class ExecutionTracker {
|
|
|
543
524
|
* (`onJobStatus`, `addJobsToRun`) so a status reply cannot interleave with the
|
|
544
525
|
* synthetic→real job swap and wedge the run in `running`.
|
|
545
526
|
*
|
|
546
|
-
* The lock is **reentrant**:
|
|
547
|
-
*
|
|
527
|
+
* The lock is **reentrant**: these paths re-enter a locked method in the same
|
|
528
|
+
* async context for the same runId — onJobStatus → scheduler hook →
|
|
548
529
|
* dispatchReadyJob → addJobsToRun; onJobStatus → enforceSchedulerInvariant →
|
|
549
|
-
* onJobStatus; runSchedulerHook → onJobStatus (skip). A
|
|
550
|
-
*
|
|
551
|
-
*
|
|
552
|
-
*
|
|
553
|
-
* correctly waits. All reentrant paths are same-runId, so there is no
|
|
530
|
+
* onJobStatus; runSchedulerHook → onJobStatus (skip). A context already
|
|
531
|
+
* holding the runId's lock (`heldRunLocks`) runs `fn` inline instead of
|
|
532
|
+
* deadlocking; a concurrent caller (a separate WS message) is a different async
|
|
533
|
+
* context and waits. Reentrant paths are all same-runId, so there is no
|
|
554
534
|
* cross-run lock-ordering deadlock.
|
|
555
535
|
*/
|
|
556
536
|
private withRunLock;
|
|
@@ -558,14 +538,8 @@ export declare class ExecutionTracker {
|
|
|
558
538
|
/** Synthetic job ID to replace (e.g. needs-pending-deploy-{uuid}). */
|
|
559
539
|
replaceSyntheticId?: string): Promise<void>;
|
|
560
540
|
private addJobsToRunImpl;
|
|
561
|
-
/**
|
|
562
|
-
* Mark a run as a test run for observer broadcasting.
|
|
563
|
-
* Called by the test pipeline after creating the execution run.
|
|
564
|
-
*/
|
|
541
|
+
/** Mark a run as a test run for observer broadcasting (called by the test pipeline). */
|
|
565
542
|
markTestRun(runId: string): void;
|
|
566
|
-
/**
|
|
567
|
-
* Check if a run is a test run.
|
|
568
|
-
*/
|
|
569
543
|
isTestRun(runId: string): boolean;
|
|
570
544
|
/**
|
|
571
545
|
* Update job status within a run.
|
|
@@ -579,30 +553,26 @@ export declare class ExecutionTracker {
|
|
|
579
553
|
/**
|
|
580
554
|
* Whether this coordinator may finalize the run now.
|
|
581
555
|
*
|
|
582
|
-
* The in-memory
|
|
583
|
-
*
|
|
584
|
-
*
|
|
585
|
-
*
|
|
586
|
-
*
|
|
587
|
-
*
|
|
588
|
-
* open, and the status computed afterwards covers every job, not a subset.
|
|
556
|
+
* The in-memory map is only this coordinator's view: a rehydrated run holds
|
|
557
|
+
* just the jobs its own agents reported, and even the owner holds a
|
|
558
|
+
* sibling-claimed job at its last-heard status. So once the map reads
|
|
559
|
+
* complete, the shared `execution_jobs` rows are folded in
|
|
560
|
+
* ({@link syncRunJobsFromRows}) and the check repeats over every job — a job a
|
|
561
|
+
* sibling still runs keeps the run open.
|
|
589
562
|
*
|
|
590
563
|
* Then the registration window: while a LIVE sibling still has jobs to
|
|
591
|
-
* register (a build-window owner
|
|
592
|
-
*
|
|
593
|
-
* to
|
|
594
|
-
*
|
|
595
|
-
* and a run it left behind is finished by the stale detector's
|
|
596
|
-
* all-terminal sweep.
|
|
564
|
+
* register (a build-window owner dispatches its real jobs after the build),
|
|
565
|
+
* finalization defers to whichever coordinator sees the last job finish, or
|
|
566
|
+
* to that owner's release, which re-runs this check. A dead holder reads as no
|
|
567
|
+
* window; the stale detector's all-terminal sweep finishes its run.
|
|
597
568
|
*/
|
|
598
569
|
private readyToFinalize;
|
|
599
570
|
/**
|
|
600
|
-
* Fold the run's
|
|
601
|
-
*
|
|
602
|
-
*
|
|
603
|
-
*
|
|
604
|
-
*
|
|
605
|
-
* memory is always at least as current as its row.
|
|
571
|
+
* Fold the run's `execution_jobs` rows into its in-memory map: an unknown row
|
|
572
|
+
* is added at its status, and a non-terminal job is lifted to the row's
|
|
573
|
+
* terminal status. Memory is never downgraded — the reported job's row is
|
|
574
|
+
* upserted before any completion check, so a terminal memory entry is at
|
|
575
|
+
* least as current as its row.
|
|
606
576
|
*/
|
|
607
577
|
private syncRunJobsFromRows;
|
|
608
578
|
/**
|
|
@@ -613,56 +583,51 @@ export declare class ExecutionTracker {
|
|
|
613
583
|
private registrationWindowHeldByLiveSibling;
|
|
614
584
|
/**
|
|
615
585
|
* Phase 1a: recover run state from the DB when in-memory tracking is empty.
|
|
616
|
-
*
|
|
617
|
-
* (in which case the caller skips this status update entirely).
|
|
586
|
+
* Null when the run is unknown to the DB; the caller then skips the update.
|
|
618
587
|
*/
|
|
619
588
|
private recoverRunFromDb;
|
|
620
589
|
/**
|
|
621
590
|
* Reopen a run rehydrated from a prematurely `failed` row (e.g. "No agents
|
|
622
591
|
* available to dispatch jobs", but a queued job is now running): reset the
|
|
623
592
|
* row to `running`, clear the failure it recorded, and raise its status
|
|
624
|
-
* generation
|
|
593
|
+
* generation; run completion then sets the final status.
|
|
625
594
|
*
|
|
626
|
-
* Called only
|
|
627
|
-
*
|
|
628
|
-
*
|
|
629
|
-
*
|
|
595
|
+
* Called only after the rehydrating job report is applied to its job row: a
|
|
596
|
+
* replay of a frame the row already records must not reopen (and roll up
|
|
597
|
+
* again) a finished run. A build failure is intentionally terminal and is
|
|
598
|
+
* never reopened.
|
|
630
599
|
*/
|
|
631
600
|
private reopenRecoveredRun;
|
|
632
601
|
/**
|
|
633
602
|
* Read the run row's status generation into the in-memory run after this
|
|
634
603
|
* coordinator's reopen matched nothing.
|
|
635
604
|
*
|
|
636
|
-
* The reopen loses when another coordinator
|
|
637
|
-
* this view is a generation behind
|
|
638
|
-
*
|
|
639
|
-
*
|
|
640
|
-
* none of them would apply.
|
|
605
|
+
* The reopen loses when another coordinator reopened the run first, or when
|
|
606
|
+
* this view is a generation behind. Either way the row's generation is
|
|
607
|
+
* current: keeping an older value would send every later frame at a
|
|
608
|
+
* generation the Platform treats as older, so none would apply.
|
|
641
609
|
*/
|
|
642
610
|
private adoptStoredStatusEpoch;
|
|
643
611
|
/**
|
|
644
|
-
* Phase 1b: recover job state from the dispatch_queue
|
|
645
|
-
*
|
|
612
|
+
* Phase 1b: recover job state from the dispatch_queue into the run's
|
|
613
|
+
* in-memory map.
|
|
646
614
|
*/
|
|
647
615
|
private recoverJobFromDispatchQueue;
|
|
648
616
|
/**
|
|
649
617
|
* Phase 2: build the upsert payload for execution_jobs and execute it.
|
|
650
618
|
*
|
|
651
|
-
* The conflict update
|
|
652
|
-
*
|
|
653
|
-
*
|
|
654
|
-
* downstream by the run-completion phase to populate run.log_bytes).
|
|
619
|
+
* The conflict update cannot rewrite a terminal row; `applied` reports the
|
|
620
|
+
* guard's verdict. The log byte total is set on a terminal state and feeds
|
|
621
|
+
* run.log_bytes at run completion.
|
|
655
622
|
*/
|
|
656
623
|
private persistJobStatusUpdate;
|
|
657
624
|
/**
|
|
658
|
-
* Phase 3a:
|
|
659
|
-
* starts executing.
|
|
625
|
+
* Phase 3a: pending -> running when the first job starts.
|
|
660
626
|
*/
|
|
661
627
|
private maybeTransitionRunToRunning;
|
|
662
628
|
/**
|
|
663
|
-
* Phase 3b:
|
|
664
|
-
*
|
|
665
|
-
* running" — the run should reflect this intermediate state.
|
|
629
|
+
* Phase 3b: an agent's 'cancelling' (graceful cancel in progress, hooks
|
|
630
|
+
* running) moves the run to 'cancelling' too.
|
|
666
631
|
*/
|
|
667
632
|
private maybeTransitionRunToCancelling;
|
|
668
633
|
/**
|
|
@@ -675,32 +640,27 @@ export declare class ExecutionTracker {
|
|
|
675
640
|
*/
|
|
676
641
|
private fireJobStatusChangeCallback;
|
|
677
642
|
/**
|
|
678
|
-
* Phase 6: reconcile orphaned steps.
|
|
679
|
-
*
|
|
680
|
-
*
|
|
681
|
-
* the dashboard shows "running" with an ever-growing duration for the lost
|
|
682
|
-
* step.
|
|
643
|
+
* Phase 6: reconcile orphaned steps. Steps still 'running' when their job
|
|
644
|
+
* is terminal (a lost step.complete IPC message) take the job's terminal
|
|
645
|
+
* state; otherwise the dashboard shows the step running forever.
|
|
683
646
|
*/
|
|
684
647
|
private reconcileOrphanedSteps;
|
|
685
648
|
/**
|
|
686
|
-
* Phase 7: drift report handling.
|
|
687
|
-
*
|
|
688
|
-
*
|
|
689
|
-
* (Phase 8) will propagate failures to any downstreams.
|
|
649
|
+
* Phase 7: drift report handling. Jobs an agent reports as droppedJobs move
|
|
650
|
+
* to the terminal drift_dropped, so the scheduler hook (Phase 8) propagates
|
|
651
|
+
* failure to their downstreams.
|
|
690
652
|
*/
|
|
691
653
|
private handleDriftDroppedJobs;
|
|
692
654
|
/**
|
|
693
|
-
* Phase 8: needs-aware scheduler hook
|
|
694
|
-
*
|
|
695
|
-
*
|
|
696
|
-
* so recursive skip propagation terminates naturally.
|
|
655
|
+
* Phase 8: needs-aware scheduler hook: evaluate a terminal job's downstreams
|
|
656
|
+
* for dispatch. The DAG is acyclic (validated at compile time L1 and eval
|
|
657
|
+
* time L2), so recursive skip propagation terminates.
|
|
697
658
|
*/
|
|
698
659
|
private runSchedulerHook;
|
|
699
660
|
/**
|
|
700
661
|
* Rolling-wave hook: fires beside the needs-scheduler when a fan-out child of
|
|
701
|
-
* a bounded wave (`maxParallel` set) reaches terminal
|
|
702
|
-
*
|
|
703
|
-
* what to do, then performs it:
|
|
662
|
+
* a bounded wave (`maxParallel` set) reaches terminal, and performs what
|
|
663
|
+
* {@link evaluateWave} decides:
|
|
704
664
|
*
|
|
705
665
|
* - `release`: clear the next held sibling's `wave_gated` flag and fire the
|
|
706
666
|
* onJobReady callback (the existing ready→dispatch path).
|
|
@@ -710,17 +670,15 @@ export declare class ExecutionTracker {
|
|
|
710
670
|
private runWaveSchedulerHook;
|
|
711
671
|
/**
|
|
712
672
|
* When a proxy job of an invoke gate reaches terminal, aggregate the gate once
|
|
713
|
-
* every sibling proxy is terminal:
|
|
714
|
-
*
|
|
715
|
-
*
|
|
673
|
+
* every sibling proxy is terminal: `failed` if any proxy failed, else
|
|
674
|
+
* `success`. The terminal gate releases its downstream `needs` through the
|
|
675
|
+
* scheduler hook. A no-op for any non-proxy job.
|
|
716
676
|
*/
|
|
717
677
|
private runInvokeGateAggregationHook;
|
|
718
678
|
/**
|
|
719
|
-
* Phase 9: stuck-jobs invariant check (Layer 3)
|
|
720
|
-
*
|
|
721
|
-
*
|
|
722
|
-
* stop (returns true) — the recursive calls will re-enter and re-check
|
|
723
|
-
* completion with fresh state.
|
|
679
|
+
* Phase 9: stuck-jobs invariant check (Layer 3), before declaring a run
|
|
680
|
+
* complete. Stuck jobs are failed via recursive onJobStatus calls, which
|
|
681
|
+
* re-check completion with fresh state; true tells the caller to stop.
|
|
724
682
|
*/
|
|
725
683
|
private enforceSchedulerInvariantOrFail;
|
|
726
684
|
/**
|
|
@@ -730,9 +688,8 @@ export declare class ExecutionTracker {
|
|
|
730
688
|
*/
|
|
731
689
|
private finalizeRunCompletion;
|
|
732
690
|
/**
|
|
733
|
-
*
|
|
734
|
-
*
|
|
735
|
-
* else (success) to `success`.
|
|
691
|
+
* The terminal job status a proxy carries for a terminal run status:
|
|
692
|
+
* `success` and `cancelled` map through; every other status is `failed`.
|
|
736
693
|
*/
|
|
737
694
|
private mapRunStatusToProxyStatus;
|
|
738
695
|
/**
|
|
@@ -743,9 +700,8 @@ export declare class ExecutionTracker {
|
|
|
743
700
|
*/
|
|
744
701
|
private readNonSecretRunOutputs;
|
|
745
702
|
/**
|
|
746
|
-
* Normalize an `outputs` JSONB cell
|
|
747
|
-
*
|
|
748
|
-
* null / empty / malformed cell yields `null`.
|
|
703
|
+
* Normalize an `outputs` JSONB cell (already-parsed object or JSON string)
|
|
704
|
+
* to a plain object; `null` for a null / empty / malformed cell.
|
|
749
705
|
*/
|
|
750
706
|
private parseOutputsCell;
|
|
751
707
|
/**
|
|
@@ -756,52 +712,42 @@ export declare class ExecutionTracker {
|
|
|
756
712
|
*/
|
|
757
713
|
private mirrorSummonedRunOntoProxy;
|
|
758
714
|
/**
|
|
759
|
-
* Set a proxy job's timeline span to the summoned run's
|
|
760
|
-
* completed_at
|
|
761
|
-
*
|
|
762
|
-
* a
|
|
763
|
-
* still renders a point-width bar rather than a status-only placeholder.
|
|
715
|
+
* Set a proxy job's timeline span to the summoned run's started_at /
|
|
716
|
+
* completed_at. A run rejected before any step has no start, so it falls back
|
|
717
|
+
* to its completion (or, with none recorded, the mirror timestamp) and still
|
|
718
|
+
* renders a point-width bar.
|
|
764
719
|
*/
|
|
765
720
|
private mirrorSummonedRunSpanOntoProxy;
|
|
766
721
|
/**
|
|
767
|
-
*
|
|
768
|
-
* terminal
|
|
769
|
-
*
|
|
770
|
-
*
|
|
771
|
-
*
|
|
772
|
-
*
|
|
773
|
-
* `onJobStatus` idempotency guard makes a double-fire with the normal finalize
|
|
774
|
-
* a no-op.
|
|
722
|
+
* Drive the proxy mirror now when a just-tagged summoned run is already
|
|
723
|
+
* terminal. Closes the tag race: a run that finalizes before
|
|
724
|
+
* `summoned_by_run_id` lands (a synchronous all-jobs-rejected completion, or
|
|
725
|
+
* an HA peer finalizing first) reads a null tag and skips, so the gate hangs
|
|
726
|
+
* until its timeout. The invoke gate calls this after tagging each spawned
|
|
727
|
+
* run; the `onJobStatus` idempotency guard makes a double-fire a no-op.
|
|
775
728
|
*/
|
|
776
729
|
reconcileSummonedRunIfTerminal(spawnedRunId: string): Promise<void>;
|
|
777
730
|
/**
|
|
778
|
-
* Mark a run
|
|
779
|
-
*
|
|
780
|
-
*
|
|
781
|
-
* onExecutionStarted but the build subsequently fails. Without this,
|
|
782
|
-
* the execution_runs row would stay in a non-terminal state.
|
|
731
|
+
* Mark a run failed when its build fails (timeout or error) after the
|
|
732
|
+
* processor tracked it early via onExecutionStarted; otherwise the row stays
|
|
733
|
+
* non-terminal.
|
|
783
734
|
*/
|
|
784
735
|
onBuildFailed(runId: string, initFailure?: InitFailure): Promise<void>;
|
|
785
736
|
/**
|
|
786
|
-
*
|
|
787
|
-
*
|
|
788
|
-
*
|
|
789
|
-
* Inserts a minimal execution_runs row with status='failed' directly so the
|
|
790
|
-
* dashboard (and any run query) can observe the failure instead of a missing run.
|
|
737
|
+
* Insert a minimal `failed` execution_runs row when the build timed out
|
|
738
|
+
* before onExecutionStarted inserted one (buildJobTrackedEarly was false), so
|
|
739
|
+
* the dashboard and run queries see the failure instead of a missing run.
|
|
791
740
|
*
|
|
792
|
-
*
|
|
793
|
-
*
|
|
794
|
-
* reads its `chain_depth` — the stamp is here for the same reason as on
|
|
741
|
+
* A pre-dispatch recording site that writes the row and returns. The row is
|
|
742
|
+
* terminal, so no invoke gate reads its `chain_depth`; it is stamped as in
|
|
795
743
|
* {@link ExecutionTracker.recordInitFailureRun}: this row is the run's ONLY
|
|
796
|
-
* record, and
|
|
797
|
-
* died inside.
|
|
744
|
+
* record, and a `0` claims it started the chain it died inside.
|
|
798
745
|
*/
|
|
799
746
|
onBuildFailedBeforeTracking(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, routingKey: string, triggerEvent?: string, commitMessage?: string, failureReason?: string, initFailure?: InitFailure,
|
|
800
747
|
/**
|
|
801
|
-
* Internal-trigger provenance,
|
|
802
|
-
*
|
|
803
|
-
* `
|
|
804
|
-
* the only caller.
|
|
748
|
+
* Internal-trigger provenance, one object rather than more positional
|
|
749
|
+
* args. Shape matches `preDispatchRunProvenance` in
|
|
750
|
+
* `dispatch-matched-workflow.ts`, the only caller.
|
|
805
751
|
*/
|
|
806
752
|
provenance?: {
|
|
807
753
|
chainDepth?: number;
|
|
@@ -809,21 +755,19 @@ export declare class ExecutionTracker {
|
|
|
809
755
|
rerunLineage?: RerunLineage;
|
|
810
756
|
}): Promise<void>;
|
|
811
757
|
/**
|
|
812
|
-
* Write a `failed` execution_runs row
|
|
813
|
-
* any in-memory run so the recorded failure
|
|
814
|
-
*
|
|
815
|
-
*
|
|
758
|
+
* Write a `failed` execution_runs row (with the structured init_failure) for
|
|
759
|
+
* an init failure, and stamp any in-memory run complete so the recorded failure
|
|
760
|
+
* is the run's final word. Forwarded via onExecutionStatusChange to the Platform
|
|
761
|
+
* projection.
|
|
816
762
|
*
|
|
817
|
-
* An existing row
|
|
818
|
-
*
|
|
819
|
-
*
|
|
820
|
-
*
|
|
821
|
-
*
|
|
822
|
-
* did not record the init failure, so it must not report one.
|
|
763
|
+
* An existing row is overwritten while live (dispatch may have registered the
|
|
764
|
+
* run before init failed) and left alone once terminal, so a finished run is
|
|
765
|
+
* never rewritten as failed. When the guard leaves it alone, the failure
|
|
766
|
+
* metric and Platform forward are suppressed too: this orchestrator did not
|
|
767
|
+
* record the init failure, so it must not report one.
|
|
823
768
|
*
|
|
824
|
-
*
|
|
825
|
-
*
|
|
826
|
-
* leave no trace on the dashboard.
|
|
769
|
+
* Without this, the secret / install-secret / all-jobs-rejected early exits in
|
|
770
|
+
* dispatch-matched-workflow leave no trace on the dashboard.
|
|
827
771
|
*/
|
|
828
772
|
recordInitFailureRun(args: {
|
|
829
773
|
runId: string;
|
|
@@ -840,25 +784,21 @@ export declare class ExecutionTracker {
|
|
|
840
784
|
commitMessage?: string;
|
|
841
785
|
/**
|
|
842
786
|
* The repository that DEFINES the workflow this run was about to execute.
|
|
843
|
-
* REQUIRED
|
|
844
|
-
*
|
|
845
|
-
*
|
|
846
|
-
* workflow lives in this run's own repository", which the whole
|
|
847
|
-
* defining-repository predicate then reads as fact.
|
|
787
|
+
* REQUIRED so a caller that omits it does not compile: a null marker means
|
|
788
|
+
* "the workflow lives in this run's own repository", not "unknown", and the
|
|
789
|
+
* defining-repository predicate reads it as fact.
|
|
848
790
|
*
|
|
849
|
-
* Narrowed to the differing case
|
|
850
|
-
*
|
|
851
|
-
* and records nothing.
|
|
791
|
+
* Narrowed to the differing case as in `onExecutionStarted`, so passing
|
|
792
|
+
* `repoIdentifier` (the per-repository case) records nothing.
|
|
852
793
|
*/
|
|
853
794
|
workflowRepoIdentifier: string;
|
|
854
795
|
/**
|
|
855
796
|
* Inherited invoke-chain depth for a run summoned by an invoke gate.
|
|
856
797
|
*
|
|
857
|
-
*
|
|
858
|
-
*
|
|
859
|
-
*
|
|
860
|
-
*
|
|
861
|
-
* have started the chain it actually died inside.
|
|
798
|
+
* Unlike a hold ({@link ExecutionTracker.recordRunHeld}), an init failure is
|
|
799
|
+
* terminal and never fires a gate, so nothing reads it to bound recursion.
|
|
800
|
+
* It is recorded because this row is the run's ONLY record, and a `0`
|
|
801
|
+
* claims it started the chain it died inside.
|
|
862
802
|
*/
|
|
863
803
|
chainDepth?: number;
|
|
864
804
|
/**
|
|
@@ -866,12 +806,10 @@ export declare class ExecutionTracker {
|
|
|
866
806
|
* the run's `trigger_decision` marker
|
|
867
807
|
* (`EventRouter.isFailureLifecycleRun`).
|
|
868
808
|
*
|
|
869
|
-
*
|
|
870
|
-
* `__workflow_complete
|
|
871
|
-
*
|
|
872
|
-
*
|
|
873
|
-
* so adding an emit here later cannot silently re-open the self-retrigger
|
|
874
|
-
* loop.
|
|
809
|
+
* For consistency, not a live reader: this path emits no
|
|
810
|
+
* `__workflow_complete`. The row then agrees with the one
|
|
811
|
+
* {@link ExecutionTracker.recordRunHeld} writes for the same dispatch, and a
|
|
812
|
+
* later emit here cannot silently re-open the self-retrigger loop.
|
|
875
813
|
*/
|
|
876
814
|
dispatchedByFailureLifecycle?: boolean;
|
|
877
815
|
/** Re-run lineage, when this run re-runs another. */
|
|
@@ -881,20 +819,16 @@ export declare class ExecutionTracker {
|
|
|
881
819
|
* Write one `failed` execution_runs row for a global eval round that never
|
|
882
820
|
* produced verdicts.
|
|
883
821
|
*
|
|
884
|
-
*
|
|
885
|
-
*
|
|
886
|
-
* that outcome is invisible: no run was ever created for those workflows, so
|
|
887
|
-
* there is nothing on the dashboard to explain why they did not appear.
|
|
822
|
+
* A failed round suppresses every organization-wide workflow it was deciding
|
|
823
|
+
* on; without this row nothing on the dashboard explains their absence.
|
|
888
824
|
*
|
|
889
|
-
* **One row for the whole round, not one per workflow
|
|
890
|
-
*
|
|
891
|
-
*
|
|
892
|
-
* affected workflow instead.
|
|
825
|
+
* **One row for the whole round, not one per workflow:** the round collapses
|
|
826
|
+
* N candidate workflows into one pre-run job, and N rows would undo that. The
|
|
827
|
+
* reason names every affected workflow instead.
|
|
893
828
|
*
|
|
894
|
-
*
|
|
895
|
-
* `dispatch_queue` row
|
|
896
|
-
* guard
|
|
897
|
-
* belongs to no other run.
|
|
829
|
+
* `runId` is the round job's (last attempt's) own, so the row, the
|
|
830
|
+
* `dispatch_queue` row and the attempt's logs share one id. `doNothing` is
|
|
831
|
+
* guard enough: the id was minted for this round and belongs to no other run.
|
|
898
832
|
*/
|
|
899
833
|
recordGlobalEvalRoundFailureRun(args: {
|
|
900
834
|
runId: string;
|
|
@@ -910,30 +844,26 @@ export declare class ExecutionTracker {
|
|
|
910
844
|
* The source `providerContext` was taken from, when a cross-provider
|
|
911
845
|
* lock-file fallback made it a different source from `routingKey`.
|
|
912
846
|
*
|
|
913
|
-
*
|
|
914
|
-
*
|
|
915
|
-
*
|
|
916
|
-
* gives one source's credentials to another source's API client.
|
|
847
|
+
* The two are a PAIR: a re-run re-drives the organization-wide pass with the
|
|
848
|
+
* bundle those credentials belong to. Pairing the stored context with the
|
|
849
|
+
* inbound routing key gives one source's credentials to another's API client.
|
|
917
850
|
*/
|
|
918
851
|
dispatchRoutingKey?: string;
|
|
919
852
|
failureReason: string;
|
|
920
853
|
triggerEvent?: string;
|
|
921
854
|
/**
|
|
922
|
-
* The repository whose global workflows the round was deciding
|
|
923
|
-
*
|
|
924
|
-
* repository from `repoIdentifier` — recorded, like everywhere else, only
|
|
925
|
-
* when the two actually differ.
|
|
855
|
+
* The repository whose global workflows the round was deciding — normally
|
|
856
|
+
* not `repoIdentifier`, and recorded only when the two differ.
|
|
926
857
|
*/
|
|
927
858
|
workflowRepoIdentifier: string;
|
|
928
859
|
}): Promise<void>;
|
|
929
860
|
/**
|
|
930
861
|
* Record a run paused at the workflow install gate (a `registries:` /
|
|
931
|
-
* `installEnv:` protection rule returned hold / wait / queue)
|
|
932
|
-
* `execution_runs` row
|
|
933
|
-
*
|
|
934
|
-
*
|
|
935
|
-
*
|
|
936
|
-
* dropped so jobs registered before the gate cannot roll it up.
|
|
862
|
+
* `installEnv:` protection rule returned hold / wait / queue) as a resumable
|
|
863
|
+
* `held` `execution_runs` row the dashboard run list shows. No jobs are
|
|
864
|
+
* tracked: the caller's held_runs row and pending workflow context keep the
|
|
865
|
+
* run from counting as complete, and any in-memory run is dropped so jobs
|
|
866
|
+
* registered before the gate cannot roll it up.
|
|
937
867
|
*
|
|
938
868
|
* An existing row for this runId is flipped to `held` while it is still live
|
|
939
869
|
* and left alone once it is terminal — a finished run is never re-held.
|
|
@@ -953,35 +883,26 @@ export declare class ExecutionTracker {
|
|
|
953
883
|
triggerEvent?: string;
|
|
954
884
|
commitMessage?: string;
|
|
955
885
|
/**
|
|
956
|
-
* Pull-request number for PR-triggered holds
|
|
957
|
-
*
|
|
958
|
-
*
|
|
959
|
-
* security hold fail-closed unreachable by the comment path.
|
|
886
|
+
* Pull-request number for PR-triggered holds. PR-scoped
|
|
887
|
+
* `/kici approve|reject` joins `execution_runs` on `pr_number`; a NULL
|
|
888
|
+
* leaves a security hold fail-closed unreachable by the comment path.
|
|
960
889
|
*/
|
|
961
890
|
prNumber?: number | null;
|
|
962
891
|
/**
|
|
963
|
-
* The repository that DEFINES the held workflow. REQUIRED
|
|
964
|
-
*
|
|
965
|
-
*
|
|
966
|
-
* whole time it sits in the queue and for the run it resumes into.
|
|
892
|
+
* The repository that DEFINES the held workflow. REQUIRED as on
|
|
893
|
+
* {@link ExecutionTracker.recordInitFailureRun}: a null marker would
|
|
894
|
+
* misattribute the hold while it waits and the run it resumes into.
|
|
967
895
|
*/
|
|
968
896
|
workflowRepoIdentifier: string;
|
|
969
|
-
/**
|
|
970
|
-
* Commit of the workflow repository the held run dispatches, recorded only
|
|
971
|
-
* when the workflow is defined in another repository.
|
|
972
|
-
*/
|
|
897
|
+
/** Workflow repository commit, recorded only for a cross-repository workflow. */
|
|
973
898
|
workflowSha?: string | null;
|
|
974
|
-
/**
|
|
975
|
-
* The workflow repository's registered branch, recorded only when the
|
|
976
|
-
* workflow is defined in another repository.
|
|
977
|
-
*/
|
|
899
|
+
/** Workflow repository registered branch, recorded only for a cross-repository workflow. */
|
|
978
900
|
workflowBranch?: string | null;
|
|
979
901
|
/**
|
|
980
|
-
* Inherited invoke-chain depth for a run summoned by an invoke gate.
|
|
981
|
-
*
|
|
982
|
-
*
|
|
983
|
-
*
|
|
984
|
-
* default, which means "this run starts a chain".
|
|
902
|
+
* Inherited invoke-chain depth for a run summoned by an invoke gate. Any
|
|
903
|
+
* caller that has one MUST thread it: the resumed run can fire its own
|
|
904
|
+
* invoke gate, and the chain-depth circuit breaker reads this column back.
|
|
905
|
+
* Absent ⇒ the `0` default, meaning "this run starts a chain".
|
|
985
906
|
*/
|
|
986
907
|
chainDepth?: number;
|
|
987
908
|
/**
|
|
@@ -993,51 +914,46 @@ export declare class ExecutionTracker {
|
|
|
993
914
|
/** Re-run lineage, when this run re-runs another. */
|
|
994
915
|
rerunLineage?: RerunLineage;
|
|
995
916
|
/**
|
|
996
|
-
* True
|
|
997
|
-
*
|
|
998
|
-
* written here because a held round is recorded through this method.
|
|
917
|
+
* True for a held pre-run global evaluation round: the marker the release
|
|
918
|
+
* and re-run paths branch on.
|
|
999
919
|
*/
|
|
1000
920
|
isGlobalEvalRound?: boolean;
|
|
1001
921
|
/**
|
|
1002
|
-
* For a held evaluation round: the workflows
|
|
1003
|
-
* its release can refuse when one
|
|
1004
|
-
*
|
|
922
|
+
* For a held evaluation round: the workflows it covers (in the
|
|
923
|
+
* `trigger_decision` blob), so its release can refuse when one lost its
|
|
924
|
+
* registration while the round waited.
|
|
1005
925
|
*/
|
|
1006
926
|
heldRoundWorkflows?: readonly string[];
|
|
1007
927
|
/**
|
|
1008
|
-
* The source `providerContext` was taken from, when
|
|
1009
|
-
*
|
|
1010
|
-
*
|
|
1011
|
-
* {@link ExecutionTracker.recordGlobalEvalRoundFailureRun}.
|
|
928
|
+
* The source `providerContext` was taken from, when not `routingKey`; a held
|
|
929
|
+
* round's release re-drives the organization-wide pass with that pair (see
|
|
930
|
+
* {@link ExecutionTracker.recordGlobalEvalRoundFailureRun}).
|
|
1012
931
|
*/
|
|
1013
932
|
dispatchRoutingKey?: string;
|
|
1014
933
|
}): Promise<void>;
|
|
1015
934
|
/**
|
|
1016
|
-
* Claim a held global evaluation round for
|
|
1017
|
-
*
|
|
1018
|
-
*
|
|
1019
|
-
* round at most once.
|
|
935
|
+
* Claim a held global evaluation round for release (`held` → `pending`).
|
|
936
|
+
* False when another release already claimed it, so a re-fired release
|
|
937
|
+
* signal dispatches the round at most once.
|
|
1020
938
|
*
|
|
1021
|
-
* `pending
|
|
1022
|
-
*
|
|
1023
|
-
*
|
|
1024
|
-
*
|
|
1025
|
-
* job rows and no workflow timeout.
|
|
939
|
+
* `pending` (as `resumeHeldRun`), not `running`: orphan recovery fails a
|
|
940
|
+
* jobless `running` row older than its stale threshold, and a round row keeps
|
|
941
|
+
* its hold's `started_at` and never has job rows. No sweeper selects a
|
|
942
|
+
* jobless `pending` row with no workflow timeout.
|
|
1026
943
|
*/
|
|
1027
944
|
claimHeldGlobalEvalRound(runId: string): Promise<boolean>;
|
|
1028
945
|
/**
|
|
1029
946
|
* Terminalize a held global evaluation round whose hold was released.
|
|
1030
947
|
*
|
|
1031
|
-
* The released round runs as
|
|
1032
|
-
*
|
|
1033
|
-
*
|
|
1034
|
-
* `
|
|
1035
|
-
*
|
|
1036
|
-
* so only the release that claimed it settles it.
|
|
948
|
+
* The released round runs as its own job and creates no run row, so nothing
|
|
949
|
+
* else settles this row. `success` means it reached a verdict for its
|
|
950
|
+
* workflow repository; `failed` that it did not, with the reason. Guarded on
|
|
951
|
+
* `pending` (set by {@link ExecutionTracker.claimHeldGlobalEvalRound}), so
|
|
952
|
+
* only the claiming release settles it.
|
|
1037
953
|
*
|
|
1038
|
-
* A settled round is not an execution:
|
|
1039
|
-
*
|
|
1040
|
-
*
|
|
954
|
+
* A settled round is not an execution: a success counts no
|
|
955
|
+
* `executions_total` sample, and the frame carries `isGlobalEvalRound` so the
|
|
956
|
+
* Platform raises no run notification or outbound webhook.
|
|
1041
957
|
*/
|
|
1042
958
|
completeReleasedGlobalEvalRound(runId: string, outcome: {
|
|
1043
959
|
status: typeof ExecutionRunStatus.enum.success;
|
|
@@ -1046,8 +962,8 @@ export declare class ExecutionTracker {
|
|
|
1046
962
|
reason: string;
|
|
1047
963
|
}): Promise<void>;
|
|
1048
964
|
/**
|
|
1049
|
-
* Flip a `held` run back to `pending`
|
|
1050
|
-
*
|
|
965
|
+
* Flip a `held` run back to `pending` for the resumed dispatch. True when a
|
|
966
|
+
* held row was updated.
|
|
1051
967
|
*/
|
|
1052
968
|
resumeHeldRun(runId: string): Promise<boolean>;
|
|
1053
969
|
/**
|
|
@@ -1058,93 +974,73 @@ export declare class ExecutionTracker {
|
|
|
1058
974
|
/**
|
|
1059
975
|
* Cancel a run that has no job rows: a run cancelled before it dispatched
|
|
1060
976
|
* any job, a pre-run evaluation round, or a `pending` row a crash left
|
|
1061
|
-
* behind. No job will
|
|
1062
|
-
*
|
|
977
|
+
* behind. No job will report a terminal status, so the cancel writes and
|
|
978
|
+
* forwards the terminal row itself, as `cancelHeldRun` does.
|
|
1063
979
|
*
|
|
1064
|
-
* Guarded on
|
|
1065
|
-
*
|
|
1066
|
-
*
|
|
1067
|
-
* The cancellation reason already stamped on the row wins over `reason`.
|
|
980
|
+
* Guarded on non-terminal, not `held`, and no job rows: a run that gained a
|
|
981
|
+
* job meanwhile is left to job-driven completion, a held run to the
|
|
982
|
+
* hold-rejection path. A reason already on the row wins over `reason`.
|
|
1068
983
|
* Returns whether a row was cancelled.
|
|
1069
984
|
*/
|
|
1070
985
|
cancelJoblessRun(runId: string, reason: string): Promise<boolean>;
|
|
1071
986
|
/**
|
|
1072
|
-
*
|
|
1073
|
-
*
|
|
1074
|
-
*
|
|
1075
|
-
*
|
|
1076
|
-
* Instead of leaving the run in 'running' for OrphanRecovery to catch after 5 min,
|
|
1077
|
-
* this fails it right away.
|
|
987
|
+
* Fail a run immediately, e.g. when no agents can take any job (cron-triggered
|
|
988
|
+
* runs on a cluster leader without matching local agents and no reachable
|
|
989
|
+
* peers), rather than leave it 'running' for OrphanRecovery to catch after
|
|
990
|
+
* 5 min.
|
|
1078
991
|
*/
|
|
1079
992
|
failRun(runId: string, reason: string, initFailure?: InitFailure): Promise<void>;
|
|
1080
993
|
/**
|
|
1081
|
-
*
|
|
994
|
+
* Upsert a step's execution_steps row (started_at on 'running';
|
|
995
|
+
* completed_at, duration_ms, exit_code on terminal), with its log_path, and
|
|
996
|
+
* forward it to the Platform.
|
|
1082
997
|
*
|
|
1083
|
-
*
|
|
1084
|
-
*
|
|
1085
|
-
*
|
|
1086
|
-
*
|
|
1087
|
-
* @param logBytesStreamed Raw byte total reported by the agent's
|
|
1088
|
-
* LogStreamer at terminal step time. Accumulated into a per-(run, job)
|
|
1089
|
-
* counter that lands in `execution_jobs.log_bytes` and
|
|
1090
|
-
* `execution_runs.log_bytes` when the job and run reach terminal state,
|
|
1091
|
-
* feeding the operator-side `kici_org_log_bytes` capacity-planning gauge
|
|
1092
|
-
* on the Platform.
|
|
998
|
+
* @param logBytesStreamed Raw byte total from the agent's LogStreamer at
|
|
999
|
+
* terminal step time; summed into `execution_jobs.log_bytes` and
|
|
1000
|
+
* `execution_runs.log_bytes`, feeding the Platform's operator-side
|
|
1001
|
+
* `kici_org_log_bytes` capacity-planning gauge.
|
|
1093
1002
|
*/
|
|
1094
1003
|
onStepStatus(runId: string, jobId: string, stepIndex: number, stepName: string, state: string, timestamp: number, data?: Record<string, unknown>, logBytesStreamed?: number): Promise<void>;
|
|
1095
|
-
/**
|
|
1096
|
-
* Check if all tracked jobs in a run have reached terminal state.
|
|
1097
|
-
*/
|
|
1004
|
+
/** Whether every tracked job of the run is terminal. */
|
|
1098
1005
|
isRunComplete(runId: string): boolean;
|
|
1099
1006
|
/**
|
|
1100
1007
|
* Take a token holding a run open while some of its jobs are still to be
|
|
1101
|
-
* registered.
|
|
1102
|
-
*
|
|
1008
|
+
* registered. False for an unknown or already-completed run.
|
|
1009
|
+
*
|
|
1010
|
+
* The registration windows: a run whose source-pack `__build__` job goes
|
|
1011
|
+
* first is registered with that job ALONE, its real jobs dispatched after the
|
|
1012
|
+
* build; a run without a build is registered with NO jobs before its dispatch
|
|
1013
|
+
* loop; a deferred init job and a deferred dynamic entry register from a
|
|
1014
|
+
* fire-and-forget task that outlives the dispatch call. Without a token, the
|
|
1015
|
+
* already-registered jobs going terminal satisfy {@link isRunComplete}, so a
|
|
1016
|
+
* terminal status is written, the provider check posted and the status
|
|
1017
|
+
* forwarded — before a single real job has run.
|
|
1103
1018
|
*
|
|
1104
|
-
*
|
|
1105
|
-
* job
|
|
1106
|
-
*
|
|
1107
|
-
*
|
|
1108
|
-
*
|
|
1109
|
-
*
|
|
1110
|
-
* the dispatch call. In every case, without a token the already
|
|
1111
|
-
* -registered jobs reaching a terminal state satisfies {@link isRunComplete},
|
|
1112
|
-
* so the run is finalized early: a terminal run status is written, the
|
|
1113
|
-
* provider check is posted, and the status is forwarded to the Platform — all
|
|
1114
|
-
* before a single real job has run.
|
|
1019
|
+
* The hold is a counter read only by {@link isRunComplete}, deliberately NOT
|
|
1020
|
+
* a synthetic job-map entry: every outward projection enumerates the job map
|
|
1021
|
+
* (the Platform `state.replay` snapshot and its `execution_jobs` mirror, the
|
|
1022
|
+
* job count, workflow-complete job results, the active-run summary), so a
|
|
1023
|
+
* marker there would be a phantom pending job and an off-by-one count. The
|
|
1024
|
+
* counter writes no `execution_jobs` row either.
|
|
1115
1025
|
*
|
|
1116
|
-
* The hold is a counter on the in-memory run, read only by
|
|
1117
|
-
* {@link isRunComplete} — deliberately NOT a synthetic entry in the run's job
|
|
1118
|
-
* map. The job map is what every outward projection enumerates (the Platform
|
|
1119
|
-
* `state.replay` snapshot and its `execution_jobs` mirror, the reported job
|
|
1120
|
-
* count, the workflow-complete job results, the active-run summary), so a
|
|
1121
|
-
* marker parked there would surface as a phantom pending job on the dashboard
|
|
1122
|
-
* and an off-by-one job count for the length of the window. A counter holds
|
|
1123
|
-
* the completion check open without being visible to any of them, and it
|
|
1124
|
-
* writes no `execution_jobs` row, so it never surfaces as a phantom job.
|
|
1125
1026
|
* A token whose holder never settles keeps the run open until the stale
|
|
1126
|
-
* detector reaps it
|
|
1127
|
-
*
|
|
1128
|
-
*
|
|
1129
|
-
* `dispatch_queue`. So a holder that registers NO jobs must terminalize the
|
|
1130
|
-
* run itself rather than rely on the sweep — which is what
|
|
1027
|
+
* detector reaps it, which reaches only a run with at least one job (every
|
|
1028
|
+
* sub-scan starts from `execution_jobs` or `dispatch_queue`). So a holder that
|
|
1029
|
+
* registers NO jobs must terminalize the run itself, as
|
|
1131
1030
|
* `dispatchMatchedWorkflow` does when its dispatch loop throws.
|
|
1132
1031
|
*
|
|
1133
1032
|
* Each token must be paired with exactly one {@link releasePendingJobsHold}.
|
|
1134
1033
|
*/
|
|
1135
1034
|
holdRunForPendingJobs(runId: string): boolean;
|
|
1136
1035
|
/**
|
|
1137
|
-
* Record this coordinator
|
|
1138
|
-
*
|
|
1139
|
-
* form, which a sibling coordinator finalizing the run consults.
|
|
1036
|
+
* Record this coordinator in `execution_runs.registration_window_instance_id`
|
|
1037
|
+
* — the token's durable form, which a sibling finalizing the run consults.
|
|
1140
1038
|
*
|
|
1141
|
-
* Chained
|
|
1142
|
-
*
|
|
1143
|
-
*
|
|
1144
|
-
*
|
|
1145
|
-
*
|
|
1146
|
-
* sibling may finish. A coordinator with no instance id holds no durable
|
|
1147
|
-
* window.
|
|
1039
|
+
* Chained, not awaited: the caller is synchronous by contract (the token must
|
|
1040
|
+
* be counted before the next `await` can drop a sibling token), and the clear
|
|
1041
|
+
* on the last release awaits the chain. `registrationWindowSettled` exposes
|
|
1042
|
+
* it to a caller that needs the window durable before waiting on a job a
|
|
1043
|
+
* sibling may finish. No instance id, no durable window.
|
|
1148
1044
|
*/
|
|
1149
1045
|
private openRegistrationWindow;
|
|
1150
1046
|
/**
|
|
@@ -1159,108 +1055,78 @@ export declare class ExecutionTracker {
|
|
|
1159
1055
|
* Drop one token taken by {@link holdRunForPendingJobs}, and finalize the run
|
|
1160
1056
|
* if dropping the LAST one left every remaining job terminal.
|
|
1161
1057
|
*
|
|
1162
|
-
*
|
|
1163
|
-
*
|
|
1164
|
-
*
|
|
1165
|
-
*
|
|
1166
|
-
* `running` forever. Only the last release may finalize — while any other
|
|
1167
|
-
* token is outstanding more jobs are still on their way.
|
|
1058
|
+
* As in `addJobsToRun`, nothing else drives a completion check once the last
|
|
1059
|
+
* token is gone, so a run whose jobs all finished meanwhile would hang in
|
|
1060
|
+
* `running` forever. Only the last release may finalize: while another token
|
|
1061
|
+
* is outstanding, more jobs are on their way.
|
|
1168
1062
|
*
|
|
1169
|
-
* Idempotent
|
|
1170
|
-
*
|
|
1171
|
-
*
|
|
1172
|
-
* holding it open.
|
|
1063
|
+
* Idempotent: a no-op with no token outstanding. The count clamps at zero, so
|
|
1064
|
+
* a stray extra release cannot go negative and stop a genuinely outstanding
|
|
1065
|
+
* token from holding the run open.
|
|
1173
1066
|
*/
|
|
1174
1067
|
releasePendingJobsHold(runId: string): Promise<void>;
|
|
1175
|
-
/**
|
|
1176
|
-
* Get the current overall status of a run.
|
|
1177
|
-
*/
|
|
1178
1068
|
getRunStatus(runId: string): Extract<ExecutionRunStatus, 'running' | 'success' | 'failed' | 'cancelled'>;
|
|
1179
|
-
/**
|
|
1180
|
-
* Look up a job name from in-memory state.
|
|
1181
|
-
* Used by LogWriter to construct log paths.
|
|
1182
|
-
*/
|
|
1069
|
+
/** In-memory job name lookup, used by LogWriter for log paths. */
|
|
1183
1070
|
getJobName(runId: string, jobId: string): string | undefined;
|
|
1184
1071
|
/**
|
|
1185
|
-
* Resolve the job name
|
|
1186
|
-
*
|
|
1187
|
-
*
|
|
1188
|
-
* `execution_steps.log_path` upsert in `onStepStatus
|
|
1189
|
-
*
|
|
1190
|
-
*
|
|
1191
|
-
*
|
|
1192
|
-
*
|
|
1193
|
-
*
|
|
1194
|
-
* `dispatch_queue.job_name` (the same source `recoverJobFromDispatchQueue`
|
|
1195
|
-
* trusts), and only `jobId` when the job is genuinely unknown.
|
|
1072
|
+
* Resolve the job name in a step-log storage path
|
|
1073
|
+
* (`executions/{runId}/job-{jobName}/…`), with a durable fallback. Both
|
|
1074
|
+
* writers — the chunk writer (`log-chunk-sink`) and the
|
|
1075
|
+
* `execution_steps.log_path` upsert in `onStepStatus` — use this, so they
|
|
1076
|
+
* cannot disagree. `getJobName` is `undefined` between dispatch and
|
|
1077
|
+
* `addJobsToRun`, which would put an early chunk under `job-{jobId}` while the
|
|
1078
|
+
* reader keys on `job-{realName}`, silently losing lines. Order: in-memory,
|
|
1079
|
+
* then `dispatch_queue.job_name` (as `recoverJobFromDispatchQueue` trusts),
|
|
1080
|
+
* then `jobId` for a genuinely unknown job.
|
|
1196
1081
|
*/
|
|
1197
1082
|
resolveJobName(runId: string, jobId: string): Promise<string>;
|
|
1198
1083
|
/**
|
|
1199
|
-
* Seed the in-memory job name
|
|
1200
|
-
*
|
|
1201
|
-
*
|
|
1202
|
-
* `
|
|
1203
|
-
*
|
|
1204
|
-
*
|
|
1205
|
-
*
|
|
1206
|
-
* reroute time, makes `resolveJobName`'s fast path correct from the first
|
|
1207
|
-
* relayed chunk, independent of when the worker's `dispatch_queue` row becomes
|
|
1208
|
-
* visible. In-memory only: the `execution_jobs` row is still created lazily by
|
|
1209
|
-
* the worker's first status update. Guarded against clobbering an existing
|
|
1210
|
-
* entry so a later terminal status is never overwritten.
|
|
1084
|
+
* Seed the in-memory job name of a worker-rerouted job at reroute time. It
|
|
1085
|
+
* never enters `run.jobs` via `addJobsToRun` and has no coordinator-owned
|
|
1086
|
+
* `dispatch_queue` row (the worker owns it under a fresh id), so
|
|
1087
|
+
* `resolveJobName` would fall back to the bare `jobId` for early log chunks
|
|
1088
|
+
* while the reader keys on the real name, losing lines. In-memory only: the
|
|
1089
|
+
* worker's first status update creates the `execution_jobs` row. An existing
|
|
1090
|
+
* entry is never clobbered, so a later terminal status survives.
|
|
1211
1091
|
*/
|
|
1212
1092
|
registerJobName(runId: string, jobId: string, jobName: string): void;
|
|
1213
1093
|
/**
|
|
1214
|
-
*
|
|
1215
|
-
*
|
|
1216
|
-
*
|
|
1217
|
-
* emitting jobId), but run context is keyed by runId. This maps job → run so
|
|
1218
|
-
* `getExecutionContext(runId)` resolves for an agent-emitted custom event.
|
|
1219
|
-
* In-memory only (matches `getExecutionContext`'s semantics); emit happens
|
|
1220
|
-
* mid-execution, so the owning run is always live in `this.runs`.
|
|
1094
|
+
* The in-memory run owning a dispatched jobId, so `getExecutionContext`
|
|
1095
|
+
* resolves for a job-scoped agent `event.emit`. In-memory only, like
|
|
1096
|
+
* `getExecutionContext`: emit happens mid-execution, so the run is live.
|
|
1221
1097
|
*/
|
|
1222
1098
|
getRunIdForJob(jobId: string): string | undefined;
|
|
1223
1099
|
/**
|
|
1224
|
-
*
|
|
1225
|
-
* Used by commit status reporting to access provider/repo/sha info.
|
|
1100
|
+
* In-memory execution context of a run, for commit status reporting.
|
|
1226
1101
|
*/
|
|
1227
1102
|
getExecutionContext(runId: string): ExecutionContext | undefined;
|
|
1228
1103
|
/**
|
|
1229
1104
|
* Stamp the run's resolved trust tier and lock-file branch onto the in-memory
|
|
1230
1105
|
* state, after `onExecutionStarted` recorded the row.
|
|
1231
1106
|
*
|
|
1232
|
-
*
|
|
1233
|
-
*
|
|
1234
|
-
*
|
|
1235
|
-
*
|
|
1236
|
-
*
|
|
1237
|
-
*
|
|
1238
|
-
* run — so the in-memory copy and the row CAN disagree, and neither is
|
|
1239
|
-
* authoritative for the other. `recoverRunFromDb` reconciles by reading the
|
|
1240
|
-
* row, so the durable value is the one that survives.
|
|
1107
|
+
* Trust resolves on its own path with its own `execution_runs` update. The
|
|
1108
|
+
* dispatch site calls this with the same values, but either write can be
|
|
1109
|
+
* lost (the DB update is fire-and-forget with its own `.catch()`, and this
|
|
1110
|
+
* call sits in a `try`/`catch`) — deliberately, as the note is worth less
|
|
1111
|
+
* than the run. So memory and the row CAN disagree; `recoverRunFromDb` reads
|
|
1112
|
+
* the row, so the durable value survives.
|
|
1241
1113
|
*
|
|
1242
|
-
*
|
|
1243
|
-
*
|
|
1114
|
+
* An unknown run is a no-op: the status updates that would read the fields
|
|
1115
|
+
* are not coming either.
|
|
1244
1116
|
*
|
|
1245
|
-
* Keep this synchronous
|
|
1246
|
-
*
|
|
1247
|
-
* back outside the guard.
|
|
1117
|
+
* Keep this synchronous: callers' `try`/`catch` catches a throw, not a
|
|
1118
|
+
* rejected promise.
|
|
1248
1119
|
*/
|
|
1249
1120
|
setRunTrustContext(runId: string, trustTier: string, lockFileSource?: string): void;
|
|
1250
1121
|
/**
|
|
1251
|
-
*
|
|
1252
|
-
*
|
|
1253
|
-
* Uses optimistic WHERE status='running' so completed jobs are not updated.
|
|
1254
|
-
* Called when the agent sends a job.heartbeat message.
|
|
1122
|
+
* Stamp a running job's heartbeat on an agent job.heartbeat; the
|
|
1123
|
+
* status='running' predicate leaves completed jobs alone.
|
|
1255
1124
|
*/
|
|
1256
1125
|
updateJobHeartbeat(runId: string, jobId: string): Promise<void>;
|
|
1257
|
-
/**
|
|
1258
|
-
* Get the count of active (non-completed) runs in memory.
|
|
1259
|
-
*/
|
|
1126
|
+
/** Count of non-completed runs in memory. */
|
|
1260
1127
|
getActiveRunCount(): number;
|
|
1261
1128
|
/**
|
|
1262
|
-
*
|
|
1263
|
-
* Returns run ID, workflow name, status, and job routing counts.
|
|
1129
|
+
* Summary of active runs (with job counts) for the cluster health API.
|
|
1264
1130
|
*/
|
|
1265
1131
|
getActiveRuns(): Array<{
|
|
1266
1132
|
runId: string;
|
|
@@ -1274,9 +1140,9 @@ export declare class ExecutionTracker {
|
|
|
1274
1140
|
};
|
|
1275
1141
|
}>;
|
|
1276
1142
|
/**
|
|
1277
|
-
*
|
|
1278
|
-
*
|
|
1279
|
-
*
|
|
1143
|
+
* Replay data for every in-memory run (active and recently completed), sent
|
|
1144
|
+
* as state.replay on reconnection so the Platform reconciles its
|
|
1145
|
+
* execution_runs and execution_jobs projections.
|
|
1280
1146
|
*/
|
|
1281
1147
|
getReplayData(): Array<{
|
|
1282
1148
|
runId: string;
|
|
@@ -1312,38 +1178,31 @@ export declare class ExecutionTracker {
|
|
|
1312
1178
|
/**
|
|
1313
1179
|
* Replay data merged with terminal runs read from the orchestrator's local DB.
|
|
1314
1180
|
*
|
|
1315
|
-
*
|
|
1316
|
-
*
|
|
1317
|
-
*
|
|
1318
|
-
*
|
|
1319
|
-
* `kici_org_executions_count` operator-aggregate gauge) silently undercounts.
|
|
1181
|
+
* After a crash/restart memory is empty, but runs completed before it are
|
|
1182
|
+
* still in `execution_runs`; without re-emitting them the Platform mirror
|
|
1183
|
+
* (and the `kici_org_executions_count` operator-aggregate gauge) silently
|
|
1184
|
+
* undercounts.
|
|
1320
1185
|
*
|
|
1321
|
-
*
|
|
1322
|
-
* then
|
|
1323
|
-
* `windowHours` (default 24h)
|
|
1324
|
-
* for in-memory entries (callers may rely on that for active-run metadata);
|
|
1325
|
-
* DB-backed entries are appended in `completed_at ASC` order.
|
|
1186
|
+
* Returns the in-memory items first, in order (callers may rely on it for
|
|
1187
|
+
* active-run metadata), then appends uncovered terminal DB runs completed
|
|
1188
|
+
* within `windowHours` (default 24h), in `completed_at ASC` order.
|
|
1326
1189
|
*
|
|
1327
|
-
* Idempotent: Platform's `state.replay` handler upserts on `run_id`, so
|
|
1328
|
-
* re-emitting
|
|
1329
|
-
* state alike.
|
|
1190
|
+
* Idempotent: the Platform's `state.replay` handler upserts on `run_id`, so
|
|
1191
|
+
* re-emitting a mirrored run is a no-op for billing and dashboard state.
|
|
1330
1192
|
*/
|
|
1331
1193
|
getReplayDataWithDb(windowHours?: number): Promise<ReturnType<ExecutionTracker['getReplayData']>>;
|
|
1332
1194
|
/**
|
|
1333
|
-
* Update a job's in-memory status
|
|
1334
|
-
*
|
|
1335
|
-
* the in-memory state kept in sync for run completion detection.
|
|
1195
|
+
* Update a job's in-memory status only, for StaleRunDetector, which writes
|
|
1196
|
+
* the DB itself and needs memory in sync for completion detection.
|
|
1336
1197
|
*/
|
|
1337
1198
|
updateInMemoryJob(runId: string, jobId: string, status: string): void;
|
|
1338
1199
|
/**
|
|
1339
|
-
* Forward a terminal job status to Platform
|
|
1340
|
-
*
|
|
1341
|
-
* but need to notify Platform so its execution_jobs projection stays in sync.
|
|
1200
|
+
* Forward a terminal job status to the Platform's execution_jobs projection,
|
|
1201
|
+
* for StaleRunDetector and OrphanRecovery, which write the DB directly.
|
|
1342
1202
|
*/
|
|
1343
1203
|
forwardJobTerminalStatus(runId: string, jobId: string, jobName: string, status: string, errorMessage?: string): void;
|
|
1344
1204
|
/**
|
|
1345
|
-
*
|
|
1346
|
-
* Public wrapper for use by StaleRunDetector and OrphanRecovery.
|
|
1205
|
+
* Public run.event emitter for StaleRunDetector and OrphanRecovery.
|
|
1347
1206
|
*/
|
|
1348
1207
|
emitInfraEvent(runId: string, eventType: string, opts?: {
|
|
1349
1208
|
jobId?: string;
|
|
@@ -1351,95 +1210,71 @@ export declare class ExecutionTracker {
|
|
|
1351
1210
|
durationMs?: number;
|
|
1352
1211
|
}): void;
|
|
1353
1212
|
/**
|
|
1354
|
-
* Cancel
|
|
1355
|
-
*
|
|
1356
|
-
*
|
|
1357
|
-
* recovery timeout), any in-progress steps remain with status='running' in the DB.
|
|
1358
|
-
* This method marks them as 'cancelled' so the dashboard doesn't show stale
|
|
1359
|
-
* running indicators for steps that are no longer executing.
|
|
1213
|
+
* Cancel a job's non-terminal steps after infrastructure terminated it (stale
|
|
1214
|
+
* detection, orphan recovery, recovery timeout), so the dashboard shows no
|
|
1215
|
+
* stale running steps.
|
|
1360
1216
|
*/
|
|
1361
1217
|
cancelStepsForJob(runId: string, jobId: string, reason: string): Promise<void>;
|
|
1362
1218
|
/**
|
|
1363
|
-
*
|
|
1364
|
-
*
|
|
1365
|
-
*
|
|
1366
|
-
* stale jobs.
|
|
1219
|
+
* Complete the run when every job is terminal, from memory or else from the
|
|
1220
|
+
* DB (crash recovery / pruned runs). Called by StaleRunDetector after
|
|
1221
|
+
* marking stale jobs.
|
|
1367
1222
|
*/
|
|
1368
1223
|
completeRunIfAllJobsTerminal(runId: string): Promise<void>;
|
|
1369
1224
|
/**
|
|
1370
|
-
* Path A helper: complete a run
|
|
1371
|
-
*
|
|
1372
|
-
* execution_runs terminal row, fires Platform-forwarding +
|
|
1373
|
-
* workflow-complete callbacks, and schedules in-memory pruning.
|
|
1225
|
+
* Path A helper: complete a locally tracked run from memory — terminal row,
|
|
1226
|
+
* Platform-forwarding and workflow-complete callbacks, pruning.
|
|
1374
1227
|
*
|
|
1375
|
-
* The terminal write is clobber-guarded.
|
|
1376
|
-
*
|
|
1377
|
-
*
|
|
1378
|
-
* the callbacks nor the completion metrics may claim it.
|
|
1228
|
+
* The terminal write is clobber-guarded. On rejection (the run finished on
|
|
1229
|
+
* the normal path) only pruning happens: the recomputed status is not this
|
|
1230
|
+
* orchestrator's record, so no callback or metric may claim it.
|
|
1379
1231
|
*/
|
|
1380
1232
|
private completeRunFromMemoryState;
|
|
1381
1233
|
/**
|
|
1382
1234
|
* Drop the in-memory state of every run another writer already finished.
|
|
1383
1235
|
*
|
|
1384
|
-
* A coordinator
|
|
1385
|
-
*
|
|
1386
|
-
*
|
|
1387
|
-
*
|
|
1388
|
-
*
|
|
1389
|
-
* run. The stale detector calls this on every tick. One query over the
|
|
1390
|
-
* non-finished runs held here; a run whose row is terminal is released the
|
|
1391
|
-
* way the finalizing path releases its own, minus the completion events and
|
|
1392
|
-
* the row write, which the finalizer already did.
|
|
1236
|
+
* A coordinator prunes a run only when it sees the last job finish. In a
|
|
1237
|
+
* cluster a sibling may see it instead, leaving this copy in memory for the
|
|
1238
|
+
* process lifetime (one `RunState` per cross-coordinator run). The stale
|
|
1239
|
+
* detector calls this every tick; a run whose row is terminal is released
|
|
1240
|
+
* like the finalizer's own, minus the completion events and row write.
|
|
1393
1241
|
*
|
|
1394
|
-
* A run
|
|
1395
|
-
*
|
|
1242
|
+
* A run whose row is non-terminal (with or without an open registration
|
|
1243
|
+
* window) is not touched.
|
|
1396
1244
|
*/
|
|
1397
1245
|
pruneRunsFinishedElsewhere(): Promise<number>;
|
|
1398
1246
|
/**
|
|
1399
|
-
*
|
|
1400
|
-
*
|
|
1401
|
-
*
|
|
1402
|
-
*
|
|
1403
|
-
* held here or is already finishing.
|
|
1247
|
+
* Finish here a run another writer finished: mark it complete so no
|
|
1248
|
+
* completion path re-enters it, shed its terminal-callback state, schedule
|
|
1249
|
+
* the prune. No event, no row write. False when not held or already
|
|
1250
|
+
* finishing.
|
|
1404
1251
|
*/
|
|
1405
1252
|
private releaseRunFinishedElsewhere;
|
|
1406
1253
|
/**
|
|
1407
1254
|
* Drop a finished run's in-memory state after a grace period, so a late
|
|
1408
1255
|
* status/heartbeat arriving just after completion still finds its run.
|
|
1409
1256
|
*
|
|
1410
|
-
*
|
|
1411
|
-
*
|
|
1412
|
-
*
|
|
1413
|
-
*
|
|
1414
|
-
* (`runLockTails` is deliberately not touched: it is keyed by runId but scoped
|
|
1415
|
-
* to a lock's holders, and `withRunLock` drops its own entry on release.)
|
|
1257
|
+
* The single owner of run-lifetime teardown, so every run-lifetime map is
|
|
1258
|
+
* cleared here — `testRunIds` too, or a stale-detector-finished run leaks its
|
|
1259
|
+
* observer marker. (`runLockTails` is deliberately skipped: scoped to a
|
|
1260
|
+
* lock's holders, `withRunLock` drops its own entry on release.)
|
|
1416
1261
|
*/
|
|
1417
1262
|
private scheduleRunPrune;
|
|
1418
1263
|
/**
|
|
1419
|
-
* Path B helper: complete a run
|
|
1420
|
-
*
|
|
1421
|
-
*
|
|
1422
|
-
* terminal row and fires Platform-forwarding callbacks.
|
|
1264
|
+
* Path B helper: complete a run from DB rows only (crash recovery / pruned
|
|
1265
|
+
* runs) — all-terminal predicate, run-state guard, terminal row,
|
|
1266
|
+
* Platform-forwarding callbacks.
|
|
1423
1267
|
*
|
|
1424
|
-
* The run-state read and the
|
|
1425
|
-
*
|
|
1426
|
-
*
|
|
1427
|
-
* completion metrics are suppressed with it.
|
|
1268
|
+
* The run-state read and the write are separate statements, so the write has
|
|
1269
|
+
* its own clobber guard; when a concurrent normal completion lands between
|
|
1270
|
+
* them, callbacks and metrics are suppressed too.
|
|
1428
1271
|
*/
|
|
1429
1272
|
private completeRunFromDbFallback;
|
|
1430
|
-
/**
|
|
1431
|
-
* Emit a run.event message to Platform via the onRunEventEmit callback.
|
|
1432
|
-
* Called at orchestrator lifecycle points for infrastructure event tracking.
|
|
1433
|
-
*/
|
|
1273
|
+
/** Emit a run.event to the Platform at an orchestrator lifecycle point. */
|
|
1434
1274
|
private emitRunEvent;
|
|
1435
1275
|
/**
|
|
1436
|
-
* Emit a scaler lifecycle event
|
|
1437
|
-
*
|
|
1438
|
-
* and writes a log line to the provisioning JSONL file.
|
|
1439
|
-
*
|
|
1440
|
-
* @param runId - The run this event belongs to
|
|
1441
|
-
* @param jobId - The job this event belongs to
|
|
1442
|
-
* @param event - The scaler event with type, detail, and timestamp
|
|
1276
|
+
* Emit a job's scaler lifecycle event: stored in event_log via the run.event
|
|
1277
|
+
* pipeline and written to the provisioning JSONL file.
|
|
1443
1278
|
*/
|
|
1444
1279
|
emitScalerEvent(runId: string, jobId: string, event: {
|
|
1445
1280
|
agentId: string;
|
|
@@ -1448,34 +1283,29 @@ export declare class ExecutionTracker {
|
|
|
1448
1283
|
timestampMs: number;
|
|
1449
1284
|
}): void;
|
|
1450
1285
|
/**
|
|
1451
|
-
* Append a line to a
|
|
1452
|
-
*
|
|
1453
|
-
* Path convention: executions/{runId}/jobs/{jobId}/orchestration.jsonl
|
|
1286
|
+
* Append a JSONL line to a job's orchestration log
|
|
1287
|
+
* (executions/{runId}/jobs/{jobId}/orchestration.jsonl).
|
|
1454
1288
|
*/
|
|
1455
1289
|
private writeOrchLog;
|
|
1456
1290
|
/**
|
|
1457
|
-
*
|
|
1458
|
-
* Lists failed job names for non-success outcomes.
|
|
1291
|
+
* Human-readable description of a non-success run (its failed job names).
|
|
1459
1292
|
*/
|
|
1460
1293
|
private buildRunDescription;
|
|
1461
1294
|
/**
|
|
1462
|
-
*
|
|
1463
|
-
*
|
|
1464
|
-
* Per locked decision:
|
|
1295
|
+
* Overall run status from job statuses:
|
|
1465
1296
|
* - success ONLY if ALL jobs pass (skipped jobs count as success)
|
|
1466
1297
|
* - failed if ANY job ended in a failure status (see `STATUS_FAILURE_CLASS`)
|
|
1467
1298
|
* - cancelled if ANY job cancelled (and none failed)
|
|
1468
1299
|
*/
|
|
1469
1300
|
private computeRunStatus;
|
|
1470
1301
|
/**
|
|
1471
|
-
* Derive the failure class from a terminal run's job statuses
|
|
1302
|
+
* Derive the failure class from a terminal run's job statuses; null for
|
|
1472
1303
|
* success (and any non-terminal status). `timed_out_stale` / `drift_dropped`
|
|
1473
|
-
* / `unroutable` jobs
|
|
1474
|
-
*
|
|
1475
|
-
*
|
|
1476
|
-
*
|
|
1477
|
-
*
|
|
1478
|
-
* own paths, not here.
|
|
1304
|
+
* / `unroutable` jobs (never dispatched, `runsOn` matched no agent, agent went
|
|
1305
|
+
* silent, or dropped by a topology reroute) are infra-class and collapse to
|
|
1306
|
+
* `timed_out`; any other `failed` run is a `step_failure`. `never_started`
|
|
1307
|
+
* (init failure) and `dead_orchestrator` (Platform-detected) are stamped by
|
|
1308
|
+
* their own paths.
|
|
1479
1309
|
*/
|
|
1480
1310
|
private computeFailureClass;
|
|
1481
1311
|
}
|