@omnitron-dev/omnitron 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/BUILD.json +5 -0
- package/dist/cli/omnitron.js +351 -29
- package/dist/cli/omnitron.js.map +1 -1
- package/dist/cluster/cluster.rpc-service.d.ts +40 -0
- package/dist/cluster/cluster.rpc-service.d.ts.map +1 -1
- package/dist/cluster/cluster.rpc-service.js +49 -3
- package/dist/cluster/cluster.rpc-service.js.map +1 -1
- package/dist/cluster/config-sync.d.ts.map +1 -1
- package/dist/cluster/config-sync.js.map +1 -1
- package/dist/cluster/leader-election.d.ts +90 -2
- package/dist/cluster/leader-election.d.ts.map +1 -1
- package/dist/cluster/leader-election.js +176 -22
- package/dist/cluster/leader-election.js.map +1 -1
- package/dist/cluster/mesh-link.d.ts +136 -0
- package/dist/cluster/mesh-link.d.ts.map +1 -0
- package/dist/cluster/mesh-link.js +212 -0
- package/dist/cluster/mesh-link.js.map +1 -0
- package/dist/cluster/mesh.d.ts +59 -0
- package/dist/cluster/mesh.d.ts.map +1 -0
- package/dist/cluster/mesh.js +117 -0
- package/dist/cluster/mesh.js.map +1 -0
- package/dist/cluster/slave-connector.d.ts +193 -9
- package/dist/cluster/slave-connector.d.ts.map +1 -1
- package/dist/cluster/slave-connector.js +409 -37
- package/dist/cluster/slave-connector.js.map +1 -1
- package/dist/commands/audit.d.ts +65 -0
- package/dist/commands/audit.d.ts.map +1 -0
- package/dist/commands/audit.js +233 -0
- package/dist/commands/audit.js.map +1 -0
- package/dist/commands/backup.d.ts +4 -0
- package/dist/commands/backup.d.ts.map +1 -1
- package/dist/commands/backup.js +255 -33
- package/dist/commands/backup.js.map +1 -1
- package/dist/commands/cluster.d.ts.map +1 -1
- package/dist/commands/cluster.js +78 -47
- package/dist/commands/cluster.js.map +1 -1
- package/dist/commands/daemon-cmd.d.ts +7 -1
- package/dist/commands/daemon-cmd.d.ts.map +1 -1
- package/dist/commands/daemon-cmd.js +87 -21
- package/dist/commands/daemon-cmd.js.map +1 -1
- package/dist/commands/daemon-required.d.ts +34 -0
- package/dist/commands/daemon-required.d.ts.map +1 -0
- package/dist/commands/daemon-required.js +92 -0
- package/dist/commands/daemon-required.js.map +1 -0
- package/dist/commands/deploy.d.ts +70 -8
- package/dist/commands/deploy.d.ts.map +1 -1
- package/dist/commands/deploy.js +99 -64
- package/dist/commands/deploy.js.map +1 -1
- package/dist/commands/discover.d.ts +25 -4
- package/dist/commands/discover.d.ts.map +1 -1
- package/dist/commands/discover.js +155 -102
- package/dist/commands/discover.js.map +1 -1
- package/dist/commands/doctor.d.ts +293 -0
- package/dist/commands/doctor.d.ts.map +1 -0
- package/dist/commands/doctor.js +1706 -0
- package/dist/commands/doctor.js.map +1 -0
- package/dist/commands/env.d.ts +8 -1
- package/dist/commands/env.d.ts.map +1 -1
- package/dist/commands/env.js +13 -8
- package/dist/commands/env.js.map +1 -1
- package/dist/commands/exec.d.ts.map +1 -1
- package/dist/commands/exec.js +3 -2
- package/dist/commands/exec.js.map +1 -1
- package/dist/commands/fleet-asking.d.ts +87 -0
- package/dist/commands/fleet-asking.d.ts.map +1 -0
- package/dist/commands/fleet-asking.js +176 -0
- package/dist/commands/fleet-asking.js.map +1 -0
- package/dist/commands/fleet.d.ts +18 -0
- package/dist/commands/fleet.d.ts.map +1 -1
- package/dist/commands/fleet.js +314 -74
- package/dist/commands/fleet.js.map +1 -1
- package/dist/commands/health-check.d.ts +8 -6
- package/dist/commands/health-check.d.ts.map +1 -1
- package/dist/commands/health-check.js +26 -23
- package/dist/commands/health-check.js.map +1 -1
- package/dist/commands/health.d.ts.map +1 -1
- package/dist/commands/health.js +22 -8
- package/dist/commands/health.js.map +1 -1
- package/dist/commands/infra.d.ts +42 -1
- package/dist/commands/infra.d.ts.map +1 -1
- package/dist/commands/infra.js +418 -49
- package/dist/commands/infra.js.map +1 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +21 -36
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/inspect-graph.d.ts.map +1 -1
- package/dist/commands/inspect-graph.js +2 -6
- package/dist/commands/inspect-graph.js.map +1 -1
- package/dist/commands/inspect.d.ts.map +1 -1
- package/dist/commands/inspect.js +110 -14
- package/dist/commands/inspect.js.map +1 -1
- package/dist/commands/k8s.d.ts.map +1 -1
- package/dist/commands/k8s.js +23 -18
- package/dist/commands/k8s.js.map +1 -1
- package/dist/commands/kb.d.ts.map +1 -1
- package/dist/commands/kb.js +88 -29
- package/dist/commands/kb.js.map +1 -1
- package/dist/commands/list.d.ts.map +1 -1
- package/dist/commands/list.js +21 -9
- package/dist/commands/list.js.map +1 -1
- package/dist/commands/log-health.d.ts +120 -0
- package/dist/commands/log-health.d.ts.map +1 -0
- package/dist/commands/log-health.js +212 -0
- package/dist/commands/log-health.js.map +1 -0
- package/dist/commands/logs.d.ts +96 -2
- package/dist/commands/logs.d.ts.map +1 -1
- package/dist/commands/logs.js +522 -196
- package/dist/commands/logs.js.map +1 -1
- package/dist/commands/metrics.d.ts.map +1 -1
- package/dist/commands/metrics.js +35 -17
- package/dist/commands/metrics.js.map +1 -1
- package/dist/commands/monit.d.ts.map +1 -1
- package/dist/commands/monit.js +3 -2
- package/dist/commands/monit.js.map +1 -1
- package/dist/commands/node.d.ts +68 -0
- package/dist/commands/node.d.ts.map +1 -1
- package/dist/commands/node.js +285 -68
- package/dist/commands/node.js.map +1 -1
- package/dist/commands/output.d.ts +10 -0
- package/dist/commands/output.d.ts.map +1 -1
- package/dist/commands/output.js +41 -1
- package/dist/commands/output.js.map +1 -1
- package/dist/commands/pipeline.d.ts.map +1 -1
- package/dist/commands/pipeline.js +26 -19
- package/dist/commands/pipeline.js.map +1 -1
- package/dist/commands/project.d.ts +19 -0
- package/dist/commands/project.d.ts.map +1 -1
- package/dist/commands/project.js +125 -27
- package/dist/commands/project.js.map +1 -1
- package/dist/commands/release-store.d.ts +28 -0
- package/dist/commands/release-store.d.ts.map +1 -0
- package/dist/commands/release-store.js +158 -0
- package/dist/commands/release-store.js.map +1 -0
- package/dist/commands/release.d.ts +61 -0
- package/dist/commands/release.d.ts.map +1 -0
- package/dist/commands/release.js +555 -0
- package/dist/commands/release.js.map +1 -0
- package/dist/commands/reload.d.ts +16 -0
- package/dist/commands/reload.d.ts.map +1 -1
- package/dist/commands/reload.js +21 -5
- package/dist/commands/reload.js.map +1 -1
- package/dist/commands/remote.d.ts +40 -0
- package/dist/commands/remote.d.ts.map +1 -1
- package/dist/commands/remote.js +146 -35
- package/dist/commands/remote.js.map +1 -1
- package/dist/commands/restart.d.ts +16 -0
- package/dist/commands/restart.d.ts.map +1 -1
- package/dist/commands/restart.js +21 -5
- package/dist/commands/restart.js.map +1 -1
- package/dist/commands/scale.d.ts.map +1 -1
- package/dist/commands/scale.js +4 -3
- package/dist/commands/scale.js.map +1 -1
- package/dist/commands/secret.d.ts +54 -0
- package/dist/commands/secret.d.ts.map +1 -1
- package/dist/commands/secret.js +236 -34
- package/dist/commands/secret.js.map +1 -1
- package/dist/commands/service-units.d.ts +181 -0
- package/dist/commands/service-units.d.ts.map +1 -0
- package/dist/commands/service-units.js +291 -0
- package/dist/commands/service-units.js.map +1 -0
- package/dist/commands/service.d.ts +39 -0
- package/dist/commands/service.d.ts.map +1 -0
- package/dist/commands/service.js +370 -0
- package/dist/commands/service.js.map +1 -0
- package/dist/commands/spinner.d.ts +29 -0
- package/dist/commands/spinner.d.ts.map +1 -0
- package/dist/commands/spinner.js +44 -0
- package/dist/commands/spinner.js.map +1 -0
- package/dist/commands/stack.d.ts +49 -1
- package/dist/commands/stack.d.ts.map +1 -1
- package/dist/commands/stack.js +325 -12
- package/dist/commands/stack.js.map +1 -1
- package/dist/commands/start.d.ts +16 -0
- package/dist/commands/start.d.ts.map +1 -1
- package/dist/commands/start.js +20 -3
- package/dist/commands/start.js.map +1 -1
- package/dist/commands/status.d.ts +9 -0
- package/dist/commands/status.d.ts.map +1 -1
- package/dist/commands/status.js +45 -29
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/stop.d.ts +16 -0
- package/dist/commands/stop.d.ts.map +1 -1
- package/dist/commands/stop.js +34 -6
- package/dist/commands/stop.js.map +1 -1
- package/dist/commands/tor.d.ts +14 -7
- package/dist/commands/tor.d.ts.map +1 -1
- package/dist/commands/tor.js +113 -33
- package/dist/commands/tor.js.map +1 -1
- package/dist/commands/up.d.ts +85 -1
- package/dist/commands/up.d.ts.map +1 -1
- package/dist/commands/up.js +180 -32
- package/dist/commands/up.js.map +1 -1
- package/dist/commands/webapp.d.ts.map +1 -1
- package/dist/commands/webapp.js +2 -1
- package/dist/commands/webapp.js.map +1 -1
- package/dist/config/defaults.d.ts +16 -0
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +26 -3
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/define-ecosystem.d.ts.map +1 -1
- package/dist/config/define-ecosystem.js +2 -0
- package/dist/config/define-ecosystem.js.map +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +1 -1
- package/dist/config/index.js.map +1 -1
- package/dist/config/loader.d.ts +45 -1
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +64 -6
- package/dist/config/loader.js.map +1 -1
- package/dist/config/omnitron-config.schema.d.ts.map +1 -1
- package/dist/config/omnitron-config.schema.js +16 -0
- package/dist/config/omnitron-config.schema.js.map +1 -1
- package/dist/config/types.d.ts +411 -42
- package/dist/config/types.d.ts.map +1 -1
- package/dist/daemon/config-reload.d.ts +27 -0
- package/dist/daemon/config-reload.d.ts.map +1 -0
- package/dist/daemon/config-reload.js +42 -0
- package/dist/daemon/config-reload.js.map +1 -0
- package/dist/daemon/daemon-client.d.ts +136 -2
- package/dist/daemon/daemon-client.d.ts.map +1 -1
- package/dist/daemon/daemon-client.js +307 -21
- package/dist/daemon/daemon-client.js.map +1 -1
- package/dist/daemon/daemon-entry.js +90 -30
- package/dist/daemon/daemon-entry.js.map +1 -1
- package/dist/daemon/daemon-scheduler.d.ts +4 -0
- package/dist/daemon/daemon-scheduler.d.ts.map +1 -1
- package/dist/daemon/daemon-scheduler.js +137 -7
- package/dist/daemon/daemon-scheduler.js.map +1 -1
- package/dist/daemon/daemon-state-store.service.d.ts +263 -0
- package/dist/daemon/daemon-state-store.service.d.ts.map +1 -0
- package/dist/daemon/daemon-state-store.service.js +446 -0
- package/dist/daemon/daemon-state-store.service.js.map +1 -0
- package/dist/daemon/daemon.d.ts +205 -1
- package/dist/daemon/daemon.d.ts.map +1 -1
- package/dist/daemon/daemon.js +1139 -139
- package/dist/daemon/daemon.js.map +1 -1
- package/dist/daemon/daemon.module.d.ts.map +1 -1
- package/dist/daemon/daemon.module.js +163 -83
- package/dist/daemon/daemon.module.js.map +1 -1
- package/dist/daemon/daemon.rpc-service.d.ts +90 -1
- package/dist/daemon/daemon.rpc-service.d.ts.map +1 -1
- package/dist/daemon/daemon.rpc-service.js +438 -96
- package/dist/daemon/daemon.rpc-service.js.map +1 -1
- package/dist/daemon/pid-manager.d.ts +16 -3
- package/dist/daemon/pid-manager.d.ts.map +1 -1
- package/dist/daemon/pid-manager.js +31 -4
- package/dist/daemon/pid-manager.js.map +1 -1
- package/dist/daemon/spawn-daemon.d.ts +73 -0
- package/dist/daemon/spawn-daemon.d.ts.map +1 -0
- package/dist/daemon/spawn-daemon.js +101 -0
- package/dist/daemon/spawn-daemon.js.map +1 -0
- package/dist/daemon/state-store.d.ts +65 -3
- package/dist/daemon/state-store.d.ts.map +1 -1
- package/dist/daemon/state-store.js +100 -58
- package/dist/daemon/state-store.js.map +1 -1
- package/dist/database/connection.d.ts +71 -0
- package/dist/database/connection.d.ts.map +1 -0
- package/dist/database/connection.js +129 -0
- package/dist/database/connection.js.map +1 -0
- package/dist/database/dialect.d.ts +60 -0
- package/dist/database/dialect.d.ts.map +1 -0
- package/dist/database/dialect.js +80 -0
- package/dist/database/dialect.js.map +1 -0
- package/dist/database/migrate.d.ts +5 -2
- package/dist/database/migrate.d.ts.map +1 -1
- package/dist/database/migrate.js +23 -33
- package/dist/database/migrate.js.map +1 -1
- package/dist/database/migration-runner.d.ts +50 -0
- package/dist/database/migration-runner.d.ts.map +1 -0
- package/dist/database/migration-runner.js +143 -0
- package/dist/database/migration-runner.js.map +1 -0
- package/dist/database/migrations/006_login_throttle.d.ts +17 -0
- package/dist/database/migrations/006_login_throttle.d.ts.map +1 -0
- package/dist/database/migrations/006_login_throttle.js +33 -0
- package/dist/database/migrations/006_login_throttle.js.map +1 -0
- package/dist/database/migrations/007_sync_ingested.d.ts +21 -0
- package/dist/database/migrations/007_sync_ingested.d.ts.map +1 -0
- package/dist/database/migrations/007_sync_ingested.js +39 -0
- package/dist/database/migrations/007_sync_ingested.js.map +1 -0
- package/dist/database/migrations/008_node_health_version_text.d.ts +24 -0
- package/dist/database/migrations/008_node_health_version_text.d.ts.map +1 -0
- package/dist/database/migrations/008_node_health_version_text.js +35 -0
- package/dist/database/migrations/008_node_health_version_text.js.map +1 -0
- package/dist/database/migrations/009_node_health_unmeasured.d.ts +40 -0
- package/dist/database/migrations/009_node_health_unmeasured.d.ts.map +1 -0
- package/dist/database/migrations/009_node_health_unmeasured.js +63 -0
- package/dist/database/migrations/009_node_health_unmeasured.js.map +1 -0
- package/dist/database/migrations/010_metrics_raw_had_no_reader.d.ts +22 -0
- package/dist/database/migrations/010_metrics_raw_had_no_reader.d.ts.map +1 -0
- package/dist/database/migrations/010_metrics_raw_had_no_reader.js +26 -0
- package/dist/database/migrations/010_metrics_raw_had_no_reader.js.map +1 -0
- package/dist/database/migrations/011_alerts_nobody_had_to_write.d.ts +56 -0
- package/dist/database/migrations/011_alerts_nobody_had_to_write.d.ts.map +1 -0
- package/dist/database/migrations/011_alerts_nobody_had_to_write.js +79 -0
- package/dist/database/migrations/011_alerts_nobody_had_to_write.js.map +1 -0
- package/dist/database/migrations/012_a_disk_nobody_watched.d.ts +26 -0
- package/dist/database/migrations/012_a_disk_nobody_watched.d.ts.map +1 -0
- package/dist/database/migrations/012_a_disk_nobody_watched.js +40 -0
- package/dist/database/migrations/012_a_disk_nobody_watched.js.map +1 -0
- package/dist/database/migrations/013_an_alarm_delivered.d.ts +14 -0
- package/dist/database/migrations/013_an_alarm_delivered.d.ts.map +1 -0
- package/dist/database/migrations/013_an_alarm_delivered.js +28 -0
- package/dist/database/migrations/013_an_alarm_delivered.js.map +1 -0
- package/dist/database/migrations/index.d.ts +21 -0
- package/dist/database/migrations/index.d.ts.map +1 -0
- package/dist/database/migrations/index.js +47 -0
- package/dist/database/migrations/index.js.map +1 -0
- package/dist/database/schema.d.ts +27 -11
- package/dist/database/schema.d.ts.map +1 -1
- package/dist/database/sqlite-date-binding.d.ts +71 -0
- package/dist/database/sqlite-date-binding.d.ts.map +1 -0
- package/dist/database/sqlite-date-binding.js +174 -0
- package/dist/database/sqlite-date-binding.js.map +1 -0
- package/dist/execution/data-channel.d.ts +48 -0
- package/dist/execution/data-channel.d.ts.map +1 -0
- package/dist/execution/data-channel.js +55 -0
- package/dist/execution/data-channel.js.map +1 -0
- package/dist/execution/execution.service.d.ts +139 -0
- package/dist/execution/execution.service.d.ts.map +1 -1
- package/dist/execution/execution.service.js +273 -10
- package/dist/execution/execution.service.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/infrastructure/bare-metal-plan.d.ts +208 -0
- package/dist/infrastructure/bare-metal-plan.d.ts.map +1 -0
- package/dist/infrastructure/bare-metal-plan.js +363 -0
- package/dist/infrastructure/bare-metal-plan.js.map +1 -0
- package/dist/infrastructure/bare-metal-runner.d.ts +65 -0
- package/dist/infrastructure/bare-metal-runner.d.ts.map +1 -0
- package/dist/infrastructure/bare-metal-runner.js +354 -0
- package/dist/infrastructure/bare-metal-runner.js.map +1 -0
- package/dist/infrastructure/config-payload.d.ts +149 -0
- package/dist/infrastructure/config-payload.d.ts.map +1 -0
- package/dist/infrastructure/config-payload.js +235 -0
- package/dist/infrastructure/config-payload.js.map +1 -0
- package/dist/infrastructure/container-runtime.d.ts +169 -1
- package/dist/infrastructure/container-runtime.d.ts.map +1 -1
- package/dist/infrastructure/container-runtime.js +525 -33
- package/dist/infrastructure/container-runtime.js.map +1 -1
- package/dist/infrastructure/gateway-reachability.d.ts +85 -0
- package/dist/infrastructure/gateway-reachability.d.ts.map +1 -0
- package/dist/infrastructure/gateway-reachability.js +116 -0
- package/dist/infrastructure/gateway-reachability.js.map +1 -0
- package/dist/infrastructure/gateway-redis-db.d.ts +29 -0
- package/dist/infrastructure/gateway-redis-db.d.ts.map +1 -0
- package/dist/infrastructure/gateway-redis-db.js +32 -0
- package/dist/infrastructure/gateway-redis-db.js.map +1 -0
- package/dist/infrastructure/host-inspection.d.ts +223 -0
- package/dist/infrastructure/host-inspection.d.ts.map +1 -0
- package/dist/infrastructure/host-inspection.js +501 -0
- package/dist/infrastructure/host-inspection.js.map +1 -0
- package/dist/infrastructure/infrastructure-gate.d.ts.map +1 -1
- package/dist/infrastructure/infrastructure.service.d.ts +206 -6
- package/dist/infrastructure/infrastructure.service.d.ts.map +1 -1
- package/dist/infrastructure/infrastructure.service.js +415 -89
- package/dist/infrastructure/infrastructure.service.js.map +1 -1
- package/dist/infrastructure/known-machines.d.ts +76 -0
- package/dist/infrastructure/known-machines.d.ts.map +1 -0
- package/dist/infrastructure/known-machines.js +94 -0
- package/dist/infrastructure/known-machines.js.map +1 -0
- package/dist/infrastructure/node-credentials.d.ts +45 -0
- package/dist/infrastructure/node-credentials.d.ts.map +1 -0
- package/dist/infrastructure/node-credentials.js +80 -0
- package/dist/infrastructure/node-credentials.js.map +1 -0
- package/dist/infrastructure/orphan-containers.d.ts +99 -0
- package/dist/infrastructure/orphan-containers.d.ts.map +1 -0
- package/dist/infrastructure/orphan-containers.js +106 -0
- package/dist/infrastructure/orphan-containers.js.map +1 -0
- package/dist/infrastructure/phantom-endpoint-janitor.d.ts +20 -0
- package/dist/infrastructure/phantom-endpoint-janitor.d.ts.map +1 -1
- package/dist/infrastructure/phantom-endpoint-janitor.js +56 -6
- package/dist/infrastructure/phantom-endpoint-janitor.js.map +1 -1
- package/dist/infrastructure/presets/gateway-openresty.d.ts.map +1 -1
- package/dist/infrastructure/presets/gateway-openresty.js +29 -3
- package/dist/infrastructure/presets/gateway-openresty.js.map +1 -1
- package/dist/infrastructure/presets/minio.d.ts.map +1 -1
- package/dist/infrastructure/presets/minio.js +36 -3
- package/dist/infrastructure/presets/minio.js.map +1 -1
- package/dist/infrastructure/presets/postgres.d.ts.map +1 -1
- package/dist/infrastructure/presets/postgres.js +69 -2
- package/dist/infrastructure/presets/postgres.js.map +1 -1
- package/dist/infrastructure/presets/redis.d.ts.map +1 -1
- package/dist/infrastructure/presets/redis.js +13 -0
- package/dist/infrastructure/presets/redis.js.map +1 -1
- package/dist/infrastructure/presets/registry.d.ts.map +1 -1
- package/dist/infrastructure/presets/registry.js +81 -12
- package/dist/infrastructure/presets/registry.js.map +1 -1
- package/dist/infrastructure/presets/tor.d.ts +38 -0
- package/dist/infrastructure/presets/tor.d.ts.map +1 -1
- package/dist/infrastructure/presets/tor.js +78 -69
- package/dist/infrastructure/presets/tor.js.map +1 -1
- package/dist/infrastructure/presets/torrc.d.ts +54 -0
- package/dist/infrastructure/presets/torrc.d.ts.map +1 -0
- package/dist/infrastructure/presets/torrc.js +153 -0
- package/dist/infrastructure/presets/torrc.js.map +1 -0
- package/dist/infrastructure/presets/types.d.ts +17 -0
- package/dist/infrastructure/presets/types.d.ts.map +1 -1
- package/dist/infrastructure/provisioning-outcome.d.ts +85 -0
- package/dist/infrastructure/provisioning-outcome.d.ts.map +1 -0
- package/dist/infrastructure/provisioning-outcome.js +122 -0
- package/dist/infrastructure/provisioning-outcome.js.map +1 -0
- package/dist/infrastructure/server-registry.d.ts +27 -0
- package/dist/infrastructure/server-registry.d.ts.map +1 -1
- package/dist/infrastructure/server-registry.js +60 -5
- package/dist/infrastructure/server-registry.js.map +1 -1
- package/dist/infrastructure/service-binding.d.ts +60 -0
- package/dist/infrastructure/service-binding.d.ts.map +1 -0
- package/dist/infrastructure/service-binding.js +90 -0
- package/dist/infrastructure/service-binding.js.map +1 -0
- package/dist/infrastructure/service-credentials.d.ts +105 -0
- package/dist/infrastructure/service-credentials.d.ts.map +1 -0
- package/dist/infrastructure/service-credentials.js +141 -0
- package/dist/infrastructure/service-credentials.js.map +1 -0
- package/dist/infrastructure/service-resolver.d.ts +160 -12
- package/dist/infrastructure/service-resolver.d.ts.map +1 -1
- package/dist/infrastructure/service-resolver.js +463 -53
- package/dist/infrastructure/service-resolver.js.map +1 -1
- package/dist/infrastructure/shipped-config.d.ts +43 -0
- package/dist/infrastructure/shipped-config.d.ts.map +1 -0
- package/dist/infrastructure/shipped-config.js +159 -0
- package/dist/infrastructure/shipped-config.js.map +1 -0
- package/dist/infrastructure/stack-infra-manager.d.ts +6 -0
- package/dist/infrastructure/stack-infra-manager.d.ts.map +1 -1
- package/dist/infrastructure/stack-infra-manager.js +31 -13
- package/dist/infrastructure/stack-infra-manager.js.map +1 -1
- package/dist/infrastructure/types.d.ts +261 -6
- package/dist/infrastructure/types.d.ts.map +1 -1
- package/dist/mcp/mcp-bridge.d.ts.map +1 -1
- package/dist/mcp/mcp-bridge.js.map +1 -1
- package/dist/mcp/tool-groups/apps.tools.d.ts +2 -1
- package/dist/mcp/tool-groups/apps.tools.d.ts.map +1 -1
- package/dist/mcp/tool-groups/apps.tools.js +2 -2
- package/dist/mcp/tool-groups/apps.tools.js.map +1 -1
- package/dist/mcp/tool-groups/infra.tools.d.ts +16 -2
- package/dist/mcp/tool-groups/infra.tools.d.ts.map +1 -1
- package/dist/mcp/tool-groups/infra.tools.js +51 -41
- package/dist/mcp/tool-groups/infra.tools.js.map +1 -1
- package/dist/mcp/tool-groups/kb.tools.d.ts.map +1 -1
- package/dist/mcp/tool-groups/kb.tools.js +4 -6
- package/dist/mcp/tool-groups/kb.tools.js.map +1 -1
- package/dist/mcp/tool-groups/management.tools.d.ts +20 -3
- package/dist/mcp/tool-groups/management.tools.d.ts.map +1 -1
- package/dist/mcp/tool-groups/management.tools.js +158 -125
- package/dist/mcp/tool-groups/management.tools.js.map +1 -1
- package/dist/mcp/tool-groups/monitoring.tools.d.ts +10 -1
- package/dist/mcp/tool-groups/monitoring.tools.d.ts.map +1 -1
- package/dist/mcp/tool-groups/monitoring.tools.js +65 -18
- package/dist/mcp/tool-groups/monitoring.tools.js.map +1 -1
- package/dist/mcp/unavailable-tools.d.ts +35 -0
- package/dist/mcp/unavailable-tools.d.ts.map +1 -0
- package/dist/mcp/unavailable-tools.js +59 -0
- package/dist/mcp/unavailable-tools.js.map +1 -0
- package/dist/monitoring/app-health.indicator.d.ts.map +1 -1
- package/dist/monitoring/app-health.indicator.js +12 -0
- package/dist/monitoring/app-health.indicator.js.map +1 -1
- package/dist/monitoring/desktop-notifier.d.ts +26 -0
- package/dist/monitoring/desktop-notifier.d.ts.map +1 -0
- package/dist/monitoring/desktop-notifier.js +49 -0
- package/dist/monitoring/desktop-notifier.js.map +1 -0
- package/dist/monitoring/docker-health.indicator.d.ts +17 -1
- package/dist/monitoring/docker-health.indicator.d.ts.map +1 -1
- package/dist/monitoring/docker-health.indicator.js +38 -8
- package/dist/monitoring/docker-health.indicator.js.map +1 -1
- package/dist/monitoring/event-loop-stall.indicator.d.ts +26 -0
- package/dist/monitoring/event-loop-stall.indicator.d.ts.map +1 -0
- package/dist/monitoring/event-loop-stall.indicator.js +47 -0
- package/dist/monitoring/event-loop-stall.indicator.js.map +1 -0
- package/dist/monitoring/event-loop-watch.d.ts +124 -0
- package/dist/monitoring/event-loop-watch.d.ts.map +1 -0
- package/dist/monitoring/event-loop-watch.js +179 -0
- package/dist/monitoring/event-loop-watch.js.map +1 -0
- package/dist/monitoring/host-disk.d.ts +6 -0
- package/dist/monitoring/host-disk.d.ts.map +1 -0
- package/dist/monitoring/host-disk.js +26 -0
- package/dist/monitoring/host-disk.js.map +1 -0
- package/dist/monitoring/log-line.d.ts +188 -0
- package/dist/monitoring/log-line.d.ts.map +1 -0
- package/dist/monitoring/log-line.js +458 -0
- package/dist/monitoring/log-line.js.map +1 -0
- package/dist/monitoring/log-manager.d.ts +89 -8
- package/dist/monitoring/log-manager.d.ts.map +1 -1
- package/dist/monitoring/log-manager.js +246 -77
- package/dist/monitoring/log-manager.js.map +1 -1
- package/dist/observability/metrics-bridge.d.ts.map +1 -1
- package/dist/observability/metrics-server.d.ts.map +1 -1
- package/dist/orchestrator/app-handle.d.ts +157 -2
- package/dist/orchestrator/app-handle.d.ts.map +1 -1
- package/dist/orchestrator/app-handle.js +165 -3
- package/dist/orchestrator/app-handle.js.map +1 -1
- package/dist/orchestrator/app-health.d.ts +33 -0
- package/dist/orchestrator/app-health.d.ts.map +1 -0
- package/dist/orchestrator/app-health.js +74 -0
- package/dist/orchestrator/app-health.js.map +1 -0
- package/dist/orchestrator/app-traffic.d.ts +14 -0
- package/dist/orchestrator/app-traffic.d.ts.map +1 -0
- package/dist/orchestrator/app-traffic.js +33 -0
- package/dist/orchestrator/app-traffic.js.map +1 -0
- package/dist/orchestrator/bootstrap-loader-child.d.ts +19 -0
- package/dist/orchestrator/bootstrap-loader-child.d.ts.map +1 -0
- package/dist/orchestrator/bootstrap-loader-child.js +43 -0
- package/dist/orchestrator/bootstrap-loader-child.js.map +1 -0
- package/dist/orchestrator/bootstrap-loader.d.ts +72 -0
- package/dist/orchestrator/bootstrap-loader.d.ts.map +1 -1
- package/dist/orchestrator/bootstrap-loader.js +350 -5
- package/dist/orchestrator/bootstrap-loader.js.map +1 -1
- package/dist/orchestrator/bootstrap-process.d.ts +99 -4
- package/dist/orchestrator/bootstrap-process.d.ts.map +1 -1
- package/dist/orchestrator/bootstrap-process.js +330 -19
- package/dist/orchestrator/bootstrap-process.js.map +1 -1
- package/dist/orchestrator/build-service.d.ts.map +1 -1
- package/dist/orchestrator/build-service.js.map +1 -1
- package/dist/orchestrator/call-exposed-service.d.ts +7 -0
- package/dist/orchestrator/call-exposed-service.d.ts.map +1 -0
- package/dist/orchestrator/call-exposed-service.js +50 -0
- package/dist/orchestrator/call-exposed-service.js.map +1 -0
- package/dist/orchestrator/classic-launcher.d.ts +1 -0
- package/dist/orchestrator/classic-launcher.d.ts.map +1 -1
- package/dist/orchestrator/classic-launcher.js +34 -3
- package/dist/orchestrator/classic-launcher.js.map +1 -1
- package/dist/orchestrator/file-watcher.d.ts +78 -1
- package/dist/orchestrator/file-watcher.d.ts.map +1 -1
- package/dist/orchestrator/file-watcher.js +181 -6
- package/dist/orchestrator/file-watcher.js.map +1 -1
- package/dist/orchestrator/module-worker-process.d.ts.map +1 -1
- package/dist/orchestrator/module-worker-process.js +10 -0
- package/dist/orchestrator/module-worker-process.js.map +1 -1
- package/dist/orchestrator/orchestrator.service.d.ts +371 -11
- package/dist/orchestrator/orchestrator.service.d.ts.map +1 -1
- package/dist/orchestrator/orchestrator.service.js +1602 -269
- package/dist/orchestrator/orchestrator.service.js.map +1 -1
- package/dist/orchestrator/owned-pids.d.ts +83 -0
- package/dist/orchestrator/owned-pids.d.ts.map +1 -0
- package/dist/orchestrator/owned-pids.js +93 -0
- package/dist/orchestrator/owned-pids.js.map +1 -0
- package/dist/orchestrator/port-conflict.d.ts +102 -0
- package/dist/orchestrator/port-conflict.d.ts.map +1 -0
- package/dist/orchestrator/port-conflict.js +131 -0
- package/dist/orchestrator/port-conflict.js.map +1 -0
- package/dist/orchestrator/process-janitor.d.ts +78 -4
- package/dist/orchestrator/process-janitor.d.ts.map +1 -1
- package/dist/orchestrator/process-janitor.js +246 -30
- package/dist/orchestrator/process-janitor.js.map +1 -1
- package/dist/orchestrator/ps-metrics.d.ts +37 -0
- package/dist/orchestrator/ps-metrics.d.ts.map +1 -0
- package/dist/orchestrator/ps-metrics.js +60 -0
- package/dist/orchestrator/ps-metrics.js.map +1 -0
- package/dist/orchestrator/service-router.d.ts +58 -2
- package/dist/orchestrator/service-router.d.ts.map +1 -1
- package/dist/orchestrator/service-router.js +157 -25
- package/dist/orchestrator/service-router.js.map +1 -1
- package/dist/orchestrator/topology-query.d.ts +49 -0
- package/dist/orchestrator/topology-query.d.ts.map +1 -0
- package/dist/orchestrator/topology-query.js +54 -0
- package/dist/orchestrator/topology-query.js.map +1 -0
- package/dist/project/artifact-builder.d.ts +174 -3
- package/dist/project/artifact-builder.d.ts.map +1 -1
- package/dist/project/artifact-builder.js +454 -42
- package/dist/project/artifact-builder.js.map +1 -1
- package/dist/project/build-decision.d.ts +58 -0
- package/dist/project/build-decision.d.ts.map +1 -0
- package/dist/project/build-decision.js +59 -0
- package/dist/project/build-decision.js.map +1 -0
- package/dist/project/config-resolver.d.ts +6 -4
- package/dist/project/config-resolver.d.ts.map +1 -1
- package/dist/project/config-resolver.js +128 -74
- package/dist/project/config-resolver.js.map +1 -1
- package/dist/project/declared-config.d.ts +42 -0
- package/dist/project/declared-config.d.ts.map +1 -0
- package/dist/project/declared-config.js +60 -0
- package/dist/project/declared-config.js.map +1 -0
- package/dist/project/dependency-analyzer.d.ts.map +1 -1
- package/dist/project/dependency-analyzer.js.map +1 -1
- package/dist/project/deploy-phases.d.ts +35 -0
- package/dist/project/deploy-phases.d.ts.map +1 -0
- package/dist/project/deploy-phases.js +75 -0
- package/dist/project/deploy-phases.js.map +1 -0
- package/dist/project/exec-in-group.d.ts +49 -0
- package/dist/project/exec-in-group.d.ts.map +1 -0
- package/dist/project/exec-in-group.js +100 -0
- package/dist/project/exec-in-group.js.map +1 -0
- package/dist/project/node-app-config.d.ts +101 -0
- package/dist/project/node-app-config.d.ts.map +1 -0
- package/dist/project/node-app-config.js +134 -0
- package/dist/project/node-app-config.js.map +1 -0
- package/dist/project/node-app-health.d.ts +76 -0
- package/dist/project/node-app-health.d.ts.map +1 -0
- package/dist/project/node-app-health.js +119 -0
- package/dist/project/node-app-health.js.map +1 -0
- package/dist/project/operator-account.d.ts +287 -0
- package/dist/project/operator-account.d.ts.map +1 -0
- package/dist/project/operator-account.js +373 -0
- package/dist/project/operator-account.js.map +1 -0
- package/dist/project/registry.d.ts +45 -8
- package/dist/project/registry.d.ts.map +1 -1
- package/dist/project/registry.js +217 -49
- package/dist/project/registry.js.map +1 -1
- package/dist/project/requirements-scanner.d.ts +1 -0
- package/dist/project/requirements-scanner.d.ts.map +1 -1
- package/dist/project/requirements-scanner.js +28 -3
- package/dist/project/requirements-scanner.js.map +1 -1
- package/dist/project/stack-env.d.ts +37 -0
- package/dist/project/stack-env.d.ts.map +1 -0
- package/dist/project/stack-env.js +85 -0
- package/dist/project/stack-env.js.map +1 -0
- package/dist/project/stand-containers.d.ts +4 -0
- package/dist/project/stand-containers.d.ts.map +1 -0
- package/dist/project/stand-containers.js +27 -0
- package/dist/project/stand-containers.js.map +1 -0
- package/dist/project/working-tree.d.ts +62 -0
- package/dist/project/working-tree.d.ts.map +1 -0
- package/dist/project/working-tree.js +101 -0
- package/dist/project/working-tree.js.map +1 -0
- package/dist/release/artifact-store.d.ts +360 -0
- package/dist/release/artifact-store.d.ts.map +1 -0
- package/dist/release/artifact-store.js +1040 -0
- package/dist/release/artifact-store.js.map +1 -0
- package/dist/release/attest-on-node.d.ts +92 -0
- package/dist/release/attest-on-node.d.ts.map +1 -0
- package/dist/release/attest-on-node.js +207 -0
- package/dist/release/attest-on-node.js.map +1 -0
- package/dist/release/attest.d.ts +135 -0
- package/dist/release/attest.d.ts.map +1 -0
- package/dist/release/attest.js +199 -0
- package/dist/release/attest.js.map +1 -0
- package/dist/release/build-run.d.ts +151 -0
- package/dist/release/build-run.d.ts.map +1 -0
- package/dist/release/build-run.js +609 -0
- package/dist/release/build-run.js.map +1 -0
- package/dist/release/builder.d.ts +57 -0
- package/dist/release/builder.d.ts.map +1 -0
- package/dist/release/builder.js +120 -0
- package/dist/release/builder.js.map +1 -0
- package/dist/release/daemon-omni.d.ts +41 -0
- package/dist/release/daemon-omni.d.ts.map +1 -0
- package/dist/release/daemon-omni.js +59 -0
- package/dist/release/daemon-omni.js.map +1 -0
- package/dist/release/definition-inputs.d.ts +34 -0
- package/dist/release/definition-inputs.d.ts.map +1 -0
- package/dist/release/definition-inputs.js +90 -0
- package/dist/release/definition-inputs.js.map +1 -0
- package/dist/release/delivered.d.ts +53 -0
- package/dist/release/delivered.d.ts.map +1 -0
- package/dist/release/delivered.js +81 -0
- package/dist/release/delivered.js.map +1 -0
- package/dist/release/layout.d.ts +61 -0
- package/dist/release/layout.d.ts.map +1 -0
- package/dist/release/layout.js +114 -0
- package/dist/release/layout.js.map +1 -0
- package/dist/release/load.d.ts +44 -0
- package/dist/release/load.d.ts.map +1 -0
- package/dist/release/load.js +84 -0
- package/dist/release/load.js.map +1 -0
- package/dist/release/manifest.d.ts +299 -0
- package/dist/release/manifest.d.ts.map +1 -0
- package/dist/release/manifest.js +176 -0
- package/dist/release/manifest.js.map +1 -0
- package/dist/release/migrations.d.ts +32 -0
- package/dist/release/migrations.d.ts.map +1 -0
- package/dist/release/migrations.js +51 -0
- package/dist/release/migrations.js.map +1 -0
- package/dist/release/publish.d.ts +71 -0
- package/dist/release/publish.d.ts.map +1 -0
- package/dist/release/publish.js +102 -0
- package/dist/release/publish.js.map +1 -0
- package/dist/release/store.d.ts +174 -0
- package/dist/release/store.d.ts.map +1 -0
- package/dist/release/store.js +257 -0
- package/dist/release/store.js.map +1 -0
- package/dist/services/alert.rpc-service.d.ts +11 -15
- package/dist/services/alert.rpc-service.d.ts.map +1 -1
- package/dist/services/alert.rpc-service.js +29 -14
- package/dist/services/alert.rpc-service.js.map +1 -1
- package/dist/services/alert.service.d.ts +96 -59
- package/dist/services/alert.service.d.ts.map +1 -1
- package/dist/services/alert.service.js +455 -70
- package/dist/services/alert.service.js.map +1 -1
- package/dist/services/anonymous-input.d.ts +26 -0
- package/dist/services/anonymous-input.d.ts.map +1 -0
- package/dist/services/anonymous-input.js +44 -0
- package/dist/services/anonymous-input.js.map +1 -0
- package/dist/services/audit.rpc-service.d.ts +27 -0
- package/dist/services/audit.rpc-service.d.ts.map +1 -0
- package/dist/services/audit.rpc-service.js +57 -0
- package/dist/services/audit.rpc-service.js.map +1 -0
- package/dist/services/audit.service.d.ts +165 -0
- package/dist/services/audit.service.d.ts.map +1 -0
- package/dist/services/audit.service.js +242 -0
- package/dist/services/audit.service.js.map +1 -0
- package/dist/services/auth-context.d.ts +21 -1
- package/dist/services/auth-context.d.ts.map +1 -1
- package/dist/services/auth-context.js +51 -5
- package/dist/services/auth-context.js.map +1 -1
- package/dist/services/auth.rpc-service.d.ts +0 -1
- package/dist/services/auth.rpc-service.d.ts.map +1 -1
- package/dist/services/auth.rpc-service.js +23 -9
- package/dist/services/auth.rpc-service.js.map +1 -1
- package/dist/services/auth.service.d.ts +47 -33
- package/dist/services/auth.service.d.ts.map +1 -1
- package/dist/services/auth.service.js +155 -23
- package/dist/services/auth.service.js.map +1 -1
- package/dist/services/backup-pipeline.d.ts +97 -0
- package/dist/services/backup-pipeline.d.ts.map +1 -0
- package/dist/services/backup-pipeline.js +286 -0
- package/dist/services/backup-pipeline.js.map +1 -0
- package/dist/services/backup-schedule.d.ts +69 -0
- package/dist/services/backup-schedule.d.ts.map +1 -0
- package/dist/services/backup-schedule.js +132 -0
- package/dist/services/backup-schedule.js.map +1 -0
- package/dist/services/backup.rpc-service.d.ts +31 -2
- package/dist/services/backup.rpc-service.d.ts.map +1 -1
- package/dist/services/backup.rpc-service.js +53 -0
- package/dist/services/backup.rpc-service.js.map +1 -1
- package/dist/services/backup.service.d.ts +458 -14
- package/dist/services/backup.service.d.ts.map +1 -1
- package/dist/services/backup.service.js +1368 -170
- package/dist/services/backup.service.js.map +1 -1
- package/dist/services/bundle-build-worker.d.ts +19 -0
- package/dist/services/bundle-build-worker.d.ts.map +1 -0
- package/dist/services/bundle-build-worker.js +40 -0
- package/dist/services/bundle-build-worker.js.map +1 -0
- package/dist/services/bundle-builder.d.ts +487 -0
- package/dist/services/bundle-builder.d.ts.map +1 -0
- package/dist/services/bundle-builder.js +1083 -0
- package/dist/services/bundle-builder.js.map +1 -0
- package/dist/services/bundle-worker-protocol.d.ts +19 -0
- package/dist/services/bundle-worker-protocol.d.ts.map +1 -0
- package/dist/services/bundle-worker-protocol.js +32 -0
- package/dist/services/bundle-worker-protocol.js.map +1 -0
- package/dist/services/deploy.rpc-service.d.ts +4 -1
- package/dist/services/deploy.rpc-service.d.ts.map +1 -1
- package/dist/services/deploy.rpc-service.js +10 -0
- package/dist/services/deploy.rpc-service.js.map +1 -1
- package/dist/services/deploy.service.d.ts +14 -24
- package/dist/services/deploy.service.d.ts.map +1 -1
- package/dist/services/deploy.service.js +46 -5
- package/dist/services/deploy.service.js.map +1 -1
- package/dist/services/discovery.rpc-service.d.ts +2 -1
- package/dist/services/discovery.rpc-service.d.ts.map +1 -1
- package/dist/services/discovery.rpc-service.js.map +1 -1
- package/dist/services/discovery.service.d.ts +32 -28
- package/dist/services/discovery.service.d.ts.map +1 -1
- package/dist/services/discovery.service.js +108 -133
- package/dist/services/discovery.service.js.map +1 -1
- package/dist/services/event-broadcaster.rpc-service.d.ts +36 -23
- package/dist/services/event-broadcaster.rpc-service.d.ts.map +1 -1
- package/dist/services/event-broadcaster.rpc-service.js +70 -32
- package/dist/services/event-broadcaster.rpc-service.js.map +1 -1
- package/dist/services/event-broadcaster.service.d.ts.map +1 -1
- package/dist/services/event-broadcaster.service.js.map +1 -1
- package/dist/services/fleet.rpc-service.d.ts +2 -1
- package/dist/services/fleet.rpc-service.d.ts.map +1 -1
- package/dist/services/fleet.rpc-service.js +3 -1
- package/dist/services/fleet.rpc-service.js.map +1 -1
- package/dist/services/fleet.service.d.ts +32 -34
- package/dist/services/fleet.service.d.ts.map +1 -1
- package/dist/services/fleet.service.js +66 -10
- package/dist/services/fleet.service.js.map +1 -1
- package/dist/services/health-check.rpc-service.d.ts +2 -1
- package/dist/services/health-check.rpc-service.d.ts.map +1 -1
- package/dist/services/health-check.rpc-service.js.map +1 -1
- package/dist/services/health-check.service.d.ts +50 -36
- package/dist/services/health-check.service.d.ts.map +1 -1
- package/dist/services/health-check.service.js +179 -182
- package/dist/services/health-check.service.js.map +1 -1
- package/dist/services/infrastructure.rpc-service.d.ts +297 -2
- package/dist/services/infrastructure.rpc-service.d.ts.map +1 -1
- package/dist/services/infrastructure.rpc-service.js +523 -11
- package/dist/services/infrastructure.rpc-service.js.map +1 -1
- package/dist/services/kubernetes.rpc-service.d.ts +2 -1
- package/dist/services/kubernetes.rpc-service.d.ts.map +1 -1
- package/dist/services/kubernetes.rpc-service.js.map +1 -1
- package/dist/services/kubernetes.service.d.ts +5 -39
- package/dist/services/kubernetes.service.d.ts.map +1 -1
- package/dist/services/kubernetes.service.js +29 -5
- package/dist/services/kubernetes.service.js.map +1 -1
- package/dist/services/local-bundle.d.ts +151 -0
- package/dist/services/local-bundle.d.ts.map +1 -0
- package/dist/services/local-bundle.js +233 -0
- package/dist/services/local-bundle.js.map +1 -0
- package/dist/services/lockout.d.ts +28 -0
- package/dist/services/lockout.d.ts.map +1 -0
- package/dist/services/lockout.js +41 -0
- package/dist/services/lockout.js.map +1 -0
- package/dist/services/log-collector.rpc-service.d.ts +16 -0
- package/dist/services/log-collector.rpc-service.d.ts.map +1 -1
- package/dist/services/log-collector.rpc-service.js +16 -0
- package/dist/services/log-collector.rpc-service.js.map +1 -1
- package/dist/services/log-collector.service.d.ts +172 -54
- package/dist/services/log-collector.service.d.ts.map +1 -1
- package/dist/services/log-collector.service.js +492 -102
- package/dist/services/log-collector.service.js.map +1 -1
- package/dist/services/log-retention.d.ts +50 -0
- package/dist/services/log-retention.d.ts.map +1 -0
- package/dist/services/log-retention.js +52 -0
- package/dist/services/log-retention.js.map +1 -0
- package/dist/services/maintenance-lock.d.ts +162 -0
- package/dist/services/maintenance-lock.d.ts.map +1 -0
- package/dist/services/maintenance-lock.js +316 -0
- package/dist/services/maintenance-lock.js.map +1 -0
- package/dist/services/master-address.d.ts +107 -0
- package/dist/services/master-address.d.ts.map +1 -0
- package/dist/services/master-address.js +154 -0
- package/dist/services/master-address.js.map +1 -0
- package/dist/services/migration-output.d.ts +16 -0
- package/dist/services/migration-output.d.ts.map +1 -0
- package/dist/services/migration-output.js +22 -0
- package/dist/services/migration-output.js.map +1 -0
- package/dist/services/netron-telemetry-transport.d.ts.map +1 -1
- package/dist/services/netron-telemetry-transport.js.map +1 -1
- package/dist/services/node-deploy-lease.d.ts +154 -0
- package/dist/services/node-deploy-lease.d.ts.map +1 -0
- package/dist/services/node-deploy-lease.js +386 -0
- package/dist/services/node-deploy-lease.js.map +1 -0
- package/dist/services/node-health.repository.d.ts +63 -4
- package/dist/services/node-health.repository.d.ts.map +1 -1
- package/dist/services/node-health.repository.js +108 -67
- package/dist/services/node-health.repository.js.map +1 -1
- package/dist/services/node-manager.rpc-service.d.ts +355 -10
- package/dist/services/node-manager.rpc-service.d.ts.map +1 -1
- package/dist/services/node-manager.rpc-service.js +730 -34
- package/dist/services/node-manager.rpc-service.js.map +1 -1
- package/dist/services/node-manager.service.d.ts +267 -95
- package/dist/services/node-manager.service.d.ts.map +1 -1
- package/dist/services/node-manager.service.js +805 -103
- package/dist/services/node-manager.service.js.map +1 -1
- package/dist/services/node-upgrade.d.ts +129 -0
- package/dist/services/node-upgrade.d.ts.map +1 -0
- package/dist/services/node-upgrade.js +161 -0
- package/dist/services/node-upgrade.js.map +1 -0
- package/dist/services/node-upgrade.service.d.ts +232 -0
- package/dist/services/node-upgrade.service.d.ts.map +1 -0
- package/dist/services/node-upgrade.service.js +468 -0
- package/dist/services/node-upgrade.service.js.map +1 -0
- package/dist/services/pipeline.rpc-service.d.ts +2 -1
- package/dist/services/pipeline.rpc-service.d.ts.map +1 -1
- package/dist/services/pipeline.rpc-service.js.map +1 -1
- package/dist/services/pipeline.service.d.ts +15 -46
- package/dist/services/pipeline.service.d.ts.map +1 -1
- package/dist/services/pipeline.service.js +72 -9
- package/dist/services/pipeline.service.js.map +1 -1
- package/dist/services/project.rpc-service.d.ts +133 -5
- package/dist/services/project.rpc-service.d.ts.map +1 -1
- package/dist/services/project.rpc-service.js +249 -22
- package/dist/services/project.rpc-service.js.map +1 -1
- package/dist/services/project.service.d.ts +677 -15
- package/dist/services/project.service.d.ts.map +1 -1
- package/dist/services/project.service.js +2798 -304
- package/dist/services/project.service.js.map +1 -1
- package/dist/services/redeploy-decision.d.ts +176 -0
- package/dist/services/redeploy-decision.d.ts.map +1 -0
- package/dist/services/redeploy-decision.js +142 -0
- package/dist/services/redeploy-decision.js.map +1 -0
- package/dist/services/release.rpc-service.d.ts +159 -0
- package/dist/services/release.rpc-service.d.ts.map +1 -0
- package/dist/services/release.rpc-service.js +395 -0
- package/dist/services/release.rpc-service.js.map +1 -0
- package/dist/services/release.service.d.ts +114 -0
- package/dist/services/release.service.d.ts.map +1 -0
- package/dist/services/release.service.js +208 -0
- package/dist/services/release.service.js.map +1 -0
- package/dist/services/remote-deployer.service.d.ts +569 -8
- package/dist/services/remote-deployer.service.d.ts.map +1 -1
- package/dist/services/remote-deployer.service.js +1618 -169
- package/dist/services/remote-deployer.service.js.map +1 -1
- package/dist/services/remote-ops.service.d.ts +51 -9
- package/dist/services/remote-ops.service.d.ts.map +1 -1
- package/dist/services/remote-ops.service.js +235 -39
- package/dist/services/remote-ops.service.js.map +1 -1
- package/dist/services/remote-provisioner.d.ts +122 -0
- package/dist/services/remote-provisioner.d.ts.map +1 -0
- package/dist/services/remote-provisioner.js +258 -0
- package/dist/services/remote-provisioner.js.map +1 -0
- package/dist/services/secrets.rpc-service.d.ts +51 -2
- package/dist/services/secrets.rpc-service.d.ts.map +1 -1
- package/dist/services/secrets.rpc-service.js +81 -2
- package/dist/services/secrets.rpc-service.js.map +1 -1
- package/dist/services/secrets.service.d.ts +108 -9
- package/dist/services/secrets.service.d.ts.map +1 -1
- package/dist/services/secrets.service.js +313 -48
- package/dist/services/secrets.service.js.map +1 -1
- package/dist/services/slave-storage.service.d.ts +107 -0
- package/dist/services/slave-storage.service.d.ts.map +1 -1
- package/dist/services/slave-storage.service.js +171 -2
- package/dist/services/slave-storage.service.js.map +1 -1
- package/dist/services/static-carry-forward.d.ts +85 -0
- package/dist/services/static-carry-forward.d.ts.map +1 -0
- package/dist/services/static-carry-forward.js +159 -0
- package/dist/services/static-carry-forward.js.map +1 -0
- package/dist/services/sync-policy.d.ts +167 -0
- package/dist/services/sync-policy.d.ts.map +1 -0
- package/dist/services/sync-policy.js +161 -0
- package/dist/services/sync-policy.js.map +1 -0
- package/dist/services/sync-summary.d.ts +39 -0
- package/dist/services/sync-summary.d.ts.map +1 -0
- package/dist/services/sync-summary.js +62 -0
- package/dist/services/sync-summary.js.map +1 -0
- package/dist/services/sync.rpc-service.d.ts +19 -7
- package/dist/services/sync.rpc-service.d.ts.map +1 -1
- package/dist/services/sync.rpc-service.js +36 -5
- package/dist/services/sync.rpc-service.js.map +1 -1
- package/dist/services/sync.service.d.ts +350 -18
- package/dist/services/sync.service.d.ts.map +1 -1
- package/dist/services/sync.service.js +864 -116
- package/dist/services/sync.service.js.map +1 -1
- package/dist/services/system-info.rpc-service.d.ts +2 -1
- package/dist/services/system-info.rpc-service.d.ts.map +1 -1
- package/dist/services/system-info.rpc-service.js.map +1 -1
- package/dist/services/system-info.service.d.ts +56 -68
- package/dist/services/system-info.service.d.ts.map +1 -1
- package/dist/services/system-info.service.js +240 -127
- package/dist/services/system-info.service.js.map +1 -1
- package/dist/services/telemetry.rpc-service.d.ts.map +1 -1
- package/dist/services/telemetry.rpc-service.js +9 -3
- package/dist/services/telemetry.rpc-service.js.map +1 -1
- package/dist/services/telemetry.service.d.ts +4 -2
- package/dist/services/telemetry.service.d.ts.map +1 -1
- package/dist/services/telemetry.service.js +4 -2
- package/dist/services/telemetry.service.js.map +1 -1
- package/dist/services/trace-collector.rpc-service.d.ts +2 -1
- package/dist/services/trace-collector.rpc-service.d.ts.map +1 -1
- package/dist/services/trace-collector.rpc-service.js.map +1 -1
- package/dist/services/trace-collector.service.d.ts +2 -41
- package/dist/services/trace-collector.service.d.ts.map +1 -1
- package/dist/services/trace-collector.service.js +27 -7
- package/dist/services/trace-collector.service.js.map +1 -1
- package/dist/services/wait-for-postgres.d.ts +28 -0
- package/dist/services/wait-for-postgres.d.ts.map +1 -0
- package/dist/services/wait-for-postgres.js +77 -0
- package/dist/services/wait-for-postgres.js.map +1 -0
- package/dist/shared/alert-expression.d.ts +60 -0
- package/dist/shared/alert-expression.d.ts.map +1 -0
- package/dist/shared/alert-expression.js +124 -0
- package/dist/shared/alert-expression.js.map +1 -0
- package/dist/shared/audit-outcome.d.ts +39 -0
- package/dist/shared/audit-outcome.d.ts.map +1 -0
- package/dist/shared/audit-outcome.js +45 -0
- package/dist/shared/audit-outcome.js.map +1 -0
- package/dist/shared/build-freshness.d.ts +59 -0
- package/dist/shared/build-freshness.d.ts.map +1 -0
- package/dist/shared/build-freshness.js +113 -0
- package/dist/shared/build-freshness.js.map +1 -0
- package/dist/shared/build-stamp.d.ts +26 -0
- package/dist/shared/build-stamp.d.ts.map +1 -0
- package/dist/shared/build-stamp.js +36 -0
- package/dist/shared/build-stamp.js.map +1 -0
- package/dist/shared/describe-error.d.ts +22 -0
- package/dist/shared/describe-error.d.ts.map +1 -0
- package/dist/shared/describe-error.js +39 -0
- package/dist/shared/describe-error.js.map +1 -0
- package/dist/shared/dto/alerts.d.ts +101 -0
- package/dist/shared/dto/alerts.d.ts.map +1 -0
- package/dist/shared/dto/alerts.js +9 -0
- package/dist/shared/dto/alerts.js.map +1 -0
- package/dist/shared/dto/auth.d.ts +49 -0
- package/dist/shared/dto/auth.d.ts.map +1 -0
- package/dist/shared/dto/auth.js +14 -0
- package/dist/shared/dto/auth.js.map +1 -0
- package/dist/shared/dto/backups.d.ts +16 -0
- package/dist/shared/dto/backups.d.ts.map +1 -0
- package/dist/shared/dto/backups.js +9 -0
- package/dist/shared/dto/backups.js.map +1 -0
- package/dist/shared/dto/deploy.d.ts +28 -0
- package/dist/shared/dto/deploy.d.ts.map +1 -0
- package/dist/shared/dto/deploy.js +8 -0
- package/dist/shared/dto/deploy.js.map +1 -0
- package/dist/shared/dto/discovery.d.ts +24 -0
- package/dist/shared/dto/discovery.d.ts.map +1 -0
- package/dist/shared/dto/discovery.js +9 -0
- package/dist/shared/dto/discovery.js.map +1 -0
- package/dist/shared/dto/fleet.d.ts +35 -0
- package/dist/shared/dto/fleet.d.ts.map +1 -0
- package/dist/shared/dto/fleet.js +8 -0
- package/dist/shared/dto/fleet.js.map +1 -0
- package/dist/shared/dto/health.d.ts +26 -0
- package/dist/shared/dto/health.d.ts.map +1 -0
- package/dist/shared/dto/health.js +9 -0
- package/dist/shared/dto/health.js.map +1 -0
- package/dist/shared/dto/kubernetes.d.ts +44 -0
- package/dist/shared/dto/kubernetes.d.ts.map +1 -0
- package/dist/shared/dto/kubernetes.js +9 -0
- package/dist/shared/dto/kubernetes.js.map +1 -0
- package/dist/shared/dto/logs.d.ts +77 -0
- package/dist/shared/dto/logs.d.ts.map +1 -0
- package/dist/shared/dto/logs.js +10 -0
- package/dist/shared/dto/logs.js.map +1 -0
- package/dist/shared/dto/metrics.d.ts +65 -0
- package/dist/shared/dto/metrics.d.ts.map +1 -0
- package/dist/shared/dto/metrics.js +17 -0
- package/dist/shared/dto/metrics.js.map +1 -0
- package/dist/shared/dto/nodes.d.ts +347 -0
- package/dist/shared/dto/nodes.d.ts.map +1 -0
- package/dist/shared/dto/nodes.js +8 -0
- package/dist/shared/dto/nodes.js.map +1 -0
- package/dist/shared/dto/pipelines.d.ts +53 -0
- package/dist/shared/dto/pipelines.d.ts.map +1 -0
- package/dist/shared/dto/pipelines.js +9 -0
- package/dist/shared/dto/pipelines.js.map +1 -0
- package/dist/shared/dto/project.d.ts +142 -8
- package/dist/shared/dto/project.d.ts.map +1 -1
- package/dist/shared/dto/services.d.ts +831 -7
- package/dist/shared/dto/services.d.ts.map +1 -1
- package/dist/shared/dto/system-info.d.ts +93 -0
- package/dist/shared/dto/system-info.d.ts.map +1 -0
- package/dist/shared/dto/system-info.js +9 -0
- package/dist/shared/dto/system-info.js.map +1 -0
- package/dist/shared/dto/traces.d.ts +49 -0
- package/dist/shared/dto/traces.d.ts.map +1 -0
- package/dist/shared/dto/traces.js +9 -0
- package/dist/shared/dto/traces.js.map +1 -0
- package/dist/shared/env-config.d.ts +74 -0
- package/dist/shared/env-config.d.ts.map +1 -0
- package/dist/shared/env-config.js +99 -0
- package/dist/shared/env-config.js.map +1 -0
- package/dist/shared/events.d.ts +45 -39
- package/dist/shared/events.d.ts.map +1 -1
- package/dist/shared/events.js +6 -0
- package/dist/shared/events.js.map +1 -1
- package/dist/shared/format.d.ts +1 -1
- package/dist/shared/format.d.ts.map +1 -1
- package/dist/shared/format.js +4 -0
- package/dist/shared/format.js.map +1 -1
- package/dist/shared/local-infra-host.d.ts +28 -0
- package/dist/shared/local-infra-host.d.ts.map +1 -0
- package/dist/shared/local-infra-host.js +28 -0
- package/dist/shared/local-infra-host.js.map +1 -0
- package/dist/shared/node-check.d.ts +37 -0
- package/dist/shared/node-check.d.ts.map +1 -0
- package/dist/shared/node-check.js +41 -0
- package/dist/shared/node-check.js.map +1 -0
- package/dist/shared/paths.d.ts +29 -0
- package/dist/shared/paths.d.ts.map +1 -0
- package/dist/shared/paths.js +54 -0
- package/dist/shared/paths.js.map +1 -0
- package/dist/shared/pnpm.d.ts +29 -0
- package/dist/shared/pnpm.d.ts.map +1 -0
- package/dist/shared/pnpm.js +57 -0
- package/dist/shared/pnpm.js.map +1 -0
- package/dist/shared/private-files.d.ts +29 -0
- package/dist/shared/private-files.d.ts.map +1 -0
- package/dist/shared/private-files.js +109 -0
- package/dist/shared/private-files.js.map +1 -0
- package/dist/shared/redact-env.d.ts +24 -0
- package/dist/shared/redact-env.d.ts.map +1 -0
- package/dist/shared/redact-env.js +36 -0
- package/dist/shared/redact-env.js.map +1 -0
- package/dist/shared/release-reading.d.ts +85 -0
- package/dist/shared/release-reading.d.ts.map +1 -0
- package/dist/shared/release-reading.js +81 -0
- package/dist/shared/release-reading.js.map +1 -0
- package/dist/shared/roles.d.ts +58 -3
- package/dist/shared/roles.d.ts.map +1 -1
- package/dist/shared/roles.js +55 -0
- package/dist/shared/roles.js.map +1 -1
- package/dist/shared/rpcauth.d.ts +13 -0
- package/dist/shared/rpcauth.d.ts.map +1 -0
- package/dist/shared/rpcauth.js +23 -0
- package/dist/shared/rpcauth.js.map +1 -0
- package/dist/shared/shell-escape.d.ts +9 -0
- package/dist/shared/shell-escape.d.ts.map +1 -0
- package/dist/shared/shell-escape.js +11 -0
- package/dist/shared/shell-escape.js.map +1 -0
- package/dist/shared/sync-reading.d.ts +45 -0
- package/dist/shared/sync-reading.d.ts.map +1 -0
- package/dist/shared/sync-reading.js +42 -0
- package/dist/shared/sync-reading.js.map +1 -0
- package/dist/shared/tokens.d.ts +19 -0
- package/dist/shared/tokens.d.ts.map +1 -1
- package/dist/shared/tokens.js +28 -0
- package/dist/shared/tokens.js.map +1 -1
- package/dist/webapp/webapp.service.d.ts +33 -2
- package/dist/webapp/webapp.service.d.ts.map +1 -1
- package/dist/webapp/webapp.service.js +143 -31
- package/dist/webapp/webapp.service.js.map +1 -1
- package/dist/workers/health-monitor-process.d.ts.map +1 -1
- package/dist/workers/health-monitor-process.js +10 -0
- package/dist/workers/health-monitor-process.js.map +1 -1
- package/dist/workers/health-monitor.service.d.ts +45 -3
- package/dist/workers/health-monitor.service.d.ts.map +1 -1
- package/dist/workers/health-monitor.service.js +152 -25
- package/dist/workers/health-monitor.service.js.map +1 -1
- package/dist/workers/system-worker-manager.d.ts +43 -0
- package/dist/workers/system-worker-manager.d.ts.map +1 -1
- package/dist/workers/system-worker-manager.js +150 -3
- package/dist/workers/system-worker-manager.js.map +1 -1
- package/dist/workers/types.d.ts +8 -2
- package/dist/workers/types.d.ts.map +1 -1
- package/package.json +64 -30
- package/webapp/dist/BUILD.json +5 -0
- package/webapp/dist/assets/FormControlLabel-BrH2diyl.js +1 -0
- package/webapp/dist/assets/Grid-BZ7xhoHQ.js +1 -0
- package/webapp/dist/assets/Switch-DGSAxQPM.js +1 -0
- package/webapp/dist/assets/alerts-sXqHJbsa.js +1 -0
- package/webapp/dist/assets/apps-B_Ah8Flh.js +1 -0
- package/webapp/dist/assets/audit-YXihwyui.js +1 -0
- package/webapp/dist/assets/client-CGbBvy9k.js +1 -0
- package/webapp/dist/assets/constants-veKGFkC6.js +1 -0
- package/webapp/dist/assets/containers-Di4NOTLf.js +2 -0
- package/webapp/dist/assets/daemon-memory-CCPZ4cV_.js +1 -0
- package/webapp/dist/assets/dashboard-Cfuf95N3.js +1 -0
- package/webapp/dist/assets/dashboard-builder-CowaYb-M.js +1 -0
- package/webapp/dist/assets/deploy-release-dialog-D6C-XcKY.js +1 -0
- package/webapp/dist/assets/deployments-ChTq5kCw.js +1 -0
- package/webapp/dist/assets/detail-BsGWv7Dj.js +1 -0
- package/webapp/dist/assets/detail-CyVf_cw5.js +2 -0
- package/webapp/dist/assets/detail-D027j6aO.js +8 -0
- package/webapp/dist/assets/formatters-BqjL73N3.js +1 -0
- package/webapp/dist/assets/index-B8AHIPc-.css +1 -0
- package/webapp/dist/assets/index-DiZbrwxT.js +414 -0
- package/webapp/dist/assets/logs-L2b8Oqmo.js +12 -0
- package/webapp/dist/assets/maplibre-gl-MlmG2kbn.js +801 -0
- package/webapp/dist/assets/metrics-CQgjz2Lz.js +2 -0
- package/webapp/dist/assets/nodes-Nwia2Ty6.js +7 -0
- package/webapp/dist/assets/pipelines-DCu_rXOr.js +4 -0
- package/webapp/dist/assets/projects-Bb4aV76h.js +1 -0
- package/webapp/dist/assets/react-CwJFpaho.js +1 -0
- package/webapp/dist/assets/react-apexcharts.esm-Dy0Fez42.js +1956 -0
- package/webapp/dist/assets/release-bits-Dj4EOhhb.js +1 -0
- package/webapp/dist/assets/release-build-panel-D6OCVlhT.js +1 -0
- package/webapp/dist/assets/releases-CnAt3yO6.js +2 -0
- package/webapp/dist/assets/rolldown-runtime-hePW80VL.js +1 -0
- package/webapp/dist/assets/settings-DU-mjiwb.js +1 -0
- package/webapp/dist/assets/settled-pair-C7SjZuns.js +1 -0
- package/webapp/dist/assets/sign-in-Bcvjrr4C.js +1 -0
- package/webapp/dist/assets/stacks-B-e8p50I.js +1 -0
- package/webapp/dist/assets/sync-reading-C3PnJ7AX.js +1 -0
- package/webapp/dist/assets/system-info-BtE3dgfw.js +1 -0
- package/webapp/dist/assets/topology-Cwch-uF1.js +12 -0
- package/webapp/dist/assets/topology-DLioOiRN.css +1 -0
- package/webapp/dist/assets/traces-BbShR-4P.js +1 -0
- package/webapp/dist/assets/use-stack-context-CZUgPq88.js +1 -0
- package/webapp/dist/index.html +6 -6
- package/dist/commands/dev.d.ts +0 -23
- package/dist/commands/dev.d.ts.map +0 -1
- package/dist/commands/dev.js +0 -76
- package/dist/commands/dev.js.map +0 -1
- package/dist/config/define-app.d.ts +0 -6
- package/dist/config/define-app.d.ts.map +0 -1
- package/dist/config/define-app.js +0 -38
- package/dist/config/define-app.js.map +0 -1
- package/dist/infrastructure/gateway-generator.d.ts +0 -63
- package/dist/infrastructure/gateway-generator.d.ts.map +0 -1
- package/dist/infrastructure/gateway-generator.js +0 -253
- package/dist/infrastructure/gateway-generator.js.map +0 -1
- package/dist/monitoring/health.service.d.ts +0 -14
- package/dist/monitoring/health.service.d.ts.map +0 -1
- package/dist/monitoring/health.service.js +0 -50
- package/dist/monitoring/health.service.js.map +0 -1
- package/dist/monitoring/metrics.service.d.ts +0 -14
- package/dist/monitoring/metrics.service.d.ts.map +0 -1
- package/dist/monitoring/metrics.service.js +0 -40
- package/dist/monitoring/metrics.service.js.map +0 -1
- package/dist/orchestrator/bootstrap-runner.d.ts +0 -15
- package/dist/orchestrator/bootstrap-runner.d.ts.map +0 -1
- package/dist/orchestrator/bootstrap-runner.js +0 -152
- package/dist/orchestrator/bootstrap-runner.js.map +0 -1
- package/dist/orchestrator/topology-proxy.d.ts +0 -65
- package/dist/orchestrator/topology-proxy.d.ts.map +0 -1
- package/dist/orchestrator/topology-proxy.js +0 -209
- package/dist/orchestrator/topology-proxy.js.map +0 -1
- package/dist/orchestrator/topology-relay.d.ts +0 -111
- package/dist/orchestrator/topology-relay.d.ts.map +0 -1
- package/dist/orchestrator/topology-relay.js +0 -255
- package/dist/orchestrator/topology-relay.js.map +0 -1
- package/dist/services/auth-middleware.d.ts +0 -47
- package/dist/services/auth-middleware.d.ts.map +0 -1
- package/dist/services/auth-middleware.js +0 -158
- package/dist/services/auth-middleware.js.map +0 -1
- package/dist/services/environment.rpc-service.d.ts +0 -22
- package/dist/services/environment.rpc-service.d.ts.map +0 -1
- package/dist/services/environment.rpc-service.js +0 -65
- package/dist/services/environment.rpc-service.js.map +0 -1
- package/dist/services/environment.service.d.ts +0 -87
- package/dist/services/environment.service.d.ts.map +0 -1
- package/dist/services/environment.service.js +0 -220
- package/dist/services/environment.service.js.map +0 -1
- package/dist/services/metrics-collector.service.d.ts +0 -103
- package/dist/services/metrics-collector.service.d.ts.map +0 -1
- package/dist/services/metrics-collector.service.js +0 -235
- package/dist/services/metrics-collector.service.js.map +0 -1
- package/dist/services/metrics.rpc-service.d.ts +0 -41
- package/dist/services/metrics.rpc-service.d.ts.map +0 -1
- package/dist/services/metrics.rpc-service.js +0 -90
- package/dist/services/metrics.rpc-service.js.map +0 -1
- package/dist/services/metrics.service.d.ts +0 -72
- package/dist/services/metrics.service.d.ts.map +0 -1
- package/dist/services/metrics.service.js +0 -185
- package/dist/services/metrics.service.js.map +0 -1
- package/dist/supervisor/omnitron-supervisor.d.ts +0 -21
- package/dist/supervisor/omnitron-supervisor.d.ts.map +0 -1
- package/dist/supervisor/omnitron-supervisor.js +0 -43
- package/dist/supervisor/omnitron-supervisor.js.map +0 -1
- package/dist/webapp/static-server.d.ts +0 -24
- package/dist/webapp/static-server.d.ts.map +0 -1
- package/dist/webapp/static-server.js +0 -152
- package/dist/webapp/static-server.js.map +0 -1
- package/webapp/dist/assets/Badge-D2ZBP5iK.js +0 -1
- package/webapp/dist/assets/Card-vO88g06B.js +0 -1
- package/webapp/dist/assets/CardActions-C9h7Q2wW.js +0 -1
- package/webapp/dist/assets/CardContent-BA_Xmmhk.js +0 -1
- package/webapp/dist/assets/CardHeader-ceIs__6x.js +0 -1
- package/webapp/dist/assets/Chip-Ckk0xUhK.js +0 -1
- package/webapp/dist/assets/DialogActions-BhHM7hov.js +0 -1
- package/webapp/dist/assets/DialogContentText-BT2JQRmK.js +0 -1
- package/webapp/dist/assets/FormControlLabel-CcQByVS8.js +0 -1
- package/webapp/dist/assets/Grid-14PETRQW.js +0 -1
- package/webapp/dist/assets/InputAdornment-B8aHIsQ7.js +0 -1
- package/webapp/dist/assets/Skeleton-B74-Ds6O.js +0 -37
- package/webapp/dist/assets/Switch-DvWD2wh4.js +0 -1
- package/webapp/dist/assets/SwitchBase-CDU4nY8O.js +0 -1
- package/webapp/dist/assets/Tab-DcCOYYp5.js +0 -1
- package/webapp/dist/assets/TableContainer-CzTh4mu-.js +0 -1
- package/webapp/dist/assets/TableRow-BySEreZR.js +0 -2
- package/webapp/dist/assets/TextField-BLf-BHz2.js +0 -1
- package/webapp/dist/assets/alerts-DnRbT41C.js +0 -1
- package/webapp/dist/assets/apps-CdR0VH2d.js +0 -1
- package/webapp/dist/assets/chunk-DtcAfS_W.js +0 -1
- package/webapp/dist/assets/client-DU8I_lDc.js +0 -2
- package/webapp/dist/assets/constants-BgnT-Xdy.js +0 -1
- package/webapp/dist/assets/containers-Dw-rRXV1.js +0 -1
- package/webapp/dist/assets/dashboard-CzgB0uZz.js +0 -1
- package/webapp/dist/assets/dashboard-builder-DsA0yg33.js +0 -1
- package/webapp/dist/assets/deployments-DtQAoIcQ.js +0 -1
- package/webapp/dist/assets/detail-BJR-G7JG.js +0 -1
- package/webapp/dist/assets/detail-nlCeSltl.js +0 -8
- package/webapp/dist/assets/formatters-Do41SRmE.js +0 -1
- package/webapp/dist/assets/index-ByQDwp2r.js +0 -566
- package/webapp/dist/assets/index-CHVXOsTS.css +0 -1
- package/webapp/dist/assets/isMuiElement-DwF2naaF.js +0 -1
- package/webapp/dist/assets/logs-Kn8o4H7k.js +0 -11
- package/webapp/dist/assets/metrics-DN8Zu5wT.js +0 -2
- package/webapp/dist/assets/nodes-Bgch7AtN.js +0 -7
- package/webapp/dist/assets/pipelines-Cry3qm-S.js +0 -4
- package/webapp/dist/assets/popper-BJFSj6rc.js +0 -1
- package/webapp/dist/assets/preload-helper-DSXbuxSR.js +0 -1
- package/webapp/dist/assets/projects-Dym2GI1U.js +0 -1
- package/webapp/dist/assets/react-apexcharts.esm-CSGK5Cxn.js +0 -958
- package/webapp/dist/assets/react-apexcharts.esm-DEAWOwx5.js +0 -1003
- package/webapp/dist/assets/settings-va3zTcow.js +0 -1
- package/webapp/dist/assets/sign-in-ik35umbp.js +0 -1
- package/webapp/dist/assets/stacks-CVFOnxbF.js +0 -1
- package/webapp/dist/assets/system-info-npIXAjX2.js +0 -1
- package/webapp/dist/assets/topology-CHpVij2M.css +0 -1
- package/webapp/dist/assets/topology-bDe70vLs.js +0 -12
- package/webapp/dist/assets/traces-DIIcT8DF.js +0 -1
- package/webapp/dist/assets/use-stack-context-DHLwZv1r.js +0 -1
- /package/webapp/dist/assets/{fs-CgZLtHiQ.js → fs-Ccl3sAcp.js} +0 -0
|
@@ -11,7 +11,13 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Classic mode:
|
|
13
13
|
* - child_process.fork() with manual exit handling (legacy compat)
|
|
14
|
-
*
|
|
14
|
+
*
|
|
15
|
+
* Metrics for both modes come from one `ps -o pid=,rss=,%cpu=` call covering
|
|
16
|
+
* every managed pid — supervisor children and pool workers alike. This line
|
|
17
|
+
* used to say "real CPU/memory metrics via pidusage"; `pidusage` is not a
|
|
18
|
+
* dependency of this package and appears nowhere else in the source. The
|
|
19
|
+
* sentence outlived the code it described, in the one area where a stale
|
|
20
|
+
* claim is hardest to catch: metrics that are wrong still look like metrics.
|
|
15
21
|
*
|
|
16
22
|
* Architecture:
|
|
17
23
|
* pm.supervisor({ children: [{ process: bootstrap-process.js, ... }] })
|
|
@@ -21,18 +27,59 @@
|
|
|
21
27
|
* → Two Netron instances: PM management plane + app HTTP data plane
|
|
22
28
|
*/
|
|
23
29
|
import fs from 'node:fs';
|
|
30
|
+
import { readDeclaredConfig, withDeclaredConfig } from '../project/declared-config.js';
|
|
24
31
|
import path from 'node:path';
|
|
25
32
|
import { fileURLToPath } from 'node:url';
|
|
26
|
-
import { PoolStrategy, SupervisionStrategy, } from '@omnitron-dev/titan-pm';
|
|
33
|
+
import { PoolStrategy, SupervisionStrategy, ProcessLifecycleQueue, LifecyclePreempted, computeBackoff, isAlive, discoverManagedProcesses, } from '@omnitron-dev/titan-pm';
|
|
27
34
|
import { EventEmitter } from '@omnitron-dev/eventemitter';
|
|
28
35
|
import { AppHandle } from './app-handle.js';
|
|
36
|
+
import { parsePsLine, parsePsBatch } from './ps-metrics.js';
|
|
29
37
|
import { launchClassic } from './classic-launcher.js';
|
|
30
38
|
import { resolveStartupOrder, resolveShutdownOrder } from './dependency-resolver.js';
|
|
31
39
|
import { buildRestartPolicy } from '../supervisor/restart-policy.js';
|
|
40
|
+
/**
|
|
41
|
+
* How long to wait for a pool worker to finish exposing its `@Service`
|
|
42
|
+
* classes before giving up on `topology.expose`. A worker reports ready
|
|
43
|
+
* before its Application starts, and exposure happens during start, so the
|
|
44
|
+
* gap is a real one — measured at ~7s for a worker that opens database
|
|
45
|
+
* connections first. The consumer that needs the service starts right after
|
|
46
|
+
* this returns, so the wait is paid once at boot and buys a feature that
|
|
47
|
+
* otherwise stays off until the next restart.
|
|
48
|
+
*/
|
|
49
|
+
/**
|
|
50
|
+
* `childName` is sometimes bare ('http') and sometimes already carries the app
|
|
51
|
+
* ('acme/dev/pricing/http'), depending on which supervisor path raised the
|
|
52
|
+
* event. Prefixing unconditionally produced
|
|
53
|
+
* `acme/dev/pricing/acme/dev/pricing/http` in the log file.
|
|
54
|
+
*/
|
|
55
|
+
function qualifyChildName(appName, childName) {
|
|
56
|
+
return childName.startsWith(`${appName}/`) || childName === appName
|
|
57
|
+
? childName
|
|
58
|
+
: `${appName}/${childName}`;
|
|
59
|
+
}
|
|
60
|
+
const TOPOLOGY_EXPOSE_TIMEOUT_MS = 30_000;
|
|
61
|
+
const TOPOLOGY_EXPOSE_POLL_MS = 250;
|
|
32
62
|
import { ServiceRouter } from './service-router.js';
|
|
33
63
|
import { loadBootstrapConfig, clearCacheFor } from './bootstrap-loader.js';
|
|
64
|
+
/** Levels a pino logger accepts. A child is spawned with nothing else. */
|
|
65
|
+
const PINO_LEVELS = new Set(['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent']);
|
|
66
|
+
/**
|
|
67
|
+
* Process-manager states in which a record no longer describes a live process.
|
|
68
|
+
*
|
|
69
|
+
* Read by `managerOwnedPids`, where the distinction decides whether a pid is
|
|
70
|
+
* protected from the janitor. Written as strings rather than imported from
|
|
71
|
+
* `ProcessStatus` because that export is a const object whose values are
|
|
72
|
+
* these same literals; the set has to hold values, not keys, and a typo here
|
|
73
|
+
* would silently protect a dead pid — so it is pinned by a test.
|
|
74
|
+
*/
|
|
75
|
+
const TERMINAL_PROCESS_STATUSES = new Set(['stopped', 'failed', 'crashed']);
|
|
34
76
|
import { BuildService } from './build-service.js';
|
|
77
|
+
import { execFile } from 'node:child_process';
|
|
35
78
|
import { ProcessJanitor } from './process-janitor.js';
|
|
79
|
+
import { addressInUse, holderCommand, parseHolders, explainConflict } from './port-conflict.js';
|
|
80
|
+
import { collectOwnedPids } from './owned-pids.js';
|
|
81
|
+
import { combineProcessHealth, notRunningHealth, poolHealth } from './app-health.js';
|
|
82
|
+
import { combineProcessTraffic } from './app-traffic.js';
|
|
36
83
|
import { CLI_VERSION } from '../config/defaults.js';
|
|
37
84
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
38
85
|
// Resolve bootstrap-process path — .ts in dev (tsx), .js when compiled
|
|
@@ -63,15 +110,77 @@ function mapStrategy(strategy) {
|
|
|
63
110
|
* stack mode + bare-name RPC paths can be deduplicated.
|
|
64
111
|
*
|
|
65
112
|
* Examples:
|
|
66
|
-
* effectiveAppName('omni/dev/
|
|
67
|
-
* effectiveAppName('
|
|
68
|
-
* effectiveAppName('a/b/c/
|
|
113
|
+
* effectiveAppName('omni/dev/payments') === 'payments'
|
|
114
|
+
* effectiveAppName('payments') === 'payments'
|
|
115
|
+
* effectiveAppName('a/b/c/payments') === 'payments' (last segment)
|
|
69
116
|
*/
|
|
70
|
-
function effectiveAppName(key) {
|
|
117
|
+
export function effectiveAppName(key) {
|
|
71
118
|
const idx = key.lastIndexOf('/');
|
|
72
119
|
return idx >= 0 ? key.slice(idx + 1) : key;
|
|
73
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* Auto-promote a bare `entry.name` to canonical `${project}/${stack}/${name}`
|
|
123
|
+
* form when the env carries OMNITRON_PROJECT + OMNITRON_STACK. The stack-mode
|
|
124
|
+
* boot path (ProjectService.startStack) already prefixes the name explicitly;
|
|
125
|
+
* this helper closes every other entry point that could otherwise register a
|
|
126
|
+
* bare-name handle which then becomes invisible to the stack-prefix-based
|
|
127
|
+
* UI lookup (ProjectService.toStackInfo filters on `${project}/${stack}/`).
|
|
128
|
+
*
|
|
129
|
+
* Idempotent: if the entry already looks namespaced (or env lacks the keys),
|
|
130
|
+
* the input is returned unchanged. An entry qualified under a DIFFERENT
|
|
131
|
+
* project/stack is preserved too — that's operator intent (cross-project
|
|
132
|
+
* administrative run), not a bug.
|
|
133
|
+
*/
|
|
134
|
+
export function ensureNamespacedEntry(entry) {
|
|
135
|
+
const project = entry.env?.['OMNITRON_PROJECT'];
|
|
136
|
+
const stack = entry.env?.['OMNITRON_STACK'];
|
|
137
|
+
if (!project || !stack)
|
|
138
|
+
return entry;
|
|
139
|
+
const canonicalPrefix = `${project}/${stack}/`;
|
|
140
|
+
if (entry.name.startsWith(canonicalPrefix))
|
|
141
|
+
return entry;
|
|
142
|
+
if (entry.name.includes('/'))
|
|
143
|
+
return entry;
|
|
144
|
+
return { ...entry, name: `${canonicalPrefix}${entry.name}` };
|
|
145
|
+
}
|
|
74
146
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
147
|
+
/**
|
|
148
|
+
* Startup budget for a child process that declares none of its own.
|
|
149
|
+
*
|
|
150
|
+
* There were TWO of these and they disagreed — `?? 60_000` on the pooled path
|
|
151
|
+
* and `?? 30_000` on the single-process path — and, worse, the pooled one did
|
|
152
|
+
* not consult `entry.startupTimeout` at all. So an operator who set
|
|
153
|
+
* `startupTimeout: 120_000` on an app had it honoured for the app and
|
|
154
|
+
* SILENTLY IGNORED for its child processes, which kept 60 seconds and could
|
|
155
|
+
* not be given more by any setting.
|
|
156
|
+
*
|
|
157
|
+
* Measured on this host: an app whose own successful boot takes 120.6 seconds
|
|
158
|
+
* carried a 120 s budget, and its `deposit-worker` child — a deposit scanner
|
|
159
|
+
* — stayed `stopped` while the app itself came up. Two copies of one decision,
|
|
160
|
+
* one of which discarded the only knob an operator has.
|
|
161
|
+
*
|
|
162
|
+
* 60 s rather than 30 s, for the reason the project's own config already
|
|
163
|
+
* writes down: an over-long ceiling costs a slower failure report, a short one
|
|
164
|
+
* costs the whole stack.
|
|
165
|
+
*/
|
|
166
|
+
const DEFAULT_CHILD_STARTUP_TIMEOUT = 60_000;
|
|
167
|
+
/**
|
|
168
|
+
* Deadline for an RPC the daemon makes to a child it supervises.
|
|
169
|
+
*
|
|
170
|
+
* A call across this connection is WORK — aggregate five minutes of candles,
|
|
171
|
+
* answer a health probe that touches a pool, hand back metrics — not a wire
|
|
172
|
+
* request, and netron's default (5000 ms) is sized for the latter. The pool
|
|
173
|
+
* path stated this number and the single-child path stated nothing, so a
|
|
174
|
+
* client built for a single child fell back to that default: measured, 94
|
|
175
|
+
* timeouts «after 5000ms» in the daemon log, every one on
|
|
176
|
+
* `callExposedService`, which lives on exactly those single children.
|
|
177
|
+
*
|
|
178
|
+
* It belongs here rather than in titan-pm because the orchestrator is what
|
|
179
|
+
* knows the nature of the work; the library's own fallback is 30 s
|
|
180
|
+
* (`process-pool.ts:212`) and applies when a caller states nothing — which,
|
|
181
|
+
* after this, no caller here does.
|
|
182
|
+
*/
|
|
183
|
+
const CHILD_REQUEST_TIMEOUT = 120_000;
|
|
75
184
|
export class OrchestratorService extends EventEmitter {
|
|
76
185
|
logger;
|
|
77
186
|
pm;
|
|
@@ -88,6 +197,30 @@ export class OrchestratorService extends EventEmitter {
|
|
|
88
197
|
* for the same name isn't permanently locked out.
|
|
89
198
|
*/
|
|
90
199
|
startingApps = new Map();
|
|
200
|
+
/**
|
|
201
|
+
* Per-app async serial-execution queue covering start / stop / restart
|
|
202
|
+
* / crash-restart-timer (P0-E + P0-I + P2-B in the audit). Pre-this-
|
|
203
|
+
* queue each path had its own coalescer:
|
|
204
|
+
* - startingApps: same-name start coalescing only — `stopApp` did
|
|
205
|
+
* NOT consult it, so stop racing in-flight start left an orphan
|
|
206
|
+
* supervisor when stop arrived before start wrote childProcess.
|
|
207
|
+
* - restartCoalescer: same-name restart coalescing; independent of
|
|
208
|
+
* stop+start.
|
|
209
|
+
* - handleClassicCrash setTimeout: a raw timer that could fire
|
|
210
|
+
* during the `await stopChild(...)` window and spawn a zombie
|
|
211
|
+
* against an app the operator just stopped.
|
|
212
|
+
* The queue collapses all four into one primitive ordered by
|
|
213
|
+
* priority (stop > restart > start > crash-restart). The same-key
|
|
214
|
+
* mutex means stop ALWAYS observes a completed start, and an
|
|
215
|
+
* explicit stop ALWAYS preempts a pending crash-restart timer.
|
|
216
|
+
*/
|
|
217
|
+
lifecycleQueue = new ProcessLifecycleQueue();
|
|
218
|
+
/**
|
|
219
|
+
* Pending post-escalation cool-down timers, keyed by app name.
|
|
220
|
+
* Held here rather than inside a lifecycle-queue slot so that a
|
|
221
|
+
* crashed app still answers operator commands while it waits.
|
|
222
|
+
*/
|
|
223
|
+
coolDownTimers = new Map();
|
|
91
224
|
config = null;
|
|
92
225
|
cwd;
|
|
93
226
|
metricsTimer = null;
|
|
@@ -129,10 +262,23 @@ export class OrchestratorService extends EventEmitter {
|
|
|
129
262
|
*/
|
|
130
263
|
static POST_ONLINE_COOLDOWN_MS = 3_000;
|
|
131
264
|
/**
|
|
132
|
-
* Orphan-process reaper. Lazily created on first
|
|
133
|
-
* tests that hit `startApp` directly don't get a periodic timer
|
|
265
|
+
* Orphan-process reaper. Lazily created on first reconcile so unit
|
|
266
|
+
* tests that hit `startApp` directly don't get a periodic timer
|
|
267
|
+
* until the first real reconcile runs (which they bypass by
|
|
268
|
+
* stubbing `bootReconciled = true`).
|
|
134
269
|
*/
|
|
135
270
|
janitor = null;
|
|
271
|
+
/**
|
|
272
|
+
* One-shot guard for cold-start reconciliation. T#63 — every code
|
|
273
|
+
* path that lands an app on this orchestrator goes through
|
|
274
|
+
* `ensureBootReconciled()`, regardless of whether the entry-point
|
|
275
|
+
* is `startAll` (daemon-config) or `startApp` (project/stack
|
|
276
|
+
* mode). Pre-T#63 the cold-start sweep only fired inside
|
|
277
|
+
* `startAll`, so project-mode boots silently inherited orphans
|
|
278
|
+
* from the previous daemon generation.
|
|
279
|
+
*/
|
|
280
|
+
bootReconciled = false;
|
|
281
|
+
bootReconcilePromise = null;
|
|
136
282
|
/** Daemon's Netron instance for native service routing */
|
|
137
283
|
daemonNetron = null;
|
|
138
284
|
/** Daemon Unix socket path for child-to-daemon Netron connections */
|
|
@@ -220,13 +366,117 @@ export class OrchestratorService extends EventEmitter {
|
|
|
220
366
|
// ============================================================================
|
|
221
367
|
// Lifecycle
|
|
222
368
|
// ============================================================================
|
|
369
|
+
/**
|
|
370
|
+
* Refresh the orchestrator's view of the ecosystem config. Called
|
|
371
|
+
* from `daemon.reloadConfig` so a subsequent `restartApp` reads the
|
|
372
|
+
* new entry definition (mode, env, dependsOn, restart policy)
|
|
373
|
+
* instead of the stale copy captured at first `startAll`. Pre-fix
|
|
374
|
+
* the two refs diverged silently after every reload — operators
|
|
375
|
+
* editing ecosystem.config.ts and running `omnitron reload`
|
|
376
|
+
* believed the daemon picked up the changes; in fact only daemon's
|
|
377
|
+
* own RPC service did, and individual app restarts kept running
|
|
378
|
+
* against the pre-reload definition until the daemon itself
|
|
379
|
+
* restarted. P0-G in the audit.
|
|
380
|
+
*/
|
|
381
|
+
setConfig(config) {
|
|
382
|
+
this.config = config;
|
|
383
|
+
// Metrics polling is armed here rather than in `startAll` alone.
|
|
384
|
+
//
|
|
385
|
+
// It used to run only from `startAll`, which is the `omnitron start`
|
|
386
|
+
// path. Apps brought up any other way — `stack start`, a single
|
|
387
|
+
// `omnitron start <app>`, or the daemon resuming a stack at boot — were
|
|
388
|
+
// never sampled, so `omnitron list`, `status` and the console's app
|
|
389
|
+
// cards reported cpu 0 / memory 0 for processes that were serving
|
|
390
|
+
// traffic. Observed live: six apps online, every one showing 0 while
|
|
391
|
+
// `ps` reported ~105 MB RSS apiece.
|
|
392
|
+
//
|
|
393
|
+
// `setConfig` is the one point every path passes through, and the call
|
|
394
|
+
// is idempotent — an existing timer is cleared first.
|
|
395
|
+
this.ensureMetricsPolling();
|
|
396
|
+
}
|
|
397
|
+
/**
|
|
398
|
+
* Start metrics sampling if it is not already running.
|
|
399
|
+
*
|
|
400
|
+
* Called from every path that can produce a managed process, so no app can
|
|
401
|
+
* end up unsampled because of how it was started.
|
|
402
|
+
*/
|
|
403
|
+
ensureMetricsPolling() {
|
|
404
|
+
if (this.metricsTimer)
|
|
405
|
+
return;
|
|
406
|
+
this.startMetricsPolling(this.config?.monitoring?.metrics?.interval ?? 5_000);
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* One-shot cold-start reconciliation: discover pre-existing managed
|
|
410
|
+
* processes, reap fork-workers left over from a previous daemon
|
|
411
|
+
* generation, and arm the periodic janitor sweep. Idempotent —
|
|
412
|
+
* concurrent callers join the in-flight promise; subsequent calls
|
|
413
|
+
* after success are no-ops.
|
|
414
|
+
*
|
|
415
|
+
* Called from BOTH `startAll` (top-level daemon-config boot) and
|
|
416
|
+
* `startApp` (project/stack mode boot), so every code path that
|
|
417
|
+
* lands an app on this orchestrator runs the cleanup. Pre-T#63 the
|
|
418
|
+
* cold-start sweep was gated on `startAll` and never ran in
|
|
419
|
+
* project mode, so an orphan from a previous daemon survived
|
|
420
|
+
* silently and held the listen port hostage from its successor.
|
|
421
|
+
*/
|
|
422
|
+
async ensureBootReconciled(config) {
|
|
423
|
+
if (this.bootReconciled)
|
|
424
|
+
return;
|
|
425
|
+
if (this.bootReconcilePromise) {
|
|
426
|
+
await this.bootReconcilePromise;
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
this.bootReconcilePromise = (async () => {
|
|
430
|
+
// Discovery: surface OMNITRON_MANAGED-tagged processes that match
|
|
431
|
+
// a configured app so the operator can see what's about to be
|
|
432
|
+
// reaped (or — once the ChildContract handshake lands — adopted).
|
|
433
|
+
// P2-A in the audit.
|
|
434
|
+
try {
|
|
435
|
+
const discovered = discoverManagedProcesses();
|
|
436
|
+
const adopted = [];
|
|
437
|
+
const knownApps = new Set(config?.apps.map((a) => a.name) ?? []);
|
|
438
|
+
for (const proc of discovered) {
|
|
439
|
+
if (proc.ppid === process.pid)
|
|
440
|
+
continue; // our own freshly-spawned child
|
|
441
|
+
const fqn = proc.fullyQualifiedName;
|
|
442
|
+
const bare = proc.appName;
|
|
443
|
+
if (knownApps.has(fqn) || knownApps.has(bare)) {
|
|
444
|
+
adopted.push(`${fqn}#${proc.pid}`);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
if (adopted.length > 0) {
|
|
448
|
+
this.logger.warn({ discovered: adopted.length, apps: adopted }, 'Found pre-existing managed processes — they will be reaped by the janitor in 60s unless adopted');
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
catch (err) {
|
|
452
|
+
this.logger.debug({ err: err.message }, 'Process discovery failed — proceeding without reconcile');
|
|
453
|
+
}
|
|
454
|
+
// Reap any fork-workers left over from a previous daemon BEFORE
|
|
455
|
+
// launching apps. Without this, a hard-killed daemon's children
|
|
456
|
+
// are still alive (now reparented to init); they hold DB
|
|
457
|
+
// connections and listen ports that the new apps will fight
|
|
458
|
+
// for. The reap is bounded by `gracefulMs` (default 3s).
|
|
459
|
+
await this.ensureJanitor().coldStartSweep();
|
|
460
|
+
// Arm the periodic sweep so runtime leaks from partially-failed
|
|
461
|
+
// restart cycles (esbuild-rebuild → spawn-fail → orphan) get
|
|
462
|
+
// collected on every interval.
|
|
463
|
+
this.ensureJanitor().start();
|
|
464
|
+
})();
|
|
465
|
+
try {
|
|
466
|
+
await this.bootReconcilePromise;
|
|
467
|
+
this.bootReconciled = true;
|
|
468
|
+
}
|
|
469
|
+
finally {
|
|
470
|
+
this.bootReconcilePromise = null;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
223
473
|
async startAll(config) {
|
|
224
474
|
this.config = config;
|
|
225
|
-
//
|
|
226
|
-
//
|
|
227
|
-
//
|
|
228
|
-
//
|
|
229
|
-
await this.
|
|
475
|
+
// Cold-start orphan reap + periodic janitor arming. Idempotent —
|
|
476
|
+
// safe to call on every boot path. Project-mode flows that call
|
|
477
|
+
// `startApp` directly trigger the same one-shot reconcile, so
|
|
478
|
+
// every boot path lands in the same post-condition. (T#63)
|
|
479
|
+
await this.ensureBootReconciled(config);
|
|
230
480
|
// Dependency-aware skip: if app A fails to start, every app that
|
|
231
481
|
// declares A in `dependsOn` (transitively) is skipped instead of
|
|
232
482
|
// attempted. Without this, a downstream app would launch, fail to
|
|
@@ -255,10 +505,8 @@ export class OrchestratorService extends EventEmitter {
|
|
|
255
505
|
if (failed.size > 0 || blocked.size > 0) {
|
|
256
506
|
this.logger.warn({ failed: [...failed], blocked: [...blocked] }, 'Startup completed with failures — some apps did not launch');
|
|
257
507
|
}
|
|
258
|
-
// Periodic sweep
|
|
259
|
-
//
|
|
260
|
-
this.ensureJanitor().start();
|
|
261
|
-
this.startMetricsPolling(config.monitoring.metrics.interval);
|
|
508
|
+
// Periodic sweep was armed by `ensureBootReconciled` above, and metrics
|
|
509
|
+
// polling by `setConfig` — no second call needed for either.
|
|
262
510
|
this.persistState();
|
|
263
511
|
}
|
|
264
512
|
/**
|
|
@@ -276,44 +524,143 @@ export class OrchestratorService extends EventEmitter {
|
|
|
276
524
|
return this.janitor;
|
|
277
525
|
}
|
|
278
526
|
/**
|
|
279
|
-
*
|
|
280
|
-
*
|
|
281
|
-
*
|
|
527
|
+
* Every OS pid the orchestrator currently considers ours. Cheap — called
|
|
528
|
+
* on every janitor sweep, and the janitor kills whatever this set omits.
|
|
529
|
+
*
|
|
530
|
+
* The rule itself lives in `owned-pids.ts`, where it can be tested: the
|
|
531
|
+
* janitor's own tests supply this set as a fixture, so they could never
|
|
532
|
+
* catch it being incomplete — and it was, by a whole class of process.
|
|
533
|
+
*/
|
|
534
|
+
/**
|
|
535
|
+
* Who is holding the port an app could not bind, in one sentence.
|
|
536
|
+
*
|
|
537
|
+
* Called where a start or restart failed, because that is the moment the
|
|
538
|
+
* answer is worth having and the moment nobody has it: the error names the
|
|
539
|
+
* address and never the holder. Measured on this daemon — four
|
|
540
|
+
* `EADDRINUSE` failures on 3005 in thirty-one seconds, `omnitron list`
|
|
541
|
+
* reporting the app crashed with no pid, and the port held the whole time
|
|
542
|
+
* by a child of this very daemon that nothing in its records claimed.
|
|
543
|
+
*
|
|
544
|
+
* Returns null when the failure was not a port clash, and says so out loud
|
|
545
|
+
* when it could not read the machine — a reading that failed and a port
|
|
546
|
+
* nobody holds are different findings.
|
|
282
547
|
*/
|
|
548
|
+
async explainPortConflict(message) {
|
|
549
|
+
const address = addressInUse(message);
|
|
550
|
+
if (!address)
|
|
551
|
+
return null;
|
|
552
|
+
const { file, args } = holderCommand(address.port, process.platform);
|
|
553
|
+
let output;
|
|
554
|
+
try {
|
|
555
|
+
output = await new Promise((resolve, reject) => {
|
|
556
|
+
execFile(file, args, { encoding: 'utf8', timeout: 5_000 }, (err, stdout) => {
|
|
557
|
+
// `ss` and `lsof` both exit non-zero when nothing matches, which is
|
|
558
|
+
// an answer rather than a failure; the empty stdout carries it.
|
|
559
|
+
if (err && !stdout)
|
|
560
|
+
reject(err);
|
|
561
|
+
else
|
|
562
|
+
resolve(stdout);
|
|
563
|
+
});
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
catch (err) {
|
|
567
|
+
return `could not read who holds port ${address.port}: ${file} ${err.message}`;
|
|
568
|
+
}
|
|
569
|
+
const holders = parseHolders(output, process.platform);
|
|
570
|
+
// One `ps` for the parents, before deciding: the decision is pure and
|
|
571
|
+
// takes the answers, so that it can be tested without a machine.
|
|
572
|
+
const parents = new Map();
|
|
573
|
+
for (const holder of holders) {
|
|
574
|
+
const ppid = await this.parentOf(holder.pid);
|
|
575
|
+
if (ppid !== undefined)
|
|
576
|
+
parents.set(holder.pid, ppid);
|
|
577
|
+
}
|
|
578
|
+
const verdict = explainConflict({
|
|
579
|
+
port: address.port,
|
|
580
|
+
holders,
|
|
581
|
+
parentOf: (pid) => parents.get(pid),
|
|
582
|
+
daemonPid: process.pid,
|
|
583
|
+
owned: this.collectOwnedPids(),
|
|
584
|
+
});
|
|
585
|
+
return verdict.because;
|
|
586
|
+
}
|
|
587
|
+
/** The parent of one process, or undefined when it cannot be read. */
|
|
588
|
+
async parentOf(pid) {
|
|
589
|
+
try {
|
|
590
|
+
const out = await new Promise((resolve, reject) => {
|
|
591
|
+
execFile('ps', ['-o', 'ppid=', '-p', String(pid)], { encoding: 'utf8', timeout: 5_000 }, (err, stdout) => {
|
|
592
|
+
if (err && !stdout)
|
|
593
|
+
reject(err);
|
|
594
|
+
else
|
|
595
|
+
resolve(stdout);
|
|
596
|
+
});
|
|
597
|
+
});
|
|
598
|
+
const parsed = Number(out.trim());
|
|
599
|
+
return Number.isInteger(parsed) ? parsed : undefined;
|
|
600
|
+
}
|
|
601
|
+
catch {
|
|
602
|
+
// A process that exited between the two readings has no parent to
|
|
603
|
+
// report, and that is not an error worth a line of its own — the
|
|
604
|
+
// verdict already says the holder was gone.
|
|
605
|
+
return undefined;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
283
608
|
collectOwnedPids() {
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
609
|
+
return collectOwnedPids(this.handles.values(), (workerId) => this.pm.getWorkerHandle(workerId)?.pid, this.managerOwnedPids());
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* Every pid the process manager currently vouches for.
|
|
613
|
+
*
|
|
614
|
+
* Two windows, and a process is only safe if both are asked. `getForkedPids`
|
|
615
|
+
* holds a claim from `fork()` until `spawn()` resolves and releases it —
|
|
616
|
+
* after that its own docstring says "the WorkerHandle registry is the
|
|
617
|
+
* authority". `listProcesses()` IS that registry, and it is the only source
|
|
618
|
+
* that answers for a process the orchestrator did not start: the daemon's
|
|
619
|
+
* system workers go through this same manager but belong to no app handle,
|
|
620
|
+
* so walking `this.handles` can never see them.
|
|
621
|
+
*
|
|
622
|
+
* Terminal records are excluded deliberately. They keep the pid they had,
|
|
623
|
+
* the OS reuses pids, and a stale one here would vouch for exactly the
|
|
624
|
+
* leaked worker the janitor exists to reap.
|
|
625
|
+
*/
|
|
626
|
+
managerOwnedPids() {
|
|
627
|
+
const pids = [...(this.pm.getForkedPids?.() ?? [])];
|
|
628
|
+
for (const info of this.pm.listProcesses?.() ?? []) {
|
|
629
|
+
if (typeof info.pid !== 'number')
|
|
288
630
|
continue;
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
continue;
|
|
293
|
-
const workerHandle = this.pm.getWorkerHandle(processId);
|
|
294
|
-
const pid = workerHandle?.pid;
|
|
295
|
-
if (typeof pid === 'number')
|
|
296
|
-
owned.add(pid);
|
|
297
|
-
}
|
|
631
|
+
if (TERMINAL_PROCESS_STATUSES.has(info.status))
|
|
632
|
+
continue;
|
|
633
|
+
pids.push(info.pid);
|
|
298
634
|
}
|
|
299
|
-
return
|
|
635
|
+
return pids;
|
|
300
636
|
}
|
|
301
637
|
async startApp(entry, config) {
|
|
302
|
-
// T#
|
|
303
|
-
//
|
|
304
|
-
//
|
|
305
|
-
//
|
|
306
|
-
//
|
|
307
|
-
//
|
|
308
|
-
//
|
|
309
|
-
|
|
310
|
-
//
|
|
638
|
+
// T#63: run the one-shot cold-start reconcile (orphan reap +
|
|
639
|
+
// janitor arming) before ANY app spawns. Project-mode boots flow
|
|
640
|
+
// through `projectService.startStack → orchestrator.startApp`
|
|
641
|
+
// and historically bypassed the reap entirely — a previous
|
|
642
|
+
// daemon's children (now reparented to init) kept their HTTP
|
|
643
|
+
// listen ports and the fresh child failed to bind silently.
|
|
644
|
+
// No-op after first call thanks to `bootReconciled`.
|
|
645
|
+
await this.ensureBootReconciled(config);
|
|
646
|
+
// Defensive auto-namespacing — promotes a bare entry.name to the
|
|
647
|
+
// canonical `${project}/${stack}/${name}` form when the env carries
|
|
648
|
+
// OMNITRON_PROJECT + OMNITRON_STACK. Catches every code path that
|
|
649
|
+
// bypasses ProjectService.startStack:
|
|
650
|
+
// - direct CLI `omnitron start <name>` (no stack context)
|
|
651
|
+
// - legacy ecosystem configs without per-app namespace
|
|
652
|
+
// - daemon startAll() flows over a not-yet-namespaced config
|
|
653
|
+
entry = ensureNamespacedEntry(entry);
|
|
311
654
|
const inflight = this.startingApps.get(entry.name);
|
|
312
655
|
if (inflight) {
|
|
313
656
|
this.logger.debug({ app: entry.name }, 'Coalescing concurrent startApp into in-flight launch (T#57)');
|
|
314
657
|
return inflight;
|
|
315
658
|
}
|
|
316
|
-
|
|
659
|
+
// Serialise against stop/restart for the same app via the
|
|
660
|
+
// lifecycle queue. A stopApp arriving while this start is queued
|
|
661
|
+
// (or in-flight) now waits its turn instead of racing through
|
|
662
|
+
// and leaving an orphan supervisor.
|
|
663
|
+
const promise = this.lifecycleQueue.enqueue(entry.name, 'start', () => this.startAppInternal(entry, config)).finally(() => {
|
|
317
664
|
this.startingApps.delete(entry.name);
|
|
318
665
|
});
|
|
319
666
|
this.startingApps.set(entry.name, promise);
|
|
@@ -339,17 +686,37 @@ export class OrchestratorService extends EventEmitter {
|
|
|
339
686
|
// - exact key already exists, online → return it (idempotent)
|
|
340
687
|
// - exact key already exists, not online → fall through and re-launch
|
|
341
688
|
// under the same key
|
|
342
|
-
// - DIFFERENT key with the same effective short name
|
|
343
|
-
//
|
|
344
|
-
//
|
|
689
|
+
// - DIFFERENT key with the same effective short name:
|
|
690
|
+
// - new is MORE specific (namespaced) and existing is bare → stop
|
|
691
|
+
// the bare one and re-launch under the namespaced key. Without
|
|
692
|
+
// this, a boot-race where bare-name auto-start beats the
|
|
693
|
+
// project/stack flow strands the app under the wrong key and
|
|
694
|
+
// webapp/CLI lookups by namespaced key see it as stopped.
|
|
695
|
+
// - new is LESS-or-equally specific → refuse: operator should
|
|
696
|
+
// restart by canonical name, not register an alias.
|
|
345
697
|
const effective = effectiveAppName(entry.name);
|
|
698
|
+
const newIsNamespaced = entry.name.includes('/');
|
|
346
699
|
for (const [key, h] of this.handles) {
|
|
347
700
|
if (key === entry.name)
|
|
348
701
|
continue;
|
|
349
702
|
if (effectiveAppName(key) !== effective)
|
|
350
703
|
continue;
|
|
704
|
+
const existingIsNamespaced = key.includes('/');
|
|
351
705
|
// Same effective app under a different key.
|
|
352
706
|
if (h.status === 'online') {
|
|
707
|
+
if (newIsNamespaced && !existingIsNamespaced) {
|
|
708
|
+
this.logger.info({ stale: key, replacing_with: entry.name, effective }, 'Stopping bare-name handle to re-register under namespaced key');
|
|
709
|
+
try {
|
|
710
|
+
await this.stopApp(key);
|
|
711
|
+
}
|
|
712
|
+
catch (err) {
|
|
713
|
+
this.logger.warn({ stale: key, error: err.message }, 'stopApp during dedup failed — clearing handle anyway');
|
|
714
|
+
}
|
|
715
|
+
this.handles.delete(key);
|
|
716
|
+
if (key !== entry.name)
|
|
717
|
+
this.metricsBridge?.evictApp(key);
|
|
718
|
+
continue;
|
|
719
|
+
}
|
|
353
720
|
this.logger.warn({ existing: key, attempted: entry.name, effective }, 'App already running under a canonical name — refusing duplicate registration');
|
|
354
721
|
return h;
|
|
355
722
|
}
|
|
@@ -357,6 +724,12 @@ export class OrchestratorService extends EventEmitter {
|
|
|
357
724
|
// failed previous launch doesn't permanently block the canonical
|
|
358
725
|
// form.
|
|
359
726
|
this.logger.info({ stale: key, replacing_with: entry.name, effective }, 'Replacing stale duplicate handle');
|
|
727
|
+
// The handle is errored or stopped, so `stopApp` never ran for it and
|
|
728
|
+
// its services are still registered on the daemon's Netron. Dropping the
|
|
729
|
+
// handle here is the last moment anything knows they exist — or that its
|
|
730
|
+
// processes still run: an errored app's other children and its pools.
|
|
731
|
+
await this.releaseExposedServices(h);
|
|
732
|
+
await this.tearDownProcesses(h);
|
|
360
733
|
this.handles.delete(key);
|
|
361
734
|
// Drop any metrics that were collected under the stale alias so it
|
|
362
735
|
// doesn't haunt future snapshots as a ghost offline app.
|
|
@@ -368,9 +741,40 @@ export class OrchestratorService extends EventEmitter {
|
|
|
368
741
|
this.logger.warn({ app: entry.name }, 'App already running');
|
|
369
742
|
return existing;
|
|
370
743
|
}
|
|
744
|
+
// A handle that is not online may still run processes. A crash restart and
|
|
745
|
+
// a cool-down recovery only flip its status before coming here; one child
|
|
746
|
+
// crashing leaves its siblings — the server among them — up; and its pools
|
|
747
|
+
// were never any supervisor's to stop. Stopped before the new handle
|
|
748
|
+
// starts, or the new one finds its own predecessor on its port: on
|
|
749
|
+
// daos/test (2026-09-25) paysys's crash restarts failed five times on
|
|
750
|
+
// `EADDRINUSE :3004`, «held by … a child this daemon still accounts for».
|
|
751
|
+
if (existing) {
|
|
752
|
+
await this.releaseExposedServices(existing);
|
|
753
|
+
await this.tearDownProcesses(existing);
|
|
754
|
+
}
|
|
371
755
|
const mode = entry.bootstrap ? 'bootstrap' : 'classic';
|
|
372
756
|
const handle = new AppHandle(entry, mode);
|
|
757
|
+
// The restart budget belongs to the APP, not to this handle instance.
|
|
758
|
+
// Every start replaces the handle, so a counter that starts at zero here
|
|
759
|
+
// can never reach `maxRestarts` — which made the budget, the escalation
|
|
760
|
+
// and the cool-down unreachable even once the restart itself worked. The
|
|
761
|
+
// window is enforced by the policy, not by forgetting.
|
|
762
|
+
if (existing)
|
|
763
|
+
handle.restarts = existing.restarts;
|
|
373
764
|
this.handles.set(entry.name, handle);
|
|
765
|
+
// Arm metrics sampling here — the one place an app enters the registry.
|
|
766
|
+
//
|
|
767
|
+
// It used to be armed only from `startAll` (the `omnitron start` path).
|
|
768
|
+
// Anything else — `stack start`, a single `omnitron start <app>`, the
|
|
769
|
+
// daemon resuming a stack at boot — was never sampled, so `list`,
|
|
770
|
+
// `status` and the console's app cards reported cpu 0 / memory 0 for
|
|
771
|
+
// processes serving traffic. Observed: six apps online, every one at
|
|
772
|
+
// zero, while `ps` showed ~105 MB RSS apiece.
|
|
773
|
+
//
|
|
774
|
+
// `setConfig` looked like the right seam but is only called on config
|
|
775
|
+
// RELOAD, so it never fired on a normal start. This is the seam that
|
|
776
|
+
// actually covers every path. Idempotent: re-arming replaces the timer.
|
|
777
|
+
this.ensureMetricsPolling();
|
|
374
778
|
this.logger.info({ app: entry.name, mode }, 'Starting app');
|
|
375
779
|
try {
|
|
376
780
|
if (mode === 'classic') {
|
|
@@ -379,43 +783,113 @@ export class OrchestratorService extends EventEmitter {
|
|
|
379
783
|
else {
|
|
380
784
|
await this.launchBootstrapMode(entry, handle, cfg);
|
|
381
785
|
}
|
|
382
|
-
|
|
786
|
+
// «App started» only for an app that is. It was logged in the same
|
|
787
|
+
// millisecond as a child's «Process failed to start», for an app whose
|
|
788
|
+
// processes never all came up.
|
|
789
|
+
if (handle.status === 'online') {
|
|
790
|
+
this.logger.info({ app: entry.name, pid: handle.pid, mode }, 'App started');
|
|
791
|
+
}
|
|
792
|
+
else {
|
|
793
|
+
this.logger.error({ app: entry.name, status: handle.status, mode }, 'App did not come up — not every declared process started');
|
|
794
|
+
}
|
|
383
795
|
this.persistState();
|
|
384
796
|
return handle;
|
|
385
797
|
}
|
|
386
798
|
catch (err) {
|
|
799
|
+
// `errored` rather than `crashed` is the right REPORT — the app never
|
|
800
|
+
// ran. It was also, until now, the end of the story: `'errored'` is read
|
|
801
|
+
// by no code in this repository outside formatters, `status` and a
|
|
802
|
+
// metrics label, so a start that failed was never attempted again. One
|
|
803
|
+
// try, then errored until a human noticed.
|
|
804
|
+
//
|
|
805
|
+
// That asymmetry is backwards. A failure to start is MORE likely to be
|
|
806
|
+
// transient than a crash after a successful boot — a dependency still
|
|
807
|
+
// coming up, a disk still cold, a machine briefly loaded. Six bootstrap
|
|
808
|
+
// apps here timed out at `config:loading` under a load average of 149
|
|
809
|
+
// and stayed down after the load passed, because nothing asked again.
|
|
810
|
+
//
|
|
811
|
+
// The counter is bumped by hand because `markErrored()` deliberately
|
|
812
|
+
// does not (an app that never ran has not "restarted"), and without it
|
|
813
|
+
// the budget below can never trip and the retry would be unbounded.
|
|
387
814
|
handle.markErrored();
|
|
815
|
+
handle.restarts++;
|
|
388
816
|
this.persistState();
|
|
817
|
+
this.scheduleRecovery(entry, handle, cfg);
|
|
389
818
|
throw err;
|
|
390
819
|
}
|
|
391
820
|
}
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
821
|
+
/**
|
|
822
|
+
* Everything a handle still runs, stopped: its supervisor's children (or its
|
|
823
|
+
* classic child process) AND its topology pools.
|
|
824
|
+
*
|
|
825
|
+
* The pools were missed. A pool's workers are not the supervisor's children
|
|
826
|
+
* — they are the daemon's, siblings of the server, recorded only in
|
|
827
|
+
* `topologyPools` — so `supervisor.stop()` never reached them, and nothing
|
|
828
|
+
* in the orchestrator destroyed a pool: only the daemon's own shutdown did.
|
|
829
|
+
* Every restart of an app with `instances > 1` left the previous workers
|
|
830
|
+
* running. On daos/test (2026-09-25): priceverse's `ohlcv-aggregator` and
|
|
831
|
+
* storage's `transform` in three generations at once.
|
|
832
|
+
*
|
|
833
|
+
* The pools go even when the supervisor's stop throws: a stop that failed
|
|
834
|
+
* half-way must not also leave the half nobody asked about.
|
|
835
|
+
*/
|
|
836
|
+
async tearDownProcesses(handle, force = false, timeout = 10_000) {
|
|
837
|
+
try {
|
|
838
|
+
if (handle.mode === 'bootstrap' && handle.supervisor) {
|
|
839
|
+
await handle.supervisor.stop();
|
|
404
840
|
}
|
|
405
|
-
|
|
406
|
-
|
|
841
|
+
else if (handle.childProcess) {
|
|
842
|
+
await this.stopChildProcess(handle, force, timeout);
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
finally {
|
|
846
|
+
for (const [name, pool] of handle.topologyPools) {
|
|
847
|
+
try {
|
|
848
|
+
await pool.destroy();
|
|
849
|
+
}
|
|
850
|
+
catch (err) {
|
|
851
|
+
this.logger.warn({ app: handle.name, pool: name, error: err.message }, 'Destroying a topology pool failed — its workers may still run');
|
|
852
|
+
}
|
|
407
853
|
}
|
|
408
|
-
handle.
|
|
854
|
+
handle.topologyPools.clear();
|
|
409
855
|
}
|
|
410
|
-
|
|
411
|
-
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Give back every service an app exposed on the daemon's Netron.
|
|
859
|
+
*
|
|
860
|
+
* A registration outlives the process behind it: the daemon keeps answering
|
|
861
|
+
* `queryInterface` with the full method list and every call through the name
|
|
862
|
+
* fails at the socket. `takeOverExisting` rescues a name the next launch
|
|
863
|
+
* re-exposes; one it does not stays advertised for the life of the daemon.
|
|
864
|
+
*/
|
|
865
|
+
async releaseExposedServices(handle) {
|
|
866
|
+
if (!handle.serviceRouter)
|
|
867
|
+
return;
|
|
868
|
+
try {
|
|
869
|
+
await handle.serviceRouter.releaseAll();
|
|
412
870
|
}
|
|
413
|
-
|
|
414
|
-
|
|
871
|
+
catch (err) {
|
|
872
|
+
this.logger.warn({ app: handle.name, error: err.message }, 'Releasing exposed services failed — the daemon may still advertise them');
|
|
415
873
|
}
|
|
416
|
-
handle.
|
|
417
|
-
|
|
418
|
-
|
|
874
|
+
handle.serviceRouter = null;
|
|
875
|
+
}
|
|
876
|
+
async stopApp(name, force = false, timeout = 10_000) {
|
|
877
|
+
const canonical = this.resolveAppName(name) ?? name;
|
|
878
|
+
// Outside the queue on purpose: the operator's intent lands the
|
|
879
|
+
// moment the command arrives, not whenever the queue drains.
|
|
880
|
+
this.clearCoolDown(canonical);
|
|
881
|
+
return this.lifecycleQueue.enqueue(canonical, 'stop', async () => {
|
|
882
|
+
const handle = this.handles.get(canonical);
|
|
883
|
+
if (!handle || handle.status === 'stopped')
|
|
884
|
+
return;
|
|
885
|
+
handle.markStopping();
|
|
886
|
+
// Unexpose topology services from daemon Netron before stopping processes
|
|
887
|
+
await this.releaseExposedServices(handle);
|
|
888
|
+
await this.tearDownProcesses(handle, force, timeout);
|
|
889
|
+
handle.markStopped();
|
|
890
|
+
this.persistState();
|
|
891
|
+
this.logger.info({ app: name }, 'App stopped');
|
|
892
|
+
});
|
|
419
893
|
}
|
|
420
894
|
async stopAll(force = false) {
|
|
421
895
|
if (!this.config)
|
|
@@ -460,6 +934,19 @@ export class OrchestratorService extends EventEmitter {
|
|
|
460
934
|
this.buildResults.clear();
|
|
461
935
|
return count;
|
|
462
936
|
}
|
|
937
|
+
/**
|
|
938
|
+
* Give a supervised app the definition its next start will use — its
|
|
939
|
+
* environment, its command. The running process is not touched; the next
|
|
940
|
+
* `restartApp` starts from this. `false` when nothing by that name runs
|
|
941
|
+
* here. See `AppHandle.redefine`.
|
|
942
|
+
*/
|
|
943
|
+
redefineApp(name, entry) {
|
|
944
|
+
const handle = this.handles.get(this.resolveAppName(name) ?? name);
|
|
945
|
+
if (!handle)
|
|
946
|
+
return false;
|
|
947
|
+
handle.redefine(entry);
|
|
948
|
+
return true;
|
|
949
|
+
}
|
|
463
950
|
/**
|
|
464
951
|
* Restart an app, coalescing concurrent triggers.
|
|
465
952
|
*
|
|
@@ -496,8 +983,58 @@ export class OrchestratorService extends EventEmitter {
|
|
|
496
983
|
async restartAppCoalesced(name, fromExternal) {
|
|
497
984
|
const canonical = this.resolveAppName(name) ?? name;
|
|
498
985
|
const handle = this.handles.get(canonical);
|
|
499
|
-
if (!handle)
|
|
986
|
+
if (!handle) {
|
|
987
|
+
// A PROJECT-QUALIFIED name with no live handle is not this daemon's to
|
|
988
|
+
// start, and matching it against the daemon's own config starts the
|
|
989
|
+
// WRONG APP.
|
|
990
|
+
//
|
|
991
|
+
// The names collide in practice. omnitron's own `omnitron.config.ts`
|
|
992
|
+
// declares `main`, `payments`, `messaging`, `storage`, `pricing` — the
|
|
993
|
+
// same five a downstream stack uses. On 2026-09-10, after payments died,
|
|
994
|
+
// `omnitron restart acme/dev/payments` matched omnitron's sample entry by
|
|
995
|
+
// its last path segment and launched `apps/payments/src/main.ts` relative
|
|
996
|
+
// to the DAEMON's cwd. It died with ERR_MODULE_NOT_FOUND on a path that
|
|
997
|
+
// exists in neither repository, and the backend stayed down while the
|
|
998
|
+
// command reported that it had restarted it.
|
|
999
|
+
//
|
|
1000
|
+
// Starting the wrong app is worse than refusing: the operator is told
|
|
1001
|
+
// their service is coming back, and the log that says otherwise is the
|
|
1002
|
+
// one they are not reading yet. `startApp` on the daemon already refuses
|
|
1003
|
+
// this exact shape and names the command that works; this is the same
|
|
1004
|
+
// refusal on the same reasoning, at the point where "no live handle" is
|
|
1005
|
+
// already known.
|
|
1006
|
+
const qualified = name.includes('/');
|
|
1007
|
+
const ownProject = this.config?.project;
|
|
1008
|
+
if (qualified && !ownProject) {
|
|
1009
|
+
const [project, stack] = name.split('/');
|
|
1010
|
+
throw new Error(`'${name}' belongs to a registered project, which this daemon restarts through its ` +
|
|
1011
|
+
`stack. Use \`omnitron stack start ${project} ${stack}\`, or restart the app by its ` +
|
|
1012
|
+
`bare name if you mean the daemon's own config.`);
|
|
1013
|
+
}
|
|
1014
|
+
// P1-F — fall through to startApp when there's no live handle but
|
|
1015
|
+
// the ecosystem config declares the app. Operators expect
|
|
1016
|
+
// `omnitron restart foo` to work even when `foo` was never started
|
|
1017
|
+
// (or was stopped, or crashed and was reaped). Previously this
|
|
1018
|
+
// threw "Unknown app", forcing the operator to first run start
|
|
1019
|
+
// explicitly.
|
|
1020
|
+
const effective = effectiveAppName(name);
|
|
1021
|
+
const entry = this.config?.apps.find((a) => a.name === name || a.name === effective || effectiveAppName(a.name) === effective);
|
|
1022
|
+
if (entry) {
|
|
1023
|
+
this.logger.info({ app: name }, 'restartApp: no live handle — falling through to startApp');
|
|
1024
|
+
// Restart under the name the OPERATOR asked for. The match above
|
|
1025
|
+
// accepts a bare entry for a qualified request, and handing it to
|
|
1026
|
+
// `startApp` unchanged re-registers the app bare —
|
|
1027
|
+
// `ensureNamespacedEntry` leaves an entry with no OMNITRON_PROJECT /
|
|
1028
|
+
// OMNITRON_STACK env alone by design. The app then silently leaves its
|
|
1029
|
+
// stack, because `ProjectService.toStackInfo` filters handles on the
|
|
1030
|
+
// `${project}/${stack}/` prefix.
|
|
1031
|
+
const requested = name.includes('/') && !entry.name.includes('/')
|
|
1032
|
+
? { ...entry, name }
|
|
1033
|
+
: entry;
|
|
1034
|
+
return this.startApp(requested);
|
|
1035
|
+
}
|
|
500
1036
|
throw new Error(`Unknown app: ${name}`);
|
|
1037
|
+
}
|
|
501
1038
|
name = canonical;
|
|
502
1039
|
const state = this.getCoalescerState(name);
|
|
503
1040
|
if (fromExternal)
|
|
@@ -569,6 +1106,26 @@ export class OrchestratorService extends EventEmitter {
|
|
|
569
1106
|
* bootstrap-config cache is cleared because changes to `bootstrap.ts`
|
|
570
1107
|
* (topology definition) need to take effect on the next start.
|
|
571
1108
|
*/
|
|
1109
|
+
/**
|
|
1110
|
+
* Start an app this daemon already supervises, using the entry its handle
|
|
1111
|
+
* carries.
|
|
1112
|
+
*
|
|
1113
|
+
* `startApp(entry)` needs an ecosystem entry, which the daemon's own config
|
|
1114
|
+
* is only one source of — a project stack registers handles this daemon
|
|
1115
|
+
* never declared. `restartAppNow` has always taken the entry off the handle
|
|
1116
|
+
* for exactly that reason; this is the same move for a start, so the name
|
|
1117
|
+
* `list` prints is a name `start` accepts.
|
|
1118
|
+
*
|
|
1119
|
+
* Returns `undefined` when the app is unknown, leaving the caller to decide
|
|
1120
|
+
* what to say about a name that names nothing.
|
|
1121
|
+
*/
|
|
1122
|
+
async startKnownApp(name) {
|
|
1123
|
+
const canonical = this.resolveAppName(name) ?? name;
|
|
1124
|
+
const handle = this.handles.get(canonical);
|
|
1125
|
+
if (!handle)
|
|
1126
|
+
return undefined;
|
|
1127
|
+
return this.startApp(handle.entry);
|
|
1128
|
+
}
|
|
572
1129
|
async restartAppNow(name) {
|
|
573
1130
|
const handle = this.handles.get(name);
|
|
574
1131
|
if (!handle)
|
|
@@ -602,10 +1159,14 @@ export class OrchestratorService extends EventEmitter {
|
|
|
602
1159
|
* Resolve a user-supplied name to a canonical handle key.
|
|
603
1160
|
*
|
|
604
1161
|
* Inputs accepted:
|
|
605
|
-
* - exact canonical key (`omni/dev/
|
|
606
|
-
* - bare short name (`
|
|
1162
|
+
* - exact canonical key (`omni/dev/payments`) → returned as-is
|
|
1163
|
+
* - bare short name (`payments`) → returns the canonical key whose
|
|
607
1164
|
* last `/`-segment matches, IF AND ONLY IF the match is unique.
|
|
608
1165
|
*
|
|
1166
|
+
* A qualified name that matches no handle exactly resolves to nothing: it
|
|
1167
|
+
* names a deployment this daemon does not have, and the nearest app with
|
|
1168
|
+
* the same last segment is a different one.
|
|
1169
|
+
*
|
|
609
1170
|
* Returns `undefined` for no match. Throws on ambiguity (multiple
|
|
610
1171
|
* stacks expose the same short name) so the caller can surface a
|
|
611
1172
|
* clear error to the operator instead of operating on the wrong
|
|
@@ -614,6 +1175,18 @@ export class OrchestratorService extends EventEmitter {
|
|
|
614
1175
|
resolveAppName(rawName) {
|
|
615
1176
|
if (this.handles.has(rawName))
|
|
616
1177
|
return rawName;
|
|
1178
|
+
// A qualified name is an ADDRESS, and the only handle that satisfies it
|
|
1179
|
+
// is the one with that address. Falling through to the short-name match
|
|
1180
|
+
// answered a question about one deployment with another's data:
|
|
1181
|
+
//
|
|
1182
|
+
// omnitron logs daos/deployed/main → daos/dev/main
|
|
1183
|
+
//
|
|
1184
|
+
// — the node's app asked for, this laptop's app printed, with nothing
|
|
1185
|
+
// said. `startApp` already refuses this for the same reason ("a
|
|
1186
|
+
// PROJECT-QUALIFIED name with no live handle is not this daemon's to
|
|
1187
|
+
// start"); the resolver every other caller goes through did not.
|
|
1188
|
+
if (rawName.includes('/'))
|
|
1189
|
+
return undefined;
|
|
617
1190
|
const effective = effectiveAppName(rawName);
|
|
618
1191
|
const matches = [];
|
|
619
1192
|
for (const key of this.handles.keys()) {
|
|
@@ -667,8 +1240,11 @@ export class OrchestratorService extends EventEmitter {
|
|
|
667
1240
|
this.logger.info({ app: name, child: childName }, 'Child reloaded successfully');
|
|
668
1241
|
}
|
|
669
1242
|
catch (err) {
|
|
670
|
-
|
|
671
|
-
|
|
1243
|
+
// `error: err`, not `error: err.message`: the logger aliases `error`
|
|
1244
|
+
// onto pino's error serializer, so the object arrives with its stack
|
|
1245
|
+
// and a string arrives as a string with nothing behind it.
|
|
1246
|
+
this.logger.error({ app: name, child: childName, error: err }, 'Failed to reload child — aborting zero-downtime reload');
|
|
1247
|
+
throw new Error(`Zero-downtime reload failed for '${name}' at child '${childName}': ${err.message}`, { cause: err });
|
|
672
1248
|
}
|
|
673
1249
|
}
|
|
674
1250
|
this.logger.info({ app: name }, 'Zero-downtime reload complete');
|
|
@@ -761,6 +1337,14 @@ export class OrchestratorService extends EventEmitter {
|
|
|
761
1337
|
getWorkerHandle(processId) {
|
|
762
1338
|
return this.pm.getWorkerHandle(processId);
|
|
763
1339
|
}
|
|
1340
|
+
/**
|
|
1341
|
+
* Surface the underlying IProcessInfo for a titan-pm processId
|
|
1342
|
+
* so callers (e.g. the daemon's `inspect` RPC) can read uptime
|
|
1343
|
+
* + status without reaching into the process-manager directly.
|
|
1344
|
+
*/
|
|
1345
|
+
getChildProcessInfo(processId) {
|
|
1346
|
+
return this.pm.getProcess(processId);
|
|
1347
|
+
}
|
|
764
1348
|
// ============================================================================
|
|
765
1349
|
// Monitoring — async, delegates to PM supervisor for bootstrap mode
|
|
766
1350
|
// ============================================================================
|
|
@@ -793,38 +1377,23 @@ export class OrchestratorService extends EventEmitter {
|
|
|
793
1377
|
}
|
|
794
1378
|
return null;
|
|
795
1379
|
}
|
|
1380
|
+
/**
|
|
1381
|
+
* `name` is resolved like every other per-app question (`main` finds
|
|
1382
|
+
* `daos/dev/main`); it was compared verbatim, so `omnitron metrics main`
|
|
1383
|
+
* printed «(no data)». An unknown name answers `{}`, and the caller says so.
|
|
1384
|
+
*/
|
|
796
1385
|
async getMetrics(name) {
|
|
1386
|
+
const wanted = name === undefined ? undefined : this.resolveAppName(name);
|
|
1387
|
+
if (name !== undefined && wanted === undefined)
|
|
1388
|
+
return {};
|
|
797
1389
|
const result = {};
|
|
798
1390
|
for (const [appName, handle] of this.handles) {
|
|
799
|
-
if (
|
|
1391
|
+
if (wanted && appName !== wanted)
|
|
800
1392
|
continue;
|
|
801
1393
|
if (handle.mode === 'bootstrap' && handle.supervisor) {
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
// Try PM-level RPC metrics first
|
|
806
|
-
const m = await handle.supervisor.getChildMetrics(childName);
|
|
807
|
-
if (m && (m.cpu > 0 || m.memory > 0)) {
|
|
808
|
-
cpu += m.cpu;
|
|
809
|
-
memory += m.memory;
|
|
810
|
-
requests += m.requests ?? 0;
|
|
811
|
-
errors += m.errors ?? 0;
|
|
812
|
-
}
|
|
813
|
-
else {
|
|
814
|
-
// Fallback: OS-level metrics via ps for the child PID
|
|
815
|
-
const processId = handle.supervisor.getChildProcessId(childName);
|
|
816
|
-
if (processId) {
|
|
817
|
-
const workerHandle = this.pm.getWorkerHandle(processId);
|
|
818
|
-
const childPid = workerHandle?.pid;
|
|
819
|
-
if (childPid) {
|
|
820
|
-
const osMetrics = await this.sampleProcessMetrics(childPid);
|
|
821
|
-
cpu += osMetrics.cpu;
|
|
822
|
-
memory += osMetrics.memory;
|
|
823
|
-
}
|
|
824
|
-
}
|
|
825
|
-
}
|
|
826
|
-
}
|
|
827
|
-
const aggregated = { cpu, memory, requests, errors };
|
|
1394
|
+
// Same sampler the poller uses, so `omnitron metrics` and `omnitron
|
|
1395
|
+
// list` cannot report different numbers for the same app.
|
|
1396
|
+
const aggregated = await this.sampleAppMetrics(handle);
|
|
828
1397
|
handle.lastMetrics = aggregated;
|
|
829
1398
|
result[appName] = aggregated;
|
|
830
1399
|
}
|
|
@@ -835,21 +1404,36 @@ export class OrchestratorService extends EventEmitter {
|
|
|
835
1404
|
}
|
|
836
1405
|
return result;
|
|
837
1406
|
}
|
|
1407
|
+
/**
|
|
1408
|
+
* Each app's health, asked of every process it runs — its supervisor
|
|
1409
|
+
* children and its worker pools — and combined (`app-health.ts`).
|
|
1410
|
+
*
|
|
1411
|
+
* `name` is resolved like every other per-app question: `main` finds
|
|
1412
|
+
* `daos/dev/main`. It was compared verbatim, so `omnitron health main`
|
|
1413
|
+
* matched nothing and the answer was empty. An unknown name answers `{}`,
|
|
1414
|
+
* and the caller says so.
|
|
1415
|
+
*/
|
|
838
1416
|
async getHealth(name) {
|
|
1417
|
+
const wanted = name === undefined ? undefined : this.resolveAppName(name);
|
|
1418
|
+
if (name !== undefined && wanted === undefined)
|
|
1419
|
+
return {};
|
|
839
1420
|
const result = {};
|
|
840
1421
|
for (const [appName, handle] of this.handles) {
|
|
841
|
-
if (
|
|
1422
|
+
if (wanted && appName !== wanted)
|
|
1423
|
+
continue;
|
|
1424
|
+
if (handle.status !== 'online') {
|
|
1425
|
+
result[appName] = notRunningHealth(handle.status);
|
|
842
1426
|
continue;
|
|
1427
|
+
}
|
|
843
1428
|
if (handle.mode === 'bootstrap' && handle.supervisor) {
|
|
844
|
-
const
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
result[appName] = health;
|
|
849
|
-
}
|
|
850
|
-
else {
|
|
851
|
-
result[appName] = handle.lastHealth;
|
|
1429
|
+
const supervisor = handle.supervisor;
|
|
1430
|
+
const answers = await Promise.all(supervisor.getChildNames().map(async (child) => [child, await supervisor.getChildHealth(child)]));
|
|
1431
|
+
for (const [pool, instance] of handle.topologyPools) {
|
|
1432
|
+
answers.push([pool, poolHealth(instance.metrics)]);
|
|
852
1433
|
}
|
|
1434
|
+
const health = answers.length > 0 ? combineProcessHealth(answers) : handle.lastHealth;
|
|
1435
|
+
handle.lastHealth = health;
|
|
1436
|
+
result[appName] = health;
|
|
853
1437
|
}
|
|
854
1438
|
else {
|
|
855
1439
|
result[appName] = handle.lastHealth;
|
|
@@ -952,7 +1536,7 @@ export class OrchestratorService extends EventEmitter {
|
|
|
952
1536
|
else {
|
|
953
1537
|
this.logger.error({ app: entry.name, code, signal }, 'App exited unexpectedly');
|
|
954
1538
|
handle.markCrashed();
|
|
955
|
-
this.
|
|
1539
|
+
this.scheduleRecovery(entry, handle, config);
|
|
956
1540
|
}
|
|
957
1541
|
this.persistState();
|
|
958
1542
|
});
|
|
@@ -967,29 +1551,56 @@ export class OrchestratorService extends EventEmitter {
|
|
|
967
1551
|
async launchBootstrapMode(entry, handle, config) {
|
|
968
1552
|
handle.markStarting();
|
|
969
1553
|
const bootstrapAbsPath = path.resolve(entry.cwd ?? this.cwd, entry.bootstrap);
|
|
970
|
-
// Load bootstrap definition to check for process topology and requirements
|
|
1554
|
+
// Load bootstrap definition to check for process topology and requirements.
|
|
1555
|
+
//
|
|
1556
|
+
// In dev, in a process of its own (`freshModuleGraph`): imported here, a
|
|
1557
|
+
// package the definition uses would be the one this daemon first saw, for
|
|
1558
|
+
// as long as it runs (bootstrap-loader.ts has the measurement). What comes
|
|
1559
|
+
// back is data, which is all the daemon reads of it — here, in the build
|
|
1560
|
+
// service and in the stack resolver; the app's own processes load the
|
|
1561
|
+
// definition themselves, functions and all.
|
|
971
1562
|
let topology;
|
|
972
1563
|
let definition;
|
|
973
1564
|
try {
|
|
974
|
-
definition = await loadBootstrapConfig(bootstrapAbsPath, { devMode: this.devMode });
|
|
1565
|
+
definition = await loadBootstrapConfig(bootstrapAbsPath, { devMode: this.devMode, freshModuleGraph: true });
|
|
975
1566
|
topology = definition.processes;
|
|
976
1567
|
}
|
|
977
1568
|
catch (err) {
|
|
978
|
-
|
|
1569
|
+
// Refused, with the reason. This fell back to «single-process mode» at
|
|
1570
|
+
// warn — but every definition declares its topology (IAppDefinition:
|
|
1571
|
+
// `processes` is required), so the fallback started the app in a shape
|
|
1572
|
+
// it was never written for. Measured 2026-09-23 on the master: paysys
|
|
1573
|
+
// and messaging, whose definitions failed to load («Parameter
|
|
1574
|
+
// decorators only work when experimental decorators are enabled»), were
|
|
1575
|
+
// started that way and died on «Dependency 'AuthModule:JWTService' not
|
|
1576
|
+
// found» and a module path that single mode could not resolve — the
|
|
1577
|
+
// operator saw those, and the cause sat one line up at warn.
|
|
1578
|
+
throw new Error(`Could not load ${entry.name}'s definition from ${bootstrapAbsPath}: ${err.message}`, { cause: err });
|
|
979
1579
|
}
|
|
980
|
-
//
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
1580
|
+
// The app's `config/default.json`, read on every start and put on a COPY
|
|
1581
|
+
// of the definition. This wrote the section onto the loader's cached
|
|
1582
|
+
// definition «if not set», so the first start's reading was every later
|
|
1583
|
+
// start's — and the logger level beside it, read in the same block, was
|
|
1584
|
+
// never read again at all (`project/declared-config.ts`).
|
|
1585
|
+
if (definition) {
|
|
1586
|
+
const declared = readDeclaredConfig(bootstrapAbsPath);
|
|
1587
|
+
if (declared.malformed) {
|
|
1588
|
+
this.logger.error({ app: entry.name, configPath: declared.configPath, error: declared.malformed }, 'config/default.json does not parse — its `omnitron` section is being ignored');
|
|
1589
|
+
}
|
|
1590
|
+
definition = withDeclaredConfig(definition, declared);
|
|
1591
|
+
// The only place the parent can learn the level the app chose; children
|
|
1592
|
+
// are spawned with it below. Validated rather than trusted: this is a
|
|
1593
|
+
// hand-edited JSON file, and a level pino does not know ("verbose", a
|
|
1594
|
+
// typo) would be handed straight to a child's logger constructor.
|
|
1595
|
+
const level = declared.loggerLevel;
|
|
1596
|
+
if (typeof level === 'string') {
|
|
1597
|
+
if (PINO_LEVELS.has(level)) {
|
|
1598
|
+
handle.appLogLevel = level;
|
|
1599
|
+
}
|
|
1600
|
+
else {
|
|
1601
|
+
this.logger.warn({ app: entry.name, configPath: declared.configPath, level }, 'config/default.json declares a logger level this runtime does not know — children keep the default');
|
|
990
1602
|
}
|
|
991
1603
|
}
|
|
992
|
-
catch { /* config file missing — skip */ }
|
|
993
1604
|
}
|
|
994
1605
|
// Resolve infrastructure config and inject as env vars.
|
|
995
1606
|
// Skip if already injected by project.service.ts (stack flow sets OMNITRON_STACK).
|
|
@@ -1102,9 +1713,12 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1102
1713
|
// fail with a cryptic "Cannot find module" downstream. Surface
|
|
1103
1714
|
// the build error directly so the operator sees the actual
|
|
1104
1715
|
// problem on `omnitron start/restart`.
|
|
1105
|
-
|
|
1716
|
+
// esbuild's error carries an `errors[]` with file, line and column.
|
|
1717
|
+
// `.message` is the summary; the locations are what an operator needs
|
|
1718
|
+
// to fix the build, and they exist only on the original.
|
|
1719
|
+
this.logger.error({ app: entry.name, error: err }, 'esbuild build failed — app cannot start without bundled output');
|
|
1106
1720
|
throw new Error(`Cannot launch '${entry.name}' in dev mode: esbuild build failed (${err.message}). ` +
|
|
1107
|
-
`Fix the build error and retry, or run with bundled output present in apps/${entry.name}/.omnitron-build
|
|
1721
|
+
`Fix the build error and retry, or run with bundled output present in apps/${entry.name}/.omnitron-build/.`, { cause: err });
|
|
1108
1722
|
}
|
|
1109
1723
|
}
|
|
1110
1724
|
const buildResult = this.buildResults.get(entry.name);
|
|
@@ -1157,14 +1771,20 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1157
1771
|
size: instances,
|
|
1158
1772
|
strategy: PoolStrategy.POWER_OF_TWO,
|
|
1159
1773
|
metrics: true,
|
|
1160
|
-
replaceUnhealthy:
|
|
1161
|
-
|
|
1774
|
+
// `replaceUnhealthy` is deliberately not set: titan-pm declares it
|
|
1775
|
+
// and reads it nowhere. An unhealthy worker is replaced by the
|
|
1776
|
+
// health path regardless, so `true` was a no-op that happened to
|
|
1777
|
+
// match the behaviour — and `false` would have been a no-op that did
|
|
1778
|
+
// not, leaving an operator believing they had turned replacement off.
|
|
1779
|
+
// What actually governs it is `healthCheck.unhealthyThreshold`.
|
|
1780
|
+
requestTimeout: CHILD_REQUEST_TIMEOUT,
|
|
1162
1781
|
maxQueueSize: 200,
|
|
1163
1782
|
spawnOptions: {
|
|
1164
1783
|
name: `${entry.name}/${procEntry.name}`,
|
|
1165
1784
|
version: '1.0.0',
|
|
1166
1785
|
allMethodsPublic: true,
|
|
1167
|
-
|
|
1786
|
+
...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
|
|
1787
|
+
startupTimeout: procEntry.startupTimeout ?? entry.startupTimeout ?? DEFAULT_CHILD_STARTUP_TIMEOUT,
|
|
1168
1788
|
...(entry.env && { env: entry.env }),
|
|
1169
1789
|
...(entry.cwd && { cwd: entry.cwd }),
|
|
1170
1790
|
dependencies: {
|
|
@@ -1187,25 +1807,31 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1187
1807
|
enabled: procEntry.health?.enabled !== false,
|
|
1188
1808
|
interval: procEntry.health?.interval ?? config.monitoring.healthCheck.interval,
|
|
1189
1809
|
},
|
|
1810
|
+
// `scaling.strategy` is what opts a pool into autoscaling; anything
|
|
1811
|
+
// else — including the default of not declaring it — means the pool
|
|
1812
|
+
// stays at the declared `instances`, which is what 'fixed' says on
|
|
1813
|
+
// the tin.
|
|
1814
|
+
//
|
|
1815
|
+
// Both arms of this ternary used to set `enabled: true`, differing
|
|
1816
|
+
// only in where the numbers came from. The setting existed, the
|
|
1817
|
+
// mechanism existed, and nothing connected them: every pool in the
|
|
1818
|
+
// platform was autoscaled whether or not it asked to be. Live effect
|
|
1819
|
+
// on two pools declared `instances: 2` — grown to their cap of eight
|
|
1820
|
+
// and held there, sixteen processes where four were configured.
|
|
1821
|
+
//
|
|
1822
|
+
// `min` follows the declared instance count rather than 1: a pool
|
|
1823
|
+
// asked for two workers should not be scaled below two.
|
|
1190
1824
|
autoScale: procEntry.scaling?.strategy === 'auto'
|
|
1191
1825
|
? {
|
|
1192
1826
|
enabled: true,
|
|
1193
|
-
min:
|
|
1827
|
+
min: instances,
|
|
1194
1828
|
max: procEntry.scaling?.maxInstances ?? instances * 4,
|
|
1195
1829
|
targetCPU: procEntry.scaling?.targetCPU ?? 70,
|
|
1196
1830
|
targetMemory: procEntry.scaling?.targetMemory ?? 80,
|
|
1197
1831
|
queueThreshold: procEntry.scaling?.queueThreshold ?? 10,
|
|
1198
1832
|
cooldownPeriod: procEntry.scaling?.cooldownPeriod ?? 30_000,
|
|
1199
1833
|
}
|
|
1200
|
-
: {
|
|
1201
|
-
enabled: true,
|
|
1202
|
-
min: 1,
|
|
1203
|
-
max: instances * 4,
|
|
1204
|
-
targetCPU: 70,
|
|
1205
|
-
targetMemory: 80,
|
|
1206
|
-
queueThreshold: 10,
|
|
1207
|
-
cooldownPeriod: 30_000,
|
|
1208
|
-
},
|
|
1834
|
+
: { enabled: false, min: instances, max: instances },
|
|
1209
1835
|
};
|
|
1210
1836
|
this.logger.info({ app: entry.name, process: procEntry.name, instances }, 'Creating topology process pool');
|
|
1211
1837
|
// pm.pool() returns a Proxy wrapping ProcessPool — execute() dispatches to pool workers
|
|
@@ -1217,24 +1843,50 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1217
1843
|
// which returns null for pool workers (they aren't supervisor
|
|
1218
1844
|
// children) and the entry is mis-reported as `status: 'stopped'`.
|
|
1219
1845
|
handle.topologyPools.set(procEntry.name, pool);
|
|
1846
|
+
this.attachPoolLogCapture(entry.name, procEntry.name, handle, pool);
|
|
1220
1847
|
// Auto-discover @Service metadata from pool workers and expose via ServiceRouter.
|
|
1221
1848
|
// No manual `exports` needed — @Service decorator IS the source of truth.
|
|
1222
1849
|
// Requires explicit `topology: { expose: true }` — opt-in for security.
|
|
1223
1850
|
if (procEntry.topology?.expose === true) {
|
|
1224
1851
|
try {
|
|
1852
|
+
// A pool worker reports ready before its Application has started, and
|
|
1853
|
+
// `@Service` classes are exposed to Netron during start — so asking
|
|
1854
|
+
// once, or once more 500ms later, asks a worker that is still booting.
|
|
1855
|
+
// What it answers then is an empty list, or a service whose method
|
|
1856
|
+
// list has not been filled in yet, and both used to end the same way:
|
|
1857
|
+
// nothing registered on the daemon, the consumer process starting
|
|
1858
|
+
// seconds later, querying, and being told the service does not exist.
|
|
1859
|
+
// The consumer resolves its topology proxy exactly once, so that
|
|
1860
|
+
// answer disabled the feature for the life of the process — in the
|
|
1861
|
+
// case that surfaced this, every OHLCV candle aggregation.
|
|
1862
|
+
//
|
|
1863
|
+
// Poll instead, and treat "a service with no methods" as not ready
|
|
1864
|
+
// rather than as a service, because an empty method list produces a
|
|
1865
|
+
// proxy that can do nothing.
|
|
1866
|
+
const usable = (list) => Array.isArray(list) && list.length > 0 && list.some((svc) => svc.methods?.length > 0);
|
|
1225
1867
|
let services = await pool.execute('getExposedServices');
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
1868
|
+
const deadline = Date.now() + TOPOLOGY_EXPOSE_TIMEOUT_MS;
|
|
1869
|
+
while (!usable(services) && Date.now() < deadline) {
|
|
1870
|
+
await new Promise((r) => setTimeout(r, TOPOLOGY_EXPOSE_POLL_MS));
|
|
1230
1871
|
services = await pool.execute('getExposedServices');
|
|
1231
1872
|
}
|
|
1232
|
-
if (!services
|
|
1233
|
-
this.logger.warn({
|
|
1873
|
+
if (!usable(services)) {
|
|
1874
|
+
this.logger.warn({
|
|
1875
|
+
app: entry.name,
|
|
1876
|
+
process: procEntry.name,
|
|
1877
|
+
waitedMs: TOPOLOGY_EXPOSE_TIMEOUT_MS,
|
|
1878
|
+
reported: services?.map((svc) => `${svc.name}(${svc.methods?.length ?? 0})`) ?? [],
|
|
1879
|
+
}, 'No usable @Service found on pool worker — topology.expose has no effect, and any ' +
|
|
1880
|
+
'process declaring topology.access on it will start without a proxy');
|
|
1234
1881
|
}
|
|
1235
|
-
for (const svc of services) {
|
|
1236
|
-
if (svc.methods.length === 0)
|
|
1882
|
+
for (const svc of services ?? []) {
|
|
1883
|
+
if (svc.methods.length === 0) {
|
|
1884
|
+
// Silently skipped before. A consumer then failed to find a
|
|
1885
|
+
// service its own declaration named, with nothing anywhere
|
|
1886
|
+
// connecting the two facts.
|
|
1887
|
+
this.logger.warn({ app: entry.name, process: procEntry.name, service: svc.name }, 'Pool service exposes no methods — not registered on the daemon');
|
|
1237
1888
|
continue;
|
|
1889
|
+
}
|
|
1238
1890
|
await serviceRouter.exposePoolService(procEntry.name, svc.name, svc.version ?? '1.0.0', pool, svc.methods);
|
|
1239
1891
|
this.logger.info({ app: entry.name, process: procEntry.name, service: svc.name, methods: svc.methods.length }, 'Auto-discovered and exposed pool service on daemon Netron');
|
|
1240
1892
|
}
|
|
@@ -1253,7 +1905,10 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1253
1905
|
name: `${entry.name}/${procEntry.name}`,
|
|
1254
1906
|
version: '1.0.0',
|
|
1255
1907
|
allMethodsPublic: true,
|
|
1256
|
-
|
|
1908
|
+
// Without this the child's client takes netron's 5000 ms; the pool
|
|
1909
|
+
// path beside it has always stated one. See CHILD_REQUEST_TIMEOUT.
|
|
1910
|
+
requestTimeout: CHILD_REQUEST_TIMEOUT,
|
|
1911
|
+
startupTimeout: procEntry.startupTimeout ?? entry.startupTimeout ?? DEFAULT_CHILD_STARTUP_TIMEOUT,
|
|
1257
1912
|
...(entry.env && { env: entry.env }),
|
|
1258
1913
|
...(entry.cwd && { cwd: entry.cwd }),
|
|
1259
1914
|
dependencies: {
|
|
@@ -1271,6 +1926,7 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1271
1926
|
timeout: procEntry.health?.timeout ?? config.monitoring.healthCheck.timeout,
|
|
1272
1927
|
},
|
|
1273
1928
|
observability: { metrics: true },
|
|
1929
|
+
...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
|
|
1274
1930
|
};
|
|
1275
1931
|
children.push({
|
|
1276
1932
|
name: `${entry.name}/${procEntry.name}`,
|
|
@@ -1295,18 +1951,11 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1295
1951
|
};
|
|
1296
1952
|
// Create supervisor WITHOUT starting — wire events first to avoid race condition
|
|
1297
1953
|
const supervisor = this.pm.createSupervisor(supervisorConfig);
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
const childPid = workerHandle?.pid ?? info?.pid;
|
|
1304
|
-
handle.markOnline(childPid ?? process.pid);
|
|
1305
|
-
}
|
|
1306
|
-
this.persistState();
|
|
1307
|
-
this.startPostOnlineCooldown(entry.name);
|
|
1308
|
-
this.emit('app:online', entry.name);
|
|
1309
|
-
});
|
|
1954
|
+
// The app's pid is its server's: the single process with a transport, the
|
|
1955
|
+
// one `handle.port` names. A topology with none has no pid to give.
|
|
1956
|
+
const server = singleEntries.find((p) => p.transports?.http) ?? singleEntries.find((p) => p.transports) ?? null;
|
|
1957
|
+
handle.declareChildren(children.map((child) => child.name), server ? `${entry.name}/${server.name}` : null);
|
|
1958
|
+
this.onChildStarted(entry, handle, supervisor);
|
|
1310
1959
|
// Set supervisor on handle BEFORE start so attachLogCapture can access it
|
|
1311
1960
|
handle.supervisor = supervisor;
|
|
1312
1961
|
handle.serviceRouter = serviceRouter;
|
|
@@ -1315,6 +1964,73 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1315
1964
|
this.wireSupervisorEvents(entry, handle, supervisor);
|
|
1316
1965
|
// Now start — events will be captured correctly
|
|
1317
1966
|
await supervisor.start();
|
|
1967
|
+
// Step 4: register the single processes that declare `topology.expose`.
|
|
1968
|
+
//
|
|
1969
|
+
// This was a pool-only feature, ignored in silence everywhere else: a
|
|
1970
|
+
// process declaring `expose: true` got no registration and no warning,
|
|
1971
|
+
// and every consumer naming it in `topology.access` started without a
|
|
1972
|
+
// proxy. Measured on priceverse, whose `collector` is a single process
|
|
1973
|
+
// holding the only objects that know whether the exchange WebSockets are
|
|
1974
|
+
// up — the server process could not ask, and answered `ready: down —
|
|
1975
|
+
// exchanges unavailable` while three were connected.
|
|
1976
|
+
//
|
|
1977
|
+
// After `supervisor.start()` because there is no child to ask before it.
|
|
1978
|
+
// A consumer among these children may therefore start before the service
|
|
1979
|
+
// is registered; consumers re-resolve their proxy on use for exactly that
|
|
1980
|
+
// reason (see priceverse's `OhlcvSchedulerService`).
|
|
1981
|
+
await this.exposeChildTopologyServices(entry, supervisor, serviceRouter, singleEntries);
|
|
1982
|
+
}
|
|
1983
|
+
/**
|
|
1984
|
+
* Register each single process's `@Service` classes on the daemon.
|
|
1985
|
+
*
|
|
1986
|
+
* Mirrors the pool path above, including its poll: a child reports ready
|
|
1987
|
+
* before its Application has started, and `@Service` classes are exposed to
|
|
1988
|
+
* Netron during start, so asking once asks a process that is still booting
|
|
1989
|
+
* and gets an empty list back.
|
|
1990
|
+
*/
|
|
1991
|
+
async exposeChildTopologyServices(entry, supervisor, serviceRouter, singleEntries) {
|
|
1992
|
+
for (const procEntry of singleEntries) {
|
|
1993
|
+
if (procEntry.topology?.expose !== true)
|
|
1994
|
+
continue;
|
|
1995
|
+
const childName = `${entry.name}/${procEntry.name}`;
|
|
1996
|
+
const getProxy = () => supervisor.getChildProxy(childName);
|
|
1997
|
+
try {
|
|
1998
|
+
const usable = (list) => Array.isArray(list) && list.length > 0 && list.some((svc) => svc.methods?.length > 0);
|
|
1999
|
+
const ask = async () => {
|
|
2000
|
+
const child = getProxy();
|
|
2001
|
+
if (!child?.getExposedServices)
|
|
2002
|
+
return undefined;
|
|
2003
|
+
return (await child.getExposedServices());
|
|
2004
|
+
};
|
|
2005
|
+
let services = await ask();
|
|
2006
|
+
const deadline = Date.now() + TOPOLOGY_EXPOSE_TIMEOUT_MS;
|
|
2007
|
+
while (!usable(services) && Date.now() < deadline) {
|
|
2008
|
+
await new Promise((r) => setTimeout(r, TOPOLOGY_EXPOSE_POLL_MS));
|
|
2009
|
+
services = await ask();
|
|
2010
|
+
}
|
|
2011
|
+
if (!usable(services)) {
|
|
2012
|
+
this.logger.warn({
|
|
2013
|
+
app: entry.name,
|
|
2014
|
+
process: procEntry.name,
|
|
2015
|
+
waitedMs: TOPOLOGY_EXPOSE_TIMEOUT_MS,
|
|
2016
|
+
reported: services?.map((svc) => `${svc.name}(${svc.methods?.length ?? 0})`) ?? [],
|
|
2017
|
+
}, 'No usable @Service found on this process — topology.expose has no effect, and any ' +
|
|
2018
|
+
'process declaring topology.access on it will start without a proxy');
|
|
2019
|
+
continue;
|
|
2020
|
+
}
|
|
2021
|
+
for (const svc of services ?? []) {
|
|
2022
|
+
if (svc.methods.length === 0) {
|
|
2023
|
+
this.logger.warn({ app: entry.name, process: procEntry.name, service: svc.name }, 'Process service exposes no methods — not registered on the daemon');
|
|
2024
|
+
continue;
|
|
2025
|
+
}
|
|
2026
|
+
await serviceRouter.exposeChildService(procEntry.name, svc.name, svc.version ?? '1.0.0', getProxy, svc.methods);
|
|
2027
|
+
this.logger.info({ app: entry.name, process: procEntry.name, service: svc.name, methods: svc.methods.length }, 'Auto-discovered and exposed process service on daemon Netron');
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
catch (err) {
|
|
2031
|
+
this.logger.error({ app: entry.name, process: procEntry.name, error: err.message }, "Failed to auto-discover/expose this process's services");
|
|
2032
|
+
}
|
|
2033
|
+
}
|
|
1318
2034
|
}
|
|
1319
2035
|
/**
|
|
1320
2036
|
* Launch single-process bootstrap app (no topology — original behavior).
|
|
@@ -1330,11 +2046,22 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1330
2046
|
const spawnOpts = {
|
|
1331
2047
|
name: entry.name,
|
|
1332
2048
|
version: '1.0.0',
|
|
2049
|
+
// As above: a single child's client otherwise runs on netron's default.
|
|
2050
|
+
requestTimeout: CHILD_REQUEST_TIMEOUT,
|
|
1333
2051
|
...(entry.env && { env: entry.env }),
|
|
1334
2052
|
dependencies: {
|
|
1335
2053
|
bootstrapPath: bootstrapAbsPath,
|
|
1336
2054
|
...(bundledModulePath && { bundledModulePath }),
|
|
1337
2055
|
},
|
|
2056
|
+
// Omitted when unset, which delegates the budget to titan-pm —
|
|
2057
|
+
// `process-spawner.ts`: `options.startupTimeout ?? resources?.timeout ??
|
|
2058
|
+
// 30_000`. Worth naming, because it is otherwise invisible from here:
|
|
2059
|
+
// an APP that declares nothing gets 30s while its CHILD processes get
|
|
2060
|
+
// `DEFAULT_CHILD_STARTUP_TIMEOUT`, 60s. Backwards if anything, since the
|
|
2061
|
+
// app usually does the heavier work — and the downstream project's own
|
|
2062
|
+
// config carries a note about a 30s default that kept a whole stack from
|
|
2063
|
+
// starting. Left as delegation rather than changed here: raising it is a
|
|
2064
|
+
// decision about titan-pm's contract, not about this call site.
|
|
1338
2065
|
...(entry.startupTimeout != null && { startupTimeout: entry.startupTimeout }),
|
|
1339
2066
|
health: {
|
|
1340
2067
|
enabled: true,
|
|
@@ -1342,6 +2069,7 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1342
2069
|
timeout: config.monitoring.healthCheck.timeout,
|
|
1343
2070
|
},
|
|
1344
2071
|
observability: { metrics: true },
|
|
2072
|
+
...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
|
|
1345
2073
|
};
|
|
1346
2074
|
const childConfig = {
|
|
1347
2075
|
name: entry.name,
|
|
@@ -1354,7 +2082,8 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1354
2082
|
size: instances,
|
|
1355
2083
|
strategy: PoolStrategy.ROUND_ROBIN,
|
|
1356
2084
|
metrics: true,
|
|
1357
|
-
|
|
2085
|
+
// See the note on the other pool: `replaceUnhealthy` is declared by
|
|
2086
|
+
// titan-pm and read by nothing.
|
|
1358
2087
|
healthCheck: {
|
|
1359
2088
|
enabled: true,
|
|
1360
2089
|
interval: config.monitoring.healthCheck.interval,
|
|
@@ -1392,20 +2121,10 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1392
2121
|
};
|
|
1393
2122
|
// Create supervisor WITHOUT starting — wire events first to avoid race condition
|
|
1394
2123
|
const supervisor = this.pm.createSupervisor(supervisorConfig);
|
|
1395
|
-
// Wire supervisor events BEFORE start so we catch the initial child:started
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
// Prefer WorkerHandle.pid (actual child OS PID), fallback to IProcessInfo.pid
|
|
1400
|
-
const workerHandle = this.pm.getWorkerHandle(processId);
|
|
1401
|
-
const info = this.pm.getProcess(processId);
|
|
1402
|
-
const childPid = workerHandle?.pid ?? info?.pid;
|
|
1403
|
-
handle.markOnline(childPid ?? process.pid);
|
|
1404
|
-
}
|
|
1405
|
-
this.persistState();
|
|
1406
|
-
this.startPostOnlineCooldown(entry.name);
|
|
1407
|
-
this.emit('app:online', entry.name);
|
|
1408
|
-
});
|
|
2124
|
+
// Wire supervisor events BEFORE start so we catch the initial child:started.
|
|
2125
|
+
// One child, and it is the whole app — so it is the server too.
|
|
2126
|
+
handle.declareChildren([childConfig.name], childConfig.name);
|
|
2127
|
+
this.onChildStarted(entry, handle, supervisor);
|
|
1409
2128
|
// Set supervisor on handle BEFORE start so attachLogCapture can access it
|
|
1410
2129
|
handle.supervisor = supervisor;
|
|
1411
2130
|
handle.instanceCount = instances;
|
|
@@ -1413,14 +2132,50 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1413
2132
|
// Now start — events will be captured correctly
|
|
1414
2133
|
await supervisor.start();
|
|
1415
2134
|
}
|
|
2135
|
+
/**
|
|
2136
|
+
* A child's start, for both launch paths: the app is online when the start
|
|
2137
|
+
* completes it, and not before (`AppHandle.childStarted`).
|
|
2138
|
+
*
|
|
2139
|
+
* Each path had its own copy of this handler, and both marked the app
|
|
2140
|
+
* online — with that child's pid — on every child's start. `app:online`
|
|
2141
|
+
* went out once per child with it: three STARTED events for one boot of a
|
|
2142
|
+
* three-process app, the first two about an app still coming up.
|
|
2143
|
+
*/
|
|
2144
|
+
onChildStarted(entry, handle, supervisor) {
|
|
2145
|
+
supervisor.on('child:started', (childName) => {
|
|
2146
|
+
// A pool child's proxy answers every property with a function —
|
|
2147
|
+
// `__processId` included — so only a string is a process id. The old
|
|
2148
|
+
// handler took that function for one, found no worker behind it, and
|
|
2149
|
+
// gave such an app the DAEMON's pid.
|
|
2150
|
+
const processId = supervisor.getChildProcessId(childName);
|
|
2151
|
+
const pid = typeof processId === 'string'
|
|
2152
|
+
? (this.pm.getWorkerHandle(processId)?.pid ?? this.pm.getProcess(processId)?.pid ?? null)
|
|
2153
|
+
: null;
|
|
2154
|
+
const whole = handle.childStarted(childName, pid);
|
|
2155
|
+
this.persistState();
|
|
2156
|
+
if (!whole)
|
|
2157
|
+
return;
|
|
2158
|
+
this.startPostOnlineCooldown(entry.name);
|
|
2159
|
+
this.emit('app:online', entry.name);
|
|
2160
|
+
});
|
|
2161
|
+
}
|
|
1416
2162
|
/**
|
|
1417
2163
|
* Wire common supervisor events (crash, restart, escalate) and log capture.
|
|
1418
2164
|
*/
|
|
1419
2165
|
wireSupervisorEvents(entry, handle, supervisor) {
|
|
1420
2166
|
supervisor.on('child:started', (childName) => {
|
|
1421
2167
|
this.attachLogCapture(entry.name, childName, handle);
|
|
2168
|
+
this.attachExitCapture(entry.name, childName, handle);
|
|
2169
|
+
// The app re-entered a live state — drop the previous crash
|
|
2170
|
+
// context so a future `omnitron inspect` shows the *next*
|
|
2171
|
+
// crash, not the last one.
|
|
2172
|
+
handle.clearLastExit();
|
|
1422
2173
|
});
|
|
2174
|
+
// A child that is down leaves the app partial until it is back — a
|
|
2175
|
+
// sibling's restart must not bring the app `online` around it.
|
|
2176
|
+
supervisor.on('child:stopped', (childName) => handle.childGone(childName));
|
|
1423
2177
|
supervisor.on('child:crash', (childName, error) => {
|
|
2178
|
+
handle.childGone(childName);
|
|
1424
2179
|
// If app never reached 'online', mark as errored (not crashed — it never ran)
|
|
1425
2180
|
if (handle.status === 'starting') {
|
|
1426
2181
|
handle.markErrored();
|
|
@@ -1428,10 +2183,94 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1428
2183
|
else {
|
|
1429
2184
|
handle.markCrashed();
|
|
1430
2185
|
}
|
|
2186
|
+
// Stash the supervisor-provided error message onto whatever
|
|
2187
|
+
// exit context the workerHandle.onExit hook already wrote. If
|
|
2188
|
+
// the exit event hasn't fired yet (race) we still need to
|
|
2189
|
+
// persist *something* — record a synthetic entry that the
|
|
2190
|
+
// exit hook below will overwrite when it does fire.
|
|
2191
|
+
if (handle.lastExit) {
|
|
2192
|
+
handle.lastExit.message = error.message;
|
|
2193
|
+
}
|
|
2194
|
+
else {
|
|
2195
|
+
handle.recordExit({ code: null, signal: null, expected: false, message: error.message });
|
|
2196
|
+
}
|
|
1431
2197
|
this.logger.error({ app: entry.name, child: childName, error: error.message }, 'Process crashed');
|
|
1432
2198
|
this.persistState();
|
|
1433
2199
|
this.emit('app:crash', entry.name, error);
|
|
1434
2200
|
});
|
|
2201
|
+
// T#60 — startup failures (child throws/rejects in waitForReady
|
|
2202
|
+
// before ever reporting ready) come through this event, NOT
|
|
2203
|
+
// child:crash. Without a handler the orchestrator just sees
|
|
2204
|
+
// status='errored' with no clue what went wrong. titan-pm
|
|
2205
|
+
// already stuffs the captured stderr into `error.details.stderr`
|
|
2206
|
+
// so we surface it on `lastExit.stderrTail`.
|
|
2207
|
+
supervisor.on('child:start-failed', (childName, error) => {
|
|
2208
|
+
const details = error.details;
|
|
2209
|
+
const stderrTail = typeof details?.stderr === 'string'
|
|
2210
|
+
? details.stderr.split('\n').filter((l) => l.length > 0).slice(-80)
|
|
2211
|
+
: [];
|
|
2212
|
+
for (const line of stderrTail)
|
|
2213
|
+
handle.appendStderr(line);
|
|
2214
|
+
// A child that never came up because WE stopped the app is not a
|
|
2215
|
+
// failure, and reporting it as one is worse than saying nothing: the
|
|
2216
|
+
// message titan-pm builds for an aborted start says the process "did not
|
|
2217
|
+
// choose to exit, so its own logs will not explain this" — true, and it
|
|
2218
|
+
// sends the reader hunting a fault in an application that was doing
|
|
2219
|
+
// nothing wrong. Every `stack stop` issued while anything was still
|
|
2220
|
+
// booting wrote one of these per child, at error level, into the app's
|
|
2221
|
+
// own log.
|
|
2222
|
+
//
|
|
2223
|
+
// `'stopping'` only — NOT `'stopped'`, which is also an AppHandle's
|
|
2224
|
+
// initial value, so accepting it would swallow the first genuine failure
|
|
2225
|
+
// of any app whose handle had not yet been marked starting.
|
|
2226
|
+
const expected = handle.status === 'stopping' ||
|
|
2227
|
+
error.stoppedDuringStartup === true;
|
|
2228
|
+
if (expected) {
|
|
2229
|
+
handle.recordExit({
|
|
2230
|
+
code: null,
|
|
2231
|
+
signal: null,
|
|
2232
|
+
expected: true,
|
|
2233
|
+
message: error.message,
|
|
2234
|
+
});
|
|
2235
|
+
this.logger.info({ app: entry.name, child: childName, err: error.message }, 'Process stopped before it finished starting');
|
|
2236
|
+
this.persistState();
|
|
2237
|
+
return;
|
|
2238
|
+
}
|
|
2239
|
+
// Put what the child printed into the app's LOG FILE, not only into the
|
|
2240
|
+
// in-memory ring `omnitron inspect` reads.
|
|
2241
|
+
//
|
|
2242
|
+
// A child that dies during startup never produces a WorkerHandle at all
|
|
2243
|
+
// — titan-pm registers one only after `waitForReady` resolves — so there
|
|
2244
|
+
// is no `onLog` to subscribe to here and nothing to replay. What does
|
|
2245
|
+
// exist is the output `waitForReady` captured while waiting, which it
|
|
2246
|
+
// attaches to the error. Feeding that to the log handlers is the only
|
|
2247
|
+
// way a failed boot reaches disk, and a failed boot is the case where
|
|
2248
|
+
// reading it matters most: an application that dies before its logger
|
|
2249
|
+
// exists has said everything it is ever going to say on these streams.
|
|
2250
|
+
// The failure itself first. A child can fail without printing a word —
|
|
2251
|
+
// it reports the error over the IPC channel and exits, which is what a
|
|
2252
|
+
// module that throws on import does — so the streams below are often
|
|
2253
|
+
// empty and the only account of what happened is on the error.
|
|
2254
|
+
this.persistChildFailure(entry.name, childName, error);
|
|
2255
|
+
this.persistChildOutput(entry.name, childName, details?.stdout, 'stdout');
|
|
2256
|
+
this.persistChildOutput(entry.name, childName, details?.stderr, 'stderr');
|
|
2257
|
+
handle.recordExit({
|
|
2258
|
+
code: null,
|
|
2259
|
+
signal: null,
|
|
2260
|
+
expected: false,
|
|
2261
|
+
message: error.message,
|
|
2262
|
+
});
|
|
2263
|
+
// The supervisor gives up on a non-critical child whose start failed (a
|
|
2264
|
+
// critical one throws, and `startApp` says so). That went unseen while
|
|
2265
|
+
// the first child's start made the app `online`; now the app would wait
|
|
2266
|
+
// for a child that is not coming, in `starting` — which `omnitron start`
|
|
2267
|
+
// takes for a start in progress and leaves alone. A declared process
|
|
2268
|
+
// that did not start is an app that did not start.
|
|
2269
|
+
if (handle.status === 'starting')
|
|
2270
|
+
handle.markErrored();
|
|
2271
|
+
this.logger.error({ app: entry.name, child: childName, err: error.message }, 'Process failed to start');
|
|
2272
|
+
this.persistState();
|
|
2273
|
+
});
|
|
1435
2274
|
supervisor.on('child:restart', (childName, attempt) => {
|
|
1436
2275
|
handle.restarts = attempt;
|
|
1437
2276
|
this.logger.info({ app: entry.name, child: childName, attempt }, 'Process restarting');
|
|
@@ -1458,9 +2297,45 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1458
2297
|
const workerHandle = this.pm.getWorkerHandle(processId);
|
|
1459
2298
|
if (!workerHandle?.onLog)
|
|
1460
2299
|
return;
|
|
1461
|
-
workerHandle.onLog((
|
|
2300
|
+
workerHandle.onLog(this.childLogForwarder(appName, handle));
|
|
2301
|
+
}
|
|
2302
|
+
/**
|
|
2303
|
+
* Attach log capture to the workers of a topology POOL.
|
|
2304
|
+
*
|
|
2305
|
+
* `attachLogCapture` above is driven by `child:started`, which only
|
|
2306
|
+
* supervisor children emit — a pool worker is not one, so nothing ever
|
|
2307
|
+
* subscribed to its output. Everything those workers print went nowhere:
|
|
2308
|
+
* the OHLCV aggregator and the image transformer have been mute since they
|
|
2309
|
+
* were written, and since titan-pm started holding pre-capture output it was
|
|
2310
|
+
* also a megabyte per worker retained for a reader that never arrived.
|
|
2311
|
+
*
|
|
2312
|
+
* Attached once, at pool creation. A worker replaced later by autoscaling or
|
|
2313
|
+
* a health restart is not covered; these pools are fixed-size, and the
|
|
2314
|
+
* alternative is an event this package does not expose yet.
|
|
2315
|
+
*/
|
|
2316
|
+
attachPoolLogCapture(appName, processName, handle, pool) {
|
|
2317
|
+
const forward = this.childLogForwarder(appName, handle);
|
|
2318
|
+
let attached = 0;
|
|
2319
|
+
for (const workerId of pool.getWorkerIds()) {
|
|
2320
|
+
const workerHandle = pool.getWorkerHandle(workerId);
|
|
2321
|
+
if (!workerHandle?.onLog)
|
|
2322
|
+
continue;
|
|
2323
|
+
workerHandle.onLog(forward);
|
|
2324
|
+
attached++;
|
|
2325
|
+
}
|
|
2326
|
+
this.logger.debug({ app: appName, process: processName, workers: attached }, 'Pool log capture attached');
|
|
2327
|
+
}
|
|
2328
|
+
/** The one place a child's line becomes an app's line. */
|
|
2329
|
+
childLogForwarder(appName, handle) {
|
|
2330
|
+
return (line, stream) => {
|
|
1462
2331
|
// In-memory ring buffer (for live CLI tailing)
|
|
1463
2332
|
handle.appendLog(line);
|
|
2333
|
+
// Separate stderr ring — feeds `lastExit.stderrTail` on death.
|
|
2334
|
+
// Without this, the operator running `omnitron inspect` after
|
|
2335
|
+
// a crash sees no hint at WHY (the in-flight pipe is gone).
|
|
2336
|
+
if (stream === 'stderr') {
|
|
2337
|
+
handle.appendStderr(line);
|
|
2338
|
+
}
|
|
1464
2339
|
// Persist to disk via registered log handlers (LogManager)
|
|
1465
2340
|
for (const handler of this.appLogHandlers) {
|
|
1466
2341
|
try {
|
|
@@ -1470,6 +2345,108 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1470
2345
|
// Log handler failure must not break the app
|
|
1471
2346
|
}
|
|
1472
2347
|
}
|
|
2348
|
+
};
|
|
2349
|
+
}
|
|
2350
|
+
/**
|
|
2351
|
+
* Write the failure a child reported before it could start to the app's log
|
|
2352
|
+
* handlers, as one structured line.
|
|
2353
|
+
*
|
|
2354
|
+
* `error.childStack` is the stack from inside the child when it managed to
|
|
2355
|
+
* send one — for a module that throws on import that is the entire
|
|
2356
|
+
* diagnosis, and it exists nowhere else once the process is gone.
|
|
2357
|
+
*/
|
|
2358
|
+
persistChildFailure(appName, childName, error) {
|
|
2359
|
+
const childStack = error.childStack;
|
|
2360
|
+
const entry = JSON.stringify({
|
|
2361
|
+
level: 50,
|
|
2362
|
+
time: new Date().toISOString(),
|
|
2363
|
+
processName: qualifyChildName(appName, childName),
|
|
2364
|
+
event: 'process.start_failed',
|
|
2365
|
+
err: { message: error.message, ...(childStack ? { stack: childStack } : {}) },
|
|
2366
|
+
msg: `Process failed to start: ${error.message}`,
|
|
2367
|
+
});
|
|
2368
|
+
for (const handler of this.appLogHandlers) {
|
|
2369
|
+
try {
|
|
2370
|
+
handler(appName, entry);
|
|
2371
|
+
}
|
|
2372
|
+
catch {
|
|
2373
|
+
// A log handler must not break failure reporting.
|
|
2374
|
+
}
|
|
2375
|
+
}
|
|
2376
|
+
}
|
|
2377
|
+
/**
|
|
2378
|
+
* Write a dead child's captured output to the app's log handlers.
|
|
2379
|
+
*
|
|
2380
|
+
* Emitted as pino-shaped JSON so it lands in the same file, at the same
|
|
2381
|
+
* level, as everything else that app ever logged — a reader tailing
|
|
2382
|
+
* `app.log` after a failed start finds the reason there rather than having
|
|
2383
|
+
* to know that this particular failure hides in a different place.
|
|
2384
|
+
*/
|
|
2385
|
+
persistChildOutput(appName, childName, output, stream) {
|
|
2386
|
+
if (!output)
|
|
2387
|
+
return;
|
|
2388
|
+
const time = new Date().toISOString();
|
|
2389
|
+
for (const line of output.split('\n')) {
|
|
2390
|
+
if (!line.trim())
|
|
2391
|
+
continue;
|
|
2392
|
+
// Already-structured lines pass through untouched; the child may well
|
|
2393
|
+
// have got far enough to produce them.
|
|
2394
|
+
const entry = line.trimStart().startsWith('{')
|
|
2395
|
+
? line
|
|
2396
|
+
: JSON.stringify({
|
|
2397
|
+
// stderr is not error. A boot script's own progress markers go
|
|
2398
|
+
// there, and a failing start emits a run of them — classifying
|
|
2399
|
+
// every one at 50 filled the log with errors that are nothing of
|
|
2400
|
+
// the sort, next to the single line that IS the failure. The
|
|
2401
|
+
// failure is reported at 50 by `persistChildFailure`; this is the
|
|
2402
|
+
// context around it, and `stream` keeps it greppable.
|
|
2403
|
+
level: stream === 'stderr' ? 40 : 30,
|
|
2404
|
+
time,
|
|
2405
|
+
processName: qualifyChildName(appName, childName),
|
|
2406
|
+
stream,
|
|
2407
|
+
msg: line,
|
|
2408
|
+
});
|
|
2409
|
+
for (const handler of this.appLogHandlers) {
|
|
2410
|
+
try {
|
|
2411
|
+
handler(appName, entry);
|
|
2412
|
+
}
|
|
2413
|
+
catch {
|
|
2414
|
+
// A log handler must not break failure reporting.
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
/**
|
|
2420
|
+
* Subscribe to the WorkerHandle's `exit` event so the orchestrator
|
|
2421
|
+
* keeps the exit code / signal / "expected" flag past the
|
|
2422
|
+
* WorkerHandle's GC. Pre-fix the supervisor's `child:crash` event
|
|
2423
|
+
* fired with an Error but never surfaced the underlying
|
|
2424
|
+
* code/signal — operators had no way to tell SIGKILL from a
|
|
2425
|
+
* thrown TypeError. The captured info lands on
|
|
2426
|
+
* `handle.lastExit` and is surfaced by `omnitron inspect`.
|
|
2427
|
+
*/
|
|
2428
|
+
attachExitCapture(appName, childName, handle) {
|
|
2429
|
+
const supervisor = handle.supervisor;
|
|
2430
|
+
if (!supervisor)
|
|
2431
|
+
return;
|
|
2432
|
+
const processId = supervisor.getChildProcessId(childName);
|
|
2433
|
+
if (!processId)
|
|
2434
|
+
return;
|
|
2435
|
+
const workerHandle = this.pm.getWorkerHandle(processId);
|
|
2436
|
+
if (!workerHandle?.onExit)
|
|
2437
|
+
return;
|
|
2438
|
+
workerHandle.onExit((info) => {
|
|
2439
|
+
// Preserve the supervisor's error message if `child:crash`
|
|
2440
|
+
// already fired before us. recordExit overwrites the synthetic
|
|
2441
|
+
// entry with the authoritative code/signal.
|
|
2442
|
+
const message = handle.lastExit?.message;
|
|
2443
|
+
handle.recordExit({
|
|
2444
|
+
code: info.code,
|
|
2445
|
+
signal: info.signal,
|
|
2446
|
+
expected: info.expected,
|
|
2447
|
+
...(message !== undefined && { message }),
|
|
2448
|
+
});
|
|
2449
|
+
this.logger.debug({ app: appName, child: childName, code: info.code, signal: info.signal, expected: info.expected }, 'Captured worker exit context');
|
|
1473
2450
|
});
|
|
1474
2451
|
}
|
|
1475
2452
|
// ============================================================================
|
|
@@ -1479,30 +2456,58 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1479
2456
|
const child = handle.childProcess;
|
|
1480
2457
|
if (!child || !child.pid)
|
|
1481
2458
|
return;
|
|
2459
|
+
const pid = child.pid;
|
|
2460
|
+
// P0-D — Signal the whole process group, not just the direct
|
|
2461
|
+
// child. classic-launcher spawns with `detached: true` so the
|
|
2462
|
+
// child becomes group leader (pgid === pid). `process.kill(-pid)`
|
|
2463
|
+
// delivers the signal to every process in the group, reaping
|
|
2464
|
+
// grandchildren (ffmpeg, docker, worker pools …) that ignore the
|
|
2465
|
+
// direct-child kill. Falls back to `child.kill` when the negative-
|
|
2466
|
+
// pid call fails (test environments, restricted permissions, the
|
|
2467
|
+
// child wasn't started detached — bootstrap-mode currently isn't).
|
|
2468
|
+
const sendSignal = (sig) => {
|
|
2469
|
+
try {
|
|
2470
|
+
process.kill(-pid, sig);
|
|
2471
|
+
return;
|
|
2472
|
+
}
|
|
2473
|
+
catch (err) {
|
|
2474
|
+
const code = err.code;
|
|
2475
|
+
if (code !== 'ESRCH' && code !== 'EPERM') {
|
|
2476
|
+
// ESRCH/EPERM = group already gone or not a group leader.
|
|
2477
|
+
// Anything else (EINVAL etc.) → log via best-effort.
|
|
2478
|
+
this.logger.debug({ app: handle.name, pid, sig, code }, 'process.kill(-pid) fallback');
|
|
2479
|
+
}
|
|
2480
|
+
}
|
|
2481
|
+
try {
|
|
2482
|
+
child.kill(sig);
|
|
2483
|
+
}
|
|
2484
|
+
catch { /* best-effort */ }
|
|
2485
|
+
};
|
|
1482
2486
|
await new Promise((resolve) => {
|
|
1483
2487
|
const timer = setTimeout(() => {
|
|
1484
|
-
|
|
1485
|
-
child.kill('SIGKILL');
|
|
1486
|
-
}
|
|
1487
|
-
catch {
|
|
1488
|
-
// SIGKILL send is best-effort
|
|
1489
|
-
}
|
|
2488
|
+
sendSignal('SIGKILL');
|
|
1490
2489
|
resolve();
|
|
1491
2490
|
}, timeout);
|
|
1492
2491
|
child.once('exit', () => {
|
|
1493
2492
|
clearTimeout(timer);
|
|
1494
2493
|
resolve();
|
|
1495
2494
|
});
|
|
1496
|
-
|
|
1497
|
-
child.kill(force ? 'SIGKILL' : 'SIGTERM');
|
|
1498
|
-
}
|
|
1499
|
-
catch {
|
|
1500
|
-
clearTimeout(timer);
|
|
1501
|
-
resolve();
|
|
1502
|
-
}
|
|
2495
|
+
sendSignal(force ? 'SIGKILL' : 'SIGTERM');
|
|
1503
2496
|
});
|
|
1504
2497
|
}
|
|
1505
|
-
|
|
2498
|
+
/**
|
|
2499
|
+
* Cancel a pending crash cool-down for `name`, if one is armed.
|
|
2500
|
+
* An explicit stop means "this app is going down now, stay down",
|
|
2501
|
+
* which the cool-down must not undo ten minutes later.
|
|
2502
|
+
*/
|
|
2503
|
+
clearCoolDown(name) {
|
|
2504
|
+
const timer = this.coolDownTimers.get(name);
|
|
2505
|
+
if (timer) {
|
|
2506
|
+
clearTimeout(timer);
|
|
2507
|
+
this.coolDownTimers.delete(name);
|
|
2508
|
+
}
|
|
2509
|
+
}
|
|
2510
|
+
async scheduleRecovery(entry, handle, config) {
|
|
1506
2511
|
const policy = buildRestartPolicy(entry, config);
|
|
1507
2512
|
if (!policy.enabled)
|
|
1508
2513
|
return;
|
|
@@ -1510,87 +2515,416 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1510
2515
|
handle.status = 'crashed';
|
|
1511
2516
|
this.logger.error({ app: entry.name }, 'Max restarts exceeded');
|
|
1512
2517
|
this.emit('app:escalated', entry.name);
|
|
2518
|
+
// P1-I — auto-recovery after a cool-down. pm2 has this as
|
|
2519
|
+
// `restart_delay` + a fresh attempt window; pre-fix omnitron
|
|
2520
|
+
// left the app permanently 'crashed' until human intervention.
|
|
2521
|
+
// We reset the counter + schedule one fresh attempt 10 minutes
|
|
2522
|
+
// out. If THAT attempt also crashes through maxRestarts we
|
|
2523
|
+
// land back here and pause another 10 minutes — bounded
|
|
2524
|
+
// self-healing without thrashing.
|
|
2525
|
+
//
|
|
2526
|
+
// The wait happens OUTSIDE the lifecycle queue, and that is the
|
|
2527
|
+
// whole point. It used to be a queued `crash-restart` whose
|
|
2528
|
+
// `run()` slept for the entire cool-down — but `enqueue` will
|
|
2529
|
+
// not preempt a RUNNING head, and with an empty queue the
|
|
2530
|
+
// sleeper always became the head immediately. So the `onPreempt`
|
|
2531
|
+
// hook meant to cancel it could never fire, and an operator's
|
|
2532
|
+
// `stop` or `restart` simply queued behind ten minutes of sleep:
|
|
2533
|
+
// the CLI's RPC timed out after 60s while the op stayed queued,
|
|
2534
|
+
// to fire whenever the cool-down finally elapsed. An app that
|
|
2535
|
+
// exhausted its restart budget was unreachable by every command
|
|
2536
|
+
// the CLI offers — `restart` hung, `stop` hung, `stack start`
|
|
2537
|
+
// skipped it — and the only recovery was restarting the daemon.
|
|
2538
|
+
const COOL_DOWN_MS = 10 * 60_000;
|
|
2539
|
+
this.clearCoolDown(entry.name);
|
|
2540
|
+
const timer = setTimeout(() => {
|
|
2541
|
+
this.coolDownTimers.delete(entry.name);
|
|
2542
|
+
// Re-read the handle: every start replaces it (see
|
|
2543
|
+
// `startAppInternal`), so the one captured above is stale the
|
|
2544
|
+
// moment anything restarts this app. Only a still-crashed app
|
|
2545
|
+
// wants this attempt — an operator who started it meanwhile
|
|
2546
|
+
// has already settled the question.
|
|
2547
|
+
const current = this.handles.get(entry.name);
|
|
2548
|
+
if (!current || current.status !== 'crashed')
|
|
2549
|
+
return;
|
|
2550
|
+
this.logger.info({ app: entry.name }, 'Cool-down elapsed — attempting fresh recovery');
|
|
2551
|
+
current.restarts = 0; // fresh attempt window
|
|
2552
|
+
current.status = 'stopped';
|
|
2553
|
+
void this.lifecycleQueue
|
|
2554
|
+
.enqueue(entry.name, 'crash-restart', () => this.startAppInternal(entry, config))
|
|
2555
|
+
.catch((err) => {
|
|
2556
|
+
if (err instanceof LifecyclePreempted)
|
|
2557
|
+
return;
|
|
2558
|
+
this.logger.error({ app: entry.name, err: err.message }, 'Cool-down recovery failed');
|
|
2559
|
+
});
|
|
2560
|
+
}, COOL_DOWN_MS);
|
|
2561
|
+
timer.unref?.();
|
|
2562
|
+
this.coolDownTimers.set(entry.name, timer);
|
|
1513
2563
|
return;
|
|
1514
2564
|
}
|
|
1515
2565
|
const backoff = policy.backoff ?? config.supervision.backoff;
|
|
1516
2566
|
const delay = this.calculateBackoff(handle.restarts, backoff);
|
|
1517
2567
|
this.logger.info({ app: entry.name, delay, attempt: handle.restarts + 1 }, 'Scheduling restart');
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
2568
|
+
// P0-I — schedule the crash-restart through the lifecycle queue
|
|
2569
|
+
// so an explicit stop preempts the pending restart before it
|
|
2570
|
+
// fires. The setTimeout is still the timing source, but the
|
|
2571
|
+
// queue.enqueue with `onPreempt` guarantees the timer is cleared
|
|
2572
|
+
// (and the promise rejected with LifecyclePreempted) the moment a
|
|
2573
|
+
// higher-priority op for the same app arrives. Pre-fix the timer
|
|
2574
|
+
// could fire during stopApp's `await stopChild(...)` window and
|
|
2575
|
+
// spawn a zombie under the same handle.
|
|
2576
|
+
let timer = null;
|
|
2577
|
+
const restartPromise = this.lifecycleQueue.enqueue(entry.name, 'crash-restart', () => new Promise((resolve, reject) => {
|
|
2578
|
+
timer = setTimeout(async () => {
|
|
2579
|
+
timer = null;
|
|
2580
|
+
// Guard: don't restart if orchestrator was stopped during backoff
|
|
2581
|
+
if (handle.status === 'stopping' || handle.status === 'stopped') {
|
|
2582
|
+
resolve();
|
|
2583
|
+
return;
|
|
2584
|
+
}
|
|
2585
|
+
try {
|
|
2586
|
+
handle.status = 'stopped';
|
|
2587
|
+
// `startAppInternal`, not `startApp`.
|
|
2588
|
+
//
|
|
2589
|
+
// This task holds the lifecycle-queue slot for `entry.name`.
|
|
2590
|
+
// `startApp` enqueues 'start' on that same key, and the queue
|
|
2591
|
+
// cannot run it until this task returns — so the await never
|
|
2592
|
+
// settles and the restart silently never happens. Measured:
|
|
2593
|
+
// `startApp:enter` with no matching exit, and the app left in
|
|
2594
|
+
// whatever state the crash put it in.
|
|
2595
|
+
//
|
|
2596
|
+
// This is why the whole policy below — bounded attempts,
|
|
2597
|
+
// exponential backoff, the ten-minute cool-down described in its
|
|
2598
|
+
// own comment as "bounded self-healing" — had never restarted
|
|
2599
|
+
// anything. The queue was introduced (P0-I) to stop a restart
|
|
2600
|
+
// timer racing a stop; it also made the restart unreachable, and
|
|
2601
|
+
// nothing failed loudly enough to say so.
|
|
2602
|
+
await this.startAppInternal(entry, config);
|
|
2603
|
+
resolve();
|
|
2604
|
+
}
|
|
2605
|
+
catch (err) {
|
|
2606
|
+
const message = err.message;
|
|
2607
|
+
this.logger.error({ app: entry.name, error: message }, 'Restart failed');
|
|
2608
|
+
// Said separately and after, because reading the process table
|
|
2609
|
+
// takes a moment and the failure itself must not wait for it.
|
|
2610
|
+
void this.explainPortConflict(message).then((held) => {
|
|
2611
|
+
if (held)
|
|
2612
|
+
this.logger.error({ app: entry.name }, `Restart failed — ${held}`);
|
|
2613
|
+
});
|
|
2614
|
+
reject(err);
|
|
2615
|
+
}
|
|
2616
|
+
}, delay);
|
|
2617
|
+
}), () => {
|
|
2618
|
+
// onPreempt — explicit stop / restart arrived. Cancel the
|
|
2619
|
+
// pending timer so the restart never fires.
|
|
2620
|
+
if (timer) {
|
|
2621
|
+
clearTimeout(timer);
|
|
2622
|
+
timer = null;
|
|
1528
2623
|
}
|
|
1529
|
-
}
|
|
1530
|
-
//
|
|
2624
|
+
});
|
|
2625
|
+
// Swallow LifecyclePreempted — that's the explicit-stop happy
|
|
2626
|
+
// path, not an error. Anything else gets logged.
|
|
2627
|
+
restartPromise.catch((err) => {
|
|
2628
|
+
if (err instanceof LifecyclePreempted)
|
|
2629
|
+
return;
|
|
2630
|
+
this.logger.warn({ app: entry.name, err: err.message }, 'Crash restart task rejected');
|
|
2631
|
+
});
|
|
2632
|
+
// Legacy field — callers still poke handle.crashRestartTimer to
|
|
2633
|
+
// check "is a restart pending"; keep it in sync with the queued
|
|
2634
|
+
// timer so existing inspect / status code paths don't break.
|
|
1531
2635
|
handle.crashRestartTimer = timer;
|
|
1532
2636
|
}
|
|
1533
2637
|
calculateBackoff(attempt, backoff) {
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
2638
|
+
// Delegate to the shared backoff library in titan-pm. Pre-fix the
|
|
2639
|
+
// inline impl here defaulted `initial=1000` while supervisor's
|
|
2640
|
+
// copy defaulted to 300 — same config produced 3.3× longer
|
|
2641
|
+
// delays in classic-mode than bootstrap-mode. Now both routes
|
|
2642
|
+
// run the same math.
|
|
2643
|
+
return computeBackoff(attempt + 1, backoff);
|
|
2644
|
+
}
|
|
2645
|
+
/**
|
|
2646
|
+
* The live OS processes behind one topology entry.
|
|
2647
|
+
*
|
|
2648
|
+
* Single-instance entries run as supervisor children; entries with
|
|
2649
|
+
* `instances > 1` run in a pool the supervisor knows nothing about. Both
|
|
2650
|
+
* shapes are resolved here, in one place, because metrics sampling and
|
|
2651
|
+
* status reporting each used to resolve them separately — and disagreed.
|
|
2652
|
+
* The pool branch asserted `online` from `pool.size > 0` before looking at
|
|
2653
|
+
* a pid at all, so a worker whose process had exited still read as online
|
|
2654
|
+
* (observed live: two pool workers reporting `online` on pids that `kill
|
|
2655
|
+
* -0` said were gone).
|
|
2656
|
+
*/
|
|
2657
|
+
resolveTopologyPids(handle, topo) {
|
|
2658
|
+
return this.resolveTopologyMembers(handle, topo).members.map((m) => m.pid);
|
|
2659
|
+
}
|
|
2660
|
+
/**
|
|
2661
|
+
* The live processes behind one topology entry, keyed by the PM process id
|
|
2662
|
+
* as well as the OS pid.
|
|
2663
|
+
*
|
|
2664
|
+
* The pid alone answers "is it alive"; the process id is what any question
|
|
2665
|
+
* ABOUT that process has to be asked with — `pm.getProcess(id)` carries the
|
|
2666
|
+
* spawn time, and the supervisor's restart counter is keyed by child name.
|
|
2667
|
+
* `resolveTopologyPids` used to discard both on the way out, which is why
|
|
2668
|
+
* the `omnitron list` DTO had nothing to report but the parent's numbers.
|
|
2669
|
+
*/
|
|
2670
|
+
resolveTopologyMembers(handle, topo) {
|
|
2671
|
+
const isPoolEntry = (topo.instances ?? 1) > 1;
|
|
2672
|
+
if (isPoolEntry) {
|
|
2673
|
+
const pool = handle.topologyPools.get(topo.name);
|
|
2674
|
+
if (!pool || pool.size === 0)
|
|
2675
|
+
return { members: [], childName: null };
|
|
2676
|
+
const members = pool
|
|
2677
|
+
.getWorkerIds()
|
|
2678
|
+
.map((processId) => ({ processId, pid: this.pm.getWorkerHandle(processId)?.pid }))
|
|
2679
|
+
.filter((m) => typeof m.pid === 'number');
|
|
2680
|
+
return { members, childName: null };
|
|
2681
|
+
}
|
|
2682
|
+
const childNames = handle.supervisor?.getChildNames() ?? [];
|
|
2683
|
+
const childName = childNames.find((cn) => cn.includes(topo.name)) ?? topo.name;
|
|
2684
|
+
const processId = handle.supervisor?.getChildProcessId(childName);
|
|
2685
|
+
if (!processId)
|
|
2686
|
+
return { members: [], childName };
|
|
2687
|
+
const pid = this.pm.getWorkerHandle(processId)?.pid;
|
|
2688
|
+
return { members: typeof pid === 'number' ? [{ processId, pid }] : [], childName };
|
|
2689
|
+
}
|
|
2690
|
+
/**
|
|
2691
|
+
* Sample every process an app owns, and record the per-entry readings on
|
|
2692
|
+
* the handle on the way through.
|
|
2693
|
+
*
|
|
2694
|
+
* A bootstrap app is several OS processes, so "the memory this app uses"
|
|
2695
|
+
* is their sum. The periodic poller used to sample the single pid stored
|
|
2696
|
+
* on the handle — which for a multi-process app is whichever child was
|
|
2697
|
+
* registered last — and overwrite `lastMetrics` with it every five
|
|
2698
|
+
* seconds, while `getMetrics()` right beside it aggregated correctly and
|
|
2699
|
+
* wrote to the same field. The poller won on frequency, so the aggregate
|
|
2700
|
+
* was never what anyone saw: `acme/dev/main` reported 122 MB against 437 MB
|
|
2701
|
+
* actually resident across its three processes.
|
|
2702
|
+
*
|
|
2703
|
+
* ## Why `ps` and not the child's own report
|
|
2704
|
+
*
|
|
2705
|
+
* The aggregating path preferred the PM-level metrics a child reports over
|
|
2706
|
+
* its RPC channel, falling back to `ps`. Those two are not the same
|
|
2707
|
+
* quantities:
|
|
2708
|
+
*
|
|
2709
|
+
* - `cpu` from the child is `(cpuUsage().user + .system) / 1e6` — CPU
|
|
2710
|
+
* SECONDS CONSUMED SINCE START, a monotonically growing counter. `ps`
|
|
2711
|
+
* `%cpu` is a percentage. The dashboard column is labelled `CPU %`,
|
|
2712
|
+
* so an app running for a day would have read `86400`. Observed
|
|
2713
|
+
* climbing live: 12.845 to 13.297 across twelve seconds of idling.
|
|
2714
|
+
* - `memory` from the child is `memoryUsage().heapUsed` — the V8 heap.
|
|
2715
|
+
* `ps` reports RSS, which is what "memory this app uses" means to
|
|
2716
|
+
* whoever is sizing a host. Observed: 66 MB heap against 149 MB
|
|
2717
|
+
* resident for the same process.
|
|
2718
|
+
*
|
|
2719
|
+
* Mixing them per-child, depending on which source answered first, gave a
|
|
2720
|
+
* column that was neither quantity. `ps` is asked for both figures here so
|
|
2721
|
+
* that every number under one label means one thing.
|
|
2722
|
+
*/
|
|
2723
|
+
async sampleAppMetrics(handle) {
|
|
2724
|
+
const topology = handle.topologyProcesses ?? [];
|
|
2725
|
+
const childNames = handle.supervisor?.getChildNames() ?? [];
|
|
2726
|
+
// Which pids belong to which topology entry. Everything is sampled in a
|
|
2727
|
+
// single `ps` call: sampling one child at a time would fork a process per
|
|
2728
|
+
// child per polling interval, and a monitor that costs more than what it
|
|
2729
|
+
// monitors is its own kind of defect.
|
|
2730
|
+
const pidsByEntry = new Map();
|
|
2731
|
+
for (const topo of topology) {
|
|
2732
|
+
pidsByEntry.set(topo.name, this.resolveTopologyPids(handle, topo));
|
|
2733
|
+
}
|
|
2734
|
+
if (topology.length === 0) {
|
|
2735
|
+
// No declared topology: fall back to the supervisor's children, and
|
|
2736
|
+
// failing that to the handle's own pid.
|
|
2737
|
+
for (const childName of childNames) {
|
|
2738
|
+
const processId = handle.supervisor?.getChildProcessId(childName);
|
|
2739
|
+
const pid = processId ? this.pm.getWorkerHandle(processId)?.pid : undefined;
|
|
2740
|
+
if (typeof pid === 'number')
|
|
2741
|
+
pidsByEntry.set(childName, [pid]);
|
|
2742
|
+
}
|
|
2743
|
+
if (pidsByEntry.size === 0 && handle.pid) {
|
|
2744
|
+
pidsByEntry.set(handle.name, [handle.pid]);
|
|
2745
|
+
}
|
|
2746
|
+
}
|
|
2747
|
+
const batch = await this.sampleProcessMetricsBatch([...pidsByEntry.values()].flat());
|
|
2748
|
+
let cpu = 0;
|
|
2749
|
+
let memory = 0;
|
|
2750
|
+
const sampled = new Map();
|
|
2751
|
+
for (const [name, pids] of pidsByEntry) {
|
|
2752
|
+
const sample = { cpu: 0, memory: 0 };
|
|
2753
|
+
for (const pid of pids) {
|
|
2754
|
+
const os = batch.get(pid);
|
|
2755
|
+
if (!os)
|
|
2756
|
+
continue;
|
|
2757
|
+
sample.cpu += os.cpu;
|
|
2758
|
+
sample.memory += os.memory;
|
|
2759
|
+
}
|
|
2760
|
+
sampled.set(name, sample);
|
|
2761
|
+
cpu += sample.cpu;
|
|
2762
|
+
memory += sample.memory;
|
|
2763
|
+
}
|
|
2764
|
+
// Traffic can only come from inside the child; `ps` knows nothing about
|
|
2765
|
+
// it. Its CPU and memory figures are deliberately NOT used — see the note
|
|
2766
|
+
// on this method. Only what a child reports as TRAFFIC is summed: the
|
|
2767
|
+
// `requests`/`errors` a child on an older runtime reports are the call
|
|
2768
|
+
// counters of its wrapper — the supervisor's own health and metrics
|
|
2769
|
+
// calls — and adding them up counted this daemon's questions as the
|
|
2770
|
+
// app's requests (`combineProcessTraffic`).
|
|
2771
|
+
const reports = [];
|
|
2772
|
+
for (const childName of childNames) {
|
|
2773
|
+
const reported = await handle.supervisor?.getChildMetrics(childName);
|
|
2774
|
+
reports.push(reported?.traffic);
|
|
2775
|
+
}
|
|
2776
|
+
const traffic = combineProcessTraffic(reports);
|
|
2777
|
+
// A sample that produced nothing for an app that HAS live processes is a
|
|
2778
|
+
// failed sample, not a measurement of zero — `ps` can time out under load,
|
|
2779
|
+
// and it did: an app was observed reporting 0 MB one second and 143 MB the
|
|
2780
|
+
// next. Overwriting the last known reading with zeros would put the exact
|
|
2781
|
+
// placeholder this method exists to remove back on the dashboard, so the
|
|
2782
|
+
// previous reading is kept instead.
|
|
2783
|
+
const sampledNothing = batch.size === 0 && [...pidsByEntry.values()].some((pids) => pids.length > 0);
|
|
2784
|
+
if (sampledNothing && handle.lastMetrics) {
|
|
2785
|
+
return handle.lastMetrics;
|
|
2786
|
+
}
|
|
2787
|
+
handle.childMetrics = sampled;
|
|
2788
|
+
return traffic
|
|
2789
|
+
? {
|
|
2790
|
+
cpu,
|
|
2791
|
+
memory,
|
|
2792
|
+
requests: traffic.requests,
|
|
2793
|
+
errors: traffic.serverErrors,
|
|
2794
|
+
traffic,
|
|
2795
|
+
...(traffic.latency
|
|
2796
|
+
? {
|
|
2797
|
+
latency: {
|
|
2798
|
+
p50: traffic.latency.p50,
|
|
2799
|
+
p75: traffic.latency.p75,
|
|
2800
|
+
p90: traffic.latency.p90,
|
|
2801
|
+
p95: traffic.latency.p95,
|
|
2802
|
+
p99: traffic.latency.p99,
|
|
2803
|
+
mean: traffic.latency.mean,
|
|
2804
|
+
},
|
|
2805
|
+
}
|
|
2806
|
+
: {}),
|
|
2807
|
+
}
|
|
2808
|
+
: { cpu, memory };
|
|
2809
|
+
}
|
|
2810
|
+
/**
|
|
2811
|
+
* Sample several processes in one `ps` call.
|
|
2812
|
+
*
|
|
2813
|
+
* Pids the OS no longer knows are simply absent from the result — an
|
|
2814
|
+
* omission the caller can see, unlike a zeroed reading it cannot tell
|
|
2815
|
+
* apart from an idle process.
|
|
2816
|
+
*/
|
|
2817
|
+
async sampleProcessMetricsBatch(pids) {
|
|
2818
|
+
const out = new Map();
|
|
2819
|
+
const unique = [...new Set(pids)];
|
|
2820
|
+
if (unique.length === 0)
|
|
2821
|
+
return out;
|
|
2822
|
+
try {
|
|
2823
|
+
const psOutput = await new Promise((resolve, reject) => {
|
|
2824
|
+
execFile('ps', ['-p', unique.join(','), '-o', 'pid=,rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
|
|
2825
|
+
// `ps` exits non-zero when NONE of the pids exist, but still
|
|
2826
|
+
// prints the ones that do — so stdout is worth reading either way.
|
|
2827
|
+
if (err && !stdout)
|
|
2828
|
+
reject(err);
|
|
2829
|
+
else
|
|
2830
|
+
resolve(stdout.trim());
|
|
2831
|
+
});
|
|
2832
|
+
});
|
|
2833
|
+
for (const [pid, sample] of parsePsBatch(psOutput))
|
|
2834
|
+
out.set(pid, sample);
|
|
2835
|
+
}
|
|
2836
|
+
catch {
|
|
2837
|
+
// Leave the map empty: no reading is honest, a zero reading is not.
|
|
2838
|
+
}
|
|
2839
|
+
return out;
|
|
2840
|
+
}
|
|
2841
|
+
/**
|
|
2842
|
+
* Sample CPU/memory for a single OS process via `ps`.
|
|
2843
|
+
*
|
|
2844
|
+
* `null` when it could not be read. This used to return
|
|
2845
|
+
* `{ cpu: 0, memory: 0 }` on any failure, and the caller wrote that
|
|
2846
|
+
* straight over `handle.lastMetrics` — so a `ps` that timed out under load
|
|
2847
|
+
* made a running classic-mode app read as idle, indistinguishable from one
|
|
2848
|
+
* that genuinely was. The batch path above already treated absence as
|
|
2849
|
+
* absence; these two now agree.
|
|
2850
|
+
*/
|
|
1544
2851
|
async sampleProcessMetrics(pid) {
|
|
1545
2852
|
try {
|
|
1546
|
-
const { execFile } = await import('node:child_process');
|
|
1547
2853
|
const psOutput = await new Promise((resolve, reject) => {
|
|
1548
|
-
execFile('ps', ['-p', String(pid), '-o', 'rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
|
|
1549
|
-
if (err)
|
|
2854
|
+
execFile('ps', ['-p', String(pid), '-o', 'pid=,rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
|
|
2855
|
+
if (err && !stdout)
|
|
1550
2856
|
reject(err);
|
|
1551
2857
|
else
|
|
1552
2858
|
resolve(stdout.trim());
|
|
1553
2859
|
});
|
|
1554
2860
|
});
|
|
1555
|
-
|
|
1556
|
-
return {
|
|
1557
|
-
cpu: parseFloat(cpuPercent ?? '0'),
|
|
1558
|
-
memory: parseInt(rssKb ?? '0', 10) * 1024,
|
|
1559
|
-
};
|
|
2861
|
+
return parsePsLine(psOutput)?.sample ?? null;
|
|
1560
2862
|
}
|
|
1561
2863
|
catch {
|
|
1562
|
-
return
|
|
2864
|
+
return null;
|
|
1563
2865
|
}
|
|
1564
2866
|
}
|
|
1565
|
-
/**
|
|
2867
|
+
/**
|
|
2868
|
+
* Periodic CPU/memory sampling for every managed process.
|
|
2869
|
+
*
|
|
2870
|
+
* Idempotent: re-arming replaces the previous timer rather than stacking a
|
|
2871
|
+
* second one, so `setConfig` can call it on every config reload.
|
|
2872
|
+
*/
|
|
1566
2873
|
startMetricsPolling(interval) {
|
|
2874
|
+
if (this.metricsTimer) {
|
|
2875
|
+
clearInterval(this.metricsTimer);
|
|
2876
|
+
this.metricsTimer = null;
|
|
2877
|
+
}
|
|
1567
2878
|
this.metricsTimer = setInterval(async () => {
|
|
1568
2879
|
for (const handle of this.handles.values()) {
|
|
1569
2880
|
if (handle.status !== 'online')
|
|
1570
2881
|
continue;
|
|
1571
2882
|
if (handle.mode === 'classic' && handle.childProcess?.pid) {
|
|
1572
2883
|
const m = await this.sampleProcessMetrics(handle.childProcess.pid);
|
|
1573
|
-
|
|
2884
|
+
// A failed read keeps the previous numbers rather than replacing
|
|
2885
|
+
// them with zeros — the same rule the bootstrap path below uses.
|
|
2886
|
+
if (m) {
|
|
2887
|
+
handle.lastMetrics = {
|
|
2888
|
+
...m,
|
|
2889
|
+
requests: handle.lastMetrics?.requests ?? 0,
|
|
2890
|
+
errors: handle.lastMetrics?.errors ?? 0,
|
|
2891
|
+
};
|
|
2892
|
+
}
|
|
1574
2893
|
}
|
|
1575
|
-
else if (handle.mode === 'bootstrap'
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
2894
|
+
else if (handle.mode === 'bootstrap') {
|
|
2895
|
+
// Every process the app owns, not just the one on the handle.
|
|
2896
|
+
// As sampled. The counts used to fall back to the previous reading
|
|
2897
|
+
// whenever a sample said 0 — which kept a stale figure alive
|
|
2898
|
+
// across a process restart (traffic counts restart from zero with
|
|
2899
|
+
// the process) and, while `requests` was the wrapper's call count,
|
|
2900
|
+
// mixed the two meanings in one field.
|
|
2901
|
+
handle.lastMetrics = await this.sampleAppMetrics(handle);
|
|
1582
2902
|
}
|
|
1583
2903
|
}
|
|
1584
2904
|
}, interval);
|
|
2905
|
+
this.metricsTimer.unref();
|
|
1585
2906
|
}
|
|
1586
2907
|
// ============================================================================
|
|
1587
2908
|
// Private — Utilities
|
|
1588
2909
|
// ============================================================================
|
|
1589
2910
|
toProcessInfo(handle) {
|
|
2911
|
+
// Ghost-online guard. The in-memory `handle.status` reflects the
|
|
2912
|
+
// last event the supervisor observed. If a child died between
|
|
2913
|
+
// the last sweep and now (kill -9 by an operator, OOM kill,
|
|
2914
|
+
// container restart) the handle still claims 'online' until the
|
|
2915
|
+
// next event lands. A cheap signal-0 verification at DTO-mapping
|
|
2916
|
+
// time keeps `omnitron list` honest: when the OS reports the pid
|
|
2917
|
+
// is gone, surface 'crashed' instead of lying to the operator.
|
|
2918
|
+
// `isAlive` covers EPERM as alive (correct — process exists, we
|
|
2919
|
+
// just can't signal it).
|
|
2920
|
+
let status = handle.status;
|
|
2921
|
+
if (status === 'online' && handle.pid != null && !isAlive(handle.pid)) {
|
|
2922
|
+
status = 'crashed';
|
|
2923
|
+
}
|
|
1590
2924
|
const info = {
|
|
1591
2925
|
name: handle.name,
|
|
1592
2926
|
pid: handle.pid,
|
|
1593
|
-
status
|
|
2927
|
+
status,
|
|
1594
2928
|
cpu: handle.cpu,
|
|
1595
2929
|
memory: handle.memory,
|
|
1596
2930
|
uptime: handle.uptime,
|
|
@@ -1605,49 +2939,48 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1605
2939
|
info.processes = [];
|
|
1606
2940
|
for (const topo of handle.topologyProcesses) {
|
|
1607
2941
|
const isPoolEntry = (topo.instances ?? 1) > 1;
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
childPid = wh?.pid ?? null;
|
|
1637
|
-
childStatus = childPid ? 'online' : 'stopped';
|
|
1638
|
-
}
|
|
1639
|
-
}
|
|
2942
|
+
// The same ghost-online guard the parent gets a few lines above.
|
|
2943
|
+
// Without it the pool branch reported `online` from the pool's own
|
|
2944
|
+
// bookkeeping, never checking whether the pid it printed alongside
|
|
2945
|
+
// still existed.
|
|
2946
|
+
const { members, childName } = this.resolveTopologyMembers(handle, topo);
|
|
2947
|
+
const alive = members.filter((m) => isAlive(m.pid));
|
|
2948
|
+
const declaredInstances = topo.instances ?? 1;
|
|
2949
|
+
const liveInstances = isPoolEntry
|
|
2950
|
+
? (handle.topologyPools.get(topo.name)?.size ?? 0)
|
|
2951
|
+
: alive.length;
|
|
2952
|
+
const chosen = alive[0] ?? members[0];
|
|
2953
|
+
const childPid = chosen?.pid ?? null;
|
|
2954
|
+
const childStatus = alive.length > 0 ? 'online' : members.length > 0 ? 'crashed' : 'stopped';
|
|
2955
|
+
// The row prints one pid, so the row's clock is that pid's clock.
|
|
2956
|
+
// `startTime` is written when the process id is registered and a
|
|
2957
|
+
// restart registers a new one, so it moves when the process does.
|
|
2958
|
+
// (`IProcessInfo.restartCount` does NOT: it is assigned 0 there and
|
|
2959
|
+
// incremented nowhere. The supervisor keeps the real count, by child
|
|
2960
|
+
// name — reading the field instead would have replaced a wrong
|
|
2961
|
+
// constant with a wrong lookup.)
|
|
2962
|
+
const chosenInfo = chosen ? this.pm.getProcess(chosen.processId) : undefined;
|
|
2963
|
+
const childUptime = typeof chosenInfo?.startTime === 'number' && chosenInfo.startTime > 0
|
|
2964
|
+
? Math.max(0, Date.now() - chosenInfo.startTime)
|
|
2965
|
+
: 0;
|
|
2966
|
+
// A pool is not a supervisor child and nothing counts its restarts.
|
|
2967
|
+
// `null` says so; `0` would be absence written as a value from the
|
|
2968
|
+
// domain, leaving "never restarted" and "not tracked" identical.
|
|
2969
|
+
const childRestarts = isPoolEntry || !childName ? null : (handle.supervisor?.getRestartCount(childName) ?? null);
|
|
1640
2970
|
// Derive process type from declarations for backward-compatible DTO
|
|
1641
2971
|
const derivedType = topo.transports ? 'server' : isPoolEntry ? 'worker' : 'custom';
|
|
2972
|
+
const sample = handle.childMetrics.get(topo.name);
|
|
1642
2973
|
info.processes.push({
|
|
1643
2974
|
name: topo.name,
|
|
1644
2975
|
type: derivedType,
|
|
1645
2976
|
pid: childPid,
|
|
1646
2977
|
status: childStatus,
|
|
1647
|
-
cpu: 0,
|
|
1648
|
-
memory: 0,
|
|
1649
|
-
uptime:
|
|
1650
|
-
restarts:
|
|
2978
|
+
cpu: sample?.cpu ?? 0,
|
|
2979
|
+
memory: sample?.memory ?? 0,
|
|
2980
|
+
uptime: childUptime,
|
|
2981
|
+
restarts: childRestarts,
|
|
2982
|
+
instances: liveInstances,
|
|
2983
|
+
declaredInstances,
|
|
1651
2984
|
});
|
|
1652
2985
|
}
|
|
1653
2986
|
}
|